python test ioc extractor

# Steps
# 1.) Drop IoCs [IP address or urls only currentl] from things below into "iocs.txt"
# - Emotet ( https://twitter.com/Cryptolaemus1 )
# - UrlHaus ( https://urlhaus.abuse.ch/downloads/csv_online/ )
# - Phish Tank ( https://www.phishtank.com/phish_search.php?page=1&active=y&valid=y&Search=Search )
# - Trickbot ( https://twitter.com/search?q=%23trickbot&src=typed_query&f=live )
# 2.) run script
import sys
import os
import re
from urllib.parse import urlparse
debug = 0
filepath = 'iocs.txt'
ipfilepath = 'ips.txt'
dnsfilepath = 'dns.txt'
ioccontainswhitelist = [
".sendgrid.net"
]
iocwhitelist = [
"google.com",
"www.google.com",
"urlhaus.abuse.ch",
"pastebin.com",
"ak.imgfarm.com",
"docs.google.com",
"drive.google.com",
"i.imgur.com",
"img.sobot.com",
"imgur.com",
"www.imgur.com",
"raw.githubusercontent.com",
"github.com",
"www.github.com",
"adobe.com",
"www.adobe.com",
"ibm.com",
"www.ibm.com",
"dell.com",
"www.dell.com",
"bing.com",
"www.bing.com",
"msn.com",
"www.msn.com",
"documentcloud.adobe.com",
"cisco.com",
"www.cisco.com",
"oshkosh.webex.com",
"l.yimg.com",
"yimg.com",
"dl.dropboxusercontent.com",
"dropbox.com",
"www.dropbox.com",
"godaddy.com",
"godaddysites.com",
"files.constantcontact.com",
"ipinfo.io",
"bit.ly",
"onedrive.live.com",
"000webhostapp.com",
"storage.googleapis.com",
"wikileaks.org",
"forms.gle",
"go2l.ink",
"capesandbox.com",
"twitter.com",
"paste.cryptolaemus.com",
"cryptolaemus.com",
"gist.githubusercontent.com",
"bitbucket.org",
"img1.wsimg.com",
"cdn.discordapp.com",
"web.mit.edu",
"bit.do",
"na3.docusign.net",
"sway.office.com",
"sites.google.com",
"aka.ms",
"login.microsoftonline.com",
"track.smtpsendmail.com",
"r20.rs6.net",
"files.gamebanana.com",
"sems.sas.com"
]
with open(filepath) as fp:
 rawLine = fp.readline().rstrip()
 ipResults = []
 dnsResults = []
 while rawLine:
  if debug:
   print("DEBUG: reviewing %s" % rawLine)
  found = 0
  ioc = None
  isIp = 0
  isDns = 0
  ipRegex = re.findall( r'(\d+\.\d+\.\d+\.\d+)', rawLine )
  if ipRegex:
   ioc = ipRegex[0]
   isIp = 1
   if debug:
    print("DEBUG: IP address %s" % ioc)
  if not ioc:
   urlRegex = re.search('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\), ]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', rawLine) 
   if urlRegex:
    parts = urlRegex.group().split('/')
    if parts and len(parts) > 2:
     ioc = urlRegex[0].split('/')[2]
     isDns = 1
     if debug:
      print("DEBUG: url with http %s" % ioc)
   if not ioc:
    if debug:
     print("DEBUG: checking url 2nd time")
    urlRegexV2 = re.search('^([a-zA-Z.]+\.([a-zA-Z]){2,7}\/.*)$', rawLine) 
    if urlRegexV2:
     parts = urlRegexV2.group().split('/')
     if parts and len(parts) > 0:
      ioc = parts[0]
      isDns = 1
      if debug:
       print("DEBUG: url without http %s" % ioc)
    else:
     if debug:
      print("DEBUG: checking url 3rd time")
     urlTest = re.search('^([a-zA-Z.]+)\.([a-zA-Z]){2,7}$', rawLine) 
     if urlTest:
      ioc = rawLine
      isDns = 1
      if debug:
       print("DEBUG: url domain only %s" % ioc)
  if ioc:
   ioc = ioc.lower()
   for ignoredIoc in iocwhitelist:
    if ioc == ignoredIoc.lower():
     found = 1
     if debug:
      print("DEBUG: ignored %s" % ioc)
     break
   for ignoredIoc in ioccontainswhitelist:
    if ignoredIoc.lower() in ioc:
     found = 1
     if debug:
      print("DEBUG: ignored %s" % ioc)
     break
   if found == 0:
    if "..." not in ioc:
     if isIp:
      ipResults.append(ioc)
     if isDns:
      dnsResults.append(ioc)
  rawLine = fp.readline()

ipResults = sorted(set(ipResults))
dnsResults = sorted(set(dnsResults))

with open(ipfilepath,"w+") as fp:
 for ioc in ipResults:
  fp.write("%s\n" % ioc)
 if len(ipResults) > 0:
  print("%s created" % ipfilepath)

with open(dnsfilepath,"w+") as fp:
 for ioc in dnsResults:
  fp.write("%s\n" % ioc)
 if len(dnsResults) > 0:
  print("%s created" % dnsfilepath)