Python IoT search with Wget and Yara Rules

# Given a list of urls, determine what type of IoT device (or any device for that matter) they are based on you plugging in Yara rules into the .yar files

import os
import subprocess
import traceback

debug = "false"
skipDownload = "false"
input = "urls.txt"
yaraSpecificRuleFile = "IoTSpecific.yar"
yaraGenericRuleFile = "IoTGeneric.yar"
wgetParams = "--quiet --no-check-certificate --timeout=2 --tries=3"
yaraParams = ""
outputExt = ".html"
urls = open(input, "r")

if skipDownload == "false":
 for url in urls:
  url = url.rstrip()
  cleanurl = url.rstrip().replace('/','_').replace('\\','_').replace(':','_').replace('.','_').replace('&','_').replace('?','_').replace('=','_').replace('%','_') + outputExt
  wgetCommand = ("wget %s --output-document=%s %s 2>/dev/null" % (wgetParams, cleanurl, url))
  if(debug == "true"):
   print(("DEBUG,Starting Download of '%s' to '%s'" % (url, cleanurl)))
  try:
   output = subprocess.check_output(wgetCommand, shell=True)
   if "error" in output:
    print(("ERROR,Unable to download '%s' error '%s'" % (wgetCommand, output)))
  except Exception:
   print(("ERROR,Unable to download '%s' error '%s'" % (wgetCommand,traceback.print_exc())))
  if(debug == "true"):
   print(("DEBUG,Finished Download of '%s' to '%s'" % (url, cleanurl)))

for htmlfile in os.listdir('.'):
 if htmlfile.endswith(".html"):
  if os.stat(htmlfile).st_size == 0:
   print(("NoResponse,%s" % (htmlfile)))
  else:
   yaraCommand = ("yara %s %s %s" % (yaraParams, yaraSpecificRuleFile, htmlfile))
   if(debug == "true"):
    print(("DEBUG,Starting Scanning: '%s'" % (htmlfile)))
   try:
    output = subprocess.check_output(yaraCommand, shell=True)
    if "error" in output:
     print(("ERROR,Unable to Scan '%s' error '%s'" % (yaraCommand, output)))
    if output:
     output = output.rstrip()
     print(output.replace(" ",","))
    else:
     yaraRescanCommand = ("yara %s %s %s" % (yaraParams, yaraGenericRuleFile, htmlfile))
     if(debug == "true"):
      print(("DEBUG,Starting ReScanning: '%s'" % (htmlfile)))
     try:
      output = subprocess.check_output(yaraRescanCommand, shell=True)
      if "error" in output:
       print(("ERROR,Unable to ReScan '%s' error '%s'" % (yaraCommand, output)))
      if output:
       output = output.rstrip()
       print(output.replace(" ",","))
      else:
       print(("NoMatch,%s" % (htmlfile)))
     except Exception:
      print(("ERROR,Unable to ReScan '%s' error '%s'" % (yaraCommand,traceback.print_exc())))
     if(debug == "true"):
      print(("DEBUG,Finished ReScanning: '%s'" % (htmlfile)))
   except Exception:
    print(("ERROR,Unable to Scan '%s' error '%s'" % (yaraCommand,traceback.print_exc())))
   if(debug == "true"):
    print(("DEBUG,Finished Scanning: '%s'" % (htmlfile)))

urls.close()