Script Query UrlHaus , OpenPhish, PhishTank and Extract Dns, IPs for Threat Intel Feed

code to pull dns & ips from urlhaus, openphish, phishtank, etc.


#usage: iex (get-content .\GetData.ps1 | out-string) > output.txt
$debug = $true
$fileOutput = "dns.csv"
$fileIpOutput = "ip.csv"
$ignoreList = @("google.com", "www.google.com", "urlhaus.abuse.ch", "pastebin.com", "ak.imgfarm.com", "docs.google.com", "drive.google.com", "i.imgur.com", "img.sobot.com", "imgur.com", "www.imgur.com", "raw.githubusercontent.com", "github.com", "www.github.com", "adobe.com", "www.adobe.com", "ibm.com", "www.ibm.com", "dell.com", "www.dell.com", "bing.com", "www.bing.com", "msn.com", "www.msn.com", "documentcloud.adobe.com", "cisco.com", "www.cisco.com",  "l.yimg.com", "yimg.com", "dl.dropboxusercontent.com", "dropbox.com", "www.dropbox.com", "godaddy.com", "godaddysites.com", "files.constantcontact.com", "ipinfo.io", "bit.ly", "onedrive.live.com", "000webhostapp.com", "storage.googleapis.com", "wikileaks.org", "forms.gle", "go2l.ink", "capesandbox.com", "twitter.com", "paste.cryptolaemus.com", "cryptolaemus.com", "gist.githubusercontent.com", "bitbucket.org", "img1.wsimg.com", "cdn.discordapp.com", "web.mit.edu", "bit.do", "na3.docusign.net", "sway.office.com", "sites.google.com", "aka.ms", "login.microsoftonline.com", "track.smtpsendmail.com", "r20.rs6.net", "files.gamebanana.com", "sems.sas.com", "www.avast.com", "1.0.0.0", "bitly.com", "instagram.com", "www.instagram.com", "1.2.0.1073", "2016.3.3.0332", "3.0.0.2013", "31.128.173.853", "4.8.0.904", "cdn.speedof.me", "codeload.github.com", "tr.im", "urlz.fr", "accounts.google.com", "t.co", "fls.doubleclick.net", "1359940.fls.doubleclick.net", "rebrand.ly", "23.4.43.27", "app.smartsheet.com", "forms.office.com", "api.whatsapp.com", "form.jotform.com", "tinyurl.com", "firebasestorage.googleapis.com", "www.google.com.au", "go.pardot.com", "goo.gl", "click.icptrack.com", "online.jimmyjohns.com", "feeds.feedburner.com", "www.google.co.uk", "event.on24.com", "www.powr.io", "protect-us.mimecast.com", "visitor.constantcontact.com", "www.questionpro.com", "click.pstmrk.it", "code.jivosite.com", "apple.co", "www.google.com.mx", "linktr.ee", "www.vcita.com", "www.evernote.com", "www.123formbuilder.com", "tiny.cc", "app.box.com", "script.google.com", "disq.us", "click.email.microsoftemail.com", "fiddle.jshell.net", "cache.nebula.phx3.secureserver.net", "lnkd.in", "www.magazineluiza.com.br", "share.hsforms.com", "fbwat.ch", "app.dialoginsight.com", "cl.s10.exct.net", "etrack05.com", "www.alaskausa.org", "vk.com", "storage.cloud.google.com", "1drv.ms", "www.imcreator.com", "172.217.21.162", "sinacloud.net", "tinyurl.com", "is.gd", "note.youdao.com", "www.surveygizmo.com", "www.tinyurl.com", "surveygizmo.com", "ow.ly", "www.eater.com", "eater.com", "www.stats.gov.cn", "stats.gov.cn", "buff.ly", "www.angelfire.com", "epl.paypal-communication.com", "forms.zohopublic.com", "objectstorage.us-ashburn-1.oraclecloud.com", "t.yesware.com", "snip.ly", "cutt.ly", "mysurveygizmo.com", "www.mysurveygizmo.com", "gitlab.com", "ht.ly", "teamapp.com", "chat.chatra.io", "id.ee.co.uk", "paste.ee","youtube.com","www.youtube.com","play.google.com","google.com.br","docsend.com","www.google.com.br","www.emailmeform.com","emailmeform.com","web.facebook.com","upload.facebook.com","te.bathandbodyworks.com","tatatechnologies.workplace.com","statis.facebook.com","protect-eu.mimecast.com","notion.so","mtouch.facebook.com","messenger.com","j.mp","images2.imgbox.com","graph.facebook.com","fbthirdpartypixel.com","es-la.facebook.com","error.facebook.com","email.secureserver.net","edge-chat.workplace.com","edge-chat.facebook.com","deref-gmx.net","cs.atdmt.com","click.mail.onedrive.com","ca.surveygizmo.com","business.facebook.com","badge.facebook.com","apps.facebook.com","api.facebook.com","an.facebook.com","about.instagram.com","yadi.sk", "157.240.2.20", "www.notion.so","static.facebook.com","www.login-bank.org", "ctt.ec", "www.teamapp.com", "t.umblr.com", "upscri.be", "www.imeipro.info", "imeipro.info", "wisegeek.com", "deref-mail.com", "app.getaccept.com", "cdn2.hubspot.net", "slack-redir.net", "www.wisegeek.com", "chime.com", "www.chime.com", "b.link" , "hyperurl.co", "s3.ap-south-1.amazonaws.com", "podio.com", "s3-us-west-2.amazonaws.com", "tfaforms.com", "www.tfaforms.com", "webservice99.com", "mediafire.com", "www.mediafire.com", "smarturl.it","s3.us-east-1.amazonaws.com","www.restaurantdive.com" ,"rawcdn.githack.com"","https","http","ttp","ttps","lasvegas.craigslist.org","clicktime.symantec.com","survey.survicate.com","t.me","clicktotweet.com", "www.wetransfer.com", "wetransfer.com", "www.geocities.ws", "geocities.ws", "wa.me", "email.godaddy.com", "emailmarketing.locaweb.com.br", "dlvr.it", "www.sendspace.com", "v.ht", "52.109.124.1", "static.wixstatic.com","docs.wixstatic.com","image.prntscr.com,"d1yjjnpx0p53s8.cloudfront.net", "canva.com", "articulo.mercadolibre.com.mx", "e-mudhra.com", "www.canva.com", "listado.mercadolibre.com.mx")
#$urlIntelThem = "https://openphish.com/feed.txt"
#$urlIntelThem = "https://data.phishtank.com/data/online-valid.csv"
#$urlIntelThem = "https://phishstats.info/phish_score.txt"
#$urlIntelThem = "https://urlhaus.abuse.ch/downloads/csv/"
#$urlIntelThem = "https://urlhaus.abuse.ch/downloads/csv_recent/"
$urlIntelThem = "https://urlhaus.abuse.ch/downloads/csv_online/"
$rawHttpThem = ""
$rawIntelThem = ""
$dnsList = ""
$ipList = ""
$first = 0
if($debug){ Write-Output ("Requesting '{0}'" -f $urlIntelThem) }
$httpResponseThem = Invoke-WebRequest -UseDefaultCredentials $urlIntelThem
$rawHttpThem = $httpResponseThem.RawContent
if($debug){ Write-Output ("Downloaded '{0}'" -f $urlIntelThem) }
if($rawHttpThem.IndexOf("abuse.ch") -gt 0){
$rawIntelThem = $rawHttpThem.SubString($rawHttpThem.LastIndexOf("# ")+2)
}elseif($rawHttpThem.IndexOf("PhishStats") -gt 0){
$rawIntelThem = "date,score,url,ip`r`n{0}" -f $rawHttpThem.SubString($rawHttpThem.LastIndexOf("# ")+2)
}elseif($rawHttpThem.IndexOf("phish_id") -gt 0){
$rawIntelThem = $rawHttpThem.SubString($rawHttpThem.IndexOf("phish_id"))
}else{
$first = $rawHttpThem.IndexOf("http")
$rawIntelThem = "url`r`n{0}" -f $rawHttpThem.SubString($first)
}
$csvThemIntel = ConvertFrom-Csv $rawIntelThem
$outputList = @()
$outputIpList = @()
$savedCount = 0
$savedIpCount = 0
$ignoredCount = 0
foreach($rowIntelThem in $csvThemIntel){
try {
$domainThem = ([System.Uri]::new($rowIntelThem.url).Host).ToString()
$ignoreIt = 0
foreach($ignoredItem in $ignoreList){
if($domainThem.ToLower() -eq $ignoredItem.ToLower()){
$ignoreIt = 1
$ignoredCount = $ignoredCount +1
break
}
}
if($ignoreIt -eq 0){
$ipThem = [IPAddress] $domainThem
$newHit = New-Object PSObject
$newHit | add-member Noteproperty ip $ipThem
$outputIpList += $newHit
$savedIpCount = $savedIpCount + 1
}
}
catch{
if($domainThem.ToLower().StartsWith("www.")){
#double count it (www.ebay.com and ebay.com)
$newHit = New-Object PSObject
$newHit | add-member Noteproperty dns $domainThem.SubString(4)
$outputList += $newHit
$savedCount = $savedCount + 1
}
$newHit = New-Object PSObject
$newHit | add-member Noteproperty dns $domainThem
$outputList += $newHit
$savedCount = $savedCount + 1
}
}
if($debug){ Write-Output ("Exporting '{0}'" -f $fileOutput) }
$outputList | Export-Csv -NoTypeInformation -Path $fileOutput
if($debug){ Write-Output ("Saved '{0}'" -f $fileOutput) }
if($debug){ Write-Output ("Exporting '{0}'" -f $fileIpOutput) }
$outputIpList | Export-Csv -NoTypeInformation -Path $fileIpOutput
if($debug){ Write-Output ("Saved '{0}'" -f $fileIpOutput) }
if($debug){ Write-Output ("Dns='{0}', Ips='{1}', Ignored='{2}'" -f $savedCount, $savedIpCount, $ignoredCount) }
if($debug){
foreach($dns in $outputList){
if($dnsList -eq ""){
$dnsList = $dns.dns
}else{
$dnsList = "{0},{1}" -f $dnsList , $dns.dns
}
}
foreach($ip in $outputIpList){
if($ipList -eq ""){
$ipList = $ip.ip
}else{
$ipList = "{0},{1}" -f $ipList , $ip.ip
}
}
Write-Output $dnsList
Write-Output $ipList
}