jake/subtake
1
Fork 0
mirror of https://github.com/jakejarvis/subtake.git synced 2025-04-26 07:35:24 -04:00
Code Releases Activity

vastly faster sonar.sh

Browse Source
This commit is contained in:
Jake Jarvis 2019-04-10 09:15:44 -04:00
parent a8d04f1beb
commit a1582ca39e
No known key found for this signature in database
GPG Key ID: D36CB66F4002B25B
2 changed files with 59 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
fingerprints.json
View File

@ -48,7 +48,8 @@
{ {
"service": "Heroku", "service": "Heroku",
"cname": [ "cname": [
"herokuapp.com" "herokuapp.com",
"herokudns.com"
], ],
"fingerprint": [ "fingerprint": [
"herokucdn.com/error-pages/no-such-app.html" "herokucdn.com/error-pages/no-such-app.html"

169
sonar.sh
View File

@ -1,129 +1,74 @@
#!/bin/bash #!/bin/bash
# Usage : ./sonar.sh <version number> <file> # Usage : ./sonar.sh <version number> <file>
# Example: ./sonar.sh 2018-10-27-1540655191-fdns_cname.json.gz cname_list.txt # Example: ./sonar.sh 2018-10-27-1540655191-fdns_cname.json.gz sonar.txt
# Progress spinner
function ech() {
spinner=( "|" "/" "-" "\\" )
while true; do
for i in ${spinner[@]}; do
echo -ne "\r[$i] $1"
sleep 0.15
done
done
}
# Joining elements together mkdir /tmp/sonar
function join_by() {
local IFS=$1
shift
echo "$*"
}
# Kill function # Gathering data from scans.io / Rapid7 Project Sonar if not already provided
function die() {
disown $1
kill -9 $1
length=$(echo -n $3 | wc -m)
Count=$(($length + 5))
Clear=$(head -c $Count < /dev/zero | tr '\0' '\040')
echo -ne "\r $Clear"
echo -e "\r[*] $2"
}
function run() {
ech "$1" &
pid=$!
eval "$2"
die $pid "$3" "$1"
}
# Gathering data from scans.io / Rapid7 Project Sonar
# Find the latest filename listed at https://opendata.rapid7.com/sonar.fdns_v2/ ending with fdns_cname.json.gz and pass in as first argument # Find the latest filename listed at https://opendata.rapid7.com/sonar.fdns_v2/ ending with fdns_cname.json.gz and pass in as first argument
# Example: 2018-10-27-1540655191-fdns_cname.json.gz # Example: 2018-10-27-1540655191-fdns_cname.json.gz
if [ ! -f $1 ]; then if [ ! -f $1 ]; then
cmd="wget -q https://opendata.rapid7.com/sonar.fdns_v2/$1" echo "Downloading $1, this may take a while..."
run "Downloading $1, this may take a while..." "$cmd" "Finished downloading $1." wget -q -O /tmp/sonar/$1 https://opendata.rapid7.com/sonar.fdns_v2/$1
echo "Finished downloading $1."
fi fi
# Parsing it into a file called cname_scanio
msg="Grepping for CNAME records."
ech $msg &
pid=$!
zcat < $1 | grep 'type":"cname' | awk -F'":"' '{print $3, $5}' | \
awk -F'"' '{print $1, $3}' | sed -e s/" type "/" "/g >> cname_scanio
die $pid "CNAME records grepped." $msg
# List of CNAMEs we're going to grep for # Parsing data into a temp file called sonar_cnames
declare -a arr=( echo "Grepping for CNAME records..."
"\.cloudfront.net" zcat < $1 | grep 'type":"cname' | awk -F'":"' '{print $3, $5}' | \
awk -F'"' '{print $1, $3}' | sed -e s/" type "/" "/g >> /tmp/sonar/sonar_cnames
echo "CNAME records grepped."
# List of fingerprints we're going to grep for
declare -a prints=(
"\.s3-website" "\.s3-website"
"\.s3.amazonaws.com" "\.s3.amazonaws.com$"
"w.amazonaws.com" "\.herokuapp.com$"
"1.amazonaws.com" "\.herokudns.com$"
"2.amazonaws.com" # "\.wordpress.com$"
"s3-external" "\.pantheonsite.io$"
"s3-accelerate.amazonaws.com" "domains.tumblr.com$"
"\.herokuapp.com" "\.zendesk.com$"
"\.herokudns.com" "\.github.com$"
"\.wordpress.com" "\.github.io$"
"\.pantheonsite.io" "\.global.fastly.net$"
"domains.tumblr.com" "\.ghost.io$"
"\.zendesk.com" # "\.myshopify.com$"
"\.github.io" "\.surge.sh$"
"\.global.fastly.net" "\.bitbucket.io$"
"\.helpjuice.com" "\.azurewebsites.net$"
"\.helpscoutdocs.com" "\.cloudapp.net$"
"\.ghost.io" "\.trafficmanager.net$"
"cargocollective.com" "\.blob.core.windows.net$"
"redirect.feedpress.me"
"\.myshopify.com"
"\.statuspage.io"
"\.uservoice.com"
"\.surge.sh"
"\.bitbucket.io"
"custom.intercom.help"
"proxy.webflow.com"
"landing.subscribepage.com"
"endpoint.mykajabi.com"
"\.teamwork.com"
"\.thinkific.com"
"clientaccess.tave.com"
"wishpond.com"
"\.aftership.com"
"ideas.aha.io"
"domains.tictail.com"
"cname.mendix.net"
"\.bcvp0rtal.com"
"\.brightcovegallery.com"
"\.gallery.video"
"\.bigcartel.com"
"\.activehosted.com"
"\.createsend.com"
"\.acquia-test.co"
"\.proposify.biz"
"simplebooklet.com"
"\.gr8.com"
"\.vendecommerce.com"
"\.azurewebsites.net"
"\.cloudapp.net"
"\.trafficmanager.net"
"\.blob.core.windows.net"
) )
# Prepare CNAME grep
DOMAINS=$(join_by '|' ${arr[@]})
# Grepping CNAMEs from the array # Grepping CNAMEs w/ matching fingerprints from the array
cmd="grep -Ei '${DOMAINS}' cname_scanio >> cname_db" echo "Grepping for fingerprints..."
run "Sorting CNAME records." "$cmd" "CNAME records sorted." grep -Ei $(echo ${prints[@]}|tr " " "|") /tmp/sonar/sonar_cnames >> /tmp/sonar/sonar_prints
echo "Fingerprints grepped."
# Sorting the CNAME list
cmd="cat cname_db | cut -d' ' -f1 | sort | uniq >> $2"
run "Cleaning up." "$cmd" "Cleaned up."
# RM files. # Output only the CNAME (not the fingerprint)
rm cname_db cname_scanio echo "Sorting CNAME records..."
echo "[+] Finished." cat /tmp/sonar/sonar_prints | awk '{print $1}' >> /tmp/sonar/sonar_records
echo "CNAME records sorted."
# Removing recursive records
echo "Removing recursive records..."
grep -v -Ei $(echo ${prints[@]}|tr " " "|") /tmp/sonar/sonar_records >> $2
echo "Removed recursive records."
# Remove temp files
echo "Cleaning up..."
rm -rf /tmp/sonar
rm $1
echo "Cleaned up."
echo "[+] Finished!"