mirror of
https://github.com/jakejarvis/subtake.git
synced 2025-04-26 01:45:25 -04:00
100 lines
2.8 KiB
Bash
Executable File
100 lines
2.8 KiB
Bash
Executable File
#!/bin/bash
|
|
# Usage : ./sonar.sh <version number> <file>
|
|
# Example: ./sonar.sh 2018-10-27-1540655191 sonar.txt
|
|
|
|
set -u
|
|
|
|
# DEBUG: Mark start time
|
|
time_start=$(date -u +%s)
|
|
|
|
# Set location for temporary junk
|
|
tempdir=/tmp/sonar
|
|
|
|
# Make sure there aren't existing temp files
|
|
rm -rf ${tempdir:?}
|
|
mkdir -p $tempdir
|
|
|
|
|
|
# Download dataset from Rapid7 if not already provided
|
|
# Find the latest timestamp listed at https://opendata.rapid7.com/sonar.fdns_v2/ (the string preceding "-fdns_cname.json.gz") and pass in as first argument
|
|
# Example: 2018-10-27-1540655191
|
|
filename="$1-fdns_cname.json.gz"
|
|
if [ ! -f "$tempdir/$filename" ]; then
|
|
SECONDS=0
|
|
echo "[-] Downloading $filename from Rapid7..."
|
|
curl -#Lo "$tempdir/$filename" "https://opendata.rapid7.com/sonar.fdns_v2/$filename"
|
|
echo "[+] Successfully downloaded $filename. Took $((SECONDS/60)) minutes."
|
|
fi
|
|
|
|
|
|
# Parse data into a temp file called sonar_cnames
|
|
SECONDS=0
|
|
echo "[-] Extracting CNAME records..."
|
|
zcat < "$tempdir/$filename" | grep 'type":"cname' | awk -F'":"' '{print $3, $5}' | \
|
|
awk -F'"' '{print $1, $3}' | sed -e s/" type "/" "/g > $tempdir/sonar_cnames
|
|
rm "${tempdir:?}/$filename"
|
|
echo "[+] CNAME records extracted. Took $((SECONDS/60)) minutes."
|
|
|
|
|
|
# List of fingerprints we're going to grep for
|
|
declare -a prints=(
|
|
"\.s3-website"
|
|
"\.s3.amazonaws.com$"
|
|
"\.herokuapp.com$"
|
|
"\.herokudns.com$"
|
|
"\.wordpress.com$"
|
|
"\.pantheonsite.io$"
|
|
"domains.tumblr.com$"
|
|
"\.zendesk.com$"
|
|
"\.github.com$"
|
|
"\.github.io$"
|
|
"\.global.fastly.net$"
|
|
"\.ghost.io$"
|
|
"\.myshopify.com$"
|
|
"\.surge.sh$"
|
|
"\.bitbucket.io$"
|
|
"\.azurewebsites.net$"
|
|
"\.cloudapp.net$"
|
|
"\.trafficmanager.net$"
|
|
"\.blob.core.windows.net$"
|
|
)
|
|
|
|
prints_array=$(echo "${prints[@]}" | tr ' ' '|')
|
|
|
|
|
|
# Grepping CNAMEs w/ matching fingerprints from the array
|
|
echo "[-] Dusting for fingerprints..."
|
|
SECONDS=0
|
|
grep -Ei "$prints_array" $tempdir/sonar_cnames > $tempdir/sonar_prints
|
|
rm ${tempdir:?}/sonar_cnames
|
|
echo "[+] Fingerprints dusted. Took $((SECONDS/60)) minutes."
|
|
|
|
|
|
# Output only the CNAME (not the target/fingerprint)
|
|
echo "[-] Isolating CNAME records..."
|
|
SECONDS=0
|
|
awk '{print $1}' $tempdir/sonar_prints > $tempdir/sonar_records
|
|
rm ${tempdir:?}/sonar_prints
|
|
echo "[+] CNAME records isloated. Took $((SECONDS/60)) minutes."
|
|
|
|
|
|
# Removing recursive records (when CNAME contains its own fingerprint; ex: abcd.herokuapp.com -> us-east-1-a.route.herokuapp.com)
|
|
echo "[-] Removing recursive records..."
|
|
SECONDS=0
|
|
grep -v -Ei "$prints_array" $tempdir/sonar_records > "$2"
|
|
rm ${tempdir:?}/sonar_records
|
|
echo "[+] Recursive records removed. Took $((SECONDS/60)) minutes."
|
|
|
|
|
|
# All done with temp files, make sure we've tidied everything up
|
|
echo "[-] Cleaning up..."
|
|
rm -rf ${tempdir:?}
|
|
echo "[+] Cleaned up."
|
|
|
|
|
|
# DEBUG: Mark finish time
|
|
time_end=$(date -u +%s)
|
|
|
|
|
|
echo "[+] Finally done! Took $(((time_end-time_start)/60)) minutes total."
|