jake/subtake
1
Fork 0
mirror of https://github.com/jakejarvis/subtake.git synced 2025-09-15 20:55:32 -04:00
Code Releases Activity

safer bash syntax

Browse Source
This commit is contained in:
Jake Jarvis
2019-04-10 16:50:39 -04:00
parent a1582ca39e
commit f73a778573
1 changed files with 59 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
sonar.sh
View File

@@ -1,25 +1,39 @@
#!/bin/bash #!/bin/bash
# Usage : ./sonar.sh <version number> <file> # Usage : ./sonar.sh <version number> <file>
# Example: ./sonar.sh 2018-10-27-1540655191-fdns_cname.json.gz sonar.txt # Example: ./sonar.sh 2018-10-27-1540655191 sonar.txt
set -u
# DEBUG: Mark start time
time_start=$(date -u +%s)
# Set location for temporary junk
tempdir=/tmp/sonar
# Make sure there aren't existing temp files
mkdir -p $tempdir
rm -rf ${tempdir:?}/*
mkdir /tmp/sonar # Download dataset from Rapid7 if not already provided
# Find the latest timestamp listed at https://opendata.rapid7.com/sonar.fdns_v2/ (the string preceding "-fdns_cname.json.gz") and pass in as first argument
# Gathering data from scans.io / Rapid7 Project Sonar if not already provided # Example: 2018-10-27-1540655191
# Find the latest filename listed at https://opendata.rapid7.com/sonar.fdns_v2/ ending with fdns_cname.json.gz and pass in as first argument filename="$1-fdns_cname.json.gz"
# Example: 2018-10-27-1540655191-fdns_cname.json.gz if [ ! -f "$tempdir/$filename" ]; then
if [ ! -f $1 ]; then SECONDS=0
echo "Downloading $1, this may take a while..." echo "[-] Downloading $filename from Rapid7..."
wget -q -O /tmp/sonar/$1 https://opendata.rapid7.com/sonar.fdns_v2/$1 curl -#Lo "$tempdir/$filename" "https://opendata.rapid7.com/sonar.fdns_v2/$filename"
echo "Finished downloading $1." echo "[+] Successfully downloaded $filename. Took $((SECONDS/60)) minutes."
fi fi
# Parsing data into a temp file called sonar_cnames # Parse data into a temp file called sonar_cnames
echo "Grepping for CNAME records..." SECONDS=0
zcat < $1 | grep 'type":"cname' | awk -F'":"' '{print $3, $5}' | \ echo "[-] Extracting CNAME records..."
awk -F'"' '{print $1, $3}' | sed -e s/" type "/" "/g >> /tmp/sonar/sonar_cnames zcat < "$tempdir/$filename" | grep 'type":"cname' | awk -F'":"' '{print $3, $5}' | \
echo "CNAME records grepped." awk -F'"' '{print $1, $3}' | sed -e s/" type "/" "/g > $tempdir/sonar_cnames
rm "${tempdir:?}/$filename"
echo "[+] CNAME records extracted. Took $((SECONDS/60)) minutes."
# List of fingerprints we're going to grep for # List of fingerprints we're going to grep for
@@ -28,7 +42,7 @@ declare -a prints=(
"\.s3.amazonaws.com$" "\.s3.amazonaws.com$"
"\.herokuapp.com$" "\.herokuapp.com$"
"\.herokudns.com$" "\.herokudns.com$"
# "\.wordpress.com$" "\.wordpress.com$"
"\.pantheonsite.io$" "\.pantheonsite.io$"
"domains.tumblr.com$" "domains.tumblr.com$"
"\.zendesk.com$" "\.zendesk.com$"
@@ -36,7 +50,7 @@ declare -a prints=(
"\.github.io$" "\.github.io$"
"\.global.fastly.net$" "\.global.fastly.net$"
"\.ghost.io$" "\.ghost.io$"
# "\.myshopify.com$" "\.myshopify.com$"
"\.surge.sh$" "\.surge.sh$"
"\.bitbucket.io$" "\.bitbucket.io$"
"\.azurewebsites.net$" "\.azurewebsites.net$"
@@ -45,30 +59,41 @@ declare -a prints=(
"\.blob.core.windows.net$" "\.blob.core.windows.net$"
) )
prints_array=$(echo "${prints[@]}" | tr ' ' '|')
# Grepping CNAMEs w/ matching fingerprints from the array # Grepping CNAMEs w/ matching fingerprints from the array
echo "Grepping for fingerprints..." echo "[-] Dusting for fingerprints..."
grep -Ei $(echo ${prints[@]}|tr " " "|") /tmp/sonar/sonar_cnames >> /tmp/sonar/sonar_prints SECONDS=0
echo "Fingerprints grepped." grep -Ei "$prints_array" $tempdir/sonar_cnames > $tempdir/sonar_prints
rm ${tempdir:?}/sonar_cnames
echo "[+] Fingerprints dusted. Took $((SECONDS/60)) minutes."
# Output only the CNAME (not the fingerprint) # Output only the CNAME (not the target/fingerprint)
echo "Sorting CNAME records..." echo "[-] Isolating CNAME records..."
cat /tmp/sonar/sonar_prints | awk '{print $1}' >> /tmp/sonar/sonar_records SECONDS=0
echo "CNAME records sorted." awk '{print $1}' $tempdir/sonar_prints > $tempdir/sonar_records
rm ${tempdir:?}/sonar_prints
echo "[+] CNAME records isloated. Took $((SECONDS/60)) minutes."
# Removing recursive records # Removing recursive records (when CNAME contains its own fingerprint; ex: abcd.herokuapp.com -> us-east-1-a.route.herokuapp.com)
echo "Removing recursive records..." echo "[-] Removing recursive records..."
grep -v -Ei $(echo ${prints[@]}|tr " " "|") /tmp/sonar/sonar_records >> $2 SECONDS=0
echo "Removed recursive records." grep -v -Ei "$prints_array" $tempdir/sonar_records > "$2"
rm ${tempdir:?}/sonar_records
echo "[+] Recursive records removed. Took $((SECONDS/60)) minutes."
# Remove temp files # All done with temp files, make sure we've tidied everything up
echo "Cleaning up..." echo "[-] Cleaning up..."
rm -rf /tmp/sonar rm -rf ${tempdir:?}
rm $1 echo "[+] Cleaned up."
echo "Cleaned up."
echo "[+] Finished!" # DEBUG: Mark finish time
time_end=$(date -u +%s)
echo "[+] Finally done! Took $(((time_end-time_start)/60)) minutes total."