jake/subtake
1
Fork 0
mirror of https://github.com/jakejarvis/subtake.git synced 2025-06-27 16:45:41 -04:00
Code Releases Activity

safer bash syntax

Browse Source
This commit is contained in:
Jake Jarvis
2019-04-10 16:50:39 -04:00
parent a1582ca39e
commit f73a778573
1 changed files with 59 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
sonar.sh
View File

@ -1,25 +1,39 @@
#!/bin/bash
# Usage : ./sonar.sh <version number> <file>
# Example: ./sonar.sh 2018-10-27-1540655191-fdns_cname.json.gz sonar.txt
# Example: ./sonar.sh 2018-10-27-1540655191 sonar.txt
set -u
# DEBUG: Mark start time
time_start=$(date -u +%s)
# Set location for temporary junk
tempdir=/tmp/sonar
# Make sure there aren't existing temp files
mkdir -p $tempdir
rm -rf ${tempdir:?}/*
mkdir /tmp/sonar
# Gathering data from scans.io / Rapid7 Project Sonar if not already provided
# Find the latest filename listed at https://opendata.rapid7.com/sonar.fdns_v2/ ending with fdns_cname.json.gz and pass in as first argument
# Example: 2018-10-27-1540655191-fdns_cname.json.gz
if [ ! -f $1 ]; then
echo "Downloading $1, this may take a while..."
wget -q -O /tmp/sonar/$1 https://opendata.rapid7.com/sonar.fdns_v2/$1
echo "Finished downloading $1."
# Download dataset from Rapid7 if not already provided
# Find the latest timestamp listed at https://opendata.rapid7.com/sonar.fdns_v2/ (the string preceding "-fdns_cname.json.gz") and pass in as first argument
# Example: 2018-10-27-1540655191
filename="$1-fdns_cname.json.gz"
if [ ! -f "$tempdir/$filename" ]; then
SECONDS=0
echo "[-] Downloading $filename from Rapid7..."
curl -#Lo "$tempdir/$filename" "https://opendata.rapid7.com/sonar.fdns_v2/$filename"
echo "[+] Successfully downloaded $filename. Took $((SECONDS/60)) minutes."
fi
# Parsing data into a temp file called sonar_cnames
echo "Grepping for CNAME records..."
zcat < $1 | grep 'type":"cname' | awk -F'":"' '{print $3, $5}' | \
awk -F'"' '{print $1, $3}' | sed -e s/" type "/" "/g >> /tmp/sonar/sonar_cnames
echo "CNAME records grepped."
# Parse data into a temp file called sonar_cnames
SECONDS=0
echo "[-] Extracting CNAME records..."
zcat < "$tempdir/$filename" | grep 'type":"cname' | awk -F'":"' '{print $3, $5}' | \
awk -F'"' '{print $1, $3}' | sed -e s/" type "/" "/g > $tempdir/sonar_cnames
rm "${tempdir:?}/$filename"
echo "[+] CNAME records extracted. Took $((SECONDS/60)) minutes."
# List of fingerprints we're going to grep for
@ -28,7 +42,7 @@ declare -a prints=(
"\.s3.amazonaws.com$"
"\.herokuapp.com$"
"\.herokudns.com$"
# "\.wordpress.com$"
"\.wordpress.com$"
"\.pantheonsite.io$"
"domains.tumblr.com$"
"\.zendesk.com$"
@ -36,7 +50,7 @@ declare -a prints=(
"\.github.io$"
"\.global.fastly.net$"
"\.ghost.io$"
# "\.myshopify.com$"
"\.myshopify.com$"
"\.surge.sh$"
"\.bitbucket.io$"
"\.azurewebsites.net$"
@ -45,30 +59,41 @@ declare -a prints=(
"\.blob.core.windows.net$"
)
prints_array=$(echo "${prints[@]}" | tr ' ' '|')
# Grepping CNAMEs w/ matching fingerprints from the array
echo "Grepping for fingerprints..."
grep -Ei $(echo ${prints[@]}|tr " " "|") /tmp/sonar/sonar_cnames >> /tmp/sonar/sonar_prints
echo "Fingerprints grepped."
echo "[-] Dusting for fingerprints..."
SECONDS=0
grep -Ei "$prints_array" $tempdir/sonar_cnames > $tempdir/sonar_prints
rm ${tempdir:?}/sonar_cnames
echo "[+] Fingerprints dusted. Took $((SECONDS/60)) minutes."
# Output only the CNAME (not the fingerprint)
echo "Sorting CNAME records..."
cat /tmp/sonar/sonar_prints | awk '{print $1}' >> /tmp/sonar/sonar_records
echo "CNAME records sorted."
# Output only the CNAME (not the target/fingerprint)
echo "[-] Isolating CNAME records..."
SECONDS=0
awk '{print $1}' $tempdir/sonar_prints > $tempdir/sonar_records
rm ${tempdir:?}/sonar_prints
echo "[+] CNAME records isloated. Took $((SECONDS/60)) minutes."
# Removing recursive records
echo "Removing recursive records..."
grep -v -Ei $(echo ${prints[@]}|tr " " "|") /tmp/sonar/sonar_records >> $2
echo "Removed recursive records."
# Removing recursive records (when CNAME contains its own fingerprint; ex: abcd.herokuapp.com -> us-east-1-a.route.herokuapp.com)
echo "[-] Removing recursive records..."
SECONDS=0
grep -v -Ei "$prints_array" $tempdir/sonar_records > "$2"
rm ${tempdir:?}/sonar_records
echo "[+] Recursive records removed. Took $((SECONDS/60)) minutes."
# Remove temp files
echo "Cleaning up..."
rm -rf /tmp/sonar
rm $1
echo "Cleaned up."
# All done with temp files, make sure we've tidied everything up
echo "[-] Cleaning up..."
rm -rf ${tempdir:?}
echo "[+] Cleaned up."
echo "[+] Finished!"
# DEBUG: Mark finish time
time_end=$(date -u +%s)
echo "[+] Finally done! Took $(((time_end-time_start)/60)) minutes total."