keep weekly and monthly backups (and rotate old ones)

README.md

@@ -51,14 +51,16 @@ cp .env.example .env
 
 #### Periodic tasks
 
-- [`backup.sh`](scripts/backup.sh): Backs up Postgres, Redis, and `.env.production` secrets to a `.tar.gz` file in `/home/mastodon/backups` — useful for a [periodic cronjob](https://github.com/jakejarvis/mastodon-utils/wiki/Cron-jobs#backups).
+- [`backup.sh`](scripts/backup.sh): Backs up Postgres, Redis, and `.env.production` secrets to a `.tar.gz` file in `$MASTODON_ROOT/backups`. Useful for a [daily cronjob](https://github.com/jakejarvis/mastodon-utils/wiki/Cron-jobs#backups).
-- [`weekly_cleanup.sh`](scripts/weekly_cleanup.sh): Runs Mastodon's built-in [cleanup commands](https://docs.joinmastodon.org/admin/setup/#cleanup), designed for a [weekly cronjob](https://github.com/jakejarvis/mastodon-utils/wiki/Cron-jobs#media-cleanup).
+  - Keeps archives for the last 5 days, last 4 weeks, and every month
+  - Optionally uploads to S3 with [`s3cmd`](https://s3tools.org/s3cmd)
+- [`purge.sh`](scripts/purge.sh): Runs Mastodon's built-in [cleanup commands](https://docs.joinmastodon.org/admin/setup/#cleanup), designed for a [weekly cronjob](https://github.com/jakejarvis/mastodon-utils/wiki/Cron-jobs#media-cleanup).
   - Keeps 14 days of media
   - Keeps 90 days of profile avatars, headers, and link preview cards
 
 #### Dangerous
 
-**The following scripts are highly opinionated, catastrophically destructive, and very specific to me.** Check them out line-by-line instead of running them.
+> **🚨 The following scripts are highly opinionated, catastrophically destructive, and very specific to me.** Check them out line-by-line instead of running them.
 
 - [`install.sh`](scripts/install.sh): Assumes an absolutely clean install of Ubuntu and installs Mastodon ***with all of the quirks from this repo.*** Configure `MASTODON_USER` and other paths in `.env` first (see [`.env.example`](.env.example)) if necessary. [Get the far less dangerous version of `install.sh` here instead.](https://github.com/jakejarvis/mastodon-installer/blob/main/install.sh)
 - [`upgrade.sh`](scripts/upgrade.sh): Upgrades Mastodon server (latest version if vanilla Mastodon, latest commit if `glitch-soc`) and ***re-applies all customizations***. [Get the far less dangerous version of `upgrade.sh` here instead.](https://github.com/jakejarvis/mastodon-installer/blob/main/upgrade.sh)

etc/nginx/sites-available/mastodon.conf

@@ -36,6 +36,14 @@ server {
 	sendfile on;
 	client_max_body_size 100m;
 
+	# reused values
+	set $hsts      "max-age=63072000";
+	set $compress_mimes "application/atom+xml application/javascript application/json application/rss+xml
+			application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype
+			application/x-font-ttf application/x-javascript application/xhtml+xml application/xml
+			font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon
+			image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml";
+
 	gzip on;
 	gzip_disable "msie6";
 	gzip_vary on;
@@ -44,11 +52,7 @@ server {
 	gzip_buffers 16 8k;
 	gzip_http_version 1.1;
 	gzip_min_length 256;
-	gzip_types	application/atom+xml application/javascript application/json application/rss+xml
+	gzip_types $compress_mimes;
-			application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype
-			application/x-font-ttf application/x-javascript application/xhtml+xml application/xml
-			font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon
-			image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml;
 
 	# https://github.com/google/ngx_brotli#sample-configuration
 	# https://github.com/jakejarvis/mastodon-utils/wiki/nginx#brotli-compression
@@ -56,37 +60,33 @@ server {
 	# brotli_comp_level 4;
 	# brotli_static on;
 	# brotli_min_length 256;
-	# brotli_types	application/atom+xml application/javascript application/json application/rss+xml
+	# brotli_types $compress_mimes;
-	# 		application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype
-	# 		application/x-font-ttf application/x-javascript application/xhtml+xml application/xml
-	# 		font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon
-	# 		image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml;
 
 	# sends most paths to the backend proxy and ignores the location blocks below, except if
 	# the file exists in /home/mastodon/live
 	location / {
-		add_header Strict-Transport-Security "max-age=63072000" always;
+		add_header Strict-Transport-Security $hsts always;
 		try_files $uri @proxy;
 	}
 
 	# condensed version of original Mastodon nginx.conf
 	location ~ ^/(?:assets|avatars|emoji|headers|packs|shortcuts|sounds)/ {
 		add_header Cache-Control "public, max-age=2419200, must-revalidate"; # 28 days
-		add_header Strict-Transport-Security "max-age=63072000" always;
+		add_header Strict-Transport-Security $hsts always;
 		try_files $uri =404;
 	}
 
 	# media uploads & cache (irrelevant if offloading to S3)
 	location ~ ^/system/ {
 		add_header Cache-Control "public, max-age=2419200, immutable"; # 28 days
-		add_header Strict-Transport-Security "max-age=63072000" always;
+		add_header Strict-Transport-Security $hsts always;
 		try_files $uri =404;
 	}
 
 	# static files *only in the root* of /public (/favicon.ico, /sw.js, /robots.txt, etc.)
 	location ~ ^/[^/]+\.(?:js|css|png|gif|jpg|txt|ico)$ {
 		add_header Cache-Control "public, max-age=604800, must-revalidate"; # 7 days
-		add_header Strict-Transport-Security "max-age=63072000" always;
+		add_header Strict-Transport-Security $hsts always;
 		try_files $uri =404;
 	}
 
@@ -107,7 +107,7 @@ server {
 		# security headers
 		proxy_hide_header Strict-Transport-Security;
 		proxy_hide_header X-Powered-By;
-		add_header Strict-Transport-Security "max-age=63072000" always;
+		add_header Strict-Transport-Security $hsts always;
 
 		# debugging headers
 		add_header Via "1.1 $proxy_host" always;
@@ -143,7 +143,7 @@ server {
 		proxy_hide_header X-Clacks-Overhead;
 		proxy_hide_header X-XSS-Protection;
 		add_header Referrer-Policy "strict-origin" always;
-		add_header Strict-Transport-Security "max-age=63072000" always;
+		add_header Strict-Transport-Security $hsts always;
 
 		# debugging headers
 		add_header Via "1.1 $proxy_host" always;

init.sh

@@ -20,7 +20,6 @@ export BACKUPS_ROOT="${BACKUPS_ROOT:="$MASTODON_ROOT/backups"}"
 export LOGS_ROOT="${LOGS_ROOT:="$MASTODON_ROOT/logs"}"
 export RBENV_ROOT="${RBENV_ROOT:="$MASTODON_ROOT/.rbenv"}"
 export NVM_DIR="${NVM_DIR:="$MASTODON_ROOT/.nvm"}"
-export MY_NAME_IS_JAKE_JARVIS="${MY_NAME_IS_JAKE_JARVIS:="false"}"
 
 # automatically detect glitch-soc
 # shellcheck disable=SC2155

scripts/backup.sh

@@ -20,16 +20,12 @@ if [ "$(systemctl is-active mastodon-web.service)" = "active" ]; then
   echo ""
 fi
 
-if [ ! -d "$BACKUPS_ROOT" ]; then
+# TODO: unsafe & ugly, do better.
-  sudo mkdir -p "$BACKUPS_ROOT"
-fi
-
-# TODO: ugly, do better
 TEMP_DIR=$(sudo mktemp -d)
 sudo chmod 777 "$TEMP_DIR"
 
 echo "* Backing up Postgres..."
-sudo -u postgres pg_dump -Fc mastodon_production -f "$TEMP_DIR/postgres.dump"
+sudo -Hiu postgres pg_dump -Fc mastodon_production -f "$TEMP_DIR/postgres.dump"
 
 echo "* Backing up Redis..."
 sudo cp /var/lib/redis/dump.rdb "$TEMP_DIR/redis.rdb"
@@ -42,22 +38,48 @@ sudo mkdir -p "$TEMP_DIR/certs"
 sudo cp -r /etc/letsencrypt/{archive,live,renewal} "$TEMP_DIR/certs/"
 
 echo "* Compressing..."
-ARCHIVE_DEST="$BACKUPS_ROOT/mastodon-$(date "+%Y.%m.%d-%H.%M.%S").tar.gz"
+TEMP_ARCHIVE="$(sudo mktemp)"
-sudo tar --owner=0 --group=0 -czvf "$ARCHIVE_DEST" -C "$TEMP_DIR" .
+sudo tar --owner=0 --group=0 -czvf "$TEMP_ARCHIVE" -C "$TEMP_DIR" .
-sudo chown "$MASTODON_USER":"$MASTODON_USER" "$ARCHIVE_DEST"
+sudo mkdir -p "$BACKUPS_ROOT"/{daily,weekly,monthly}
+ARCHIVE_FILENAME="mastodon-$(date "+%Y.%m.%d").tar.gz"
+# weekly backup (every Sunday)
+if [ "$(date +"%u")" -eq 7 ]; then
+  WEEKLY_DEST="$BACKUPS_ROOT/weekly/$ARCHIVE_FILENAME"
+  sudo cp -f "$TEMP_ARCHIVE" "$WEEKLY_DEST"
+  echo "* Saved weekly backup to '$WEEKLY_DEST'"
+fi
+# monthly backup (first day of the month)
+if [ "$(date +"%d")" -eq 1 ]; then
+  MONTHLY_DEST="$BACKUPS_ROOT/monthly/$ARCHIVE_FILENAME"
+  sudo cp -f "$TEMP_ARCHIVE" "$MONTHLY_DEST"
+  echo "* Saved monthly backup to '$MONTHLY_DEST'"
+fi
+# daily backup (always)
+DAILY_DEST="$BACKUPS_ROOT/daily/$ARCHIVE_FILENAME"
+sudo cp -f "$TEMP_ARCHIVE" "$DAILY_DEST"
+echo "* Saved daily backup to '$DAILY_DEST'"
 
-echo "* Fixing permissions..."
+echo "* Rotating old backups..."
-sudo chown -R "$MASTODON_USER":"$MASTODON_USER" "$BACKUPS_ROOT"
+# NOTE: keep all monthly backups for now
+# keep last 4 weekly backups
+find "$BACKUPS_ROOT/weekly" -mindepth 1 -type f -mtime +3 -delete
+# keep last 5 daily backups
+find "$BACKUPS_ROOT/daily" -mindepth 1 -type f -mtime +4 -delete
+
+# sync backups dir with s3 bucket if s3cmd is installed & BACKUP_S3_BUCKET env var is set
+# https://www.linode.com/docs/products/storage/object-storage/guides/s3cmd/
+if [ -n "${BACKUP_S3_BUCKET:+x}" ] && command -v s3cmd >/dev/null 2>&1; then
+  echo "* Uploading to S3..."
+  sudo s3cmd sync --delete-removed "$BACKUPS_ROOT/" "s3://$BACKUP_S3_BUCKET" || :
+else
+  echo "⚠ Skipping S3 upload; check that 's3cmd' is present in \$PATH, and \$BACKUP_S3_BUCKET is set."
+fi
 
 echo "* Removing temp files..."
 sudo rm -rf --preserve-root "$TEMP_DIR"
 
-echo "* Saved archive to '$ARCHIVE_DEST'"
+echo "* Fixing permissions..."
-
+sudo chown -R "$MASTODON_USER":"$MASTODON_USER" "$BACKUPS_ROOT"
-if [ -s /usr/local/bin/linode-cli ]; then
-  echo "* Uploading to S3..."
-  sudo /usr/local/bin/linode-cli obj put "$ARCHIVE_DEST" jarvis-backup
-fi
 
 echo "* 🎉 done! (keep this archive safe!)"
 

scripts/install.sh

@@ -3,17 +3,17 @@
 # exit when any step fails
 set -euo pipefail
 
+# initialize paths (and silence warnings about things not existing yet because that's why we're running the installer.)
+# shellcheck disable=SC1091
+. "$(dirname "${BASH_SOURCE[0]}")"/../init.sh >/dev/null
+
 # can't say you weren't warned :)
-if [ "$MY_NAME_IS_JAKE_JARVIS" != "pinky promise" ]; then
+if [ "${MY_NAME_IS_JAKE_JARVIS:=}" != "pinky promise" ]; then
   echo "🚨 LISTEN UP!!!! YOU PROBABLY WANT THIS SCRIPT INSTEAD:"
   echo "https://github.com/jakejarvis/mastodon-installer/blob/main/install.sh"
   exit 69
 fi
 
-# initialize paths (and silence warnings about things not existing yet because that's why we're running the installer.)
-# shellcheck disable=SC1091
-. "$(dirname "${BASH_SOURCE[0]}")"/../init.sh >/dev/null
-
 # check for existing installation
 if [ -d "$APP_ROOT" ]; then
   echo "⚠ $APP_ROOT already exists. Are you sure Mastodon isn't already installed?"
@@ -288,7 +288,7 @@ as_mastodon touch "$LOGS_ROOT"/cron.log
 (
   sudo crontab -l
   echo -e "\n$INSTALLER_WUZ_HERE
-@weekly  bash -c \"$UTILS_ROOT/scripts/weekly_cleanup.sh >> $LOGS_ROOT/cron.log 2>&1\"
+@weekly  bash -c \"$UTILS_ROOT/scripts/purge.sh >> $LOGS_ROOT/cron.log 2>&1\"
 @weekly  bash -c \"$UTILS_ROOT/scripts/backup.sh >> $LOGS_ROOT/cron.log 2>&1\"
 
 # automatically renew Let's Encrypt certificates

scripts/purge.sh

@@ -2,12 +2,12 @@
 
 # cronjob ran once per week at 3 AM on Sunday; see https://crontab.guru/#0_3_*_*_0
 # syntax for crontab -e:
-#   0 3 * * 0  bash -c "/home/mastodon/utils/scripts/weekly_cleanup.sh >> /home/mastodon/logs/cron.log 2>&1"
+#   0 3 * * 0  bash -c "/home/mastodon/utils/scripts/purge.sh >> /home/mastodon/logs/cron.log 2>&1"
 
 # exit when any step fails
 set -o pipefail
 
-echo -e "\n===== weekly_cleanup.sh: started at $(date '+%Y-%m-%d %H:%M:%S') =====\n"
+echo -e "\n===== purge.sh: started at $(date '+%Y-%m-%d %H:%M:%S') =====\n"
 
 # initialize paths
 # shellcheck disable=SC1091
@@ -17,4 +17,4 @@ tootctl media remove --days 14
 tootctl media remove --prune-profiles --days 90
 tootctl preview_cards remove --days 90
 
-echo -e "\n===== weekly_cleanup.sh: finished at $(date '+%Y-%m-%d %H:%M:%S') =====\n"
+echo -e "\n===== purge.sh: finished at $(date '+%Y-%m-%d %H:%M:%S') =====\n"

scripts/upgrade.sh

@@ -3,17 +3,17 @@
 # exit when any step fails
 set -euo pipefail
 
+# initialize paths
+# shellcheck disable=SC1091
+. "$(dirname "${BASH_SOURCE[0]}")"/../init.sh
+
 # can't say you weren't warned :)
-if [ "$MY_NAME_IS_JAKE_JARVIS" != "pinky promise" ]; then
+if [ "${MY_NAME_IS_JAKE_JARVIS:=}" != "pinky promise" ]; then
   echo "🚨 LISTEN UP!!!! YOU PROBABLY WANT THIS SCRIPT INSTEAD:"
   echo "https://github.com/jakejarvis/mastodon-installer/blob/main/upgrade.sh"
   exit 69
 fi
 
-# initialize paths
-# shellcheck disable=SC1091
-. "$(dirname "${BASH_SOURCE[0]}")"/../init.sh
-
 # pull latest mastodon source
 cd "$APP_ROOT"
 as_mastodon git fetch --all
@@ -43,7 +43,7 @@ as_mastodon nvm install
 as_mastodon nvm use
 as_mastodon npm install --global yarn
 # install deps
-as_mastodon bundle install --jobs "$(getconf _NPROCESSORS_ONLN)"
+as_mastodon bundle check || as_mastodon bundle install --jobs "$(getconf _NPROCESSORS_ONLN)"
 as_mastodon yarn install --pure-lockfile
 
 # compile new assets