From 352a57c1dc170aa3a1375525a6b880f776025cf0 Mon Sep 17 00:00:00 2001 From: Jake Jarvis Date: Sat, 7 Jan 2023 12:18:36 -0500 Subject: [PATCH] keep weekly and monthly backups (and rotate old ones) --- README.md | 8 ++-- etc/nginx/sites-available/mastodon.conf | 32 +++++++------- init.sh | 1 - scripts/backup.sh | 56 +++++++++++++++++-------- scripts/install.sh | 12 +++--- scripts/{weekly_cleanup.sh => purge.sh} | 6 +-- scripts/upgrade.sh | 12 +++--- 7 files changed, 75 insertions(+), 52 deletions(-) rename scripts/{weekly_cleanup.sh => purge.sh} (57%) diff --git a/README.md b/README.md index 3a97e0e..40e5732 100644 --- a/README.md +++ b/README.md @@ -51,14 +51,16 @@ cp .env.example .env #### Periodic tasks -- [`backup.sh`](scripts/backup.sh): Backs up Postgres, Redis, and `.env.production` secrets to a `.tar.gz` file in `/home/mastodon/backups` — useful for a [periodic cronjob](https://github.com/jakejarvis/mastodon-utils/wiki/Cron-jobs#backups). -- [`weekly_cleanup.sh`](scripts/weekly_cleanup.sh): Runs Mastodon's built-in [cleanup commands](https://docs.joinmastodon.org/admin/setup/#cleanup), designed for a [weekly cronjob](https://github.com/jakejarvis/mastodon-utils/wiki/Cron-jobs#media-cleanup). +- [`backup.sh`](scripts/backup.sh): Backs up Postgres, Redis, and `.env.production` secrets to a `.tar.gz` file in `$MASTODON_ROOT/backups`. Useful for a [daily cronjob](https://github.com/jakejarvis/mastodon-utils/wiki/Cron-jobs#backups). + - Keeps archives for the last 5 days, last 4 weeks, and every month + - Optionally uploads to S3 with [`s3cmd`](https://s3tools.org/s3cmd) +- [`purge.sh`](scripts/purge.sh): Runs Mastodon's built-in [cleanup commands](https://docs.joinmastodon.org/admin/setup/#cleanup), designed for a [weekly cronjob](https://github.com/jakejarvis/mastodon-utils/wiki/Cron-jobs#media-cleanup). - Keeps 14 days of media - Keeps 90 days of profile avatars, headers, and link preview cards #### Dangerous -**The following scripts are highly opinionated, catastrophically destructive, and very specific to me.** Check them out line-by-line instead of running them. +> **🚨 The following scripts are highly opinionated, catastrophically destructive, and very specific to me.** Check them out line-by-line instead of running them. - [`install.sh`](scripts/install.sh): Assumes an absolutely clean install of Ubuntu and installs Mastodon ***with all of the quirks from this repo.*** Configure `MASTODON_USER` and other paths in `.env` first (see [`.env.example`](.env.example)) if necessary. [Get the far less dangerous version of `install.sh` here instead.](https://github.com/jakejarvis/mastodon-installer/blob/main/install.sh) - [`upgrade.sh`](scripts/upgrade.sh): Upgrades Mastodon server (latest version if vanilla Mastodon, latest commit if `glitch-soc`) and ***re-applies all customizations***. [Get the far less dangerous version of `upgrade.sh` here instead.](https://github.com/jakejarvis/mastodon-installer/blob/main/upgrade.sh) diff --git a/etc/nginx/sites-available/mastodon.conf b/etc/nginx/sites-available/mastodon.conf index 0e8e8ef..15a9f14 100644 --- a/etc/nginx/sites-available/mastodon.conf +++ b/etc/nginx/sites-available/mastodon.conf @@ -36,6 +36,14 @@ server { sendfile on; client_max_body_size 100m; + # reused values + set $hsts "max-age=63072000"; + set $compress_mimes "application/atom+xml application/javascript application/json application/rss+xml + application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype + application/x-font-ttf application/x-javascript application/xhtml+xml application/xml + font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon + image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml"; + gzip on; gzip_disable "msie6"; gzip_vary on; @@ -44,11 +52,7 @@ server { gzip_buffers 16 8k; gzip_http_version 1.1; gzip_min_length 256; - gzip_types application/atom+xml application/javascript application/json application/rss+xml - application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype - application/x-font-ttf application/x-javascript application/xhtml+xml application/xml - font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon - image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml; + gzip_types $compress_mimes; # https://github.com/google/ngx_brotli#sample-configuration # https://github.com/jakejarvis/mastodon-utils/wiki/nginx#brotli-compression @@ -56,37 +60,33 @@ server { # brotli_comp_level 4; # brotli_static on; # brotli_min_length 256; - # brotli_types application/atom+xml application/javascript application/json application/rss+xml - # application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype - # application/x-font-ttf application/x-javascript application/xhtml+xml application/xml - # font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon - # image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml; + # brotli_types $compress_mimes; # sends most paths to the backend proxy and ignores the location blocks below, except if # the file exists in /home/mastodon/live location / { - add_header Strict-Transport-Security "max-age=63072000" always; + add_header Strict-Transport-Security $hsts always; try_files $uri @proxy; } # condensed version of original Mastodon nginx.conf location ~ ^/(?:assets|avatars|emoji|headers|packs|shortcuts|sounds)/ { add_header Cache-Control "public, max-age=2419200, must-revalidate"; # 28 days - add_header Strict-Transport-Security "max-age=63072000" always; + add_header Strict-Transport-Security $hsts always; try_files $uri =404; } # media uploads & cache (irrelevant if offloading to S3) location ~ ^/system/ { add_header Cache-Control "public, max-age=2419200, immutable"; # 28 days - add_header Strict-Transport-Security "max-age=63072000" always; + add_header Strict-Transport-Security $hsts always; try_files $uri =404; } # static files *only in the root* of /public (/favicon.ico, /sw.js, /robots.txt, etc.) location ~ ^/[^/]+\.(?:js|css|png|gif|jpg|txt|ico)$ { add_header Cache-Control "public, max-age=604800, must-revalidate"; # 7 days - add_header Strict-Transport-Security "max-age=63072000" always; + add_header Strict-Transport-Security $hsts always; try_files $uri =404; } @@ -107,7 +107,7 @@ server { # security headers proxy_hide_header Strict-Transport-Security; proxy_hide_header X-Powered-By; - add_header Strict-Transport-Security "max-age=63072000" always; + add_header Strict-Transport-Security $hsts always; # debugging headers add_header Via "1.1 $proxy_host" always; @@ -143,7 +143,7 @@ server { proxy_hide_header X-Clacks-Overhead; proxy_hide_header X-XSS-Protection; add_header Referrer-Policy "strict-origin" always; - add_header Strict-Transport-Security "max-age=63072000" always; + add_header Strict-Transport-Security $hsts always; # debugging headers add_header Via "1.1 $proxy_host" always; diff --git a/init.sh b/init.sh index 624c71b..0a821b5 100755 --- a/init.sh +++ b/init.sh @@ -20,7 +20,6 @@ export BACKUPS_ROOT="${BACKUPS_ROOT:="$MASTODON_ROOT/backups"}" export LOGS_ROOT="${LOGS_ROOT:="$MASTODON_ROOT/logs"}" export RBENV_ROOT="${RBENV_ROOT:="$MASTODON_ROOT/.rbenv"}" export NVM_DIR="${NVM_DIR:="$MASTODON_ROOT/.nvm"}" -export MY_NAME_IS_JAKE_JARVIS="${MY_NAME_IS_JAKE_JARVIS:="false"}" # automatically detect glitch-soc # shellcheck disable=SC2155 diff --git a/scripts/backup.sh b/scripts/backup.sh index b83de5b..8b3b218 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -20,16 +20,12 @@ if [ "$(systemctl is-active mastodon-web.service)" = "active" ]; then echo "" fi -if [ ! -d "$BACKUPS_ROOT" ]; then - sudo mkdir -p "$BACKUPS_ROOT" -fi - -# TODO: ugly, do better +# TODO: unsafe & ugly, do better. TEMP_DIR=$(sudo mktemp -d) sudo chmod 777 "$TEMP_DIR" echo "* Backing up Postgres..." -sudo -u postgres pg_dump -Fc mastodon_production -f "$TEMP_DIR/postgres.dump" +sudo -Hiu postgres pg_dump -Fc mastodon_production -f "$TEMP_DIR/postgres.dump" echo "* Backing up Redis..." sudo cp /var/lib/redis/dump.rdb "$TEMP_DIR/redis.rdb" @@ -42,22 +38,48 @@ sudo mkdir -p "$TEMP_DIR/certs" sudo cp -r /etc/letsencrypt/{archive,live,renewal} "$TEMP_DIR/certs/" echo "* Compressing..." -ARCHIVE_DEST="$BACKUPS_ROOT/mastodon-$(date "+%Y.%m.%d-%H.%M.%S").tar.gz" -sudo tar --owner=0 --group=0 -czvf "$ARCHIVE_DEST" -C "$TEMP_DIR" . -sudo chown "$MASTODON_USER":"$MASTODON_USER" "$ARCHIVE_DEST" +TEMP_ARCHIVE="$(sudo mktemp)" +sudo tar --owner=0 --group=0 -czvf "$TEMP_ARCHIVE" -C "$TEMP_DIR" . +sudo mkdir -p "$BACKUPS_ROOT"/{daily,weekly,monthly} +ARCHIVE_FILENAME="mastodon-$(date "+%Y.%m.%d").tar.gz" +# weekly backup (every Sunday) +if [ "$(date +"%u")" -eq 7 ]; then + WEEKLY_DEST="$BACKUPS_ROOT/weekly/$ARCHIVE_FILENAME" + sudo cp -f "$TEMP_ARCHIVE" "$WEEKLY_DEST" + echo "* Saved weekly backup to '$WEEKLY_DEST'" +fi +# monthly backup (first day of the month) +if [ "$(date +"%d")" -eq 1 ]; then + MONTHLY_DEST="$BACKUPS_ROOT/monthly/$ARCHIVE_FILENAME" + sudo cp -f "$TEMP_ARCHIVE" "$MONTHLY_DEST" + echo "* Saved monthly backup to '$MONTHLY_DEST'" +fi +# daily backup (always) +DAILY_DEST="$BACKUPS_ROOT/daily/$ARCHIVE_FILENAME" +sudo cp -f "$TEMP_ARCHIVE" "$DAILY_DEST" +echo "* Saved daily backup to '$DAILY_DEST'" -echo "* Fixing permissions..." -sudo chown -R "$MASTODON_USER":"$MASTODON_USER" "$BACKUPS_ROOT" +echo "* Rotating old backups..." +# NOTE: keep all monthly backups for now +# keep last 4 weekly backups +find "$BACKUPS_ROOT/weekly" -mindepth 1 -type f -mtime +3 -delete +# keep last 5 daily backups +find "$BACKUPS_ROOT/daily" -mindepth 1 -type f -mtime +4 -delete + +# sync backups dir with s3 bucket if s3cmd is installed & BACKUP_S3_BUCKET env var is set +# https://www.linode.com/docs/products/storage/object-storage/guides/s3cmd/ +if [ -n "${BACKUP_S3_BUCKET:+x}" ] && command -v s3cmd >/dev/null 2>&1; then + echo "* Uploading to S3..." + sudo s3cmd sync --delete-removed "$BACKUPS_ROOT/" "s3://$BACKUP_S3_BUCKET" || : +else + echo "⚠ Skipping S3 upload; check that 's3cmd' is present in \$PATH, and \$BACKUP_S3_BUCKET is set." +fi echo "* Removing temp files..." sudo rm -rf --preserve-root "$TEMP_DIR" -echo "* Saved archive to '$ARCHIVE_DEST'" - -if [ -s /usr/local/bin/linode-cli ]; then - echo "* Uploading to S3..." - sudo /usr/local/bin/linode-cli obj put "$ARCHIVE_DEST" jarvis-backup -fi +echo "* Fixing permissions..." +sudo chown -R "$MASTODON_USER":"$MASTODON_USER" "$BACKUPS_ROOT" echo "* 🎉 done! (keep this archive safe!)" diff --git a/scripts/install.sh b/scripts/install.sh index 5cc9000..26e484b 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3,17 +3,17 @@ # exit when any step fails set -euo pipefail +# initialize paths (and silence warnings about things not existing yet because that's why we're running the installer.) +# shellcheck disable=SC1091 +. "$(dirname "${BASH_SOURCE[0]}")"/../init.sh >/dev/null + # can't say you weren't warned :) -if [ "$MY_NAME_IS_JAKE_JARVIS" != "pinky promise" ]; then +if [ "${MY_NAME_IS_JAKE_JARVIS:=}" != "pinky promise" ]; then echo "🚨 LISTEN UP!!!! YOU PROBABLY WANT THIS SCRIPT INSTEAD:" echo "https://github.com/jakejarvis/mastodon-installer/blob/main/install.sh" exit 69 fi -# initialize paths (and silence warnings about things not existing yet because that's why we're running the installer.) -# shellcheck disable=SC1091 -. "$(dirname "${BASH_SOURCE[0]}")"/../init.sh >/dev/null - # check for existing installation if [ -d "$APP_ROOT" ]; then echo "⚠ $APP_ROOT already exists. Are you sure Mastodon isn't already installed?" @@ -288,7 +288,7 @@ as_mastodon touch "$LOGS_ROOT"/cron.log ( sudo crontab -l echo -e "\n$INSTALLER_WUZ_HERE -@weekly bash -c \"$UTILS_ROOT/scripts/weekly_cleanup.sh >> $LOGS_ROOT/cron.log 2>&1\" +@weekly bash -c \"$UTILS_ROOT/scripts/purge.sh >> $LOGS_ROOT/cron.log 2>&1\" @weekly bash -c \"$UTILS_ROOT/scripts/backup.sh >> $LOGS_ROOT/cron.log 2>&1\" # automatically renew Let's Encrypt certificates diff --git a/scripts/weekly_cleanup.sh b/scripts/purge.sh similarity index 57% rename from scripts/weekly_cleanup.sh rename to scripts/purge.sh index 93d41db..5f7dc1b 100755 --- a/scripts/weekly_cleanup.sh +++ b/scripts/purge.sh @@ -2,12 +2,12 @@ # cronjob ran once per week at 3 AM on Sunday; see https://crontab.guru/#0_3_*_*_0 # syntax for crontab -e: -# 0 3 * * 0 bash -c "/home/mastodon/utils/scripts/weekly_cleanup.sh >> /home/mastodon/logs/cron.log 2>&1" +# 0 3 * * 0 bash -c "/home/mastodon/utils/scripts/purge.sh >> /home/mastodon/logs/cron.log 2>&1" # exit when any step fails set -o pipefail -echo -e "\n===== weekly_cleanup.sh: started at $(date '+%Y-%m-%d %H:%M:%S') =====\n" +echo -e "\n===== purge.sh: started at $(date '+%Y-%m-%d %H:%M:%S') =====\n" # initialize paths # shellcheck disable=SC1091 @@ -17,4 +17,4 @@ tootctl media remove --days 14 tootctl media remove --prune-profiles --days 90 tootctl preview_cards remove --days 90 -echo -e "\n===== weekly_cleanup.sh: finished at $(date '+%Y-%m-%d %H:%M:%S') =====\n" +echo -e "\n===== purge.sh: finished at $(date '+%Y-%m-%d %H:%M:%S') =====\n" diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh index b6d4b3b..a47fd90 100755 --- a/scripts/upgrade.sh +++ b/scripts/upgrade.sh @@ -3,17 +3,17 @@ # exit when any step fails set -euo pipefail +# initialize paths +# shellcheck disable=SC1091 +. "$(dirname "${BASH_SOURCE[0]}")"/../init.sh + # can't say you weren't warned :) -if [ "$MY_NAME_IS_JAKE_JARVIS" != "pinky promise" ]; then +if [ "${MY_NAME_IS_JAKE_JARVIS:=}" != "pinky promise" ]; then echo "🚨 LISTEN UP!!!! YOU PROBABLY WANT THIS SCRIPT INSTEAD:" echo "https://github.com/jakejarvis/mastodon-installer/blob/main/upgrade.sh" exit 69 fi -# initialize paths -# shellcheck disable=SC1091 -. "$(dirname "${BASH_SOURCE[0]}")"/../init.sh - # pull latest mastodon source cd "$APP_ROOT" as_mastodon git fetch --all @@ -43,7 +43,7 @@ as_mastodon nvm install as_mastodon nvm use as_mastodon npm install --global yarn # install deps -as_mastodon bundle install --jobs "$(getconf _NPROCESSORS_ONLN)" +as_mastodon bundle check || as_mastodon bundle install --jobs "$(getconf _NPROCESSORS_ONLN)" as_mastodon yarn install --pure-lockfile # compile new assets