mirror of
https://github.com/jakejarvis/mastodon-utils.git
synced 2025-04-26 01:05:22 -04:00
keep weekly and monthly backups (and rotate old ones)
This commit is contained in:
parent
6461633e18
commit
352a57c1dc
GPG Key ID:
2B0C9CF251E69A39
@ -51,14 +51,16 @@ cp .env.example .env
|
||||
|
||||
#### Periodic tasks
|
||||
|
||||
- [`backup.sh`](scripts/backup.sh): Backs up Postgres, Redis, and `.env.production` secrets to a `.tar.gz` file in `/home/mastodon/backups` — useful for a [periodic cronjob](https://github.com/jakejarvis/mastodon-utils/wiki/Cron-jobs#backups).
|
||||
- [`weekly_cleanup.sh`](scripts/weekly_cleanup.sh): Runs Mastodon's built-in [cleanup commands](https://docs.joinmastodon.org/admin/setup/#cleanup), designed for a [weekly cronjob](https://github.com/jakejarvis/mastodon-utils/wiki/Cron-jobs#media-cleanup).
|
||||
- [`backup.sh`](scripts/backup.sh): Backs up Postgres, Redis, and `.env.production` secrets to a `.tar.gz` file in `$MASTODON_ROOT/backups`. Useful for a [daily cronjob](https://github.com/jakejarvis/mastodon-utils/wiki/Cron-jobs#backups).
|
||||
- Keeps archives for the last 5 days, last 4 weeks, and every month
|
||||
- Optionally uploads to S3 with [`s3cmd`](https://s3tools.org/s3cmd)
|
||||
- [`purge.sh`](scripts/purge.sh): Runs Mastodon's built-in [cleanup commands](https://docs.joinmastodon.org/admin/setup/#cleanup), designed for a [weekly cronjob](https://github.com/jakejarvis/mastodon-utils/wiki/Cron-jobs#media-cleanup).
|
||||
- Keeps 14 days of media
|
||||
- Keeps 90 days of profile avatars, headers, and link preview cards
|
||||
|
||||
#### Dangerous
|
||||
|
||||
**The following scripts are highly opinionated, catastrophically destructive, and very specific to me.** Check them out line-by-line instead of running them.
|
||||
> **🚨 The following scripts are highly opinionated, catastrophically destructive, and very specific to me.** Check them out line-by-line instead of running them.
|
||||
|
||||
- [`install.sh`](scripts/install.sh): Assumes an absolutely clean install of Ubuntu and installs Mastodon ***with all of the quirks from this repo.*** Configure `MASTODON_USER` and other paths in `.env` first (see [`.env.example`](.env.example)) if necessary. [Get the far less dangerous version of `install.sh` here instead.](https://github.com/jakejarvis/mastodon-installer/blob/main/install.sh)
|
||||
- [`upgrade.sh`](scripts/upgrade.sh): Upgrades Mastodon server (latest version if vanilla Mastodon, latest commit if `glitch-soc`) and ***re-applies all customizations***. [Get the far less dangerous version of `upgrade.sh` here instead.](https://github.com/jakejarvis/mastodon-installer/blob/main/upgrade.sh)
|
||||
|
||||
@ -36,6 +36,14 @@ server {
|
||||
sendfile on;
|
||||
client_max_body_size 100m;
|
||||
|
||||
# reused values
|
||||
set $hsts "max-age=63072000";
|
||||
set $compress_mimes "application/atom+xml application/javascript application/json application/rss+xml
|
||||
application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype
|
||||
application/x-font-ttf application/x-javascript application/xhtml+xml application/xml
|
||||
font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon
|
||||
image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml";
|
||||
|
||||
gzip on;
|
||||
gzip_disable "msie6";
|
||||
gzip_vary on;
|
||||
@ -44,11 +52,7 @@ server {
|
||||
gzip_buffers 16 8k;
|
||||
gzip_http_version 1.1;
|
||||
gzip_min_length 256;
|
||||
gzip_types application/atom+xml application/javascript application/json application/rss+xml
|
||||
application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype
|
||||
application/x-font-ttf application/x-javascript application/xhtml+xml application/xml
|
||||
font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon
|
||||
image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml;
|
||||
gzip_types $compress_mimes;
|
||||
|
||||
# https://github.com/google/ngx_brotli#sample-configuration
|
||||
# https://github.com/jakejarvis/mastodon-utils/wiki/nginx#brotli-compression
|
||||
@ -56,37 +60,33 @@ server {
|
||||
# brotli_comp_level 4;
|
||||
# brotli_static on;
|
||||
# brotli_min_length 256;
|
||||
# brotli_types application/atom+xml application/javascript application/json application/rss+xml
|
||||
# application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype
|
||||
# application/x-font-ttf application/x-javascript application/xhtml+xml application/xml
|
||||
# font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon
|
||||
# image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml;
|
||||
# brotli_types $compress_mimes;
|
||||
|
||||
# sends most paths to the backend proxy and ignores the location blocks below, except if
|
||||
# the file exists in /home/mastodon/live
|
||||
location / {
|
||||
add_header Strict-Transport-Security "max-age=63072000" always;
|
||||
add_header Strict-Transport-Security $hsts always;
|
||||
try_files $uri @proxy;
|
||||
}
|
||||
|
||||
# condensed version of original Mastodon nginx.conf
|
||||
location ~ ^/(?:assets|avatars|emoji|headers|packs|shortcuts|sounds)/ {
|
||||
add_header Cache-Control "public, max-age=2419200, must-revalidate"; # 28 days
|
||||
add_header Strict-Transport-Security "max-age=63072000" always;
|
||||
add_header Strict-Transport-Security $hsts always;
|
||||
try_files $uri =404;
|
||||
}
|
||||
|
||||
# media uploads & cache (irrelevant if offloading to S3)
|
||||
location ~ ^/system/ {
|
||||
add_header Cache-Control "public, max-age=2419200, immutable"; # 28 days
|
||||
add_header Strict-Transport-Security "max-age=63072000" always;
|
||||
add_header Strict-Transport-Security $hsts always;
|
||||
try_files $uri =404;
|
||||
}
|
||||
|
||||
# static files *only in the root* of /public (/favicon.ico, /sw.js, /robots.txt, etc.)
|
||||
location ~ ^/[^/]+\.(?:js|css|png|gif|jpg|txt|ico)$ {
|
||||
add_header Cache-Control "public, max-age=604800, must-revalidate"; # 7 days
|
||||
add_header Strict-Transport-Security "max-age=63072000" always;
|
||||
add_header Strict-Transport-Security $hsts always;
|
||||
try_files $uri =404;
|
||||
}
|
||||
|
||||
@ -107,7 +107,7 @@ server {
|
||||
# security headers
|
||||
proxy_hide_header Strict-Transport-Security;
|
||||
proxy_hide_header X-Powered-By;
|
||||
add_header Strict-Transport-Security "max-age=63072000" always;
|
||||
add_header Strict-Transport-Security $hsts always;
|
||||
|
||||
# debugging headers
|
||||
add_header Via "1.1 $proxy_host" always;
|
||||
@ -143,7 +143,7 @@ server {
|
||||
proxy_hide_header X-Clacks-Overhead;
|
||||
proxy_hide_header X-XSS-Protection;
|
||||
add_header Referrer-Policy "strict-origin" always;
|
||||
add_header Strict-Transport-Security "max-age=63072000" always;
|
||||
add_header Strict-Transport-Security $hsts always;
|
||||
|
||||
# debugging headers
|
||||
add_header Via "1.1 $proxy_host" always;
|
||||
|
||||
1
init.sh
1
init.sh
@ -20,7 +20,6 @@ export BACKUPS_ROOT="${BACKUPS_ROOT:="$MASTODON_ROOT/backups"}"
|
||||
export LOGS_ROOT="${LOGS_ROOT:="$MASTODON_ROOT/logs"}"
|
||||
export RBENV_ROOT="${RBENV_ROOT:="$MASTODON_ROOT/.rbenv"}"
|
||||
export NVM_DIR="${NVM_DIR:="$MASTODON_ROOT/.nvm"}"
|
||||
export MY_NAME_IS_JAKE_JARVIS="${MY_NAME_IS_JAKE_JARVIS:="false"}"
|
||||
|
||||
# automatically detect glitch-soc
|
||||
# shellcheck disable=SC2155
|
||||
|
||||
@ -20,16 +20,12 @@ if [ "$(systemctl is-active mastodon-web.service)" = "active" ]; then
|
||||
echo ""
|
||||
fi
|
||||
|
||||
if [ ! -d "$BACKUPS_ROOT" ]; then
|
||||
sudo mkdir -p "$BACKUPS_ROOT"
|
||||
fi
|
||||
|
||||
# TODO: ugly, do better
|
||||
# TODO: unsafe & ugly, do better.
|
||||
TEMP_DIR=$(sudo mktemp -d)
|
||||
sudo chmod 777 "$TEMP_DIR"
|
||||
|
||||
echo "* Backing up Postgres..."
|
||||
sudo -u postgres pg_dump -Fc mastodon_production -f "$TEMP_DIR/postgres.dump"
|
||||
sudo -Hiu postgres pg_dump -Fc mastodon_production -f "$TEMP_DIR/postgres.dump"
|
||||
|
||||
echo "* Backing up Redis..."
|
||||
sudo cp /var/lib/redis/dump.rdb "$TEMP_DIR/redis.rdb"
|
||||
@ -42,22 +38,48 @@ sudo mkdir -p "$TEMP_DIR/certs"
|
||||
sudo cp -r /etc/letsencrypt/{archive,live,renewal} "$TEMP_DIR/certs/"
|
||||
|
||||
echo "* Compressing..."
|
||||
ARCHIVE_DEST="$BACKUPS_ROOT/mastodon-$(date "+%Y.%m.%d-%H.%M.%S").tar.gz"
|
||||
sudo tar --owner=0 --group=0 -czvf "$ARCHIVE_DEST" -C "$TEMP_DIR" .
|
||||
sudo chown "$MASTODON_USER":"$MASTODON_USER" "$ARCHIVE_DEST"
|
||||
TEMP_ARCHIVE="$(sudo mktemp)"
|
||||
sudo tar --owner=0 --group=0 -czvf "$TEMP_ARCHIVE" -C "$TEMP_DIR" .
|
||||
sudo mkdir -p "$BACKUPS_ROOT"/{daily,weekly,monthly}
|
||||
ARCHIVE_FILENAME="mastodon-$(date "+%Y.%m.%d").tar.gz"
|
||||
# weekly backup (every Sunday)
|
||||
if [ "$(date +"%u")" -eq 7 ]; then
|
||||
WEEKLY_DEST="$BACKUPS_ROOT/weekly/$ARCHIVE_FILENAME"
|
||||
sudo cp -f "$TEMP_ARCHIVE" "$WEEKLY_DEST"
|
||||
echo "* Saved weekly backup to '$WEEKLY_DEST'"
|
||||
fi
|
||||
# monthly backup (first day of the month)
|
||||
if [ "$(date +"%d")" -eq 1 ]; then
|
||||
MONTHLY_DEST="$BACKUPS_ROOT/monthly/$ARCHIVE_FILENAME"
|
||||
sudo cp -f "$TEMP_ARCHIVE" "$MONTHLY_DEST"
|
||||
echo "* Saved monthly backup to '$MONTHLY_DEST'"
|
||||
fi
|
||||
# daily backup (always)
|
||||
DAILY_DEST="$BACKUPS_ROOT/daily/$ARCHIVE_FILENAME"
|
||||
sudo cp -f "$TEMP_ARCHIVE" "$DAILY_DEST"
|
||||
echo "* Saved daily backup to '$DAILY_DEST'"
|
||||
|
||||
echo "* Fixing permissions..."
|
||||
sudo chown -R "$MASTODON_USER":"$MASTODON_USER" "$BACKUPS_ROOT"
|
||||
echo "* Rotating old backups..."
|
||||
# NOTE: keep all monthly backups for now
|
||||
# keep last 4 weekly backups
|
||||
find "$BACKUPS_ROOT/weekly" -mindepth 1 -type f -mtime +3 -delete
|
||||
# keep last 5 daily backups
|
||||
find "$BACKUPS_ROOT/daily" -mindepth 1 -type f -mtime +4 -delete
|
||||
|
||||
# sync backups dir with s3 bucket if s3cmd is installed & BACKUP_S3_BUCKET env var is set
|
||||
# https://www.linode.com/docs/products/storage/object-storage/guides/s3cmd/
|
||||
if [ -n "${BACKUP_S3_BUCKET:+x}" ] && command -v s3cmd >/dev/null 2>&1; then
|
||||
echo "* Uploading to S3..."
|
||||
sudo s3cmd sync --delete-removed "$BACKUPS_ROOT/" "s3://$BACKUP_S3_BUCKET" || :
|
||||
else
|
||||
echo "⚠ Skipping S3 upload; check that 's3cmd' is present in \$PATH, and \$BACKUP_S3_BUCKET is set."
|
||||
fi
|
||||
|
||||
echo "* Removing temp files..."
|
||||
sudo rm -rf --preserve-root "$TEMP_DIR"
|
||||
|
||||
echo "* Saved archive to '$ARCHIVE_DEST'"
|
||||
|
||||
if [ -s /usr/local/bin/linode-cli ]; then
|
||||
echo "* Uploading to S3..."
|
||||
sudo /usr/local/bin/linode-cli obj put "$ARCHIVE_DEST" jarvis-backup
|
||||
fi
|
||||
echo "* Fixing permissions..."
|
||||
sudo chown -R "$MASTODON_USER":"$MASTODON_USER" "$BACKUPS_ROOT"
|
||||
|
||||
echo "* 🎉 done! (keep this archive safe!)"
|
||||
|
||||
|
||||
@ -3,17 +3,17 @@
|
||||
# exit when any step fails
|
||||
set -euo pipefail
|
||||
|
||||
# initialize paths (and silence warnings about things not existing yet because that's why we're running the installer.)
|
||||
# shellcheck disable=SC1091
|
||||
. "$(dirname "${BASH_SOURCE[0]}")"/../init.sh >/dev/null
|
||||
|
||||
# can't say you weren't warned :)
|
||||
if [ "$MY_NAME_IS_JAKE_JARVIS" != "pinky promise" ]; then
|
||||
if [ "${MY_NAME_IS_JAKE_JARVIS:=}" != "pinky promise" ]; then
|
||||
echo "🚨 LISTEN UP!!!! YOU PROBABLY WANT THIS SCRIPT INSTEAD:"
|
||||
echo "https://github.com/jakejarvis/mastodon-installer/blob/main/install.sh"
|
||||
exit 69
|
||||
fi
|
||||
|
||||
# initialize paths (and silence warnings about things not existing yet because that's why we're running the installer.)
|
||||
# shellcheck disable=SC1091
|
||||
. "$(dirname "${BASH_SOURCE[0]}")"/../init.sh >/dev/null
|
||||
|
||||
# check for existing installation
|
||||
if [ -d "$APP_ROOT" ]; then
|
||||
echo "⚠ $APP_ROOT already exists. Are you sure Mastodon isn't already installed?"
|
||||
@ -288,7 +288,7 @@ as_mastodon touch "$LOGS_ROOT"/cron.log
|
||||
(
|
||||
sudo crontab -l
|
||||
echo -e "\n$INSTALLER_WUZ_HERE
|
||||
@weekly bash -c \"$UTILS_ROOT/scripts/weekly_cleanup.sh >> $LOGS_ROOT/cron.log 2>&1\"
|
||||
@weekly bash -c \"$UTILS_ROOT/scripts/purge.sh >> $LOGS_ROOT/cron.log 2>&1\"
|
||||
@weekly bash -c \"$UTILS_ROOT/scripts/backup.sh >> $LOGS_ROOT/cron.log 2>&1\"
|
||||
|
||||
# automatically renew Let's Encrypt certificates
|
||||
|
||||
@ -2,12 +2,12 @@
|
||||
|
||||
# cronjob ran once per week at 3 AM on Sunday; see https://crontab.guru/#0_3_*_*_0
|
||||
# syntax for crontab -e:
|
||||
# 0 3 * * 0 bash -c "/home/mastodon/utils/scripts/weekly_cleanup.sh >> /home/mastodon/logs/cron.log 2>&1"
|
||||
# 0 3 * * 0 bash -c "/home/mastodon/utils/scripts/purge.sh >> /home/mastodon/logs/cron.log 2>&1"
|
||||
|
||||
# exit when any step fails
|
||||
set -o pipefail
|
||||
|
||||
echo -e "\n===== weekly_cleanup.sh: started at $(date '+%Y-%m-%d %H:%M:%S') =====\n"
|
||||
echo -e "\n===== purge.sh: started at $(date '+%Y-%m-%d %H:%M:%S') =====\n"
|
||||
|
||||
# initialize paths
|
||||
# shellcheck disable=SC1091
|
||||
@ -17,4 +17,4 @@ tootctl media remove --days 14
|
||||
tootctl media remove --prune-profiles --days 90
|
||||
tootctl preview_cards remove --days 90
|
||||
|
||||
echo -e "\n===== weekly_cleanup.sh: finished at $(date '+%Y-%m-%d %H:%M:%S') =====\n"
|
||||
echo -e "\n===== purge.sh: finished at $(date '+%Y-%m-%d %H:%M:%S') =====\n"
|
||||
@ -3,17 +3,17 @@
|
||||
# exit when any step fails
|
||||
set -euo pipefail
|
||||
|
||||
# initialize paths
|
||||
# shellcheck disable=SC1091
|
||||
. "$(dirname "${BASH_SOURCE[0]}")"/../init.sh
|
||||
|
||||
# can't say you weren't warned :)
|
||||
if [ "$MY_NAME_IS_JAKE_JARVIS" != "pinky promise" ]; then
|
||||
if [ "${MY_NAME_IS_JAKE_JARVIS:=}" != "pinky promise" ]; then
|
||||
echo "🚨 LISTEN UP!!!! YOU PROBABLY WANT THIS SCRIPT INSTEAD:"
|
||||
echo "https://github.com/jakejarvis/mastodon-installer/blob/main/upgrade.sh"
|
||||
exit 69
|
||||
fi
|
||||
|
||||
# initialize paths
|
||||
# shellcheck disable=SC1091
|
||||
. "$(dirname "${BASH_SOURCE[0]}")"/../init.sh
|
||||
|
||||
# pull latest mastodon source
|
||||
cd "$APP_ROOT"
|
||||
as_mastodon git fetch --all
|
||||
@ -43,7 +43,7 @@ as_mastodon nvm install
|
||||
as_mastodon nvm use
|
||||
as_mastodon npm install --global yarn
|
||||
# install deps
|
||||
as_mastodon bundle install --jobs "$(getconf _NPROCESSORS_ONLN)"
|
||||
as_mastodon bundle check || as_mastodon bundle install --jobs "$(getconf _NPROCESSORS_ONLN)"
|
||||
as_mastodon yarn install --pure-lockfile
|
||||
|
||||
# compile new assets
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user