jake/mastodon-utils
1
Fork 0
mirror of https://github.com/jakejarvis/mastodon-utils.git synced 2025-04-26 07:55:22 -04:00
Code Issues Releases Wiki Activity

specifically block Twitterbot from robots.txt

Browse Source
This commit is contained in:
Jake Jarvis 2022-12-16 09:11:15 -05:00
parent 2b1b586d88
commit 30f958e1c5
Signed by: jake
GPG Key ID: 2B0C9CF251E69A39
4 changed files with 226 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

234
etc/grafana/public-dashboard.json
View File

File diff suppressed because one or more lines are too long

8
etc/nginx/sites-available/mastodon.conf
View File

@ -44,24 +44,24 @@ server {
gzip_comp_level 6; gzip_comp_level 6;
gzip_buffers 16 8k; gzip_buffers 16 8k;
gzip_http_version 1.1; gzip_http_version 1.1;
gzip_types application/atom+xml application/javascript application/json application/rss+xml gzip_min_length 256;
gzip_types application/atom+xml application/javascript application/json application/rss+xml
application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype
application/x-font-ttf application/x-javascript application/xhtml+xml application/xml application/x-font-ttf application/x-javascript application/xhtml+xml application/xml
font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon
image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml; image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml;
gzip_min_length 256;
# https://github.com/google/ngx_brotli#sample-configuration # https://github.com/google/ngx_brotli#sample-configuration
# https://github.com/jakejarvis/mastodon-utils/wiki/Brotli-compression-for-nginx # https://github.com/jakejarvis/mastodon-utils/wiki/Brotli-compression-for-nginx
brotli on; brotli on;
brotli_comp_level 4; brotli_comp_level 4;
brotli_static on; brotli_static on;
brotli_types application/atom+xml application/javascript application/json application/rss+xml brotli_min_length 256;
brotli_types application/atom+xml application/javascript application/json application/rss+xml
application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype
application/x-font-ttf application/x-javascript application/xhtml+xml application/xml application/x-font-ttf application/x-javascript application/xhtml+xml application/xml
font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon
image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml; image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml;
brotli_min_length 256;
# add shortcut to public Grafana dashboard # add shortcut to public Grafana dashboard
location ~ ^/dashboard/?$ { location ~ ^/dashboard/?$ {

10
patches/robots.patch
View File

@ -1,11 +1,17 @@
diff --git a/public/robots.txt b/public/robots.txt diff --git a/public/robots.txt b/public/robots.txt
index 771bf2160..1f53798bb 100644 index 771bf2160b..af16ddb0ce 100644
--- a/public/robots.txt --- a/public/robots.txt
+++ b/public/robots.txt +++ b/public/robots.txt
@@ -1,5 +1,2 @@ @@ -1,5 +1,8 @@
-# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file -# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
- -
+# block everything except About page
User-agent: * User-agent: *
-Disallow: /media_proxy/ -Disallow: /media_proxy/
-Disallow: /interact/ -Disallow: /interact/
+Allow: /about
+Disallow: /
+
+# sorry, Elon
+User-agent: Twitterbot
+Disallow: / +Disallow: /

2
scripts/upgrade.sh
View File

@ -34,7 +34,7 @@ fi
# create blank custom.css (this overrides any CSS set in the admin panel, but if that's not being used, then # create blank custom.css (this overrides any CSS set in the admin panel, but if that's not being used, then
# this quickly saves a request to the backend) # this quickly saves a request to the backend)
sudo -u mastodon "$APP_ROOT/public/custom.css" sudo -u mastodon touch "$APP_ROOT/public/custom.css"
# set new ruby version # set new ruby version
RUBY_VERSION="$(sudo -u mastodon cat $APP_ROOT/.ruby-version)" RUBY_VERSION="$(sudo -u mastodon cat $APP_ROOT/.ruby-version)"