specifically block Twitterbot from robots.txt

2026-06-13 02:05:27 -04:00 · 2022-12-16 09:11:15 -05:00
parent 2b1b586d88
commit 30f958e1c5
4 changed files with 226 additions and 28 deletions
@@ -1,11 +1,17 @@
 diff --git a/public/robots.txt b/public/robots.txt
-index 771bf2160..1f53798bb 100644
+index 771bf2160b..af16ddb0ce 100644
 --- a/public/robots.txt
 +++ b/public/robots.txt
-@@ -1,5 +1,2 @@
+@@ -1,5 +1,8 @@
 -# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
 -
+# block everything except About page
 User-agent: *
 -Disallow: /media_proxy/
 -Disallow: /interact/
+Allow: /about
+Disallow: /
+
+# sorry, Elon
+User-agent: Twitterbot
 +Disallow: /