mirror of
https://github.com/jakejarvis/mastodon-utils.git
synced 2025-04-26 02:15:22 -04:00
18 lines
434 B
Diff
18 lines
434 B
Diff
diff --git a/public/robots.txt b/public/robots.txt
|
|
index 771bf2160b..af16ddb0ce 100644
|
|
--- a/public/robots.txt
|
|
+++ b/public/robots.txt
|
|
@@ -1,5 +1,8 @@
|
|
-# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
|
|
-
|
|
+# block everything except About page
|
|
User-agent: *
|
|
-Disallow: /media_proxy/
|
|
-Disallow: /interact/
|
|
+Allow: /about
|
|
+Disallow: /
|
|
+
|
|
+# sorry, Elon
|
|
+User-agent: Twitterbot
|
|
+Disallow: /
|