From 00165a5871ac1ecc822c61d66c072574049cd560 Mon Sep 17 00:00:00 2001
From: Jake Jarvis <jake@jarv.is>
Date: Sun, 23 Jun 2024 13:23:15 -0400
Subject: [PATCH] block even more AI bullsh*t

---
 pages/robots.txt.ts | 56 +++++++++++++++++++++++++++------------------
 1 file changed, 34 insertions(+), 22 deletions(-)

diff --git a/pages/robots.txt.ts b/pages/robots.txt.ts
index e0004b18..7ffaedb1 100644
--- a/pages/robots.txt.ts
+++ b/pages/robots.txt.ts
@@ -6,31 +6,43 @@ export const getServerSideProps: GetServerSideProps<Record<string, never>> = asy
   // this production check should be unnecessary because "noindex" and "nofollow" are also set in a meta tag (see
   // DefaultSeo's props in pages/_app.tsx), but it doesn't hurt...
   const robots = `User-agent: *
-${process.env.NEXT_PUBLIC_VERCEL_ENV !== "production" ? `Disallow: /` : `Allow: /`}
+${
+  process.env.NEXT_PUBLIC_VERCEL_ENV !== "production"
+    ? `Disallow: /`
+    : `Allow: /
 
-# Block CommonCrawl
-User-agent: CCBot
-Disallow: /
-
-# Block Google Bard
-User-agent: Google-Extended
-Disallow: /
-
-# Block OpenAI & ChatGPT
-User-agent: GPTBot
-Disallow: /
-User-agent: ChatGPT-User
-Disallow: /
-
-# Block Anthropic AI
+# I'm already _so_ over this shit...
+# https://github.com/ai-robots-txt/ai.robots.txt/blob/main/robots.txt
+User-agent: AdsBot-Google
+User-agent: Amazonbot
 User-agent: anthropic-ai
-Disallow: /
-User-agent: ClaudeBot
-Disallow: /
-
-# Block ByteDance
+User-agent: Applebot-Extended
 User-agent: Bytespider
-Disallow: /
+User-agent: CCBot
+User-agent: ChatGPT-User
+User-agent: Claude-Web
+User-agent: ClaudeBot
+User-agent: cohere-ai
+User-agent: Diffbot
+User-agent: FacebookBot
+User-agent: FriendlyCrawler
+User-agent: Google-Extended
+User-agent: GPTBot
+User-agent: img2dataset
+User-agent: omgili
+User-agent: omgilibot
+User-agent: peer39_crawler
+User-agent: peer39_crawler/1.0
+User-agent: PerplexityBot
+User-agent: YouBot
+User-agent: AhrefsBot
+User-agent: BLEXBot
+User-agent: DataForSeoBot
+User-agent: magpie-crawler
+User-agent: MJ12bot
+User-agent: TurnitinBot
+Disallow: /`
+}
 
 Sitemap: ${process.env.NEXT_PUBLIC_BASE_URL || ""}/sitemap.xml
 `;