jake/jarv.is
1
Fork 0
mirror of https://github.com/jakejarvis/jarv.is.git synced 2025-10-28 12:05:50 -04:00
Code Issues Activity

block AI crap via robots.txt

Browse Source
This commit is contained in:
Jake Jarvis
2024-02-27 14:23:14 -05:00
parent 815059a9e2
commit 2de3914449
1 changed files with 26 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
pages/robots.txt.ts
View File

@@ -6,13 +6,33 @@ export const getServerSideProps: GetServerSideProps<Record<string, never>> = asy
// this production check should be unnecessary because "noindex" and "nofollow" are also set in a meta tag (see // this production check should be unnecessary because "noindex" and "nofollow" are also set in a meta tag (see
// DefaultSeo's props in pages/_app.tsx), but it doesn't hurt... // DefaultSeo's props in pages/_app.tsx), but it doesn't hurt...
const robots = `User-agent: * const robots = `User-agent: *
${ ${process.env.NEXT_PUBLIC_VERCEL_ENV !== "production" ? `Disallow: /` : `Allow: /`}
process.env.NEXT_PUBLIC_VERCEL_ENV !== "production"
? `Disallow: /`
: `Allow: /
Sitemap: ${process.env.NEXT_PUBLIC_BASE_URL || ""}/sitemap.xml` # Block CommonCrawl
} User-agent: CCBot
Disallow: /
# Block Google Bard
User-agent: Google-Extended
Disallow: /
# Block OpenAI & ChatGPT
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
# Block Anthropic AI
User-agent: anthropic-ai
Disallow: /
User-agent: ClaudeBot
Disallow: /
# Block ByteDance
User-agent: Bytespider
Disallow: /
Sitemap: ${process.env.NEXT_PUBLIC_BASE_URL || ""}/sitemap.xml
`; `;
res.setHeader("content-type", "text/plain; charset=utf-8"); res.setHeader("content-type", "text/plain; charset=utf-8");