jake/jarv.is
jake/jarv.is
1
Fork 0
mirror of https://github.com/jakejarvis/jarv.is.git synced 2025-04-26 04:45:22 -04:00
Code Issues

block AI crap via robots.txt

Browse Source
This commit is contained in:
Jake Jarvis 2024-02-27 14:23:14 -05:00
parent 815059a9e2
commit 2de3914449
Signed by: jake
SSH Key Fingerprint: SHA256:nCkvAjYA6XaSPUqc4TfbBQTpzr8Xj7ritg/sGInCdkc
1 changed files with 26 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
pages/robots.txt.ts
View File

@ -6,13 +6,33 @@ export const getServerSideProps: GetServerSideProps<Record<string, never>> = asy
// this production check should be unnecessary because "noindex" and "nofollow" are also set in a meta tag (see
// DefaultSeo's props in pages/_app.tsx), but it doesn't hurt...
const robots = `User-agent: *
${
process.env.NEXT_PUBLIC_VERCEL_ENV !== "production"
? `Disallow: /`
: `Allow: /
${process.env.NEXT_PUBLIC_VERCEL_ENV !== "production" ? `Disallow: /` : `Allow: /`}
Sitemap: ${process.env.NEXT_PUBLIC_BASE_URL || ""}/sitemap.xml`
}
# Block CommonCrawl
User-agent: CCBot
Disallow: /
# Block Google Bard
User-agent: Google-Extended
Disallow: /
# Block OpenAI & ChatGPT
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
# Block Anthropic AI
User-agent: anthropic-ai
Disallow: /
User-agent: ClaudeBot
Disallow: /
# Block ByteDance
User-agent: Bytespider
Disallow: /
Sitemap: ${process.env.NEXT_PUBLIC_BASE_URL || ""}/sitemap.xml
`;
res.setHeader("content-type", "text/plain; charset=utf-8");