feat: add /llms.txt, /llms-full.txt, and Accept-header content negotiation for LLM-friendly docs

- Add `/llms.txt` route using Fumadocs' `llms().index()` helper (follows the https://llmstxt.org convention) and `/llms-full.txt` that concatenates the processed Markdown of every docs page into a single response
- Enable `includeProcessedMarkdown` on the docs collection and add `getLLMText` + `markdownPathToSlugs` helpers in `lib/source.ts`
- Add `src/start.ts` with a `llmMiddleware` that intercepts `/docs/**` requests: serves raw Markdown when the path ends in `.md` or when the `Accept` header prefers Markdown (via `fumadocs-core/negotiation`), short-circuiting TanStack Start's normal rendering

apps/web/source.config.ts

@@ -2,4 +2,9 @@ import { defineDocs } from "fumadocs-mdx/config";
 
 export const docs = defineDocs({
   dir: "content/docs",
+  docs: {
+    postprocess: {
+      includeProcessedMarkdown: true,
+    },
+  },
 });

apps/web/src/lib/source.ts

@@ -5,3 +5,27 @@ export const source = loader({
   baseUrl: "/docs",
   source: docs.toFumadocsSource(),
 });
+
+/**
+ * Convert a `/docs/*.md` URL path into loader slugs: strip the trailing `.md`
+ * and collapse a lone `index` to the docs root. Used by the per-page markdown
+ * route and the `Accept`-header content negotiation middleware.
+ */
+export function markdownPathToSlugs(segs: string[]) {
+  if (segs.length === 0) return [];
+  const out = [...segs];
+  const last = out.length - 1;
+  out[last] = (out[last] ?? "").replace(/\.md$/, "");
+  if (out.length === 1 && out[0] === "index") out.pop();
+  return out;
+}
+
+/**
+ * Render a single docs page as LLM-friendly Markdown. Requires
+ * `includeProcessedMarkdown` to be enabled on the docs collection
+ * above, which exposes `getText("processed")`.
+ */
+export async function getLLMText(page: (typeof source)["$inferPage"]) {
+  const processed = await page.data.getText("processed");
+  return `# ${page.data.title} (${page.url})\n\n${processed}`;
+}

apps/web/src/routeTree.gen.ts

@@ -10,6 +10,8 @@
 
 import { Route as rootRouteImport } from './routes/__root'
 import { Route as SitemapDotxmlRouteImport } from './routes/sitemap[.]xml'
+import { Route as LlmsDottxtRouteImport } from './routes/llms[.]txt'
+import { Route as LlmsFullDottxtRouteImport } from './routes/llms-full[.]txt'
 import { Route as IndexRouteImport } from './routes/index'
 import { Route as OgIndexRouteImport } from './routes/og.index'
 import { Route as DocsSplatRouteImport } from './routes/docs.$'
@@ -22,6 +24,16 @@ const SitemapDotxmlRoute = SitemapDotxmlRouteImport.update({
   path: '/sitemap.xml',
   getParentRoute: () => rootRouteImport,
 } as any)
+const LlmsDottxtRoute = LlmsDottxtRouteImport.update({
+  id: '/llms.txt',
+  path: '/llms.txt',
+  getParentRoute: () => rootRouteImport,
+} as any)
+const LlmsFullDottxtRoute = LlmsFullDottxtRouteImport.update({
+  id: '/llms-full.txt',
+  path: '/llms-full.txt',
+  getParentRoute: () => rootRouteImport,
+} as any)
 const IndexRoute = IndexRouteImport.update({
   id: '/',
   path: '/',
@@ -55,6 +67,8 @@ const MSlotIdRoute = MSlotIdRouteImport.update({
 
 export interface FileRoutesByFullPath {
   '/': typeof IndexRoute
+  '/llms-full.txt': typeof LlmsFullDottxtRoute
+  '/llms.txt': typeof LlmsDottxtRoute
   '/sitemap.xml': typeof SitemapDotxmlRoute
   '/api/events': typeof ApiEventsRoute
   '/docs/$': typeof DocsSplatRoute
@@ -64,6 +78,8 @@ export interface FileRoutesByFullPath {
 }
 export interface FileRoutesByTo {
   '/': typeof IndexRoute
+  '/llms-full.txt': typeof LlmsFullDottxtRoute
+  '/llms.txt': typeof LlmsDottxtRoute
   '/sitemap.xml': typeof SitemapDotxmlRoute
   '/api/events': typeof ApiEventsRoute
   '/docs/$': typeof DocsSplatRoute
@@ -74,6 +90,8 @@ export interface FileRoutesByTo {
 export interface FileRoutesById {
   __root__: typeof rootRouteImport
   '/': typeof IndexRoute
+  '/llms-full.txt': typeof LlmsFullDottxtRoute
+  '/llms.txt': typeof LlmsDottxtRoute
   '/sitemap.xml': typeof SitemapDotxmlRoute
   '/api/events': typeof ApiEventsRoute
   '/docs/$': typeof DocsSplatRoute
@@ -85,6 +103,8 @@ export interface FileRouteTypes {
   fileRoutesByFullPath: FileRoutesByFullPath
   fullPaths:
     | '/'
+    | '/llms-full.txt'
+    | '/llms.txt'
     | '/sitemap.xml'
     | '/api/events'
     | '/docs/$'
@@ -94,6 +114,8 @@ export interface FileRouteTypes {
   fileRoutesByTo: FileRoutesByTo
   to:
     | '/'
+    | '/llms-full.txt'
+    | '/llms.txt'
     | '/sitemap.xml'
     | '/api/events'
     | '/docs/$'
@@ -103,6 +125,8 @@ export interface FileRouteTypes {
   id:
     | '__root__'
     | '/'
+    | '/llms-full.txt'
+    | '/llms.txt'
     | '/sitemap.xml'
     | '/api/events'
     | '/docs/$'
@@ -113,6 +137,8 @@ export interface FileRouteTypes {
 }
 export interface RootRouteChildren {
   IndexRoute: typeof IndexRoute
+  LlmsFullDottxtRoute: typeof LlmsFullDottxtRoute
+  LlmsDottxtRoute: typeof LlmsDottxtRoute
   SitemapDotxmlRoute: typeof SitemapDotxmlRoute
   ApiEventsRoute: typeof ApiEventsRoute
   DocsSplatRoute: typeof DocsSplatRoute
@@ -130,6 +156,20 @@ declare module '@tanstack/react-router' {
       preLoaderRoute: typeof SitemapDotxmlRouteImport
       parentRoute: typeof rootRouteImport
     }
+    '/llms.txt': {
+      id: '/llms.txt'
+      path: '/llms.txt'
+      fullPath: '/llms.txt'
+      preLoaderRoute: typeof LlmsDottxtRouteImport
+      parentRoute: typeof rootRouteImport
+    }
+    '/llms-full.txt': {
+      id: '/llms-full.txt'
+      path: '/llms-full.txt'
+      fullPath: '/llms-full.txt'
+      preLoaderRoute: typeof LlmsFullDottxtRouteImport
+      parentRoute: typeof rootRouteImport
+    }
     '/': {
       id: '/'
       path: '/'
@@ -177,6 +217,8 @@ declare module '@tanstack/react-router' {
 
 const rootRouteChildren: RootRouteChildren = {
   IndexRoute: IndexRoute,
+  LlmsFullDottxtRoute: LlmsFullDottxtRoute,
+  LlmsDottxtRoute: LlmsDottxtRoute,
   SitemapDotxmlRoute: SitemapDotxmlRoute,
   ApiEventsRoute: ApiEventsRoute,
   DocsSplatRoute: DocsSplatRoute,

apps/web/src/routes/llms-full[.]txt.ts

@@ -0,0 +1,23 @@
+import { createFileRoute } from "@tanstack/react-router";
+
+import { getLLMText, source } from "@/lib/source";
+
+/**
+ * `/llms-full.txt` — the full processed Markdown of every docs page,
+ * concatenated, for LLMs that ingest the entire corpus in one request.
+ */
+export const Route = createFileRoute("/llms-full.txt")({
+  server: {
+    handlers: {
+      GET: async () => {
+        const scanned = await Promise.all(source.getPages().map(getLLMText));
+        return new Response(scanned.join("\n\n"), {
+          headers: {
+            "content-type": "text/plain; charset=utf-8",
+            "cache-control": "public, max-age=3600, s-maxage=86400",
+          },
+        });
+      },
+    },
+  },
+});

apps/web/src/routes/llms[.]txt.ts

@@ -0,0 +1,22 @@
+import { createFileRoute } from "@tanstack/react-router";
+import { llms } from "fumadocs-core/source";
+
+import { source } from "@/lib/source";
+
+/**
+ * `/llms.txt` — a Markdown index of every docs page, following the
+ * https://llmstxt.org convention. Generated by Fumadocs' `llms()` helper.
+ */
+export const Route = createFileRoute("/llms.txt")({
+  server: {
+    handlers: {
+      GET: () =>
+        new Response(llms(source).index(), {
+          headers: {
+            "content-type": "text/plain; charset=utf-8",
+            "cache-control": "public, max-age=3600, s-maxage=86400",
+          },
+        }),
+    },
+  },
+});

apps/web/src/start.ts

@@ -0,0 +1,33 @@
+import { createCsrfMiddleware, createMiddleware, createStart } from "@tanstack/react-start";
+import { isMarkdownPreferred } from "fumadocs-core/negotiation";
+
+import { getLLMText, markdownPathToSlugs, source } from "@/lib/source";
+
+const csrfMiddleware = createCsrfMiddleware({
+  filter: (ctx) => ctx.handlerType === "serverFn",
+});
+
+const llmMiddleware = createMiddleware({ type: "request" }).server(async ({ request, next }) => {
+  const { pathname } = new URL(request.url);
+
+  if (pathname.startsWith("/docs")) {
+    if (pathname.endsWith(".md") || isMarkdownPreferred(request)) {
+      const segs = pathname.slice("/docs".length).split("/").filter(Boolean);
+      const page = source.getPage(markdownPathToSlugs(segs));
+      if (page) {
+        return new Response(await getLLMText(page), {
+          headers: {
+            "content-type": "text/markdown; charset=utf-8",
+            "cache-control": "public, max-age=3600, s-maxage=86400",
+          },
+        });
+      }
+    }
+  }
+
+  return next();
+});
+
+export const startInstance = createStart(() => ({
+  requestMiddleware: [csrfMiddleware, llmMiddleware],
+}));