feat: llms.txt/llms-full.txt route handlers

app/llms-full.txt/route.ts

@@ -0,0 +1,27 @@
+import {NextResponse} from 'next/server';
+import {getBaseUrl} from '@/lib/docs-utils';
+import {generateLlmsFullTxt} from '@/lib/llms-text';
+
+export const revalidate = false;
+export const dynamic = 'force-static';
+
+const TEXT_HEADERS = {
+  'Content-Type': 'text/plain; charset=utf-8',
+  'Cache-Control': 'public, max-age=0, must-revalidate',
+};
+
+export async function GET() {
+  try {
+    const baseUrl = getBaseUrl();
+    const body = (await generateLlmsFullTxt(baseUrl)).trim();
+
+    return new NextResponse(body, {headers: TEXT_HEADERS});
+  } catch (err) {
+    console.error('Error generating /llms-full.txt:', err);
+
+    return new NextResponse('Internal server error', {
+      status: 500,
+      headers: TEXT_HEADERS,
+    });
+  }
+}

app/llms.txt/route.ts

@@ -0,0 +1,27 @@
+import {NextResponse} from 'next/server';
+import {getBaseUrl} from '@/lib/docs-utils';
+import {generateLlmsTxt} from '@/lib/llms-text';
+
+export const revalidate = false;
+export const dynamic = 'force-static';
+
+const TEXT_HEADERS = {
+  'Content-Type': 'text/plain; charset=utf-8',
+  'Cache-Control': 'public, max-age=0, must-revalidate',
+};
+
+export async function GET() {
+  try {
+    const baseUrl = getBaseUrl();
+    const body = (await generateLlmsTxt(baseUrl)).trim();
+
+    return new NextResponse(body, {headers: TEXT_HEADERS});
+  } catch (err) {
+    console.error('Error generating /llms.txt:', err);
+
+    return new NextResponse('Internal server error', {
+      status: 500,
+      headers: TEXT_HEADERS,
+    });
+  }
+}

assets/search-index.json

@@ -94,7 +94,7 @@
     "id": "13-llms",
     "title": "Welcome, 🤖!",
     "url": "/docs/llms",
-    "content": "Are you an LLM? Do you like long walks through vector space and late-night tokenization? Or maybe you're a friend of an LLM, just trying to make life a little easier for the contextually challenged? Either way, you're in the right place! Stream on over to llms.txt for the text-only version of these docs.",
+    "content": "Are you an LLM? Do you like long walks through vector space and late-night tokenization? Or maybe you're a friend of an LLM, just trying to make life a little easier for the contextually challenged? Either way, you're in the right place! Here are some ways to get started: llms.txt - index of all documentation pages with links to individual pages llms-full.txt - full corpus of all documentation (for models that can't follow links) /docs/\\<slug>.md - individual pages as markdown (e.g. /docs/llms.md)",
     "headings": []
   },
   {

contents/docs/llms.mdx

@@ -8,4 +8,8 @@ Do you like long walks through vector space and late-night tokenization?
 
 Or maybe you're a friend of an LLM, just trying to make life a little easier for the contextually challenged?
 
-Either way, you're in the right place! Stream on over to [llms.txt](/llms.txt) for the text-only version of these docs.
+Either way, you're in the right place! Here are some ways to get started:
+
+- [llms.txt](/llms.txt) - index of all documentation pages with links to individual pages
+- [llms-full.txt](/llms-full.txt) - full corpus of all documentation (for models that can't follow links)
+- `/docs/<slug>.md` - individual pages as markdown (e.g. `/docs/llms.md`)

lib/llms-text.ts

@@ -0,0 +1,98 @@
+import {promises as fs} from 'fs';
+import matter from 'gray-matter';
+import {ROUTES} from './routes-config';
+import {getDocsContentPath} from './docs-utils';
+import {getMarkdownForSlug} from './mdx-to-markdown';
+
+const OUTPUT_BASE = `# Zero
+
+> Zero is a new kind of sync engine powered by queries.
+
+`;
+
+function normalizeSlug(href: string) {
+  return href.replace(/^\//, '').replace(/^docs\//, '');
+}
+
+export async function generateLlmsTxt(baseUrl: string): Promise<string> {
+  let output = OUTPUT_BASE;
+
+  for (const section of ROUTES) {
+    output += `## ${section.title}\n\n`;
+
+    if (section.items) {
+      for (const item of section.items) {
+        if (!item.href) continue;
+
+        const slug = normalizeSlug(item.href);
+        const url = `${baseUrl}/docs/${slug}.md`;
+
+        try {
+          const contentPath = await getDocsContentPath(slug);
+          const rawMdx = await fs.readFile(contentPath, 'utf-8');
+          const {data} = matter(rawMdx);
+          const description = data.description ?? '';
+          const descSuffix = description ? `: ${description}` : '';
+
+          output += `- [${item.title}](${url})${descSuffix}\n`;
+        } catch (err) {
+          console.warn(`warning: could not process route ${item.href}:`, err);
+        }
+      }
+    }
+
+    output += '\n';
+  }
+
+  return output;
+}
+
+export async function generateLlmsFullTxt(baseUrl: string): Promise<string> {
+  let output = OUTPUT_BASE;
+
+  for (const section of ROUTES) {
+    if (section.items) {
+      for (const item of section.items) {
+        if (!item.href) continue;
+
+        const slug = normalizeSlug(item.href);
+
+        try {
+          const markdown = await getMarkdownForSlug(slug);
+
+          if (!markdown) {
+            console.warn(`warning: no markdown generated for ${slug}`);
+            continue;
+          }
+
+          const url = `${baseUrl}/docs/${slug}.md`;
+
+          output += '---\n\n';
+
+          if (markdown.startsWith('# ')) {
+            const firstNewline = markdown.indexOf('\n');
+            const title =
+              firstNewline === -1 ? markdown : markdown.slice(0, firstNewline);
+            const rest =
+              firstNewline === -1
+                ? ''
+                : markdown.slice(firstNewline).trimStart();
+
+            output += `${title}\n\n`;
+            output += `Source: ${url}\n\n`;
+            if (rest) {
+              output += `${rest}\n\n`;
+            }
+          } else {
+            output += `Source: ${url}\n\n`;
+            output += `${markdown}\n\n`;
+          }
+        } catch (err) {
+          console.warn(`warning: error processing ${slug}:`, err);
+        }
+      }
+    }
+  }
+
+  return output;
+}

lib/mdx-to-markdown.ts

@@ -85,11 +85,27 @@ function normalizeLineBreaks() {
 
       for (let index = 0; index < children.length - 1; index++) {
         const current = children[index];
+        const next = children[index + 1];
+
         if (current?.type === 'text') {
           const textNode = current as Text;
           if (textNode.value && !/[\s([{]$/.test(textNode.value.trimEnd())) {
             textNode.value = textNode.value.trimEnd() + ' ';
           }
+        } else if (
+          (current?.type === 'link' || current?.type === 'image') &&
+          next?.type === 'text'
+        ) {
+          // ensure links/images have proper spacing before following text
+          // only add space if missing and next text starts with a letter (not punctuation)
+          const nextTextNode = next as Text;
+          if (
+            nextTextNode.value &&
+            !/^\s/.test(nextTextNode.value) &&
+            /^[a-zA-Z]/.test(nextTextNode.value)
+          ) {
+            nextTextNode.value = ' ' + nextTextNode.value;
+          }
         }
       }
     });
@@ -330,7 +346,13 @@ function resolveDocumentHref(href: string, currentSlug: string) {
   if (docSlug !== undefined) {
     const canonicalSlug = DOC_ROUTE_LOOKUP.get(docSlug) ?? docSlug;
     const clean = canonicalSlug.replace(/^\/+/, '');
-    absolutePath = clean ? `/docs/${clean}` : '/docs';
+    absolutePath = clean ? `/docs/${clean}.md` : '/docs';
+  } else if (
+    absolutePath &&
+    absolutePath.startsWith('/docs/') &&
+    !absolutePath.endsWith('.md')
+  ) {
+    absolutePath = `${absolutePath}.md`;
   }
 
   if (!absolutePath) {

package.json

@@ -4,9 +4,8 @@
   "private": true,
   "scripts": {
     "dev": "next dev",
-    "build": "npm run build:search && npm run build:llms && next build",
+    "build": "npm run build:search && next build",
     "build:search": "tsx lib/generateSearchIndex.ts",
-    "build:llms": "npx --yes tsdown lib/generate-llms.ts --outDir dist --skipLibCheck && node dist/generate-llms.mjs",
     "start": "npm run build:search && next start",
     "lint": "next lint",
     "format": "prettier --write **/*.{ts,tsx,js,json,md,mdx}",