gbrain/scripts/build-llms.ts

#!/usr/bin/env bun
/**
 * build-llms — generate llms.txt + llms-full.txt from scripts/llms-config.ts.
 *
 * Run: `bun run build:llms` (or `bun run scripts/build-llms.ts`).
 *
 * Outputs:
 *   - llms.txt       — llmstxt.org-spec index (H1 / blockquote / H2 sections).
 *   - llms-full.txt  — concatenated full content of non-optional entries.
 *
 * Deterministic: no timestamps, sorted within categories by config order.
 * Warns (does not fail) if llms-full.txt exceeds FULL_SIZE_BUDGET. CI catches
 * drift via test/build-llms.test.ts.
 *
 * Fork override: set LLMS_REPO_BASE to regenerate with a different URL base.
 */

import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
import { join, dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";

import {
  FULL_SIZE_BUDGET,
  INLINE_TIPS,
  PROJECT,
  SECTIONS,
  type DocEntry,
  type DocSection,
} from "./llms-config";

const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");

function urlFor(entry: DocEntry): string {
  return `${PROJECT.rawBaseUrl}/${entry.path}`;
}

function isDirectoryPath(path: string): boolean {
  return path.endsWith("/");
}

function renderLlmsTxt(): string {
  const lines: string[] = [];
  lines.push(`# ${PROJECT.name}`);
  lines.push("");
  lines.push(`> ${PROJECT.summary}`);
  lines.push("");
  lines.push(`Repo: ${PROJECT.repoUrl}`);
  lines.push("");

  for (const section of SECTIONS) {
    lines.push(`## ${section.heading}`);
    lines.push("");
    for (const entry of section.entries) {
      lines.push(
        `- [${entry.title}](${urlFor(entry)}): ${entry.description}`,
      );
    }
    lines.push("");
  }

  lines.push("## Operational tips");
  lines.push("");
  for (const tip of INLINE_TIPS) {
    lines.push(`- ${tip}`);
  }
  lines.push("");

  return lines.join("\n");
}

function renderLlmsFullTxt(): { content: string; sizes: Array<{ path: string; bytes: number }> } {
  const lines: string[] = [];
  const sizes: Array<{ path: string; bytes: number }> = [];

  lines.push(`# ${PROJECT.name} — Full Context`);
  lines.push("");
  lines.push(`> ${PROJECT.summary}`);
  lines.push("");
  lines.push(
    `This file concatenates core GBrain documentation for single-fetch ingestion.`,
  );
  lines.push(
    `For the link-only index, see \`llms.txt\`. Source of truth: ${PROJECT.repoUrl}.`,
  );
  lines.push("");

  for (const section of SECTIONS) {
    if (section.optional) continue;
    lines.push(`# ${section.heading}`);
    lines.push("");
    for (const entry of section.entries) {
      if (entry.includeInFull === false) continue;
      if (isDirectoryPath(entry.path)) continue;

      const absPath = join(repoRoot, entry.path);
      if (!existsSync(absPath)) {
        // build-llms won't silently skip — surface the problem. Test case 1
        // catches this too, but fail fast for manual runs.
        throw new Error(
          `llms-config references missing file: ${entry.path}`,
        );
      }

      const body = readFileSync(absPath, "utf8");
      const bytes = Buffer.byteLength(body, "utf8");
      sizes.push({ path: entry.path, bytes });

      lines.push(`## ${entry.path}`);
      lines.push("");
      lines.push(`Source: ${urlFor(entry)}`);
      lines.push("");
      lines.push(body.trimEnd());
      lines.push("");
      lines.push("---");
      lines.push("");
    }
  }

  return { content: lines.join("\n"), sizes };
}

function validateConfig(): void {
  for (const section of SECTIONS) {
    for (const entry of section.entries) {
      const absPath = join(repoRoot, entry.path);
      if (!existsSync(absPath)) {
        throw new Error(
          `llms-config references missing path: ${entry.path}`,
        );
      }
      const st = statSync(absPath);
      if (isDirectoryPath(entry.path) && !st.isDirectory()) {
        throw new Error(
          `llms-config path ends with '/' but is a file: ${entry.path}`,
        );
      }
      if (!isDirectoryPath(entry.path) && !st.isFile()) {
        throw new Error(
          `llms-config path is a directory but missing trailing '/': ${entry.path}`,
        );
      }
    }
  }
}

export function buildLlmsFiles(): {
  llmsTxt: string;
  llmsFullTxt: string;
  sizes: Array<{ path: string; bytes: number }>;
} {
  validateConfig();
  const llmsTxt = renderLlmsTxt();
  const { content: llmsFullTxt, sizes } = renderLlmsFullTxt();
  return { llmsTxt, llmsFullTxt, sizes };
}

function main(): void {
  const { llmsTxt, llmsFullTxt, sizes } = buildLlmsFiles();

  const llmsPath = join(repoRoot, "llms.txt");
  const llmsFullPath = join(repoRoot, "llms-full.txt");

  writeFileSync(llmsPath, llmsTxt);
  writeFileSync(llmsFullPath, llmsFullTxt);

  const fullBytes = Buffer.byteLength(llmsFullTxt, "utf8");
  console.log(`wrote ${llmsPath} (${Buffer.byteLength(llmsTxt, "utf8")} bytes)`);
  console.log(`wrote ${llmsFullPath} (${fullBytes} bytes)`);

  if (fullBytes > FULL_SIZE_BUDGET) {
    console.warn("");
    console.warn(
      `WARN: llms-full.txt (${fullBytes} bytes) exceeds FULL_SIZE_BUDGET (${FULL_SIZE_BUDGET} bytes).`,
    );
    console.warn(
      "Add `includeInFull: false` to the biggest entries in scripts/llms-config.ts:",
    );
    const sorted = [...sizes].sort((a, b) => b.bytes - a.bytes);
    for (const entry of sorted.slice(0, 5)) {
      console.warn(`  ${entry.bytes} bytes  ${entry.path}`);
    }
  }
}

const isMainModule = fileURLToPath(import.meta.url) === process.argv[1];
if (isMainModule) {
  try {
    main();
  } catch (err) {
    console.error(err instanceof Error ? err.message : err);
    process.exit(1);
  }
}