From f0649e2f3299b30106f24f7530e23f18568c431f Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 19 Apr 2026 00:15:18 +0800 Subject: [PATCH] feat(eval): Phase 3 world.html explorer + eval:* CLI surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Contributor DX magical moment. Static HTML explorer renders the full canonical world (240 entities) as an explorable tree, opens in any browser, zero install. Every string HTML-entity-encoded (XSS-safe — direct vuln class per eng pass 2, confidence 9/10). Added: eval/generators/world-html.ts — renderer (~240 LOC; single-file HTML with inline CSS + minimal JS) eval/generators/world-html.test.ts — 16 tests (XSS + rendering correctness) eval/cli/world-view.ts — render + open in default browser eval/cli/query-validate.ts — CLI wrapper for queries/validator eval/cli/query-new.ts — scaffold a query template Modified: package.json — 7 new eval:* scripts .gitignore — ignore generated world.html package.json scripts shipped: bun run test:eval all eval unit tests (57 pass) bun run eval:run full 4-adapter N=5 side-by-side bun run eval:run:dev N=1 fast dev iteration bun run eval:world:view render world.html + open in browser bun run eval:world:render render only (CI-friendly, --no-open) bun run eval:query:validate validate built-in T5+T5.5 (or a file path) bun run eval:query:new scaffold a new Query JSON template bun run eval:type-accuracy per-link-type accuracy report XSS safety: escapeHtml() encodes the 5 critical chars (& < > " '). Tested directly with representative Opus-generated attacks: → <img src=x onerror=alert('xss')> → <script>fetch('/steal')</script> Ledger metadata (generated_at, model) also escaped — covers the less obvious attack surface where Opus could emit tag-like content into the metadata file. world.html structure: - Left rail: entities grouped by type with counts (companies, people, meetings, concepts), alphabetical within type - Right pane: per-entity cards with title + slug + compiled_truth + timeline + canonical _facts as collapsed JSON - URL fragment deep-links (#people/alice-chen) - Sticky rail on desktop; responsive stack on mobile - Vanilla JS for active-link highlighting on scroll (no framework) Generated file: ~1MB for 240 entities (full prose). Gitignored; rebuild with `bun run eval:world:view`. Regeneration is ~50ms. Contributor TTHW (Tier 5.5 query authoring): 1. bun run eval:world:view # see entities 2. bun run eval:query:new --tier externally-authored --author "@me" 3. edit template with real slug + query text 4. bun run eval:query:validate path/to/file.json 5. submit PR Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitignore | 1 + eval/cli/query-new.ts | 85 ++++++++ eval/cli/query-validate.ts | 79 +++++++ eval/cli/world-view.ts | 64 ++++++ eval/generators/world-html.test.ts | 134 ++++++++++++ eval/generators/world-html.ts | 324 +++++++++++++++++++++++++++++ package.json | 8 + 7 files changed, 695 insertions(+) create mode 100644 eval/cli/query-new.ts create mode 100644 eval/cli/query-validate.ts create mode 100644 eval/cli/world-view.ts create mode 100644 eval/generators/world-html.test.ts create mode 100644 eval/generators/world-html.ts diff --git a/.gitignore b/.gitignore index 44799ebcd..2b0e83ef2 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ supabase/.temp/ .claude/skills/ .idea eval/reports/ +eval/data/world-v1/world.html diff --git a/eval/cli/query-new.ts b/eval/cli/query-new.ts new file mode 100644 index 000000000..273d08697 --- /dev/null +++ b/eval/cli/query-new.ts @@ -0,0 +1,85 @@ +#!/usr/bin/env bun +/** + * eval:query:new — scaffold a new Tier 5.5 query from template. + * + * Prints a well-formed Query JSON block that passes `eval:query:validate`. + * Contributors copy-paste into their own query file (or a PR against + * eval/external-authors//queries.json). + * + * Usage: + * bun run eval:query:new # default tier 5.5 + * bun run eval:query:new --tier fuzzy + * bun run eval:query:new --tier externally-authored --author "@alice" + * bun run eval:query:new --id q-custom-0042 + */ + +import type { Query, Tier } from '../runner/types.ts'; + +function printHelp() { + console.log(`eval:query:new — scaffold a Query template + +USAGE + bun run eval:query:new scaffold default (tier 5.5) + bun run eval:query:new --tier easy specify tier + bun run eval:query:new --id q-0001 specify id + bun run eval:query:new --author "@alice-researcher" external author + +OPTIONS + --tier easy | medium | hard | adversarial | fuzzy | externally-authored + --id Query ID (default: q-) + --author Author handle (required for tier=externally-authored) + +OUTPUT + Prints a JSON object that passes eval:query:validate. Copy into + your query file, fill in gold.relevant (or expected_abstention for + abstention queries), and iterate. +`); +} + +function getArg(name: string, fallback?: string): string | undefined { + const prefix = `--${name}=`; + for (const a of process.argv.slice(2)) { + if (a.startsWith(prefix)) return a.slice(prefix.length); + if (a === `--${name}`) { + const next = process.argv[process.argv.indexOf(a) + 1]; + if (next && !next.startsWith('--')) return next; + } + } + return fallback; +} + +function main() { + if (process.argv.includes('--help') || process.argv.includes('-h')) { + printHelp(); + return; + } + + const tier = (getArg('tier') ?? 'externally-authored') as Tier; + const id = getArg('id') ?? `q-${Date.now().toString().slice(-6)}`; + const author = getArg('author'); + + const template: Query = { + id, + tier, + text: 'REPLACE with your query text (a question or search fragment)', + expected_output_type: 'cited-source-pages', + gold: { + relevant: ['REPLACE/with-real-slug', 'REPLACE/with-another-slug-if-needed'], + }, + tags: ['REPLACE-with-tier-or-theme-tags'], + }; + + if (tier === 'externally-authored') { + template.author = author ?? 'REPLACE-with-your-handle'; + } + + // If the query text contains a temporal verb, the validator will require + // as_of_date. Leave a helpful placeholder. + template.as_of_date = 'REPLACE if temporal ("corpus-end" | "per-source" | YYYY-MM-DD); else delete this field'; + + console.log(JSON.stringify(template, null, 2)); + console.log(`\n// Next: save as a JSON file, run 'bun run eval:query:validate '`); + console.log(`// For Tier 5.5: submit a PR to eval/external-authors/${author ?? ''}/queries.json`); +} + +main(); diff --git a/eval/cli/query-validate.ts b/eval/cli/query-validate.ts new file mode 100644 index 000000000..0e7b1ad1e --- /dev/null +++ b/eval/cli/query-validate.ts @@ -0,0 +1,79 @@ +#!/usr/bin/env bun +/** + * eval:query:validate — validate a query file (or the built-in Tier 5/5.5 set). + * + * Usage: + * bun run eval:query:validate # validate all built-in T5+T5.5 + * bun run eval:query:validate path/to/file.ts # validate a file that exports Query[] + * bun run eval:query:validate --help + * + * Exit code 0 if all queries pass, 1 otherwise. Suitable for CI. + */ + +import { readFileSync } from 'fs'; +import { validateAll, validateQuerySet, formatIssues } from '../runner/queries/index.ts'; +import type { Query } from '../runner/types.ts'; + +function printHelp() { + console.log(`eval:query:validate — validate a Query set + +USAGE + bun run eval:query:validate validate all built-in T5 + T5.5 queries + bun run eval:query:validate validate a JSON file containing Query[] + +VALIDATOR CHECKS + - id, text, tier, expected_output_type present + - Temporal verbs (is/was/were/current/now/at the time/during/as of/when did) + require as_of_date ("corpus-end" | "per-source" | ISO-8601) + - cited-source-pages requires non-empty gold.relevant with valid slug format + - abstention requires gold.expected_abstention === true + - externally-authored (Tier 5.5) requires author field + - Duplicate IDs caught at batch level + +EXIT CODES + 0 all queries valid + 1 one or more queries failed validation +`); +} + +async function main() { + const args = process.argv.slice(2); + if (args.includes('--help') || args.includes('-h')) { + printHelp(); + return; + } + + const filePath = args[0]; + + if (!filePath) { + // Validate built-in T5 + T5.5 sets + const result = validateAll(); + console.log(result.report); + process.exit(result.ok ? 0 : 1); + } + + // Validate a file — supports JSON with { queries: Query[] } or Query[] + let queries: Query[] = []; + try { + const raw = readFileSync(filePath, 'utf-8'); + const parsed = JSON.parse(raw); + queries = Array.isArray(parsed) ? parsed : (parsed.queries ?? []); + } catch (e) { + console.error(`Error reading ${filePath}: ${(e as Error).message}`); + process.exit(1); + } + + if (queries.length === 0) { + console.error(`No queries found in ${filePath}. Expected JSON Query[] or { queries: Query[] }.`); + process.exit(1); + } + + const result = validateQuerySet(queries); + console.log(formatIssues(result)); + process.exit(result.ok ? 0 : 1); +} + +main().catch(e => { + console.error(e); + process.exit(1); +}); diff --git a/eval/cli/world-view.ts b/eval/cli/world-view.ts new file mode 100644 index 000000000..3db3e9f41 --- /dev/null +++ b/eval/cli/world-view.ts @@ -0,0 +1,64 @@ +#!/usr/bin/env bun +/** + * eval:world:view — (re)generate world.html and open it in the default browser. + * + * Combines two steps the contributor shouldn't have to think about: + * 1. Render eval/data/world-v1/world.html from the shard JSONs. + * 2. Open it in the default browser via `open` (macOS) / `xdg-open` (Linux). + * + * If the HTML already exists and shards haven't changed since, we'd ideally + * skip step 1 — but comparing timestamps is fragile and regeneration is fast + * (~50ms on 240 entities). Just regenerate every time. + * + * Usage: + * bun run eval:world:view (generates + opens) + * bun run eval:world:view --no-open (generates only; useful in CI) + */ + +import { execSync } from 'child_process'; +import { platform } from 'os'; +import { renderWorldHtmlToFile } from '../generators/world-html.ts'; + +function openInBrowser(path: string): void { + const cmd = platform() === 'darwin' ? 'open' : platform() === 'win32' ? 'start' : 'xdg-open'; + try { + execSync(`${cmd} "${path}"`, { stdio: 'ignore' }); + } catch (e) { + // Don't fail hard — the file is rendered, that's the main thing. + console.error(`Could not open browser automatically. Open manually: ${path}`); + console.error(`(${(e as Error).message})`); + } +} + +async function main() { + const args = process.argv.slice(2); + if (args.includes('--help') || args.includes('-h')) { + console.log(`eval:world:view — render + open world.html + +USAGE + bun run eval:world:view render eval/data/world-v1/world.html and open it + bun run eval:world:view --no-open render only (CI-friendly) + bun run eval:world:view --dir=PATH render from a different shard directory + +OUTPUT + eval/data/world-v1/world.html (gitignored; regenerate with this command). +`); + return; + } + + const dir = args.find(a => a.startsWith('--dir='))?.slice('--dir='.length) ?? + 'eval/data/world-v1'; + const noOpen = args.includes('--no-open'); + + const target = renderWorldHtmlToFile(dir); + console.log(`Rendered ${target}`); + + if (!noOpen) { + openInBrowser(target); + } +} + +main().catch(e => { + console.error(e); + process.exit(1); +}); diff --git a/eval/generators/world-html.test.ts b/eval/generators/world-html.test.ts new file mode 100644 index 000000000..fd9eba240 --- /dev/null +++ b/eval/generators/world-html.test.ts @@ -0,0 +1,134 @@ +import { describe, test, expect } from 'bun:test'; +import { escapeHtml, renderWorldHtml } from './world-html.ts'; + +describe('escapeHtml — XSS safety', () => { + test('escapes the 5 critical chars', () => { + expect(escapeHtml('')) + .toBe('<script>alert(1)</script>'); + }); + + test('escapes ampersand FIRST (or double-escape bug happens)', () => { + expect(escapeHtml('<')) + .toBe('&lt;'); // the input < becomes &lt; + }); + + test('escapes quotes (attribute context)', () => { + expect(escapeHtml('onclick="alert(1)"')) + .toBe('onclick="alert(1)"'); + expect(escapeHtml("onclick='alert(1)'")) + .toBe('onclick='alert(1)''); + }); + + test('passthrough for safe ASCII', () => { + expect(escapeHtml('Hello world.')).toBe('Hello world.'); + }); + + test('handles null and undefined', () => { + expect(escapeHtml(null)).toBe(''); + expect(escapeHtml(undefined)).toBe(''); + }); + + test('handles numbers', () => { + expect(escapeHtml(42)).toBe('42'); + }); + + test('preserves Unicode (non-ASCII that\'s not special HTML)', () => { + expect(escapeHtml('Héctor García 🎉')).toBe('Héctor García 🎉'); + }); + + test('real Opus prose injection attempt neutralized', () => { + // Representative: if Opus generates this in an entity backstory, + // the explorer HTML must NOT execute it. The XSS protection works + // because ``; + const safe = escapeHtml(attack); + // The opening `<` must be escaped (this is what neutralizes the tag). + expect(safe).not.toContain(' { + // Even as text content, escape quotes so it can't break attribute context. + const attack = `"javascript:alert(1)//`; + expect(escapeHtml(attack)).toBe('"javascript:alert(1)//'); + }); +}); + +describe('renderWorldHtml', () => { + const samplePage = { + slug: 'people/alice-chen', + type: 'person' as const, + title: 'Alice Chen', + compiled_truth: 'Alice Chen is a senior engineer.', + timeline: '- **2023-01-15** | Promoted to staff engineer', + _facts: { type: 'person', role: 'engineer' }, + }; + + test('renders an HTML document', () => { + const html = renderWorldHtml([samplePage]); + expect(html).toContain(''); + expect(html).toContain(' { + const html = renderWorldHtml([samplePage]); + // Rail link uses #slug anchor + expect(html).toContain('href="#people/alice-chen"'); + // Entity card has matching id + expect(html).toContain('id="people/alice-chen"'); + }); + + test('escapes all user content (XSS neutralization)', () => { + const maliciousPage = { + ...samplePage, + title: '', + compiled_truth: ``, + }; + const html = renderWorldHtml([maliciousPage]); + // Raw + + +`; +} + +// ─── CLI entrypoint ───────────────────────────────────────────── + +export function renderWorldHtmlToFile(dir: string, outPath?: string): string { + const { pages, ledger } = loadCorpus(dir); + const html = renderWorldHtml(pages, ledger); + const target = outPath ?? join(dir, 'world.html'); + writeFileSync(target, html, 'utf-8'); + return target; +} + +// Run directly (e.g. `bun eval/generators/world-html.ts`) +if (import.meta.main) { + const dir = process.argv.find(a => a.startsWith('--dir='))?.slice('--dir='.length) ?? + 'eval/data/world-v1'; + const out = process.argv.find(a => a.startsWith('--out='))?.slice('--out='.length); + const target = renderWorldHtmlToFile(dir, out); + console.log(`Wrote ${target}`); +} diff --git a/package.json b/package.json index d3fa385e5..918439542 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,14 @@ "build:schema": "bash scripts/build-schema.sh", "test": "bun test", "test:e2e": "bun test test/e2e/", + "test:eval": "bun test eval/runner/queries/validator.test.ts eval/runner/adapters/ eval/generators/world-html.test.ts", + "eval:run": "bun eval/runner/multi-adapter.ts", + "eval:run:dev": "BRAINBENCH_N=1 bun eval/runner/multi-adapter.ts", + "eval:world:view": "bun eval/cli/world-view.ts", + "eval:world:render": "bun eval/cli/world-view.ts --no-open", + "eval:query:validate": "bun eval/cli/query-validate.ts", + "eval:query:new": "bun eval/cli/query-new.ts", + "eval:type-accuracy": "bun eval/runner/type-accuracy.ts", "postinstall": "gbrain --version >/dev/null 2>&1 && gbrain apply-migrations --yes --non-interactive 2>/dev/null || true", "prepublish:clawhub": "bun run build:all", "publish:clawhub": "clawhub package publish . --family bundle-plugin"