mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-28 14:59:47 +00:00
Garry TanandClaude Opus 4.7
f0649e2f32
Contributor DX magical moment. Static HTML explorer renders the full
canonical world (240 entities) as an explorable tree, opens in any browser,
zero install. Every string HTML-entity-encoded (XSS-safe — direct vuln
class per eng pass 2, confidence 9/10).
Added:
eval/generators/world-html.ts — renderer (~240 LOC; single-file
HTML with inline CSS + minimal JS)
eval/generators/world-html.test.ts — 16 tests (XSS + rendering correctness)
eval/cli/world-view.ts — render + open in default browser
eval/cli/query-validate.ts — CLI wrapper for queries/validator
eval/cli/query-new.ts — scaffold a query template
Modified:
package.json — 7 new eval:* scripts
.gitignore — ignore generated world.html
package.json scripts shipped:
bun run test:eval all eval unit tests (57 pass)
bun run eval:run full 4-adapter N=5 side-by-side
bun run eval:run:dev N=1 fast dev iteration
bun run eval:world:view render world.html + open in browser
bun run eval:world:render render only (CI-friendly, --no-open)
bun run eval:query:validate validate built-in T5+T5.5 (or a file path)
bun run eval:query:new scaffold a new Query JSON template
bun run eval:type-accuracy per-link-type accuracy report
XSS safety:
escapeHtml() encodes the 5 critical chars (& < > " '). Tested directly
with representative Opus-generated attacks:
<img src=x onerror=alert('xss')> → <img src=x onerror=alert('xss')>
<script>fetch('/steal')</script> → <script>fetch('/steal')</script>
Ledger metadata (generated_at, model) also escaped — covers the less
obvious attack surface where Opus could emit tag-like content into the
metadata file.
world.html structure:
- Left rail: entities grouped by type with counts (companies, people,
meetings, concepts), alphabetical within type
- Right pane: per-entity cards with title + slug + compiled_truth +
timeline + canonical _facts as collapsed JSON
- URL fragment deep-links (#people/alice-chen)
- Sticky rail on desktop; responsive stack on mobile
- Vanilla JS for active-link highlighting on scroll (no framework)
Generated file: ~1MB for 240 entities (full prose). Gitignored; rebuild
with `bun run eval:world:view`. Regeneration is ~50ms.
Contributor TTHW (Tier 5.5 query authoring):
1. bun run eval:world:view # see entities
2. bun run eval:query:new --tier externally-authored --author "@me"
3. edit template with real slug + query text
4. bun run eval:query:validate path/to/file.json
5. submit PR
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
80 lines
2.4 KiB
TypeScript
80 lines
2.4 KiB
TypeScript
#!/usr/bin/env bun
|
|
/**
|
|
* eval:query:validate — validate a query file (or the built-in Tier 5/5.5 set).
|
|
*
|
|
* Usage:
|
|
* bun run eval:query:validate # validate all built-in T5+T5.5
|
|
* bun run eval:query:validate path/to/file.ts # validate a file that exports Query[]
|
|
* bun run eval:query:validate --help
|
|
*
|
|
* Exit code 0 if all queries pass, 1 otherwise. Suitable for CI.
|
|
*/
|
|
|
|
import { readFileSync } from 'fs';
|
|
import { validateAll, validateQuerySet, formatIssues } from '../runner/queries/index.ts';
|
|
import type { Query } from '../runner/types.ts';
|
|
|
|
function printHelp() {
|
|
console.log(`eval:query:validate — validate a Query set
|
|
|
|
USAGE
|
|
bun run eval:query:validate validate all built-in T5 + T5.5 queries
|
|
bun run eval:query:validate <path> validate a JSON file containing Query[]
|
|
|
|
VALIDATOR CHECKS
|
|
- id, text, tier, expected_output_type present
|
|
- Temporal verbs (is/was/were/current/now/at the time/during/as of/when did)
|
|
require as_of_date ("corpus-end" | "per-source" | ISO-8601)
|
|
- cited-source-pages requires non-empty gold.relevant with valid slug format
|
|
- abstention requires gold.expected_abstention === true
|
|
- externally-authored (Tier 5.5) requires author field
|
|
- Duplicate IDs caught at batch level
|
|
|
|
EXIT CODES
|
|
0 all queries valid
|
|
1 one or more queries failed validation
|
|
`);
|
|
}
|
|
|
|
async function main() {
|
|
const args = process.argv.slice(2);
|
|
if (args.includes('--help') || args.includes('-h')) {
|
|
printHelp();
|
|
return;
|
|
}
|
|
|
|
const filePath = args[0];
|
|
|
|
if (!filePath) {
|
|
// Validate built-in T5 + T5.5 sets
|
|
const result = validateAll();
|
|
console.log(result.report);
|
|
process.exit(result.ok ? 0 : 1);
|
|
}
|
|
|
|
// Validate a file — supports JSON with { queries: Query[] } or Query[]
|
|
let queries: Query[] = [];
|
|
try {
|
|
const raw = readFileSync(filePath, 'utf-8');
|
|
const parsed = JSON.parse(raw);
|
|
queries = Array.isArray(parsed) ? parsed : (parsed.queries ?? []);
|
|
} catch (e) {
|
|
console.error(`Error reading ${filePath}: ${(e as Error).message}`);
|
|
process.exit(1);
|
|
}
|
|
|
|
if (queries.length === 0) {
|
|
console.error(`No queries found in ${filePath}. Expected JSON Query[] or { queries: Query[] }.`);
|
|
process.exit(1);
|
|
}
|
|
|
|
const result = validateQuerySet(queries);
|
|
console.log(formatIssues(result));
|
|
process.exit(result.ok ? 0 : 1);
|
|
}
|
|
|
|
main().catch(e => {
|
|
console.error(e);
|
|
process.exit(1);
|
|
});
|