mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
Garry TanandClaude Opus 4.7
f0649e2f32
Contributor DX magical moment. Static HTML explorer renders the full
canonical world (240 entities) as an explorable tree, opens in any browser,
zero install. Every string HTML-entity-encoded (XSS-safe — direct vuln
class per eng pass 2, confidence 9/10).
Added:
eval/generators/world-html.ts — renderer (~240 LOC; single-file
HTML with inline CSS + minimal JS)
eval/generators/world-html.test.ts — 16 tests (XSS + rendering correctness)
eval/cli/world-view.ts — render + open in default browser
eval/cli/query-validate.ts — CLI wrapper for queries/validator
eval/cli/query-new.ts — scaffold a query template
Modified:
package.json — 7 new eval:* scripts
.gitignore — ignore generated world.html
package.json scripts shipped:
bun run test:eval all eval unit tests (57 pass)
bun run eval:run full 4-adapter N=5 side-by-side
bun run eval:run:dev N=1 fast dev iteration
bun run eval:world:view render world.html + open in browser
bun run eval:world:render render only (CI-friendly, --no-open)
bun run eval:query:validate validate built-in T5+T5.5 (or a file path)
bun run eval:query:new scaffold a new Query JSON template
bun run eval:type-accuracy per-link-type accuracy report
XSS safety:
escapeHtml() encodes the 5 critical chars (& < > " '). Tested directly
with representative Opus-generated attacks:
<img src=x onerror=alert('xss')> → <img src=x onerror=alert('xss')>
<script>fetch('/steal')</script> → <script>fetch('/steal')</script>
Ledger metadata (generated_at, model) also escaped — covers the less
obvious attack surface where Opus could emit tag-like content into the
metadata file.
world.html structure:
- Left rail: entities grouped by type with counts (companies, people,
meetings, concepts), alphabetical within type
- Right pane: per-entity cards with title + slug + compiled_truth +
timeline + canonical _facts as collapsed JSON
- URL fragment deep-links (#people/alice-chen)
- Sticky rail on desktop; responsive stack on mobile
- Vanilla JS for active-link highlighting on scroll (no framework)
Generated file: ~1MB for 240 entities (full prose). Gitignored; rebuild
with `bun run eval:world:view`. Regeneration is ~50ms.
Contributor TTHW (Tier 5.5 query authoring):
1. bun run eval:world:view # see entities
2. bun run eval:query:new --tier externally-authored --author "@me"
3. edit template with real slug + query text
4. bun run eval:query:validate path/to/file.json
5. submit PR
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
86 lines
2.9 KiB
TypeScript
86 lines
2.9 KiB
TypeScript
#!/usr/bin/env bun
|
|
/**
|
|
* eval:query:new — scaffold a new Tier 5.5 query from template.
|
|
*
|
|
* Prints a well-formed Query JSON block that passes `eval:query:validate`.
|
|
* Contributors copy-paste into their own query file (or a PR against
|
|
* eval/external-authors/<author-slug>/queries.json).
|
|
*
|
|
* Usage:
|
|
* bun run eval:query:new # default tier 5.5
|
|
* bun run eval:query:new --tier fuzzy
|
|
* bun run eval:query:new --tier externally-authored --author "@alice"
|
|
* bun run eval:query:new --id q-custom-0042
|
|
*/
|
|
|
|
import type { Query, Tier } from '../runner/types.ts';
|
|
|
|
function printHelp() {
|
|
console.log(`eval:query:new — scaffold a Query template
|
|
|
|
USAGE
|
|
bun run eval:query:new scaffold default (tier 5.5)
|
|
bun run eval:query:new --tier easy specify tier
|
|
bun run eval:query:new --id q-0001 specify id
|
|
bun run eval:query:new --author "@alice-researcher" external author
|
|
|
|
OPTIONS
|
|
--tier easy | medium | hard | adversarial | fuzzy | externally-authored
|
|
--id Query ID (default: q-<timestamp>)
|
|
--author Author handle (required for tier=externally-authored)
|
|
|
|
OUTPUT
|
|
Prints a JSON object that passes eval:query:validate. Copy into
|
|
your query file, fill in gold.relevant (or expected_abstention for
|
|
abstention queries), and iterate.
|
|
`);
|
|
}
|
|
|
|
function getArg(name: string, fallback?: string): string | undefined {
|
|
const prefix = `--${name}=`;
|
|
for (const a of process.argv.slice(2)) {
|
|
if (a.startsWith(prefix)) return a.slice(prefix.length);
|
|
if (a === `--${name}`) {
|
|
const next = process.argv[process.argv.indexOf(a) + 1];
|
|
if (next && !next.startsWith('--')) return next;
|
|
}
|
|
}
|
|
return fallback;
|
|
}
|
|
|
|
function main() {
|
|
if (process.argv.includes('--help') || process.argv.includes('-h')) {
|
|
printHelp();
|
|
return;
|
|
}
|
|
|
|
const tier = (getArg('tier') ?? 'externally-authored') as Tier;
|
|
const id = getArg('id') ?? `q-${Date.now().toString().slice(-6)}`;
|
|
const author = getArg('author');
|
|
|
|
const template: Query = {
|
|
id,
|
|
tier,
|
|
text: 'REPLACE with your query text (a question or search fragment)',
|
|
expected_output_type: 'cited-source-pages',
|
|
gold: {
|
|
relevant: ['REPLACE/with-real-slug', 'REPLACE/with-another-slug-if-needed'],
|
|
},
|
|
tags: ['REPLACE-with-tier-or-theme-tags'],
|
|
};
|
|
|
|
if (tier === 'externally-authored') {
|
|
template.author = author ?? 'REPLACE-with-your-handle';
|
|
}
|
|
|
|
// If the query text contains a temporal verb, the validator will require
|
|
// as_of_date. Leave a helpful placeholder.
|
|
template.as_of_date = 'REPLACE if temporal ("corpus-end" | "per-source" | YYYY-MM-DD); else delete this field';
|
|
|
|
console.log(JSON.stringify(template, null, 2));
|
|
console.log(`\n// Next: save as a JSON file, run 'bun run eval:query:validate <path>'`);
|
|
console.log(`// For Tier 5.5: submit a PR to eval/external-authors/${author ?? '<your-slug>'}/queries.json`);
|
|
}
|
|
|
|
main();
|