mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
Garry Tan
18baca6629
Adds the v1→v2 contract boundary for BrainBench. 6 JSON schemas at
eval/schemas/ pin the shape of every artifact a stack must emit to be
scorable: corpus-manifest, public-probe (PublicQuery with gold stripped),
tool-schema (12 read + 3 dry_run tools, 32K tool-output cap), transcript,
scorecard (N ∈ {1, 5, 10}), evidence-contract (structured judge input).
8 gold file templates at eval/data/gold/ scaffold the sealed qrels,
contradictions, poison items, and citation labels. Empty-but-valid
skeletons; Day 3b fills them with real content once the amara-life-v1
corpus generates.
48 tests validate schema syntax, $schema/$id/title/type headers,
round-trip stability, and cross-schema coherence (new Page types in
manifest enum, tool counts, token cap, N enum).
When v2 ports to Python + Inspect AI + Docker, these schemas are the
boundary. Same fixtures, same tool contracts, zero rework.
86 lines
3.2 KiB
JSON
86 lines
3.2 KiB
JSON
{
|
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
"$id": "https://brainbench.dev/schemas/transcript.schema.json",
|
|
"title": "BrainBench Flight-Recorder Transcript",
|
|
"description": "Full tool-call + model-call trace for a single probe run. Written by eval/runner/recorder.ts. Consumed by judge (via evidence-contract) and reviewers.",
|
|
"type": "object",
|
|
"required": ["schema_version", "probe_id", "adapter", "started_at", "ended_at", "turns"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"schema_version": {"type": "integer", "const": 1},
|
|
"probe_id": {"type": "string", "pattern": "^q-[0-9]{4}$"},
|
|
"adapter": {
|
|
"type": "object",
|
|
"required": ["name", "stack_id"],
|
|
"properties": {
|
|
"name": {"type": "string"},
|
|
"stack_id": {"type": "string"}
|
|
}
|
|
},
|
|
"started_at": {"type": "string", "format": "date-time"},
|
|
"ended_at": {"type": "string", "format": "date-time"},
|
|
"turns": {
|
|
"type": "array",
|
|
"description": "Ordered list of model turns + tool executions.",
|
|
"items": {
|
|
"type": "object",
|
|
"required": ["turn_index", "kind"],
|
|
"properties": {
|
|
"turn_index": {"type": "integer", "minimum": 0},
|
|
"kind": {
|
|
"type": "string",
|
|
"enum": ["model_call", "tool_call", "tool_result", "final_answer"]
|
|
},
|
|
"model_call": {
|
|
"type": "object",
|
|
"description": "Present when kind=model_call.",
|
|
"properties": {
|
|
"model_id": {"type": "string"},
|
|
"input_tokens": {"type": "integer"},
|
|
"output_tokens": {"type": "integer"},
|
|
"stop_reason": {"type": "string"}
|
|
}
|
|
},
|
|
"tool_call": {
|
|
"type": "object",
|
|
"description": "Present when kind=tool_call.",
|
|
"properties": {
|
|
"tool_name": {"type": "string"},
|
|
"tool_input": {"type": "object"}
|
|
}
|
|
},
|
|
"tool_result": {
|
|
"type": "object",
|
|
"description": "Present when kind=tool_result. Raw content is NOT passed to judge — structured summary is.",
|
|
"properties": {
|
|
"tool_name": {"type": "string"},
|
|
"content": {"description": "Raw tool output (opaque shape)."},
|
|
"truncated": {"type": "boolean"},
|
|
"matched_poison_fixture_ids": {
|
|
"type": "array",
|
|
"items": {"type": "string"},
|
|
"description": "Bridge-computed: slugs in result that match gold/poison.json fixture_ids."
|
|
}
|
|
}
|
|
},
|
|
"final_answer": {
|
|
"type": "object",
|
|
"description": "Present when kind=final_answer (last turn only).",
|
|
"properties": {
|
|
"text": {"type": "string"},
|
|
"evidence_refs": {
|
|
"type": "array",
|
|
"items": {"type": "string"},
|
|
"description": "Slugs the agent cited in the final answer."
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"total_input_tokens": {"type": "integer", "minimum": 0},
|
|
"total_output_tokens": {"type": "integer", "minimum": 0},
|
|
"elapsed_ms": {"type": "integer", "minimum": 0}
|
|
}
|
|
}
|