mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
Garry Tan
18baca6629
Adds the v1→v2 contract boundary for BrainBench. 6 JSON schemas at
eval/schemas/ pin the shape of every artifact a stack must emit to be
scorable: corpus-manifest, public-probe (PublicQuery with gold stripped),
tool-schema (12 read + 3 dry_run tools, 32K tool-output cap), transcript,
scorecard (N ∈ {1, 5, 10}), evidence-contract (structured judge input).
8 gold file templates at eval/data/gold/ scaffold the sealed qrels,
contradictions, poison items, and citation labels. Empty-but-valid
skeletons; Day 3b fills them with real content once the amara-life-v1
corpus generates.
48 tests validate schema syntax, $schema/$id/title/type headers,
round-trip stability, and cross-schema coherence (new Page types in
manifest enum, tool counts, token cap, N enum).
When v2 ports to Python + Inspect AI + Docker, these schemas are the
boundary. Same fixtures, same tool contracts, zero rework.
85 lines
2.9 KiB
JSON
85 lines
2.9 KiB
JSON
{
|
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
"$id": "https://brainbench.dev/schemas/tool-schema.schema.json",
|
|
"title": "BrainBench Stack Tool Surface",
|
|
"description": "The contract a stack (gbrain / mem0 / supermemory / ...) must expose to be scorable by BrainBench. 12 read tools + 3 dry_run write tools. Dry-run writes record intent to the flight-recorder but MUST NOT mutate brain state. This schema is stack-agnostic.",
|
|
"type": "object",
|
|
"required": ["schema_version", "stack_id", "read_tools", "dry_run_tools"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"schema_version": {"type": "integer", "const": 1},
|
|
"stack_id": {
|
|
"type": "string",
|
|
"description": "Stack identifier, e.g. 'gbrain-0.12.1', 'mem0-local'."
|
|
},
|
|
"read_tools": {
|
|
"type": "array",
|
|
"minItems": 12,
|
|
"maxItems": 12,
|
|
"description": "Exactly 12 read tools. Each implements the named semantics; input_schema is Anthropic tool-use compatible.",
|
|
"items": {
|
|
"type": "object",
|
|
"required": ["name", "description", "input_schema"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"name": {
|
|
"type": "string",
|
|
"enum": [
|
|
"search",
|
|
"query",
|
|
"get_page",
|
|
"list_pages",
|
|
"get_backlinks",
|
|
"get_links",
|
|
"get_timeline",
|
|
"get_tags",
|
|
"traverse_graph",
|
|
"resolve_slugs",
|
|
"get_chunks",
|
|
"get_stats"
|
|
]
|
|
},
|
|
"description": {"type": "string"},
|
|
"input_schema": {
|
|
"type": "object",
|
|
"description": "JSON Schema for the tool input (Anthropic tool_use compatible)."
|
|
},
|
|
"expand_disabled_if_present": {
|
|
"type": "boolean",
|
|
"description": "True for `query` tool — bridge hard-sets expand=false to prevent nested LLM calls.",
|
|
"default": false
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"dry_run_tools": {
|
|
"type": "array",
|
|
"minItems": 3,
|
|
"maxItems": 3,
|
|
"description": "Exactly 3 dry_run write tools. Record intent to flight-recorder transcript; do NOT mutate engine state.",
|
|
"items": {
|
|
"type": "object",
|
|
"required": ["name", "description", "input_schema"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"name": {
|
|
"type": "string",
|
|
"enum": [
|
|
"dry_run_put_page",
|
|
"dry_run_add_link",
|
|
"dry_run_add_timeline_entry"
|
|
]
|
|
},
|
|
"description": {"type": "string"},
|
|
"input_schema": {"type": "object"}
|
|
}
|
|
}
|
|
},
|
|
"tool_output_max_tokens": {
|
|
"type": "integer",
|
|
"const": 32768,
|
|
"description": "Cap per tool result. Truncation appends the literal suffix '…[truncated]'."
|
|
}
|
|
}
|
|
}
|