mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-29 19:01:39 +00:00
Garry Tan
a1476af084
Deterministic generator (Mulberry32 seed 42, whole-cloth fictional universe, template prose with controlled difficulty incl. documented v1 reflex limits) emits 135 fixtures + sealed gold across 7 categories with a 15% holdout split; 6 hand-authored spike fixtures froze the schema. Published JSON Schemas are the foreign-runner contract. Gold validated by blind double-label (96.4% agreement; findings fixed, receipt in _ledger.json). Privacy guard extended over the new corpus dirs.
75 lines
2.7 KiB
JSON
75 lines
2.7 KiB
JSON
{
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"$id": "https://github.com/garrytan/gbrain/evals/brainbench/schema/fixture.schema.json",
|
|
"title": "BrainBench fixture (schema_version 1)",
|
|
"description": "Adapter-visible conversation fixture. Gold annotations are SEALED: they live in a separate <fixture_id>.gold.json (see gold.schema.json definitions here), never inline — a `gold` key inside a turn is a validation error.",
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["schema_version", "fixture_id", "suites", "turns"],
|
|
"properties": {
|
|
"schema_version": { "const": 1 },
|
|
"fixture_id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]*$" },
|
|
"suites": {
|
|
"type": "array",
|
|
"minItems": 1,
|
|
"items": { "enum": ["know-to-ask", "push", "write-back", "continuity"] }
|
|
},
|
|
"category": { "type": "string" },
|
|
"holdout": { "type": "boolean", "description": "Excluded from the CI gate; scored in published runs only." },
|
|
"sources": { "type": "array", "items": { "type": "string" } },
|
|
"active_source": { "type": "string", "default": "default" },
|
|
"seed_pages": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["slug", "content"],
|
|
"properties": {
|
|
"slug": { "type": "string" },
|
|
"content": { "type": "string" },
|
|
"source_id": { "type": "string" }
|
|
}
|
|
}
|
|
},
|
|
"seed_facts": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["fact"],
|
|
"properties": {
|
|
"fact": { "type": "string" },
|
|
"entity_slug": { "type": ["string", "null"] },
|
|
"source": { "type": "string" },
|
|
"source_session": { "type": ["string", "null"] },
|
|
"source_id": { "type": "string" }
|
|
}
|
|
}
|
|
},
|
|
"turns": {
|
|
"type": "array",
|
|
"minItems": 1,
|
|
"items": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["turn_id", "role", "text"],
|
|
"properties": {
|
|
"turn_id": { "type": "integer" },
|
|
"role": { "enum": ["user", "assistant"] },
|
|
"text": { "type": "string", "minLength": 1 },
|
|
"ts": { "type": "string", "format": "date-time", "description": "Required on every turn of write-back fixtures and continuity writers (segmentation is time-based)." }
|
|
}
|
|
}
|
|
},
|
|
"continuity": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["pair_id", "pair_role"],
|
|
"properties": {
|
|
"pair_id": { "type": "string" },
|
|
"pair_role": { "enum": ["writer", "reader"] }
|
|
}
|
|
}
|
|
}
|
|
}
|