mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
Garry Tan
a1476af084
Deterministic generator (Mulberry32 seed 42, whole-cloth fictional universe, template prose with controlled difficulty incl. documented v1 reflex limits) emits 135 fixtures + sealed gold across 7 categories with a 15% holdout split; 6 hand-authored spike fixtures froze the schema. Published JSON Schemas are the foreign-runner contract. Gold validated by blind double-label (96.4% agreement; findings fixed, receipt in _ledger.json). Privacy guard extended over the new corpus dirs.
36 lines
1.6 KiB
JSON
36 lines
1.6 KiB
JSON
{
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"$id": "https://github.com/garrytan/gbrain/evals/brainbench/schema/baseline.schema.json",
|
|
"title": "BrainBench committed baseline (schema_version 1)",
|
|
"description": "The diff-stable committed gate baseline (evals/brainbench/baselines/main.json). Metrics rounded to 4 decimals, keys sorted, receipts excluded — every byte in this file is a reviewable delta. CI compares HEAD's run against MAIN's copy of this file, never the PR's (decision 4).",
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["schema_version", "fixtures_hash", "cells", "counts"],
|
|
"properties": {
|
|
"schema_version": { "const": 1 },
|
|
"fixtures_hash": { "type": "string" },
|
|
"justification": {
|
|
"type": "string",
|
|
"description": "REQUIRED (by the gate, not the schema) when this update regresses any metric vs the prior committed baseline — the reviewable reason."
|
|
},
|
|
"cells": {
|
|
"type": "object",
|
|
"description": "`${harness}/${suite}` → metric name → value (4-decimal rounded).",
|
|
"additionalProperties": { "type": "object", "additionalProperties": { "type": "number" } }
|
|
},
|
|
"counts": {
|
|
"type": "object",
|
|
"description": "`${harness}/${suite}` → count-aware gate inputs.",
|
|
"additionalProperties": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["gold_total", "gold_failed"],
|
|
"properties": {
|
|
"gold_total": { "type": "integer", "minimum": 0 },
|
|
"gold_failed": { "type": "integer", "minimum": 0 }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|