mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
Garry Tan
08383b2943
Five red-team criticals, all closed and test-pinned: (1) two-PR gate poisoning: a same-hash committed-baseline edit must now be receipts-backed (byte-match THIS run) or the gate is inconclusive — and a receipts-backed same-hash regression still needs a justification; (2) corpus hollowing: bless-mode gold_total shrink (deleting failing fixtures / flipping holdout) now requires a justification like any adverse move; (3) continuity cross-source detection was structurally blind to writer-seeded slugs (the zero-gate was vacuous) — writer+reader slug→source maps now merge; (4) --update-baseline writes BEFORE compare (the one-shot bless flow no longer trains users to ignore exit 2) and REFUSES to write when seed failures invalidate the run; (5) 15 continuity writers' write-back gold was computed and dropped — now accumulated, so the write-back cell is flag-independent (gold 46 → 61). Plus: baselines bind their run config (holdout/llm/harness/suite — fixtures_hash covers files only), the CI gate script refuses a working-tree baseline deletion, and renderInlineTime no longer straddles UTC midnight for non-Z offsets.
48 lines
2.1 KiB
JSON
48 lines
2.1 KiB
JSON
{
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"$id": "https://github.com/garrytan/gbrain/evals/brainbench/schema/baseline.schema.json",
|
|
"title": "BrainBench committed baseline (schema_version 1)",
|
|
"description": "The diff-stable committed gate baseline (evals/brainbench/baselines/main.json). Metrics rounded to 4 decimals, keys sorted, receipts excluded — every byte in this file is a reviewable delta. CI compares HEAD's run against MAIN's copy of this file, never the PR's (decision 4).",
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["schema_version", "fixtures_hash", "config", "cells", "counts"],
|
|
"properties": {
|
|
"schema_version": { "const": 1 },
|
|
"fixtures_hash": { "type": "string" },
|
|
"config": {
|
|
"type": "object",
|
|
"description": "Run configuration the numbers were produced under — fixtures_hash covers files only; comparisons require matching config.",
|
|
"additionalProperties": false,
|
|
"required": ["include_holdout", "llm", "harnesses", "suites"],
|
|
"properties": {
|
|
"include_holdout": { "type": "boolean" },
|
|
"llm": { "type": "boolean" },
|
|
"harnesses": { "type": "array", "items": { "type": "string" } },
|
|
"suites": { "type": "array", "items": { "type": "string" } }
|
|
}
|
|
},
|
|
"justification": {
|
|
"type": "string",
|
|
"description": "REQUIRED (by the gate, not the schema) when this update regresses any metric vs the prior committed baseline — the reviewable reason."
|
|
},
|
|
"cells": {
|
|
"type": "object",
|
|
"description": "`${harness}/${suite}` → metric name → value (4-decimal rounded).",
|
|
"additionalProperties": { "type": "object", "additionalProperties": { "type": "number" } }
|
|
},
|
|
"counts": {
|
|
"type": "object",
|
|
"description": "`${harness}/${suite}` → count-aware gate inputs.",
|
|
"additionalProperties": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["gold_total", "gold_failed"],
|
|
"properties": {
|
|
"gold_total": { "type": "integer", "minimum": 0 },
|
|
"gold_failed": { "type": "integer", "minimum": 0 }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|