mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-29 19:01:39 +00:00
Garry Tan
b3fd66bfc6
(1) test-status never read needs.brainbench.result: a failing gate job left the branch-protection aggregate green — merge-bypassable. Now in the loop. (2) The first-landing path verifies the run against the COMMITTED baseline (an unverified initial merge could seed a doctored baseline for every future PR). (3) One stored fact row may satisfy at most one gold probe (merged/ broad extractions inflated fidelity via unconstrained find()). (4) The slug-granularity limit of source-isolation detection is disclosed in the methodology doc with a pointer to the engine-layer fuzz that covers the content-leak case. Plus 30s timeouts on engine-touching tests (observed 5s default flaking at load avg 20).
68 lines
3.2 KiB
Bash
Executable File
68 lines
3.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# BrainBench CI gate (Cathedral 2, decision 4) — local parity with the
|
|
# .github/workflows/test.yml `brainbench` job.
|
|
#
|
|
# Governance: the gate compares HEAD's run against MAIN's copy of the
|
|
# committed baseline (git show origin/master:...), NEVER the working tree's —
|
|
# a PR cannot rewrite the thing it is compared against. Two modes resolve
|
|
# automatically inside `eval brainbench --compare`:
|
|
# same fixtures_hash → count-aware gate (any newly-failed gold item fails)
|
|
# different hash → corpus-bless (the PR's committed baseline must
|
|
# exactly match HEAD's run; regressions vs main need
|
|
# a `justification` in the committed baseline)
|
|
#
|
|
# Exit codes pass through: 0 pass · 1 regression · 2 error/inconclusive.
|
|
|
|
set -euo pipefail
|
|
|
|
BASELINE_PATH="evals/brainbench/baselines/main.json"
|
|
MAIN_REF="${BRAINBENCH_MAIN_REF:-origin/master}"
|
|
# mktemp default (review finding): a fixed world-writable /tmp path is a
|
|
# symlink-planting target on shared hosts. CI overrides via BRAINBENCH_OUT.
|
|
if [ -n "${BRAINBENCH_OUT:-}" ]; then
|
|
OUT="$BRAINBENCH_OUT"
|
|
OUT_IS_TEMP=0
|
|
else
|
|
OUT="$(mktemp /tmp/brainbench-result-XXXXXX.json)"
|
|
OUT_IS_TEMP=1
|
|
fi
|
|
MAIN_BASELINE="$(mktemp /tmp/brainbench-main-baseline-XXXXXX.json)"
|
|
cleanup() {
|
|
rm -f "$MAIN_BASELINE"
|
|
[ "$OUT_IS_TEMP" = "1" ] && rm -f "$OUT" || true
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
# Fail HARD when the ref itself is broken — only a genuinely-absent baseline
|
|
# may take the ungated first-landing path (review finding: an unfetched ref
|
|
# or typo'd BRAINBENCH_MAIN_REF must not silently disable the gate).
|
|
if ! git rev-parse --verify --quiet "${MAIN_REF}^{commit}" > /dev/null; then
|
|
echo "[brainbench-gate] ERROR: ref ${MAIN_REF} does not resolve — fetch it or fix BRAINBENCH_MAIN_REF" >&2
|
|
exit 2
|
|
fi
|
|
|
|
if git show "${MAIN_REF}:${BASELINE_PATH}" > "$MAIN_BASELINE" 2>/dev/null; then
|
|
# Deletion defense (red-team finding): if main carries a baseline but the
|
|
# working tree deleted it, every FUTURE PR would take the ungated
|
|
# first-landing path once this one merges. Refuse.
|
|
if [ ! -f "$BASELINE_PATH" ]; then
|
|
echo "[brainbench-gate] ERROR: ${BASELINE_PATH} exists on ${MAIN_REF} but is deleted in this tree — restore it or re-run --update-baseline" >&2
|
|
exit 2
|
|
fi
|
|
echo "[brainbench-gate] comparing against ${MAIN_REF}:${BASELINE_PATH}"
|
|
bun src/cli.ts eval brainbench --compare "$MAIN_BASELINE" --out "$OUT"
|
|
else
|
|
# First landing: the ref exists but carries no baseline yet. The COMMITTED
|
|
# baseline still gets verified against the actual run (codex adversarial
|
|
# finding: an unverified first landing could seed a doctored baseline for
|
|
# every future PR to compare against). Same-hash + committed==run logic
|
|
# inside --compare does the verification.
|
|
if [ -f "$BASELINE_PATH" ]; then
|
|
echo "[brainbench-gate] no baseline on ${MAIN_REF} yet — verifying the run against the COMMITTED baseline (first-landing path)"
|
|
bun src/cli.ts eval brainbench --compare "$BASELINE_PATH" --out "$OUT"
|
|
else
|
|
echo "[brainbench-gate] no baseline on ${MAIN_REF} and none committed — running ungated (pre-baseline tree)"
|
|
bun src/cli.ts eval brainbench --out "$OUT"
|
|
fi
|
|
fi
|