mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
feat(v0.25.0): gbrain eval replay + contributor doc + CONTRIBUTING link
Closes the gap between "session capture works" (this PR's core) and "contributors actually use it before merging." Three artifacts: - src/commands/eval-replay.ts (~340 LOC) — reads NDJSON from `gbrain eval export`, re-runs each captured query/search against the current brain, computes set-Jaccard@k, top-1 stability, and latency delta. Stable JSON shape (schema_version:1) for CI gating; human mode prints a regression table sorted worst-first. Pure Bun, zero new deps. Stub-engine tests cover Jaccard math, NDJSON parser (including v2 forward-compat rejection + line-numbered errors), --limit, --verbose, --json, and graceful per-row error handling. 16/16 passing. - docs/eval-bench.md (~80 lines) — contributor guide. The 4-command loop (export → change → replay → diff), metric definitions with healthy ranges (Jaccard ≥0.85, top-1 ≥85%, latency Δ within ±50ms), trigger paths, CI integration snippet, hand-crafted NDJSON corpus path for fresh installs, and the off-switch. Pairs with the existing docs/eval-capture.md which is the consumer-facing wire format. - CONTRIBUTING.md gains a "Running real-world eval benchmarks (touching retrieval code)" section with the trigger paths and a link to docs/eval-bench.md. Reviewers now have a one-line ask: "did you run replay?" CLAUDE.md key files updated. CHANGELOG bullets added. llms.txt regenerated. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
ec32a56978
commit
a5c375dd15
@@ -68,6 +68,9 @@ Measured on this branch's diff against v0.21.0:
|
||||
- PII scrubber in `src/core/eval-capture-scrub.ts` (6 regex families, adversarial-input safe)
|
||||
- Cross-process audit via `eval_capture_failures` + `gbrain doctor` 24h breakdown
|
||||
- `gbrain eval export` (NDJSON, schema_version:1, EPIPE-safe) + `gbrain eval prune` (explicit retention)
|
||||
- `gbrain eval replay --against <ndjson>` — contributor-facing dev loop. Reads a captured snapshot, re-runs every `query` / `search` against the current brain, prints mean Jaccard@k between captured + current `retrieved_slugs`, top-1 stability rate, and latency Δ. JSON mode (`schema_version: 1`) for CI gating, top-regressions table for human eyeballs. Closes the gap between "data captured" and "data used to gate a PR."
|
||||
- `docs/eval-bench.md` — contributor guide that walks the 4-command loop (export → change → replay → diff), defines metrics (Jaccard@k, top-1 stability, latency Δ) with healthy ranges, lists the source paths that should trigger a re-run, and shows how to wire it into CI. Linked from CONTRIBUTING.md.
|
||||
- `CONTRIBUTING.md` adds a "Running real-world eval benchmarks (touching retrieval code)" section so PRs that change retrieval have an obvious replay path. Maintainer reviews can ask "did you run replay?" instead of writing a custom benchmark per PR.
|
||||
- `hybridSearch` adds `onMeta?: (m) => void` to opts (Cathedral II callers unaffected)
|
||||
- `BrainEngine` gains 5 methods (breaking-interface for custom engines, drives v0.25.0 minor bump)
|
||||
- `test/public-exports.test.ts` + `scripts/check-exports-count.sh` lock the 17-subpath public surface
|
||||
|
||||
@@ -49,9 +49,11 @@ strict behavior when unset.
|
||||
- `src/core/search/eval.ts` — Retrieval eval harness: P@k, R@k, MRR, nDCG@k metrics + runEval() orchestrator
|
||||
- `src/core/search/source-boost.ts` (v0.22.0) — Source-type boost map keyed by slug prefix. `DEFAULT_SOURCE_BOOSTS` (originals/ 1.5, concepts/ 1.3, writing/ 1.4, people/companies/deals/ 1.2, daily/ 0.8, media/x/ 0.7, wintermute/chat/ 0.5) and `DEFAULT_HARD_EXCLUDES` (test/, archive/, attachments/, .raw/). `parseSourceBoostEnv` / `parseHardExcludesEnv` parse comma-separated `prefix:factor` pairs from `GBRAIN_SOURCE_BOOST` / `GBRAIN_SEARCH_EXCLUDE` env vars. `resolveBoostMap` and `resolveHardExcludes` merge defaults + env + caller `SearchOpts.exclude_slug_prefixes`/`include_slug_prefixes`.
|
||||
- `src/core/search/sql-ranking.ts` (v0.22.0) — Pure SQL string builders. `buildSourceFactorCase(slugColumn, boostMap, detail)` emits a CASE expression with longest-prefix-match wins (returns literal `'1.0'` when `detail === 'high'` for temporal-bypass parity with COMPILED_TRUTH_BOOST). `buildHardExcludeClause(slugColumn, prefixes)` emits `NOT (col LIKE 'p1%' OR col LIKE 'p2%')` — OR-chain wrapped in NOT, NOT `NOT LIKE ALL/ANY` (those quantifiers don't express set-exclusion). LIKE meta-character escape covers all three of `%`, `_`, AND `\` (backslash matters because it's Postgres LIKE's default escape char). Single-quote doubling on SQL string literals so injection-style inputs are inert text.
|
||||
- `src/commands/eval.ts` — `gbrain eval` command: single-run table + A/B config comparison. v0.25.0 adds sub-subcommand dispatch on `args[0]` so `gbrain eval export` + `gbrain eval prune` route into session-capture handlers; bare `gbrain eval --qrels …` fall-through preserves the legacy IR-metrics flow.
|
||||
- `src/commands/eval.ts` — `gbrain eval` command: single-run table + A/B config comparison. v0.25.0 adds sub-subcommand dispatch on `args[0]` so `gbrain eval export` + `gbrain eval prune` + `gbrain eval replay` route into session-capture handlers; bare `gbrain eval --qrels …` fall-through preserves the legacy IR-metrics flow.
|
||||
- `src/commands/eval-export.ts` (v0.25.0) — streams `eval_candidates` rows as NDJSON to stdout with `schema_version: 1` prefix on every line. EPIPE-safe, progress heartbeats on stderr, stable id-desc tiebreaker so `--since` windows never dupe/miss rows.
|
||||
- `src/commands/eval-prune.ts` (v0.25.0) — explicit retention cleanup. Requires `--older-than DUR`. `--dry-run` reports would-delete count.
|
||||
- `src/commands/eval-replay.ts` (v0.25.0) — contributor-facing replay tool. Reads NDJSON from `gbrain eval export`, re-runs each captured `query` / `search` op against the current brain, computes set-Jaccard@k between captured + current `retrieved_slugs`, top-1 stability rate, and latency Δ. Stable JSON shape (`schema_version: 1`) for CI gating; human mode prints a regression table. Pure Bun, zero new deps. The dev-loop half of BrainBench-Real that closes the gap between "data captured" and "data used to gate a PR." See `docs/eval-bench.md` for the workflow.
|
||||
- `docs/eval-bench.md` (v0.25.0) — contributor guide for using captured data to benchmark retrieval changes before merging. Linked from CONTRIBUTING.md under "Running real-world eval benchmarks (touching retrieval code)".
|
||||
- `src/core/eval-capture.ts` (v0.25.0) — op-layer capture wrapper called from `src/core/operations.ts` `query` + `search` handlers. Catches MCP + CLI + subagent tool-bridge from one site. Fire-and-forget; failures route to `engine.logEvalCaptureFailure` so `gbrain doctor` sees drops cross-process.
|
||||
- `src/core/eval-capture-scrub.ts` (v0.25.0) — zero-deps PII scrubber: emails, phones, SSN, Luhn-verified credit cards, JWT-shaped tokens, bearer tokens.
|
||||
- `src/core/search/hybrid.ts` — Cathedral II `Promise<SearchResult[]>` return shape unchanged in v0.25.0. Adds `onMeta?: (m: HybridSearchMeta) => void` callback so op-layer capture can record what hybridSearch actually did. Existing callers leave it undefined.
|
||||
|
||||
@@ -124,6 +124,40 @@ See `docs/ENGINES.md` for the full guide. In short:
|
||||
|
||||
The SQLite engine is designed and ready for implementation. See `docs/SQLITE_ENGINE.md`.
|
||||
|
||||
## Running real-world eval benchmarks (touching retrieval code)
|
||||
|
||||
If your PR touches retrieval — search ranking, RRF fusion, embeddings,
|
||||
intent classification, query expansion, source boost, or the `query` /
|
||||
`search` op handlers — run `gbrain eval replay` against a snapshot of
|
||||
real traffic before merging.
|
||||
|
||||
Quick loop:
|
||||
|
||||
```bash
|
||||
gbrain eval export --since 7d > baseline.ndjson # snapshot before your change
|
||||
# ... make your change ...
|
||||
gbrain eval replay --against baseline.ndjson # diff retrieval, get Jaccard@k
|
||||
```
|
||||
|
||||
Three numbers come back: mean Jaccard@k between captured and current slug
|
||||
sets, top-1 stability, and mean latency Δ. The replay tool flags the worst
|
||||
regressions so you can eyeball whether the change is hurting real queries.
|
||||
|
||||
Trigger paths (rerun if your diff touches any of these):
|
||||
|
||||
- `src/core/search/hybrid.ts`
|
||||
- `src/core/search/source-boost.ts`, `sql-ranking.ts`
|
||||
- `src/core/search/intent.ts`, `expansion.ts`, `dedup.ts`
|
||||
- `src/core/embedding.ts`
|
||||
- `src/core/operations.ts` (query / search handlers)
|
||||
- `src/core/postgres-engine.ts` / `pglite-engine.ts` (searchKeyword /
|
||||
searchVector SQL)
|
||||
|
||||
See [`docs/eval-bench.md`](./docs/eval-bench.md) for the full guide
|
||||
including CI integration, hand-crafted NDJSON corpora, and the cost
|
||||
considerations. The NDJSON wire format is documented in
|
||||
[`docs/eval-capture.md`](./docs/eval-capture.md).
|
||||
|
||||
## Welcome PRs
|
||||
|
||||
- SQLite engine implementation
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
# Running real-world eval benchmarks against your gbrain changes
|
||||
|
||||
Audience: gbrain maintainers and contributors. If you're touching retrieval
|
||||
(search, ranking, embeddings, intent classification, query expansion, source
|
||||
boost, hybrid fusion), this is the doc.
|
||||
|
||||
For the **NDJSON wire format** consumed by gbrain-evals, see
|
||||
[`eval-capture.md`](./eval-capture.md). This doc is the human dev loop
|
||||
that lives on top of that format.
|
||||
|
||||
## The 4-command loop
|
||||
|
||||
```bash
|
||||
# ① Capture: already happening on every MCP / CLI / subagent query, capture is on by default.
|
||||
# Inspect what's been collected:
|
||||
gbrain doctor # surfaces capture failures
|
||||
psql $DATABASE_URL -c 'SELECT count(*) FROM eval_candidates'
|
||||
|
||||
# ② Snapshot: freeze a baseline before your code change.
|
||||
gbrain eval export --since 7d > baseline.ndjson
|
||||
|
||||
# ③ Code change: do whatever you want — tune RRF_K, swap embed model, edit
|
||||
# hybrid.ts, add a new boost source, change the intent classifier.
|
||||
|
||||
# ④ Replay: re-run every captured query against the current build.
|
||||
gbrain eval replay --against baseline.ndjson
|
||||
```
|
||||
|
||||
Output:
|
||||
|
||||
```
|
||||
Replaying 247 captured queries…
|
||||
...25/247
|
||||
...50/247
|
||||
...
|
||||
Replayed 247 of 247 captured queries (0 skipped, 0 errored)
|
||||
Mean Jaccard@k: 0.927
|
||||
Top-1 stability: 91.5%
|
||||
Mean latency Δ: +14ms (current vs captured)
|
||||
|
||||
Top 5 regression(s):
|
||||
jaccard=0.20 captured=12 current=3 "find every reference to widget-co"
|
||||
jaccard=0.43 captured=14 current=8 "show me everything tagged for review"
|
||||
jaccard=0.50 captured=8 current=4 "what did alice say about the spec"
|
||||
...
|
||||
```
|
||||
|
||||
Three numbers tell you whether the change is safe to land:
|
||||
|
||||
| Metric | What it means | Healthy range |
|
||||
|---|---|---|
|
||||
| **Mean Jaccard@k** | Average overlap between captured retrieved slugs and current run's slugs. 1.0 = identical sets. | ≥0.85 for "neutral" changes. <0.7 means major retrieval shift. |
|
||||
| **Top-1 stability** | Fraction of queries whose #1 result didn't change. | ≥85% for tuning passes. <70% means top-of-funnel broke. |
|
||||
| **Mean latency Δ** | Current minus captured. Positive = slower now. | Within ±50ms of captured. >2× anywhere = regression alarm. |
|
||||
|
||||
## What it actually does
|
||||
|
||||
`gbrain eval replay` reads your NDJSON snapshot and, for each row:
|
||||
|
||||
1. Re-executes the same op (`searchKeyword` for `tool_name='search'`,
|
||||
`hybridSearch` for `tool_name='query'`) with the captured `detail` and
|
||||
`expand_enabled` values threaded back in.
|
||||
2. Captures the current `retrieved_slugs` (deduped, in result order).
|
||||
3. Computes set-Jaccard between captured and current slug sets.
|
||||
4. Records top-1 match (was the #1 result the same slug?).
|
||||
5. Records latency delta vs captured `latency_ms`.
|
||||
|
||||
It does NOT compute MRR or nDCG — those need ground-truth relevance labels,
|
||||
not a baseline comparison. For metric-against-truth eval, use
|
||||
`gbrain eval --qrels <path>` (the legacy IR-eval path, still supported). The
|
||||
replay tool answers a different question: "did my code change move
|
||||
retrieval, and which queries did it move most?"
|
||||
|
||||
## Best-effort by design
|
||||
|
||||
Replay is not pure. Three things can drift between capture and replay:
|
||||
|
||||
1. **Brain state** — your brain probably has more pages now than when the
|
||||
snapshot was taken. Unless you explicitly seed a fixed corpus, mean
|
||||
Jaccard will drop simply because new pages are eligible.
|
||||
2. **Embedding source** — if you changed `OPENAI_API_KEY` between capture
|
||||
and replay (or the embedding model rotated), vector-path results drift
|
||||
even with identical code.
|
||||
3. **Capture cap** — captured `retrieved_slugs` is a deduped set; it doesn't
|
||||
preserve internal ranking metadata. Two tools can return the same slug
|
||||
set with different scores — Jaccard will say 1.0, but a downstream
|
||||
consumer that orders by score may behave differently.
|
||||
|
||||
The metrics are **regression alarms on real queries**, not a hash check.
|
||||
Pair them with manual inspection of the top regressions.
|
||||
|
||||
## Cost
|
||||
|
||||
Every `query` row in the snapshot embeds the query string via OpenAI to run
|
||||
the vector half of `hybridSearch`. Cost is identical to a normal `gbrain
|
||||
query` invocation — text-embedding-3-large at OpenAI list price, batched
|
||||
inside a single replay row.
|
||||
|
||||
If you're iterating locally and don't want to pay per change, use
|
||||
`--limit 50` to cap rows replayed. The 50 most recent rows are usually
|
||||
enough to catch direction; expand for the final pre-merge run.
|
||||
|
||||
```bash
|
||||
# Iteration mode — 50 most recent queries
|
||||
gbrain eval replay --against baseline.ndjson --limit 50
|
||||
|
||||
# Pre-merge — full snapshot
|
||||
gbrain eval replay --against baseline.ndjson --top-regressions 20
|
||||
```
|
||||
|
||||
## CI integration
|
||||
|
||||
```bash
|
||||
gbrain eval replay --against baseline.ndjson --json > replay.json
|
||||
jq -e '.summary.mean_jaccard >= 0.85' replay.json || exit 1
|
||||
jq -e '.summary.top1_stability_rate >= 0.85' replay.json || exit 1
|
||||
```
|
||||
|
||||
Stable JSON shape (schema_version: 1):
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": 1,
|
||||
"summary": {
|
||||
"rows_total": 247,
|
||||
"rows_replayed": 247,
|
||||
"rows_skipped": 0,
|
||||
"rows_errored": 0,
|
||||
"mean_jaccard": 0.927,
|
||||
"top1_stability_rate": 0.915,
|
||||
"mean_latency_delta_ms": 14,
|
||||
"rows_over_2x_latency": 0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`--verbose` adds a `results: [...]` array with one entry per replayed row
|
||||
(useful for piping into jq or a notebook for deeper analysis).
|
||||
|
||||
## When to run this
|
||||
|
||||
Before merging anything that touches:
|
||||
|
||||
- `src/core/search/hybrid.ts` (RRF, fusion, dedup, two-pass retrieval)
|
||||
- `src/core/search/source-boost.ts` / `sql-ranking.ts` (per-source ranking)
|
||||
- `src/core/search/intent.ts` (auto-detail classification)
|
||||
- `src/core/search/expansion.ts` (Haiku query expansion)
|
||||
- `src/core/search/dedup.ts` (cross-page result collapse)
|
||||
- `src/core/embedding.ts` or any embedding model swap
|
||||
- `src/core/operations.ts` `query` or `search` op handlers (capture surface)
|
||||
- `src/core/postgres-engine.ts` / `pglite-engine.ts` `searchKeyword` /
|
||||
`searchVector` SQL
|
||||
|
||||
Skip for: schema-only migrations, doc changes, tests-only PRs, CLI ergonomics
|
||||
that don't touch retrieval.
|
||||
|
||||
## Building your own corpus
|
||||
|
||||
If you don't have captured traffic yet (fresh install, can't dogfood for a
|
||||
week before merging), you can hand-author an NDJSON file:
|
||||
|
||||
```jsonl
|
||||
{"schema_version":1,"id":1,"tool_name":"query","query":"who is alice","retrieved_slugs":["people/alice","people/alice-bio"],"expand_enabled":false,"detail":null,"latency_ms":0,"remote":false}
|
||||
{"schema_version":1,"id":2,"tool_name":"search","query":"acme deal","retrieved_slugs":["deals/acme-seed","companies/acme"],"latency_ms":0,"remote":false}
|
||||
```
|
||||
|
||||
Then run `gbrain eval replay --against handcrafted.ndjson` to confirm the
|
||||
authoritative slugs come back. This is the seam between the BrainBench-Real
|
||||
pipeline (replay against live captures) and the BrainBench fixed-fixture
|
||||
pipeline (`gbrain eval --qrels` with the sibling
|
||||
[gbrain-evals](https://github.com/garrytan/gbrain-evals) corpus).
|
||||
|
||||
## Off-switch
|
||||
|
||||
If you don't want capture at all (privacy concern, dev environment), edit
|
||||
`~/.gbrain/config.json`:
|
||||
|
||||
```json
|
||||
{"eval": {"capture": false}}
|
||||
```
|
||||
|
||||
Capture stops; existing `eval_candidates` rows stay until you `gbrain eval
|
||||
prune --older-than 0d` (or just drop the table).
|
||||
|
||||
## Failure modes
|
||||
|
||||
| What you see | What it means |
|
||||
|---|---|
|
||||
| `Mean Jaccard@k: 0.4`, top regressions all in one source dir | Source boost or hard-exclude regression on that prefix |
|
||||
| `Top-1 stability: 30%`, mean Jaccard still high | RRF tuning shifted the rank order without changing the set — re-tune `rrfK` |
|
||||
| `Mean latency Δ: +500ms`, jaccard high | Vector path got slower; check embedding API or HNSW probes |
|
||||
| `rows_errored > 0` | One or more queries threw. Inspect first 3 in human output, or `--json` to see all `error_message` fields |
|
||||
| Many `skipped: empty query` | Capture ran on rows where someone passed empty `query` — check why those were captured |
|
||||
+3
-1
@@ -135,9 +135,11 @@ strict behavior when unset.
|
||||
- `src/core/search/eval.ts` — Retrieval eval harness: P@k, R@k, MRR, nDCG@k metrics + runEval() orchestrator
|
||||
- `src/core/search/source-boost.ts` (v0.22.0) — Source-type boost map keyed by slug prefix. `DEFAULT_SOURCE_BOOSTS` (originals/ 1.5, concepts/ 1.3, writing/ 1.4, people/companies/deals/ 1.2, daily/ 0.8, media/x/ 0.7, wintermute/chat/ 0.5) and `DEFAULT_HARD_EXCLUDES` (test/, archive/, attachments/, .raw/). `parseSourceBoostEnv` / `parseHardExcludesEnv` parse comma-separated `prefix:factor` pairs from `GBRAIN_SOURCE_BOOST` / `GBRAIN_SEARCH_EXCLUDE` env vars. `resolveBoostMap` and `resolveHardExcludes` merge defaults + env + caller `SearchOpts.exclude_slug_prefixes`/`include_slug_prefixes`.
|
||||
- `src/core/search/sql-ranking.ts` (v0.22.0) — Pure SQL string builders. `buildSourceFactorCase(slugColumn, boostMap, detail)` emits a CASE expression with longest-prefix-match wins (returns literal `'1.0'` when `detail === 'high'` for temporal-bypass parity with COMPILED_TRUTH_BOOST). `buildHardExcludeClause(slugColumn, prefixes)` emits `NOT (col LIKE 'p1%' OR col LIKE 'p2%')` — OR-chain wrapped in NOT, NOT `NOT LIKE ALL/ANY` (those quantifiers don't express set-exclusion). LIKE meta-character escape covers all three of `%`, `_`, AND `\` (backslash matters because it's Postgres LIKE's default escape char). Single-quote doubling on SQL string literals so injection-style inputs are inert text.
|
||||
- `src/commands/eval.ts` — `gbrain eval` command: single-run table + A/B config comparison. v0.25.0 adds sub-subcommand dispatch on `args[0]` so `gbrain eval export` + `gbrain eval prune` route into session-capture handlers; bare `gbrain eval --qrels …` fall-through preserves the legacy IR-metrics flow.
|
||||
- `src/commands/eval.ts` — `gbrain eval` command: single-run table + A/B config comparison. v0.25.0 adds sub-subcommand dispatch on `args[0]` so `gbrain eval export` + `gbrain eval prune` + `gbrain eval replay` route into session-capture handlers; bare `gbrain eval --qrels …` fall-through preserves the legacy IR-metrics flow.
|
||||
- `src/commands/eval-export.ts` (v0.25.0) — streams `eval_candidates` rows as NDJSON to stdout with `schema_version: 1` prefix on every line. EPIPE-safe, progress heartbeats on stderr, stable id-desc tiebreaker so `--since` windows never dupe/miss rows.
|
||||
- `src/commands/eval-prune.ts` (v0.25.0) — explicit retention cleanup. Requires `--older-than DUR`. `--dry-run` reports would-delete count.
|
||||
- `src/commands/eval-replay.ts` (v0.25.0) — contributor-facing replay tool. Reads NDJSON from `gbrain eval export`, re-runs each captured `query` / `search` op against the current brain, computes set-Jaccard@k between captured + current `retrieved_slugs`, top-1 stability rate, and latency Δ. Stable JSON shape (`schema_version: 1`) for CI gating; human mode prints a regression table. Pure Bun, zero new deps. The dev-loop half of BrainBench-Real that closes the gap between "data captured" and "data used to gate a PR." See `docs/eval-bench.md` for the workflow.
|
||||
- `docs/eval-bench.md` (v0.25.0) — contributor guide for using captured data to benchmark retrieval changes before merging. Linked from CONTRIBUTING.md under "Running real-world eval benchmarks (touching retrieval code)".
|
||||
- `src/core/eval-capture.ts` (v0.25.0) — op-layer capture wrapper called from `src/core/operations.ts` `query` + `search` handlers. Catches MCP + CLI + subagent tool-bridge from one site. Fire-and-forget; failures route to `engine.logEvalCaptureFailure` so `gbrain doctor` sees drops cross-process.
|
||||
- `src/core/eval-capture-scrub.ts` (v0.25.0) — zero-deps PII scrubber: emails, phones, SSN, Luhn-verified credit cards, JWT-shaped tokens, bearer tokens.
|
||||
- `src/core/search/hybrid.ts` — Cathedral II `Promise<SearchResult[]>` return shape unchanged in v0.25.0. Adds `onMeta?: (m: HybridSearchMeta) => void` callback so op-layer capture can record what hybridSearch actually did. Existing callers leave it undefined.
|
||||
|
||||
@@ -0,0 +1,424 @@
|
||||
/**
|
||||
* gbrain eval replay — replay captured eval_candidates against current brain (v0.25.0).
|
||||
*
|
||||
* The contributor-facing half of BrainBench-Real:
|
||||
*
|
||||
* 1. capture some real traffic (default-on, lands in eval_candidates)
|
||||
* 2. snapshot it (gbrain eval export --since 7d > baseline.ndjson)
|
||||
* 3. make a code change (tune RRF_K, edit hybrid.ts, swap an embed model)
|
||||
* 4. replay against the snapshot (gbrain eval replay --against baseline.ndjson)
|
||||
*
|
||||
* Outputs three numbers a contributor can read at a glance:
|
||||
*
|
||||
* - mean Jaccard@k between captured retrieved_slugs and current run's slugs
|
||||
* - top-1 stability rate (was the #1 result the same?)
|
||||
* - mean latency delta (current - captured), positive = slower now
|
||||
*
|
||||
* Best-effort by design. Replay is NOT pure — your brain has more pages than
|
||||
* when the capture was taken, embeddings may have drifted, and the OPENAI key
|
||||
* may be different. The metrics describe "did this change hurt retrieval on
|
||||
* the queries you actually serve" not "do these match the baseline byte for
|
||||
* byte." Use it before merging anything that touches src/core/search/ or the
|
||||
* query/search op handlers.
|
||||
*
|
||||
* Usage:
|
||||
* gbrain eval replay --against captured.ndjson [--limit N] [--json]
|
||||
* [--top-regressions K] [--verbose]
|
||||
*/
|
||||
|
||||
import { readFileSync, existsSync } from 'fs';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import type { SearchResult } from '../core/types.ts';
|
||||
import { hybridSearch } from '../core/search/hybrid.ts';
|
||||
|
||||
interface ReplayOpts {
|
||||
help?: boolean;
|
||||
against?: string;
|
||||
limit?: number;
|
||||
json?: boolean;
|
||||
verbose?: boolean;
|
||||
topRegressions?: number;
|
||||
}
|
||||
|
||||
interface RowResult {
|
||||
/** Captured row's id, for back-referencing into the source NDJSON. */
|
||||
id: number;
|
||||
tool_name: 'query' | 'search';
|
||||
query: string;
|
||||
/** Set-overlap score in [0, 1]. 1.0 = identical retrieved set. */
|
||||
jaccard: number;
|
||||
/** True when current top result matches captured top result. */
|
||||
top1Match: boolean;
|
||||
/** Captured retrieved_slugs (as-is from NDJSON). */
|
||||
captured_slugs: string[];
|
||||
/** Current run's slugs (deduped, in result order). */
|
||||
current_slugs: string[];
|
||||
/** Wall-clock latency (ms) of the current re-run. */
|
||||
current_latency_ms: number;
|
||||
/** latency delta = current - captured. Positive = slower now. */
|
||||
latency_delta_ms: number;
|
||||
/** True if the row was skipped (e.g. captured query was empty). */
|
||||
skipped?: boolean;
|
||||
/** Reason the row was skipped, if any. */
|
||||
skip_reason?: string;
|
||||
/** True if the row threw during replay; current_slugs is empty. */
|
||||
errored?: boolean;
|
||||
error_message?: string;
|
||||
}
|
||||
|
||||
function parseArgs(args: string[]): ReplayOpts {
|
||||
const opts: ReplayOpts = {};
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i]!;
|
||||
const next = args[i + 1];
|
||||
switch (arg) {
|
||||
case '--help':
|
||||
case '-h':
|
||||
opts.help = true;
|
||||
break;
|
||||
case '--against':
|
||||
if (!next) break;
|
||||
opts.against = next;
|
||||
i++;
|
||||
break;
|
||||
case '--limit':
|
||||
if (!next) break;
|
||||
opts.limit = parseInt(next, 10);
|
||||
i++;
|
||||
break;
|
||||
case '--json':
|
||||
opts.json = true;
|
||||
break;
|
||||
case '--verbose':
|
||||
opts.verbose = true;
|
||||
break;
|
||||
case '--top-regressions':
|
||||
if (!next) break;
|
||||
opts.topRegressions = parseInt(next, 10);
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return opts;
|
||||
}
|
||||
|
||||
function printHelp(): void {
|
||||
console.error(`gbrain eval replay — replay captured queries against current brain
|
||||
|
||||
USAGE:
|
||||
gbrain eval replay --against FILE.ndjson [flags]
|
||||
|
||||
FLAGS:
|
||||
--against FILE NDJSON file from \`gbrain eval export\` (required).
|
||||
--limit N Replay at most N rows (default: replay all).
|
||||
Each row hits OpenAI once for query embedding —
|
||||
cap aggressively when iterating locally.
|
||||
--top-regressions K Print the K rows with the worst Jaccard scores.
|
||||
Default 5 in human mode, 0 in --json.
|
||||
--json Emit one JSON object on stdout instead of a table.
|
||||
Stable shape for CI consumption.
|
||||
--verbose Include every row's per-row diff (large output).
|
||||
--help, -h Show this help.
|
||||
|
||||
OUTPUT (human mode):
|
||||
Replayed N captured queries (M skipped, K errored)
|
||||
Mean Jaccard@k: 0.873
|
||||
Top-1 stability: 87% (N=87 / 100)
|
||||
Mean latency Δ: +12ms (current slower)
|
||||
|
||||
Top 5 regressions:
|
||||
0.20 "find every reference to widget-co" captured=12 current=3
|
||||
...
|
||||
|
||||
EXIT CODE:
|
||||
0 — replay completed (regardless of regression magnitude).
|
||||
1 — invalid args, --against not found, or NDJSON parse failure.
|
||||
|
||||
NOTES:
|
||||
Replay is best-effort. Your brain has more pages than when the snapshot
|
||||
was taken; embeddings may have drifted; OPENAI_API_KEY may be different.
|
||||
Use the metrics to spot regressions on REAL queries, not as a hash check.
|
||||
`);
|
||||
}
|
||||
|
||||
interface CapturedRow {
|
||||
schema_version: number;
|
||||
id: number;
|
||||
tool_name: 'query' | 'search';
|
||||
query: string;
|
||||
retrieved_slugs: string[];
|
||||
retrieved_chunk_ids?: number[];
|
||||
source_ids?: string[];
|
||||
expand_enabled?: boolean | null;
|
||||
detail?: 'low' | 'medium' | 'high' | null;
|
||||
detail_resolved?: 'low' | 'medium' | 'high' | null;
|
||||
vector_enabled?: boolean;
|
||||
expansion_applied?: boolean;
|
||||
latency_ms: number;
|
||||
remote?: boolean;
|
||||
job_id?: number | null;
|
||||
subagent_id?: number | null;
|
||||
created_at?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse NDJSON. One object per non-blank line. Single bad line throws — it's
|
||||
* a corrupt export and silently dropping rows would mask real bugs.
|
||||
*/
|
||||
function parseNdjson(content: string): CapturedRow[] {
|
||||
const lines = content.split('\n');
|
||||
const rows: CapturedRow[] = [];
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i]!.trim();
|
||||
if (!line) continue;
|
||||
let row: CapturedRow;
|
||||
try {
|
||||
row = JSON.parse(line);
|
||||
} catch (err) {
|
||||
throw new Error(`NDJSON parse error on line ${i + 1}: ${(err as Error).message}`);
|
||||
}
|
||||
if (typeof row.schema_version !== 'number') {
|
||||
throw new Error(`Line ${i + 1} missing schema_version — not from \`gbrain eval export\`?`);
|
||||
}
|
||||
if (row.schema_version !== 1) {
|
||||
throw new Error(
|
||||
`Line ${i + 1} has schema_version=${row.schema_version}; this replay only supports v1. ` +
|
||||
`Upgrade gbrain or re-export.`,
|
||||
);
|
||||
}
|
||||
rows.push(row);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set-Jaccard between two slug arrays. Order ignored, dupes collapsed.
|
||||
* Both empty → 1.0 (identical empty sets, no information lost).
|
||||
*/
|
||||
function jaccardSlugs(a: string[], b: string[]): number {
|
||||
const setA = new Set(a);
|
||||
const setB = new Set(b);
|
||||
if (setA.size === 0 && setB.size === 0) return 1.0;
|
||||
let intersection = 0;
|
||||
for (const s of setA) if (setB.has(s)) intersection++;
|
||||
const union = setA.size + setB.size - intersection;
|
||||
return union === 0 ? 1.0 : intersection / union;
|
||||
}
|
||||
|
||||
async function replayRow(engine: BrainEngine, row: CapturedRow): Promise<RowResult> {
|
||||
const captured_slugs = row.retrieved_slugs ?? [];
|
||||
const startedAt = Date.now();
|
||||
|
||||
// Default replay limit matches hybridSearch's default (20).
|
||||
const limit = Math.max(captured_slugs.length, 20);
|
||||
|
||||
// search → bare keyword path. query → hybrid path (vector + keyword + RRF).
|
||||
// detail and expansion are threaded in from the captured row so the same
|
||||
// logic runs that produced the original retrieval.
|
||||
let current: SearchResult[];
|
||||
try {
|
||||
if (row.tool_name === 'search') {
|
||||
const dedupedRaw = await engine.searchKeyword(row.query, { limit });
|
||||
current = dedupedRaw;
|
||||
} else {
|
||||
current = await hybridSearch(engine, row.query, {
|
||||
limit,
|
||||
detail: row.detail ?? undefined,
|
||||
expansion: row.expand_enabled ?? false,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
return {
|
||||
id: row.id,
|
||||
tool_name: row.tool_name,
|
||||
query: row.query,
|
||||
jaccard: 0,
|
||||
top1Match: false,
|
||||
captured_slugs,
|
||||
current_slugs: [],
|
||||
current_latency_ms: Date.now() - startedAt,
|
||||
latency_delta_ms: Date.now() - startedAt - row.latency_ms,
|
||||
errored: true,
|
||||
error_message: (err as Error).message ?? String(err),
|
||||
};
|
||||
}
|
||||
|
||||
const current_latency_ms = Date.now() - startedAt;
|
||||
// Dedup slugs while preserving order — same convention as search results.
|
||||
const seen = new Set<string>();
|
||||
const current_slugs: string[] = [];
|
||||
for (const r of current) {
|
||||
if (!seen.has(r.slug)) {
|
||||
seen.add(r.slug);
|
||||
current_slugs.push(r.slug);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
id: row.id,
|
||||
tool_name: row.tool_name,
|
||||
query: row.query,
|
||||
jaccard: jaccardSlugs(captured_slugs, current_slugs),
|
||||
top1Match: captured_slugs[0] !== undefined && current_slugs[0] === captured_slugs[0],
|
||||
captured_slugs,
|
||||
current_slugs,
|
||||
current_latency_ms,
|
||||
latency_delta_ms: current_latency_ms - row.latency_ms,
|
||||
};
|
||||
}
|
||||
|
||||
interface ReplaySummary {
|
||||
rows_total: number;
|
||||
rows_replayed: number;
|
||||
rows_skipped: number;
|
||||
rows_errored: number;
|
||||
/** Mean Jaccard across non-skipped, non-errored rows. */
|
||||
mean_jaccard: number;
|
||||
top1_stability_rate: number;
|
||||
mean_latency_delta_ms: number;
|
||||
/** Rows where current latency is more than 2x captured (regression alarm). */
|
||||
rows_over_2x_latency: number;
|
||||
}
|
||||
|
||||
function summarize(results: RowResult[]): ReplaySummary {
|
||||
const eligible = results.filter(r => !r.skipped && !r.errored);
|
||||
const meanJaccard = eligible.length === 0
|
||||
? 0
|
||||
: eligible.reduce((a, r) => a + r.jaccard, 0) / eligible.length;
|
||||
const top1Rate = eligible.length === 0
|
||||
? 0
|
||||
: eligible.filter(r => r.top1Match).length / eligible.length;
|
||||
const meanLatencyDelta = eligible.length === 0
|
||||
? 0
|
||||
: eligible.reduce((a, r) => a + r.latency_delta_ms, 0) / eligible.length;
|
||||
const over2x = eligible.filter(r => {
|
||||
const captured = results.find(x => x.id === r.id);
|
||||
return captured && captured.current_latency_ms > 2 * (captured.current_latency_ms - captured.latency_delta_ms);
|
||||
}).length;
|
||||
|
||||
return {
|
||||
rows_total: results.length,
|
||||
rows_replayed: eligible.length,
|
||||
rows_skipped: results.filter(r => r.skipped).length,
|
||||
rows_errored: results.filter(r => r.errored).length,
|
||||
mean_jaccard: meanJaccard,
|
||||
top1_stability_rate: top1Rate,
|
||||
mean_latency_delta_ms: meanLatencyDelta,
|
||||
rows_over_2x_latency: over2x,
|
||||
};
|
||||
}
|
||||
|
||||
function printHumanSummary(summary: ReplaySummary, results: RowResult[], topRegressions: number): void {
|
||||
const total = summary.rows_total;
|
||||
const eligible = summary.rows_replayed;
|
||||
console.log(`Replayed ${eligible} of ${total} captured queries (${summary.rows_skipped} skipped, ${summary.rows_errored} errored)`);
|
||||
console.log(`Mean Jaccard@k: ${summary.mean_jaccard.toFixed(3)}`);
|
||||
console.log(`Top-1 stability: ${(summary.top1_stability_rate * 100).toFixed(1)}%`);
|
||||
const sign = summary.mean_latency_delta_ms >= 0 ? '+' : '';
|
||||
console.log(`Mean latency Δ: ${sign}${summary.mean_latency_delta_ms.toFixed(0)}ms (current vs captured)`);
|
||||
if (summary.rows_over_2x_latency > 0) {
|
||||
console.log(`⚠ ${summary.rows_over_2x_latency} row(s) ran more than 2× slower than captured`);
|
||||
}
|
||||
|
||||
if (topRegressions > 0) {
|
||||
const sorted = [...results]
|
||||
.filter(r => !r.skipped && !r.errored)
|
||||
.sort((a, b) => a.jaccard - b.jaccard)
|
||||
.slice(0, topRegressions);
|
||||
if (sorted.length > 0 && sorted[0]!.jaccard < 1.0) {
|
||||
console.log(`\nTop ${sorted.length} regression(s):`);
|
||||
for (const r of sorted) {
|
||||
const truncQuery = r.query.length > 60 ? r.query.slice(0, 57) + '...' : r.query;
|
||||
console.log(
|
||||
` jaccard=${r.jaccard.toFixed(2)} captured=${r.captured_slugs.length} current=${r.current_slugs.length} ` +
|
||||
`"${truncQuery}"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (summary.rows_errored > 0) {
|
||||
const errors = results.filter(r => r.errored).slice(0, 3);
|
||||
console.log(`\n${summary.rows_errored} row(s) errored. First ${errors.length}:`);
|
||||
for (const r of errors) {
|
||||
const truncQuery = r.query.length > 60 ? r.query.slice(0, 57) + '...' : r.query;
|
||||
console.log(` id=${r.id} "${truncQuery}" ${r.error_message ?? ''}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function runEvalReplay(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const opts = parseArgs(args);
|
||||
if (opts.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
if (!opts.against) {
|
||||
console.error('Error: --against FILE.ndjson is required\n');
|
||||
printHelp();
|
||||
process.exit(1);
|
||||
}
|
||||
if (!existsSync(opts.against)) {
|
||||
console.error(`Error: file not found: ${opts.against}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let rows: CapturedRow[];
|
||||
try {
|
||||
const content = readFileSync(opts.against, 'utf-8');
|
||||
rows = parseNdjson(content);
|
||||
} catch (err) {
|
||||
console.error(`Error: ${(err as Error).message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (rows.length === 0) {
|
||||
console.error(`Error: ${opts.against} is empty (no NDJSON rows)`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const capped = opts.limit && opts.limit > 0 ? rows.slice(0, opts.limit) : rows;
|
||||
if (!opts.json) {
|
||||
console.error(
|
||||
`Replaying ${capped.length}${capped.length < rows.length ? ` of ${rows.length}` : ''} captured queries…`,
|
||||
);
|
||||
}
|
||||
|
||||
const results: RowResult[] = [];
|
||||
for (const row of capped) {
|
||||
if (!row.query || row.query.length === 0) {
|
||||
results.push({
|
||||
id: row.id,
|
||||
tool_name: row.tool_name,
|
||||
query: row.query ?? '',
|
||||
jaccard: 0,
|
||||
top1Match: false,
|
||||
captured_slugs: row.retrieved_slugs ?? [],
|
||||
current_slugs: [],
|
||||
current_latency_ms: 0,
|
||||
latency_delta_ms: 0,
|
||||
skipped: true,
|
||||
skip_reason: 'empty query',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const r = await replayRow(engine, row);
|
||||
results.push(r);
|
||||
if (!opts.json && results.length % 25 === 0) {
|
||||
process.stderr.write(` ...${results.length}/${capped.length}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
const summary = summarize(results);
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify({
|
||||
schema_version: 1,
|
||||
summary,
|
||||
results: opts.verbose ? results : undefined,
|
||||
}, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
const topN = opts.topRegressions ?? 5;
|
||||
printHumanSummary(summary, results, topN);
|
||||
}
|
||||
@@ -33,6 +33,10 @@ export async function runEvalCommand(engine: BrainEngine, args: string[]): Promi
|
||||
const { runEvalPrune } = await import('./eval-prune.ts');
|
||||
return runEvalPrune(engine, args.slice(1));
|
||||
}
|
||||
if (sub === 'replay') {
|
||||
const { runEvalReplay } = await import('./eval-replay.ts');
|
||||
return runEvalReplay(engine, args.slice(1));
|
||||
}
|
||||
|
||||
const opts = parseArgs(args);
|
||||
|
||||
|
||||
@@ -0,0 +1,421 @@
|
||||
/**
|
||||
* Tests for `gbrain eval replay` (v0.25.0).
|
||||
*
|
||||
* Three layers:
|
||||
* 1. Pure: Jaccard math + NDJSON parser via re-export.
|
||||
* 2. CLI happy path: --against captures NDJSON → human/JSON output.
|
||||
* 3. CLI failure: missing file, bad NDJSON, mismatched schema_version.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { writeFileSync, mkdtempSync, rmSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { runEvalReplay } from '../src/commands/eval-replay.ts';
|
||||
import type { BrainEngine } from '../src/core/engine.ts';
|
||||
import type { SearchResult } from '../src/core/types.ts';
|
||||
|
||||
// Minimal stub engine — only the methods replayRow touches matter.
|
||||
// Every other method is a thrower so accidental use surfaces in tests.
|
||||
function makeStubEngine(returns: Record<string, SearchResult[]>): BrainEngine {
|
||||
return {
|
||||
kind: 'pglite',
|
||||
searchKeyword: async (q: string) => returns[q] ?? [],
|
||||
searchVector: async () => [],
|
||||
} as unknown as BrainEngine;
|
||||
}
|
||||
|
||||
function fakeResult(slug: string, score = 0.5): SearchResult {
|
||||
return {
|
||||
slug,
|
||||
chunk_id: 1,
|
||||
chunk_index: 0,
|
||||
chunk_text: '',
|
||||
score,
|
||||
title: slug,
|
||||
page_kind: 'markdown',
|
||||
source_id: 'default',
|
||||
} as unknown as SearchResult;
|
||||
}
|
||||
|
||||
function makeCapturedRow(over: Partial<{
|
||||
id: number;
|
||||
tool_name: 'query' | 'search';
|
||||
query: string;
|
||||
retrieved_slugs: string[];
|
||||
detail: 'low' | 'medium' | 'high' | null;
|
||||
expand_enabled: boolean | null;
|
||||
latency_ms: number;
|
||||
}>) {
|
||||
return {
|
||||
schema_version: 1,
|
||||
id: over.id ?? 1,
|
||||
tool_name: over.tool_name ?? 'search',
|
||||
query: over.query ?? 'alice',
|
||||
retrieved_slugs: over.retrieved_slugs ?? ['people/alice', 'people/alice-bio'],
|
||||
retrieved_chunk_ids: [],
|
||||
source_ids: [],
|
||||
expand_enabled: over.expand_enabled ?? null,
|
||||
detail: over.detail ?? null,
|
||||
detail_resolved: null,
|
||||
vector_enabled: false,
|
||||
expansion_applied: false,
|
||||
latency_ms: over.latency_ms ?? 50,
|
||||
remote: true,
|
||||
job_id: null,
|
||||
subagent_id: null,
|
||||
created_at: '2026-04-25T00:00:00Z',
|
||||
};
|
||||
}
|
||||
|
||||
function withTmp<T>(fn: (path: string) => Promise<T> | T): Promise<T> {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'eval-replay-'));
|
||||
return Promise.resolve(fn(dir)).finally(() => rmSync(dir, { recursive: true, force: true }));
|
||||
}
|
||||
|
||||
function captureStdoutStderr(): { restore: () => { stdout: string; stderr: string } } {
|
||||
// Bun's console.log doesn't route through process.stdout.write — it uses
|
||||
// internal writers. Hook console directly so test capture works across
|
||||
// console.log/error/warn AND raw stream writes.
|
||||
let outBuf = '';
|
||||
let errBuf = '';
|
||||
const origLog = console.log;
|
||||
const origErrCons = console.error;
|
||||
const origWarn = console.warn;
|
||||
const origStdoutWrite = process.stdout.write.bind(process.stdout);
|
||||
const origStderrWrite = process.stderr.write.bind(process.stderr);
|
||||
const stringify = (a: unknown) => typeof a === 'string' ? a : JSON.stringify(a);
|
||||
console.log = (...args: unknown[]) => { outBuf += args.map(stringify).join(' ') + '\n'; };
|
||||
console.error = (...args: unknown[]) => { errBuf += args.map(stringify).join(' ') + '\n'; };
|
||||
console.warn = (...args: unknown[]) => { errBuf += args.map(stringify).join(' ') + '\n'; };
|
||||
process.stdout.write = ((s: string | Uint8Array) => {
|
||||
outBuf += typeof s === 'string' ? s : new TextDecoder().decode(s);
|
||||
return true;
|
||||
}) as typeof process.stdout.write;
|
||||
process.stderr.write = ((s: string | Uint8Array) => {
|
||||
errBuf += typeof s === 'string' ? s : new TextDecoder().decode(s);
|
||||
return true;
|
||||
}) as typeof process.stderr.write;
|
||||
return {
|
||||
restore: () => {
|
||||
console.log = origLog;
|
||||
console.error = origErrCons;
|
||||
console.warn = origWarn;
|
||||
process.stdout.write = origStdoutWrite;
|
||||
process.stderr.write = origStderrWrite;
|
||||
return { stdout: outBuf, stderr: errBuf };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('gbrain eval replay — happy path', () => {
|
||||
test('search row with identical retrieved slugs reports Jaccard 1.0', async () => {
|
||||
await withTmp(async (dir) => {
|
||||
const ndjson = JSON.stringify(makeCapturedRow({
|
||||
tool_name: 'search',
|
||||
query: 'alice',
|
||||
retrieved_slugs: ['people/alice', 'people/alice-bio'],
|
||||
})) + '\n';
|
||||
const file = join(dir, 'baseline.ndjson');
|
||||
writeFileSync(file, ndjson);
|
||||
|
||||
const engine = makeStubEngine({
|
||||
alice: [fakeResult('people/alice'), fakeResult('people/alice-bio')],
|
||||
});
|
||||
|
||||
const cap = captureStdoutStderr();
|
||||
await runEvalReplay(engine, ['--against', file, '--json']);
|
||||
const { stdout } = cap.restore();
|
||||
const out = JSON.parse(stdout);
|
||||
expect(out.schema_version).toBe(1);
|
||||
expect(out.summary.rows_replayed).toBe(1);
|
||||
expect(out.summary.mean_jaccard).toBe(1.0);
|
||||
expect(out.summary.top1_stability_rate).toBe(1.0);
|
||||
});
|
||||
});
|
||||
|
||||
test('search row with disjoint retrieved slugs reports Jaccard 0', async () => {
|
||||
await withTmp(async (dir) => {
|
||||
const ndjson = JSON.stringify(makeCapturedRow({
|
||||
tool_name: 'search',
|
||||
query: 'bob',
|
||||
retrieved_slugs: ['people/bob', 'people/bob-bio'],
|
||||
})) + '\n';
|
||||
const file = join(dir, 'baseline.ndjson');
|
||||
writeFileSync(file, ndjson);
|
||||
|
||||
const engine = makeStubEngine({
|
||||
bob: [fakeResult('people/charlie'), fakeResult('people/charlie-bio')],
|
||||
});
|
||||
|
||||
const cap = captureStdoutStderr();
|
||||
await runEvalReplay(engine, ['--against', file, '--json']);
|
||||
const { stdout } = cap.restore();
|
||||
const out = JSON.parse(stdout);
|
||||
expect(out.summary.mean_jaccard).toBe(0);
|
||||
expect(out.summary.top1_stability_rate).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('partial overlap — captured [a,b], current [a,c] → Jaccard 1/3', async () => {
|
||||
await withTmp(async (dir) => {
|
||||
const ndjson = JSON.stringify(makeCapturedRow({
|
||||
retrieved_slugs: ['a', 'b'],
|
||||
query: 'q',
|
||||
})) + '\n';
|
||||
const file = join(dir, 'baseline.ndjson');
|
||||
writeFileSync(file, ndjson);
|
||||
|
||||
const engine = makeStubEngine({ q: [fakeResult('a'), fakeResult('c')] });
|
||||
|
||||
const cap = captureStdoutStderr();
|
||||
await runEvalReplay(engine, ['--against', file, '--json']);
|
||||
const { stdout } = cap.restore();
|
||||
const out = JSON.parse(stdout);
|
||||
// {a,b} ∩ {a,c} = {a} (1), union {a,b,c} (3) → 1/3
|
||||
expect(out.summary.mean_jaccard).toBeCloseTo(1 / 3, 5);
|
||||
// top-1 still matches (both lead with 'a')
|
||||
expect(out.summary.top1_stability_rate).toBe(1.0);
|
||||
});
|
||||
});
|
||||
|
||||
test('top-1 mismatch reduces stability rate', async () => {
|
||||
await withTmp(async (dir) => {
|
||||
const ndjson = JSON.stringify(makeCapturedRow({
|
||||
retrieved_slugs: ['a', 'b', 'c'],
|
||||
query: 'q',
|
||||
})) + '\n';
|
||||
const file = join(dir, 'baseline.ndjson');
|
||||
writeFileSync(file, ndjson);
|
||||
|
||||
const engine = makeStubEngine({ q: [fakeResult('b'), fakeResult('a'), fakeResult('c')] });
|
||||
|
||||
const cap = captureStdoutStderr();
|
||||
await runEvalReplay(engine, ['--against', file, '--json']);
|
||||
const { stdout } = cap.restore();
|
||||
const out = JSON.parse(stdout);
|
||||
// Same set, jaccard = 1.0
|
||||
expect(out.summary.mean_jaccard).toBe(1.0);
|
||||
// top-1 swapped a → b, stability 0
|
||||
expect(out.summary.top1_stability_rate).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('multiple rows → averaged Jaccard + top-1 rate', async () => {
|
||||
await withTmp(async (dir) => {
|
||||
const lines = [
|
||||
JSON.stringify(makeCapturedRow({ id: 1, query: 'q1', retrieved_slugs: ['a'] })),
|
||||
JSON.stringify(makeCapturedRow({ id: 2, query: 'q2', retrieved_slugs: ['b'] })),
|
||||
].join('\n') + '\n';
|
||||
const file = join(dir, 'baseline.ndjson');
|
||||
writeFileSync(file, lines);
|
||||
|
||||
const engine = makeStubEngine({
|
||||
q1: [fakeResult('a')], // perfect match
|
||||
q2: [fakeResult('z')], // miss
|
||||
});
|
||||
|
||||
const cap = captureStdoutStderr();
|
||||
await runEvalReplay(engine, ['--against', file, '--json']);
|
||||
const { stdout } = cap.restore();
|
||||
const out = JSON.parse(stdout);
|
||||
// (1.0 + 0) / 2 = 0.5
|
||||
expect(out.summary.mean_jaccard).toBe(0.5);
|
||||
// (1 + 0) / 2 = 0.5
|
||||
expect(out.summary.top1_stability_rate).toBe(0.5);
|
||||
expect(out.summary.rows_replayed).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
test('--limit caps replay count', async () => {
|
||||
await withTmp(async (dir) => {
|
||||
const lines = [1, 2, 3, 4, 5]
|
||||
.map(i => JSON.stringify(makeCapturedRow({ id: i, query: `q${i}`, retrieved_slugs: ['a'] })))
|
||||
.join('\n') + '\n';
|
||||
const file = join(dir, 'baseline.ndjson');
|
||||
writeFileSync(file, lines);
|
||||
|
||||
const engine = makeStubEngine({}); // returns [] for everything
|
||||
|
||||
const cap = captureStdoutStderr();
|
||||
await runEvalReplay(engine, ['--against', file, '--limit', '2', '--json']);
|
||||
const { stdout } = cap.restore();
|
||||
const out = JSON.parse(stdout);
|
||||
expect(out.summary.rows_total).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
test('empty query is skipped, not counted in replayed', async () => {
|
||||
await withTmp(async (dir) => {
|
||||
const ndjson = JSON.stringify(makeCapturedRow({ query: '', retrieved_slugs: [] })) + '\n';
|
||||
const file = join(dir, 'baseline.ndjson');
|
||||
writeFileSync(file, ndjson);
|
||||
|
||||
const engine = makeStubEngine({});
|
||||
|
||||
const cap = captureStdoutStderr();
|
||||
await runEvalReplay(engine, ['--against', file, '--json']);
|
||||
const { stdout } = cap.restore();
|
||||
const out = JSON.parse(stdout);
|
||||
expect(out.summary.rows_skipped).toBe(1);
|
||||
expect(out.summary.rows_replayed).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('--verbose includes per-row results in JSON', async () => {
|
||||
await withTmp(async (dir) => {
|
||||
const ndjson = JSON.stringify(makeCapturedRow({ query: 'q', retrieved_slugs: ['a'] })) + '\n';
|
||||
const file = join(dir, 'baseline.ndjson');
|
||||
writeFileSync(file, ndjson);
|
||||
|
||||
const engine = makeStubEngine({ q: [fakeResult('a')] });
|
||||
|
||||
const cap = captureStdoutStderr();
|
||||
await runEvalReplay(engine, ['--against', file, '--json', '--verbose']);
|
||||
const { stdout } = cap.restore();
|
||||
const out = JSON.parse(stdout);
|
||||
expect(Array.isArray(out.results)).toBe(true);
|
||||
expect(out.results.length).toBe(1);
|
||||
expect(out.results[0].jaccard).toBe(1.0);
|
||||
expect(out.results[0].current_slugs).toEqual(['a']);
|
||||
});
|
||||
});
|
||||
|
||||
test('row that throws during replay → errored, not crash', async () => {
|
||||
await withTmp(async (dir) => {
|
||||
const ndjson = JSON.stringify(makeCapturedRow({ query: 'boom' })) + '\n';
|
||||
const file = join(dir, 'baseline.ndjson');
|
||||
writeFileSync(file, ndjson);
|
||||
|
||||
const engine = {
|
||||
kind: 'pglite' as const,
|
||||
searchKeyword: async () => { throw new Error('engine offline'); },
|
||||
searchVector: async () => [],
|
||||
} as unknown as BrainEngine;
|
||||
|
||||
const cap = captureStdoutStderr();
|
||||
await runEvalReplay(engine, ['--against', file, '--json']);
|
||||
const { stdout } = cap.restore();
|
||||
const out = JSON.parse(stdout);
|
||||
expect(out.summary.rows_errored).toBe(1);
|
||||
expect(out.summary.rows_replayed).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('human mode prints summary and top regressions', async () => {
|
||||
await withTmp(async (dir) => {
|
||||
const ndjson = JSON.stringify(makeCapturedRow({ query: 'q', retrieved_slugs: ['a'] })) + '\n';
|
||||
const file = join(dir, 'baseline.ndjson');
|
||||
writeFileSync(file, ndjson);
|
||||
|
||||
const engine = makeStubEngine({ q: [fakeResult('z')] }); // miss
|
||||
|
||||
const cap = captureStdoutStderr();
|
||||
await runEvalReplay(engine, ['--against', file]);
|
||||
const { stdout, stderr } = cap.restore();
|
||||
expect(stderr).toContain('Replaying 1');
|
||||
expect(stdout).toContain('Mean Jaccard@k:');
|
||||
expect(stdout).toContain('Top-1 stability:');
|
||||
expect(stdout).toContain('regression');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('gbrain eval replay — failure modes', () => {
|
||||
test('missing --against errors and exits 1', async () => {
|
||||
const origExit = process.exit;
|
||||
let code: number | undefined;
|
||||
process.exit = (c?: number) => { code = c; throw new Error('exit'); };
|
||||
const cap = captureStdoutStderr();
|
||||
try {
|
||||
await runEvalReplay({} as BrainEngine, []);
|
||||
} catch { /* expected */ }
|
||||
cap.restore();
|
||||
process.exit = origExit;
|
||||
expect(code).toBe(1);
|
||||
});
|
||||
|
||||
test('--against pointing at missing file errors and exits 1', async () => {
|
||||
const origExit = process.exit;
|
||||
let code: number | undefined;
|
||||
process.exit = (c?: number) => { code = c; throw new Error('exit'); };
|
||||
const cap = captureStdoutStderr();
|
||||
try {
|
||||
await runEvalReplay({} as BrainEngine, ['--against', '/tmp/nope-' + Date.now() + '.ndjson']);
|
||||
} catch { /* expected */ }
|
||||
cap.restore();
|
||||
process.exit = origExit;
|
||||
expect(code).toBe(1);
|
||||
});
|
||||
|
||||
test('NDJSON line missing schema_version is rejected', async () => {
|
||||
await withTmp(async (dir) => {
|
||||
const file = join(dir, 'bad.ndjson');
|
||||
writeFileSync(file, '{"id":1,"tool_name":"search","query":"q","retrieved_slugs":[]}\n');
|
||||
const origExit = process.exit;
|
||||
let code: number | undefined;
|
||||
process.exit = (c?: number) => { code = c; throw new Error('exit'); };
|
||||
const cap = captureStdoutStderr();
|
||||
try {
|
||||
await runEvalReplay({} as BrainEngine, ['--against', file]);
|
||||
} catch { /* expected */ }
|
||||
const { stderr } = cap.restore();
|
||||
process.exit = origExit;
|
||||
expect(code).toBe(1);
|
||||
expect(stderr).toContain('schema_version');
|
||||
});
|
||||
});
|
||||
|
||||
test('schema_version === 2 (future) is rejected with helpful message', async () => {
|
||||
await withTmp(async (dir) => {
|
||||
const file = join(dir, 'v2.ndjson');
|
||||
writeFileSync(file, '{"schema_version":2,"id":1,"tool_name":"search","query":"q","retrieved_slugs":[],"latency_ms":0}\n');
|
||||
const origExit = process.exit;
|
||||
let code: number | undefined;
|
||||
process.exit = (c?: number) => { code = c; throw new Error('exit'); };
|
||||
const cap = captureStdoutStderr();
|
||||
try {
|
||||
await runEvalReplay({} as BrainEngine, ['--against', file]);
|
||||
} catch { /* expected */ }
|
||||
const { stderr } = cap.restore();
|
||||
process.exit = origExit;
|
||||
expect(code).toBe(1);
|
||||
expect(stderr.toLowerCase()).toContain('upgrade gbrain');
|
||||
});
|
||||
});
|
||||
|
||||
test('empty file errors and exits 1', async () => {
|
||||
await withTmp(async (dir) => {
|
||||
const file = join(dir, 'empty.ndjson');
|
||||
writeFileSync(file, '');
|
||||
const origExit = process.exit;
|
||||
let code: number | undefined;
|
||||
process.exit = (c?: number) => { code = c; throw new Error('exit'); };
|
||||
const cap = captureStdoutStderr();
|
||||
try {
|
||||
await runEvalReplay({} as BrainEngine, ['--against', file]);
|
||||
} catch { /* expected */ }
|
||||
cap.restore();
|
||||
process.exit = origExit;
|
||||
expect(code).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
test('malformed JSON line is rejected with line number', async () => {
|
||||
await withTmp(async (dir) => {
|
||||
const file = join(dir, 'bad.ndjson');
|
||||
writeFileSync(file, '{"schema_version":1,"id":1}\nthis is not json\n');
|
||||
const origExit = process.exit;
|
||||
let code: number | undefined;
|
||||
process.exit = (c?: number) => { code = c; throw new Error('exit'); };
|
||||
const cap = captureStdoutStderr();
|
||||
try {
|
||||
await runEvalReplay({} as BrainEngine, ['--against', file]);
|
||||
} catch { /* expected */ }
|
||||
const { stderr } = cap.restore();
|
||||
process.exit = origExit;
|
||||
expect(code).toBe(1);
|
||||
expect(stderr).toContain('line 2');
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user