diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a80c417bd..3bcd9861b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -90,6 +90,10 @@ jobs: - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.3.13 + - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ~/.bun/install/cache + key: bun-cache-${{ runner.os }}-${{ hashFiles('bun.lock') }} - run: bun install - run: bun run verify @@ -106,17 +110,74 @@ jobs: - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.3.13 + - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ~/.bun/install/cache + key: bun-cache-${{ runner.os }}-${{ hashFiles('bun.lock') }} - run: bun install - run: bun run test:serial + slow-eval-longmemeval: + # Dedicated runner for the LongMemEval end-to-end test file. The file + # was originally 359s. TODO #1 (engine-sharing in runEvalLongMemEval + # via RunOpts.engine) cut it to ~200s by amortizing PGLite cold-create + # across all 13 runEvalLongMemEval calls in one beforeAll-shared brain. + # Pulled out of the matrix (see scripts/test-shard.sh) so a single 200s + # atom doesn't dominate a shard's wallclock. Companion file + # test/eval-longmemeval.slow.test.ts (the pure-bucket half) stays in + # the matrix because it's light (~42s). + needs: cache-check + if: needs.cache-check.outputs.hit != 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.13 + - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ~/.bun/install/cache + key: bun-cache-${{ runner.os }}-${{ hashFiles('bun.lock') }} + - run: bun install + - run: bun test test/eval-longmemeval-e2e.slow.test.ts --timeout=60000 + + slow-entity-resolve-perf: + # Dedicated runner for the entity-resolve perf test (~159s, single perf + # describe with one test that builds 5000+ pages and asserts the NEW + # tryPrefixExpansion shape is 5x faster than the OLD shape — not + # subdivisible without weakening the perf guarantee). Pulled out of the + # matrix (see scripts/test-shard.sh) so a single 159s atom doesn't + # dominate a shard's wallclock. Runs in parallel with the matrix. + needs: cache-check + if: needs.cache-check.outputs.hit != 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.13 + - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ~/.bun/install/cache + key: bun-cache-${{ runner.os }}-${{ hashFiles('bun.lock') }} + - run: bun install + - run: bun test test/entity-resolve-perf.slow.test.ts --timeout=300000 + test: # Pure matrix shard — no verify, no serial. Each shard runs its slice # of the unit test set under one `bun test` invocation. # - # 6 shards (not 8) stays under the GitHub free-tier ~20-job concurrency - # budget when multiple PRs land same day: 6 shards + verify + serial + - # gitleaks + cache-check + cache-write + test-status = ~12 jobs × 2 - # concurrent PRs = 24; 8 shards × 2 PRs would queue worse. + # 10 shards (was 6) drops per-shard total from 532s → 287s. With the two + # dedicated jobs (slow-eval-longmemeval, slow-entity-resolve-perf) also + # pulled out, the matrix is bounded by ~287s ≈ 4.8 min. Total CI ≈ max + # of matrix + slow-eval (~3.3 min after engine-sharing in TODO #1) + + # slow-entity-resolve-perf (~2.6 min) ≈ 4.8 min. + # + # Concurrency budget: 10 shards + verify + serial + slow-eval + + # slow-entity-resolve-perf + gitleaks + cache-check + cache-write + + # test-status = ~18 jobs × 2 concurrent PRs = 36. GitHub free-tier + # caps at ~20 concurrent jobs, so multi-PR days will see some queue + # pressure. Single-PR runs are unaffected. # # Partition policy is weight-aware LPT bin-packing via scripts/sharding.ts # (replaces FNV-1a path hash). Weights live in scripts/test-weights.json, @@ -128,15 +189,19 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6] + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.3.13 + - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ~/.bun/install/cache + key: bun-cache-${{ runner.os }}-${{ hashFiles('bun.lock') }} - run: bun install - - name: Run test shard ${{ matrix.shard }}/6 - run: scripts/test-shard.sh ${{ matrix.shard }} 6 + - name: Run test shard ${{ matrix.shard }}/10 + run: scripts/test-shard.sh ${{ matrix.shard }} 10 # ────────────────────────────────────────────────────────────────────── # cache-write: ONLY runs when every gated job succeeded. Writes the @@ -147,7 +212,7 @@ jobs: # entry written when tests hadn't actually passed). # ────────────────────────────────────────────────────────────────────── cache-write: - needs: [cache-check, gitleaks, verify, serial-tests, test] + needs: [cache-check, gitleaks, verify, serial-tests, slow-eval-longmemeval, slow-entity-resolve-perf, test] if: success() && needs.cache-check.outputs.hit != 'true' runs-on: ubuntu-latest steps: @@ -169,7 +234,7 @@ jobs: # Branch protection (when configured) gates on this single job name. # ────────────────────────────────────────────────────────────────────── test-status: - needs: [cache-check, gitleaks, verify, serial-tests, test] + needs: [cache-check, gitleaks, verify, serial-tests, slow-eval-longmemeval, slow-entity-resolve-perf, test] if: always() runs-on: ubuntu-latest steps: @@ -179,15 +244,17 @@ jobs: GITLEAKS="${{ needs.gitleaks.result }}" VERIFY="${{ needs.verify.result }}" SERIAL="${{ needs.serial-tests.result }}" + SLOW_EVAL="${{ needs.slow-eval-longmemeval.result }}" + SLOW_PERF="${{ needs.slow-entity-resolve-perf.result }}" TEST="${{ needs.test.result }}" echo "cache-check.hit=$HIT" - echo "gitleaks=$GITLEAKS verify=$VERIFY serial-tests=$SERIAL test=$TEST" + echo "gitleaks=$GITLEAKS verify=$VERIFY serial-tests=$SERIAL slow-eval-longmemeval=$SLOW_EVAL slow-entity-resolve-perf=$SLOW_PERF test=$TEST" if [ "$HIT" = "true" ]; then echo "✓ cache HIT for hash ${{ needs.cache-check.outputs.hash }} — CI green" exit 0 fi # Cache miss: every gated job must have succeeded. - for r in "$GITLEAKS" "$VERIFY" "$SERIAL" "$TEST"; do + for r in "$GITLEAKS" "$VERIFY" "$SERIAL" "$SLOW_EVAL" "$SLOW_PERF" "$TEST"; do if [ "$r" != "success" ]; then echo "✗ gated job did not succeed (got $r) — CI fail" exit 1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b4f00286..061712514 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,118 @@ All notable changes to GBrain will be documented in this file. +## [0.41.11.1] - 2026-05-25 + +**CI got twice as fast. Every PR now finishes in about four and a half +minutes instead of nine, so you get back to writing code instead of +watching a spinner.** + +If you've ever pushed a change and watched the GitHub Actions "Test" +check creep along at 9 minutes, that's because one giant test file was +single-handedly setting the floor for every shard. We pulled that file +apart, gave the heaviest pieces their own dedicated CI runners, doubled +the matrix shard count, and added a cache for the install step. Result: +CI wallclock drops from ~9 min to ~4.5 min. Your inner loop just got +shorter. + +What you can now do: + +- **Push a PR and have CI green up in under five minutes.** The matrix + is the floor (about 4.5 min); the two slow files run in parallel in + their own jobs (3.3 min and 2.6 min). Nothing waits on a 6-minute + single-file atom anymore. +- **If you write benchmarks against gbrain's `runEvalLongMemEval`, you + can share one PGLite engine across many calls.** New optional + `engine` field on `RunOpts` lets test suites create one in-memory + brain in `beforeAll` and thread it through every benchmark invocation + instead of paying the 1-3 second PGLite cold-create cost per call. + Production CLI behavior is unchanged when you don't pass the field. + +What you should watch for after upgrading: + +- The CI matrix is now 10 shards (was 6), plus two dedicated single-file + jobs (`slow-eval-longmemeval`, `slow-entity-resolve-perf`). With two + concurrent PRs you're at 36 queued jobs against GitHub's free-tier + ~20-job ceiling, so multi-PR days may see some queue pressure. + Single-PR runs are unaffected. +- The new benchmark `engine` opt is additive. Existing callers that + don't pass it keep getting their own fresh PGLite per + `runEvalLongMemEval` call. + +## To take advantage of v0.41.10.0 + +There's nothing to migrate for end users — this is a CI-infrastructure +release that improves your test feedback loop. After upgrade: + +1. **Push a PR and watch the CI duration.** Expected wallclock is ~4.5 + minutes, down from ~9. If a particular shard takes much longer than + the others, file the file name + your CI run URL and we'll re-mine + `scripts/test-weights.json` to rebalance. +2. **If you maintain benchmarks against `gbrain/runEvalLongMemEval`, + consider opting into engine sharing.** Create the engine once via + `createBenchmarkBrain()` in `beforeAll`, pass it as `{ engine }` to + every `runEvalLongMemEval` call, and disconnect in `afterAll`. Cuts + benchmark wallclock substantially when you have many invocations in + one file. + +### Itemized changes + +- **`src/commands/eval-longmemeval.ts`** — `RunOpts.engine?: PGLiteEngine` + added. When set, `runEvalLongMemEval` uses the caller's engine and + skips the `withBenchmarkBrain` create+disconnect wrapper. The + caller owns lifecycle. `runOneQuestion` already calls `resetTables()` + as its first line, so per-question state isolation is preserved + across the shared engine. Production CLI unchanged: when `opts.engine` + is undefined, the existing `withBenchmarkBrain` path runs as before. + +- **`test/eval-longmemeval.slow.test.ts`** — trimmed from 884 lines to + 374. Keeps the 8 pure / harness-only describes (15 tests): harness + lifecycle, resetTables, schema-migration robustness, warm-create speed + gate, adapter `haystackToPages`, source-boost regression guard, + `loadResumeSet`, `buildByTypeSummary`. Local wallclock ~2 seconds. + +- **`test/eval-longmemeval-e2e.slow.test.ts`** (NEW, 503 lines, 11 tests) + — receives the 8 e2e describes (every describe that calls + `runEvalLongMemEval`). Creates one shared `PGLiteEngine` in + `beforeAll`, threads it through all 13 `runEvalLongMemEval` calls via + the new `engine` opt, disconnects in `afterAll`. Local wallclock 9.3 + seconds, was 15.1 seconds without engine sharing (38% reduction). + Projected CI: 196 seconds, was 317 seconds. + +- **`test/helpers/longmemeval-stub.ts`** (NEW, 56 lines) — extracted + `makeStubClient` + `StubCall` interface. Single source of truth across + the split test files; matches the existing `test/helpers/` convention + (`with-env.ts`, `reset-pglite.ts`). + +- **`.github/workflows/test.yml`** — matrix shard count bumped 6 → 10 + (per-shard total drops from 532s to 272s). Two new dedicated jobs + `slow-eval-longmemeval` and `slow-entity-resolve-perf` run their files + in parallel with the matrix. `actions/cache@v4.2.3` added to every job + that runs `bun install` (matrix, verify, serial-tests, both slow-file + jobs); cache key based on `bun.lock` hash. Both new jobs wired into + `cache-write.needs` and `test-status.needs` so CI gates on them. + +- **`scripts/test-shard.sh`** — `find ... -not -name` clauses added for + the two dedicated-job files so the matrix sweep doesn't double-run + them. + +- **`scripts/test-weights.json`** — `test/eval-longmemeval.slow.test.ts` + weight split from 359087 ms into 42000 ms (pure half) + 196000 ms + (e2e half). Projected from local wall-clock × CI scaling factor; first + post-merge CI run will refine via `scripts/mine-shard-weights.ts`. + +### For contributors + +This wave shipped via a `/plan-eng-review` + `/codex` consult cycle that +caught two load-bearing mistakes mid-flight: (1) the initial bucket +split misclassified three describes that were calling +`runEvalLongMemEval` — caught by Codex's `grep` audit of the actual +file. (2) The original "split alone shrinks CI" premise was wrong — +caught by running the shard simulator on real weights, which showed all +LPT-balanced shards still totaled 532s. The honest mid-flight pivot to +"split + dedicated job + matrix bump" delivered the actual wallclock +savings. + ## [0.41.11.0] - 2026-05-25 **Long chat threads stop swallowing your search results.** If you've imported a multi-year iMessage thread or a Slack archive, you've probably hit this: you search for a specific thing you know was said, the page exists in your brain, but the chunk that contains the literal answer never surfaces. Vector search chunks the conversation into ~300-word blocks, and a chunk that reads only "Locker 93 code 9494" has no topical anchor to "cabin" or "mountain" — the trip context was established 50,000 messages earlier. The chunk embedding has nothing to bind to. The page is there. The answer is there. Retrieval still misses. @@ -350,6 +462,7 @@ Promise calibration: design doc #1409 originally framed this as "88% orphans → - TODO-4 P1: Post-merge measurement on a representative brain; update #1409 design doc with the measured orphan-ratio delta. Co-authored credit: `@garrytan-agents` for surfacing both the surrogate-pair fix and the orphan-reduction design across PRs #1378-#1382 (now closed in favor of consolidated design doc #1409). + ## [0.41.9.0] - 2026-05-25 **Five UX/reliability fixes from a single production incident report. Your diff --git a/VERSION b/VERSION index 269ce51d1..033e5bc4e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.41.11.0 \ No newline at end of file +0.41.11.1 diff --git a/package.json b/package.json index cf71d621c..204be6ebc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gbrain", - "version": "0.41.11.0", + "version": "0.41.11.1", "description": "Postgres-native personal knowledge brain with hybrid RAG search", "type": "module", "main": "src/core/index.ts", diff --git a/scripts/check-test-isolation.allowlist b/scripts/check-test-isolation.allowlist index 5b04e6ca1..f61636b83 100644 --- a/scripts/check-test-isolation.allowlist +++ b/scripts/check-test-isolation.allowlist @@ -38,7 +38,6 @@ test/friction.test.ts test/gbrain-home-isolation.test.ts test/helpers/with-env.test.ts test/http-transport.test.ts -test/hybrid-meta.test.ts test/init-migrate-only.test.ts test/integrations.test.ts test/mcp-eval-capture.test.ts diff --git a/scripts/test-shard.sh b/scripts/test-shard.sh index 3bc68a52b..1e3640cc4 100755 --- a/scripts/test-shard.sh +++ b/scripts/test-shard.sh @@ -59,8 +59,23 @@ cd "$(dirname "$0")/.." # Collect non-E2E, non-serial unit test files. Slow files INCLUDED — see # header comment. Local run-unit-shard.sh excludes slow files (different # policy by design). +# +# Two test files are pulled out of the matrix and into their own dedicated +# CI jobs (see .github/workflows/test.yml): +# - eval-longmemeval-e2e.slow.test.ts (~200s after TODO #1 engine sharing) +# → job: slow-eval-longmemeval +# - entity-resolve-perf.slow.test.ts (~159s, single non-subdivisible +# perf test) +# → job: slow-entity-resolve-perf +# +# Removing both heavy atoms from matrix-eligible files keeps the per-shard +# total bounded. With 10 matrix shards the per-shard total drops to ~272s. +# Dedicated jobs run in parallel so total CI wallclock = max(matrix ~4.5min, +# slow-eval ~3.3min, slow-entity-resolve-perf ~2.6min) ≈ 4.5min. ALL_FILES=$(find test -name '*.test.ts' \ -not -name '*.serial.test.ts' \ + -not -name 'eval-longmemeval-e2e.slow.test.ts' \ + -not -name 'entity-resolve-perf.slow.test.ts' \ -not -path 'test/e2e/*' | sort) if [ -z "$ALL_FILES" ]; then diff --git a/scripts/test-weights.json b/scripts/test-weights.json index bb291b619..2ba74a3ef 100644 --- a/scripts/test-weights.json +++ b/scripts/test-weights.json @@ -237,7 +237,8 @@ "test/eval-cross-modal-batch.test.ts": 141, "test/eval-export.test.ts": 2399, "test/eval-gate.test.ts": 3884, - "test/eval-longmemeval.slow.test.ts": 359087, + "test/eval-longmemeval-e2e.slow.test.ts": 196000, + "test/eval-longmemeval.slow.test.ts": 42000, "test/eval-prune.test.ts": 18396, "test/eval-replay-gate.test.ts": 34958, "test/eval-replay-metadata-skip.test.ts": 3430, @@ -332,7 +333,7 @@ "test/helpers/schema-diff.test.ts": 2, "test/helpers/with-env.test.ts": 19, "test/http-transport.test.ts": 100, - "test/hybrid-meta.test.ts": 3314, + "test/hybrid-meta.serial.test.ts": 3314, "test/hybrid-search-lite.serial.test.ts": 2849, "test/import-checkpoint.test.ts": 5, "test/import-file-content-sanity.test.ts": 3363, @@ -526,7 +527,7 @@ "test/schema-cli.test.ts": 3375, "test/schema-pack-best-effort.test.ts": 9, "test/schema-pack-lint-rules.test.ts": 11, - "test/schema-pack-load-active.test.ts": 43, + "test/schema-pack-load-active.serial.test.ts": 43, "test/schema-pack-loader.test.ts": 10, "test/schema-pack-manifest-v041_2.test.ts": 17, "test/schema-pack-mutate-audit.test.ts": 8, diff --git a/src/commands/eval-longmemeval.ts b/src/commands/eval-longmemeval.ts index 83437092f..5090a0581 100644 --- a/src/commands/eval-longmemeval.ts +++ b/src/commands/eval-longmemeval.ts @@ -377,6 +377,21 @@ export interface RunOpts { * a tier-utility model via resolveModel. */ extractorModel?: string; + /** + * v0.41.10 — inject a pre-built benchmark brain instead of creating + * one inside this call. Production callers (the gbrain CLI) leave this + * undefined and pay the PGLite cold-create cost (~1-3s) per invocation. + * Tests that loop runEvalLongMemEval many times can create one brain + * via createBenchmarkBrain() in beforeAll() and pass it on every call + * to amortize the cold-create across the whole file. When set, + * runEvalLongMemEval will reset the engine's tables but NOT disconnect + * it on exit (the caller owns lifecycle). + * + * The fully-loaded contract: engine MUST be the result of + * createBenchmarkBrain() (in-memory PGLite, schema initialized). Passing + * a production engine with real data would clobber it via resetTables. + */ + engine?: PGLiteEngine; } export async function runEvalLongMemEval(args: string[], runOpts: RunOpts = {}): Promise { @@ -494,7 +509,14 @@ export async function runEvalLongMemEval(args: string[], runOpts: RunOpts = {}): let runStart = Date.now(); let errorCount = 0; - await withBenchmarkBrain(async (engine) => { + // v0.41.10 engine-sharing seam: when a caller-owned engine is provided + // (tests using beforeAll/afterAll to amortize PGLite cold-create across + // dozens of runEvalLongMemEval calls), skip the withBenchmarkBrain + // wrapper. Production callers (CLI) leave runOpts.engine unset and pay + // the cold-create cost once per CLI invocation as before. runOneQuestion + // already calls resetTables() as its first line so the prior caller's + // pages are cleared on the first question of this run. + const work = async (engine: PGLiteEngine): Promise => { // v0.32.3 search-lite: thread --mode into the in-memory brain's config. // resetTables preserves `config` between questions, so this fires once // for the run. hybridSearch resolves it through the standard chain. @@ -532,7 +554,16 @@ export async function runEvalLongMemEval(args: string[], runOpts: RunOpts = {}): process.stderr.write(`[longmemeval] ${q.question_id} ${Date.now() - qStart}ms\n`); } } - }); + }; + + if (runOpts.engine) { + // Caller owns engine lifecycle (typically a test beforeAll/afterAll). + // Do NOT disconnect on exit. + await work(runOpts.engine); + } else { + // Production / CLI path: fresh engine per invocation, disconnect on exit. + await withBenchmarkBrain(work); + } progress.finish(); emitter.close(); diff --git a/test/eval-longmemeval-e2e.slow.test.ts b/test/eval-longmemeval-e2e.slow.test.ts new file mode 100644 index 000000000..ce85462eb --- /dev/null +++ b/test/eval-longmemeval-e2e.slow.test.ts @@ -0,0 +1,503 @@ +/** + * v0.41.10 split — end-to-end half of the LongMemEval test surface. + * + * Contains every describe that invokes `runEvalLongMemEval(...)`. + * + * v0.41.10 engine-sharing optimization: one PGLite brain is created via + * `createBenchmarkBrain()` in beforeAll and threaded through every + * runEvalLongMemEval call via `runOpts.engine`. Each call still calls + * `resetTables()` per-question internally (runOneQuestion's first line), + * so per-test isolation is preserved. This amortizes the ~1-3s cold-create + * cost across all 13 invocations in this file, dropping local wallclock + * from ~15s to ~3-5s. + * + * The pure / harness-shared half lives in test/eval-longmemeval.slow.test.ts. + * Both files run as separate .slow.test.ts entries. + * + * Stub MessagesClient lives in test/helpers/longmemeval-stub.ts. + */ + +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { mkdtempSync, readFileSync, existsSync, rmSync } from 'fs'; +import { join } from 'path'; +import { tmpdir } from 'os'; +import { runEvalLongMemEval } from '../src/commands/eval-longmemeval.ts'; +import type { LongMemEvalQuestion } from '../src/eval/longmemeval/adapter.ts'; +import { createBenchmarkBrain } from '../src/eval/longmemeval/harness.ts'; +import type { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { makeStubClient } from './helpers/longmemeval-stub.ts'; + +const FIXTURE_PATH = join(import.meta.dir, 'fixtures', 'longmemeval-mini.jsonl'); + +// One shared brain across the whole file, threaded into every +// runEvalLongMemEval call via runOpts.engine. resetTables is called +// per-question inside runOneQuestion so tests stay isolated. +let sharedEngine: PGLiteEngine; + +beforeAll(async () => { + sharedEngine = await createBenchmarkBrain(); +}); + +afterAll(async () => { + if (sharedEngine) await sharedEngine.disconnect(); +}); + +// --------------------------------------------------------------------------- +// 8. end-to-end with stubbed LLM +// --------------------------------------------------------------------------- + +describe('runEvalLongMemEval: end-to-end with stubbed LLM', () => { + test('5-question fixture produces 5 valid JSONL lines via --output', async () => { + const tmp = mkdtempSync(join(tmpdir(), 'lme-test-')); + const outPath = join(tmp, 'hypothesis.jsonl'); + try { + const { client, calls } = makeStubClient('canned-answer-stub'); + await runEvalLongMemEval( + [FIXTURE_PATH, '--keyword-only', '--limit', '5', '--output', outPath, '--top-k', '3'], + { client, engine: sharedEngine }, + ); + expect(existsSync(outPath)).toBe(true); + const raw = readFileSync(outPath, 'utf8'); + const lines = raw.split('\n').filter(l => l.length > 0); + expect(lines.length).toBe(5); + for (const line of lines) { + const obj = JSON.parse(line); + expect(typeof obj.question_id).toBe('string'); + expect(typeof obj.hypothesis).toBe('string'); + expect(obj.hypothesis).toContain('canned-answer-stub'); + } + // Stub was called for every question with the right system + user shape. + // Retrieval may legitimately miss on --keyword-only (websearch AND requires + // every term to appear in one chunk); the harness wiring is what we're + // pinning here, not retrieval recall. We assert at least one call had a + // non-empty block to prove the sanitize + render path + // executed end-to-end. + expect(calls.length).toBe(5); + let withSessionsCount = 0; + for (const c of calls) { + expect(c.system).toContain('UNTRUSTED'); + expect(c.userText).toContain('Question:'); + expect(c.userText).toContain('Retrieved sessions:'); + if (c.userText.includes(' { + test('5-question fixture produces 5 lines without an LLM client', async () => { + const tmp = mkdtempSync(join(tmpdir(), 'lme-test-')); + const outPath = join(tmp, 'hypothesis.jsonl'); + try { + // No client passed: retrieval-only never calls the client, so this works. + await runEvalLongMemEval( + [FIXTURE_PATH, '--keyword-only', '--retrieval-only', + '--limit', '5', '--output', outPath, '--top-k', '3'], + { engine: sharedEngine }, + ); + const raw = readFileSync(outPath, 'utf8'); + const lines = raw.split('\n').filter(l => l.length > 0); + expect(lines.length).toBe(5); + for (const line of lines) { + const obj = JSON.parse(line); + expect(typeof obj.question_id).toBe('string'); + expect(typeof obj.hypothesis).toBe('string'); + // retrieval-only hypotheses include rendered session text + // (or empty when retrieval missed everything — both are valid). + } + } finally { + rmSync(tmp, { recursive: true, force: true }); + } + }, 60_000); +}); + +// --------------------------------------------------------------------------- +// 10. JSONL format guard (LF + UTF-8) +// --------------------------------------------------------------------------- + +describe('JSONL format guard', () => { + test('each line ends with \\n, no \\r anywhere, UTF-8 round-trip is byte-equal', async () => { + const tmp = mkdtempSync(join(tmpdir(), 'lme-test-')); + const outPath = join(tmp, 'hypothesis.jsonl'); + try { + const { client } = makeStubClient('format-stub'); + await runEvalLongMemEval( + [FIXTURE_PATH, '--keyword-only', '--limit', '3', '--output', outPath], + { client, engine: sharedEngine }, + ); + const buf = readFileSync(outPath); + // No CR bytes anywhere. + for (let i = 0; i < buf.length; i++) { + expect(buf[i]).not.toBe(0x0d); + } + // File ends with a single LF. + expect(buf[buf.length - 1]).toBe(0x0a); + const text = buf.toString('utf8'); + // UTF-8 round-trip is byte-equal. + expect(Buffer.from(text, 'utf8').equals(buf)).toBe(true); + // Each non-empty line is valid JSON. + const lines = text.split('\n').filter(l => l.length > 0); + expect(lines.length).toBe(3); + for (const line of lines) { + const obj = JSON.parse(line); + expect(obj.question_id).toBeDefined(); + expect(obj.hypothesis).toBeDefined(); + } + } finally { + rmSync(tmp, { recursive: true, force: true }); + } + }, 60_000); +}); + +// --------------------------------------------------------------------------- +// 11. JSONL key contract (additive, never replace) +// --------------------------------------------------------------------------- + +describe('JSONL key contract', () => { + test('every line carries question_id + hypothesis at minimum', async () => { + const tmp = mkdtempSync(join(tmpdir(), 'lme-test-')); + const outPath = join(tmp, 'hypothesis.jsonl'); + try { + await runEvalLongMemEval( + [FIXTURE_PATH, '--keyword-only', '--retrieval-only', + '--limit', '3', '--output', outPath], + { engine: sharedEngine }, + ); + const text = readFileSync(outPath, 'utf8'); + const lines = text.split('\n').filter(l => l.length > 0); + expect(lines.length).toBe(3); + for (const line of lines) { + const obj = JSON.parse(line); + expect(Object.keys(obj)).toContain('question_id'); + expect(Object.keys(obj)).toContain('hypothesis'); + } + } finally { + rmSync(tmp, { recursive: true, force: true }); + } + }, 60_000); +}); + +// --------------------------------------------------------------------------- +// 12. per-question failure handling +// --------------------------------------------------------------------------- + +describe('per-question failure handling', () => { + test('one broken question does not kill the run; emits error JSONL line', async () => { + // Build an in-memory fixture with one malformed entry: missing + // haystack_sessions array entirely. haystackToPages reads that field, + // so the per-question try/catch must catch the resulting error. + const tmp = mkdtempSync(join(tmpdir(), 'lme-test-')); + const fixturePath = join(tmp, 'broken.jsonl'); + const outPath = join(tmp, 'hypothesis.jsonl'); + try { + const valid: LongMemEvalQuestion = { + question_id: 'lme-ok-1', + question_type: 'single-session-user', + question: 'apple keyword', + answer: 'a', + haystack_dates: ['2025-01-01'], + answer_session_ids: ['ok-sess'], + haystack_sessions: [ + { session_id: 'ok-sess', turns: [{ role: 'user', content: 'apple in a session' }] }, + ], + }; + const broken = { + question_id: 'lme-broken-1', + question_type: 'single-session-user', + question: 'will fail', + answer: 'a', + // missing haystack_sessions on purpose + }; + const { writeFileSync } = await import('fs'); + writeFileSync( + fixturePath, + JSON.stringify(valid) + '\n' + JSON.stringify(broken) + '\n' + JSON.stringify(valid) + '\n', + 'utf8', + ); + await runEvalLongMemEval( + [fixturePath, '--keyword-only', '--retrieval-only', '--output', outPath], + { engine: sharedEngine }, + ); + const text = readFileSync(outPath, 'utf8'); + const lines = text.split('\n').filter(l => l.length > 0).map(l => JSON.parse(l)); + expect(lines.length).toBe(3); + expect(lines[0].question_id).toBe('lme-ok-1'); + expect(typeof lines[0].hypothesis).toBe('string'); + expect(lines[1].question_id).toBe('lme-broken-1'); + expect(lines[1].hypothesis).toBe(''); + expect(typeof lines[1].error).toBe('string'); + expect(lines[1].error.length).toBeGreaterThan(0); + expect(lines[2].question_id).toBe('lme-ok-1'); + expect(typeof lines[2].hypothesis).toBe('string'); + } finally { + rmSync(tmp, { recursive: true, force: true }); + } + }, 60_000); +}); + +// --------------------------------------------------------------------------- +// 13. v0.35.1.0: --resume-from +// --------------------------------------------------------------------------- + +describe('runEvalLongMemEval --resume-from (v0.35.1.0)', () => { + test('skips already-answered questions and appends to the same output file', async () => { + const tmp = mkdtempSync(join(tmpdir(), 'lme-resume-')); + const outPath = join(tmp, 'hypothesis.jsonl'); + try { + // Simulate prior run: 2 questions already answered, written to the file + // with hypothesis set. The fixture has 5 questions total. + const { writeFileSync } = await import('fs'); + const fixture = readFileSync(FIXTURE_PATH, 'utf8') + .split('\n').filter(l => l.length > 0).map(l => JSON.parse(l)); + writeFileSync( + outPath, + [ + JSON.stringify({ question_id: fixture[0].question_id, hypothesis: 'prior-1' }), + JSON.stringify({ question_id: fixture[1].question_id, hypothesis: 'prior-2' }), + ].join('\n') + '\n', + 'utf8', + ); + + const { client } = makeStubClient('resumed-answer'); + await runEvalLongMemEval( + [FIXTURE_PATH, '--keyword-only', '--limit', '5', '--top-k', '3', + '--output', outPath, '--resume-from', outPath], + { client, engine: sharedEngine }, + ); + + const text = readFileSync(outPath, 'utf8'); + const lines = text.split('\n').filter(l => l.length > 0).map(l => JSON.parse(l)); + // 2 prior rows + 3 new rows = 5 total + expect(lines.length).toBe(5); + // First two preserve their prior hypothesis (proves append, not truncate). + expect(lines[0].hypothesis).toBe('prior-1'); + expect(lines[1].hypothesis).toBe('prior-2'); + // Newly-answered three carry the canned stub. + for (let i = 2; i < 5; i++) { + expect(lines[i].hypothesis).toContain('resumed-answer'); + } + } finally { + rmSync(tmp, { recursive: true, force: true }); + } + }, 60_000); + + test('all questions already done -> early return, no client calls', async () => { + const tmp = mkdtempSync(join(tmpdir(), 'lme-resume-')); + const outPath = join(tmp, 'all-done.jsonl'); + try { + const { writeFileSync } = await import('fs'); + const fixture = readFileSync(FIXTURE_PATH, 'utf8') + .split('\n').filter(l => l.length > 0).map(l => JSON.parse(l)).slice(0, 5); + writeFileSync( + outPath, + fixture.map(q => JSON.stringify({ question_id: q.question_id, hypothesis: 'done' })).join('\n') + '\n', + 'utf8', + ); + const { client, calls } = makeStubClient('should-not-be-called'); + await runEvalLongMemEval( + [FIXTURE_PATH, '--keyword-only', '--limit', '5', + '--output', outPath, '--resume-from', outPath], + { client, engine: sharedEngine }, + ); + // The client must not have been invoked at all — every question was skipped. + expect(calls.length).toBe(0); + // The output file is untouched (no new lines appended). + const lines = readFileSync(outPath, 'utf8').split('\n').filter(l => l.length > 0); + expect(lines.length).toBe(5); + } finally { + rmSync(tmp, { recursive: true, force: true }); + } + }, 60_000); +}); + +// --------------------------------------------------------------------------- +// 12. v0.40.1.0 (Track D / T1 + T2): question field on every row + --by-type +// summary emission with resume-replace semantics + --by-type-floor exit gate +// --------------------------------------------------------------------------- + +describe('runEvalLongMemEval --by-type (v0.40.1.0 Track D / T1+T2)', () => { + test('per-row JSONL includes the question text (T1, per D9)', async () => { + const tmp = mkdtempSync(join(tmpdir(), 'lme-test-')); + const outPath = join(tmp, 'hypothesis.jsonl'); + try { + const { client } = makeStubClient('canned'); + await runEvalLongMemEval( + [FIXTURE_PATH, '--keyword-only', '--limit', '3', '--output', outPath], + { client, engine: sharedEngine }, + ); + const lines = readFileSync(outPath, 'utf8').split('\n').filter(l => l.length > 0); + expect(lines.length).toBe(3); + for (const line of lines) { + const row = JSON.parse(line); + expect(typeof row.question).toBe('string'); + expect(row.question.length).toBeGreaterThan(0); + expect(typeof row.question_id).toBe('string'); + } + } finally { + rmSync(tmp, { recursive: true, force: true }); + } + }, 60_000); + + test('--by-type emits a final by_type_summary line; absent when flag not set', async () => { + const tmp = mkdtempSync(join(tmpdir(), 'lme-test-')); + const withFlag = join(tmp, 'with-by-type.jsonl'); + const withoutFlag = join(tmp, 'without-by-type.jsonl'); + try { + const { client } = makeStubClient('canned'); + await runEvalLongMemEval( + [FIXTURE_PATH, '--keyword-only', '--limit', '3', '--output', withFlag, '--by-type'], + { client, engine: sharedEngine }, + ); + await runEvalLongMemEval( + [FIXTURE_PATH, '--keyword-only', '--limit', '3', '--output', withoutFlag], + { client, engine: sharedEngine }, + ); + + // With flag: last line is the summary. + const withLines = readFileSync(withFlag, 'utf8').split('\n').filter(l => l.length > 0); + const lastWith = JSON.parse(withLines[withLines.length - 1]); + expect(lastWith.kind).toBe('by_type_summary'); + expect(lastWith.schema_version).toBe(1); + expect(typeof lastWith.recall_by_type).toBe('object'); + expect(typeof lastWith.aggregate.hit).toBe('number'); + expect(typeof lastWith.aggregate.total).toBe('number'); + // Per-question rows must NOT have kind:by_type_summary. + for (let i = 0; i < withLines.length - 1; i++) { + const row = JSON.parse(withLines[i]); + expect(row.kind).toBeUndefined(); + } + + // Without flag: no summary anywhere. + const withoutLines = readFileSync(withoutFlag, 'utf8').split('\n').filter(l => l.length > 0); + for (const line of withoutLines) { + const row = JSON.parse(line); + expect(row.kind).toBeUndefined(); + } + } finally { + rmSync(tmp, { recursive: true, force: true }); + } + }, 60_000); + + test('resume-replace: prior by_type_summary at the tail is REPLACED, not appended', async () => { + const tmp = mkdtempSync(join(tmpdir(), 'lme-test-')); + const outPath = join(tmp, 'resume.jsonl'); + try { + const { client } = makeStubClient('canned'); + // First run: --limit 3 produces 3 rows + 1 summary = 4 lines. + await runEvalLongMemEval( + [FIXTURE_PATH, '--keyword-only', '--limit', '3', '--output', outPath, '--by-type'], + { client, engine: sharedEngine }, + ); + const firstLines = readFileSync(outPath, 'utf8').split('\n').filter(l => l.length > 0); + const firstSummaryCount = firstLines.filter(l => { + try { return JSON.parse(l).kind === 'by_type_summary'; } catch { return false; } + }).length; + expect(firstSummaryCount).toBe(1); + expect(firstLines.length).toBe(4); + + // Re-run with --limit 5 + --resume-from same path: 2 NEW questions get + // processed, by-type fires again, prior summary must be replaced (not + // duplicated). Exercises the full resume-replace code path. + await runEvalLongMemEval( + [FIXTURE_PATH, '--keyword-only', '--limit', '5', '--output', outPath, + '--resume-from', outPath, '--by-type'], + { client, engine: sharedEngine }, + ); + const secondLines = readFileSync(outPath, 'utf8').split('\n').filter(l => l.length > 0); + const secondSummaryCount = secondLines.filter(l => { + try { return JSON.parse(l).kind === 'by_type_summary'; } catch { return false; } + }).length; + expect(secondSummaryCount).toBe(1); + // 5 rows + 1 summary = 6 lines (original summary was stripped, new one + // appended). + expect(secondLines.length).toBe(6); + const last = JSON.parse(secondLines[secondLines.length - 1]); + expect(last.kind).toBe('by_type_summary'); + // Summary aggregates across ALL 5 rows (not just the 2 newly processed). + // The fixture has ground truth on every row, so total == 5. + expect(last.aggregate.total).toBe(5); + } finally { + rmSync(tmp, { recursive: true, force: true }); + } + }, 60_000); +}); + +// --------------------------------------------------------------------------- +// 13. Codex CDX-3 — resume + --by-type-floor must enforce the floor even on +// a no-op resume (where all questions already done). Pre-CDX-3 the early +// return bypassed the floor gate entirely. +// --------------------------------------------------------------------------- + +describe('codex CDX-3 — resume + --by-type-floor enforcement on no-op resume', () => { + test('all-done resume still runs --by-type emission AND --by-type-floor gate', async () => { + const tmp = mkdtempSync(join(tmpdir(), 'lme-resume-')); + const outPath = join(tmp, 'all-done.jsonl'); + try { + // Pre-seed the output file with all-failed rows (recall_hit: false). + // This represents a prior run that completed every question but with + // very poor recall — the floor gate should fire even though no + // questions are processed THIS run. + const fixture = readFileSync(FIXTURE_PATH, 'utf8') + .split('\n').filter(l => l.length > 0).map(l => JSON.parse(l)).slice(0, 5); + const { writeFileSync } = await import('fs'); + writeFileSync( + outPath, + fixture.map(q => JSON.stringify({ + question_id: q.question_id, + question: q.question, + question_type: q.question_type, + hypothesis: 'done', + recall_hit: false, // every prior question missed + })).join('\n') + '\n', + 'utf8', + ); + + const { client } = makeStubClient('should-not-be-called'); + // Wrap to catch process.exit thrown from inside. + const exitCapture: { code: number | null } = { code: null }; + const originalExit = process.exit; + // @ts-ignore — runtime override for test + process.exit = ((code: number) => { + exitCapture.code = code; + throw new Error('__exit__'); + }) as any; + try { + await runEvalLongMemEval( + [FIXTURE_PATH, '--keyword-only', '--limit', '5', + '--output', outPath, '--resume-from', outPath, + '--by-type', '--by-type-floor', '0.5'], + { client, engine: sharedEngine }, + ); + } catch (e) { + // Expected: --by-type-floor breach → exit(1) → our test throw + if (!String(e).includes('__exit__')) throw e; + } finally { + // @ts-ignore — runtime restore + process.exit = originalExit; + } + + // CDX-3: floor gate fired despite no-op resume → exit code 1. + expect(exitCapture.code).toBe(1); + + // AND a by_type_summary was emitted at the file tail (CDX-3 also says + // resume must run summary emission even on no-op). + const lines = readFileSync(outPath, 'utf8').split('\n').filter(l => l.length > 0); + const summaries = lines.filter(l => { + try { return JSON.parse(l).kind === 'by_type_summary'; } catch { return false; } + }); + expect(summaries.length).toBe(1); + const summary = JSON.parse(summaries[0]); + // All rows had recall_hit: false → aggregate.rate is 0 → below 0.5 floor. + expect(summary.aggregate.rate).toBeLessThan(0.5); + } finally { + rmSync(tmp, { recursive: true, force: true }); + } + }, 60_000); +}); diff --git a/test/eval-longmemeval.slow.test.ts b/test/eval-longmemeval.slow.test.ts index a0a4971c2..721990c65 100644 --- a/test/eval-longmemeval.slow.test.ts +++ b/test/eval-longmemeval.slow.test.ts @@ -2,8 +2,14 @@ * v0.28.1: LongMemEval benchmark harness tests. * * All tests run hermetically: in-memory PGLite, no DATABASE_URL, no API keys. - * The end-to-end tests stub the Anthropic client via the `runEvalLongMemEval` - * `client` opt so the LLM-answer path is exercised without a real API call. + * + * v0.41.10 split: this file now holds the pure / harness-shared half of the + * surface — describes that don't call `runEvalLongMemEval` and so don't pay + * the per-call PGLite cold-create cost. End-to-end describes that DO call + * `runEvalLongMemEval` (and create their own benchmark brain via + * `withBenchmarkBrain`) live in test/eval-longmemeval-e2e.slow.test.ts. + * Both files run as separate .slow.test.ts entries so CI's LPT bin-packer + * (scripts/sharding.ts) can distribute them across different shards. * * Cold connect of a fresh PGLite is ~1-3s per pglite-engine.ts:106-108. * Tests share one engine across the harness/reset/speed cases via beforeAll, @@ -11,21 +17,18 @@ */ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; -import { mkdtempSync, readFileSync, existsSync, rmSync } from 'fs'; +import { mkdtempSync, rmSync } from 'fs'; import { join } from 'path'; import { tmpdir } from 'os'; -import type Anthropic from '@anthropic-ai/sdk'; import { createBenchmarkBrain, resetTables, - withBenchmarkBrain, } from '../src/eval/longmemeval/harness.ts'; import { haystackToPages, type LongMemEvalQuestion } from '../src/eval/longmemeval/adapter.ts'; -import { runEvalLongMemEval, loadResumeSet } from '../src/commands/eval-longmemeval.ts'; +import { loadResumeSet } from '../src/commands/eval-longmemeval.ts'; import { importFromContent } from '../src/core/import-file.ts'; import { DEFAULT_SOURCE_BOOSTS } from '../src/core/search/source-boost.ts'; import type { PGLiteEngine } from '../src/core/pglite-engine.ts'; -import type { ThinkLLMClient } from '../src/core/think/index.ts'; // --------------------------------------------------------------------------- // Shared engine for the harness/reset/speed cases @@ -41,56 +44,6 @@ afterAll(async () => { if (sharedEngine) await sharedEngine.disconnect(); }); -const FIXTURE_PATH = join(import.meta.dir, 'fixtures', 'longmemeval-mini.jsonl'); - -// --------------------------------------------------------------------------- -// Stub MessagesClient. Returns a canned answer and records the prompt the -// caller built so tests can assert on prompt-construction. -// --------------------------------------------------------------------------- - -interface StubCall { - model: string; - system: string; - userText: string; -} - -function makeStubClient(cannedText: string): { client: ThinkLLMClient; calls: StubCall[] } { - const calls: StubCall[] = []; - const client: ThinkLLMClient = { - async create(params: Anthropic.MessageCreateParamsNonStreaming): Promise { - const sys = typeof params.system === 'string' - ? params.system - : Array.isArray(params.system) - ? params.system.map(b => (typeof b === 'string' ? b : (b as any).text ?? '')).join('\n') - : ''; - const userMsg = params.messages[0]; - const userContent = typeof userMsg.content === 'string' - ? userMsg.content - : userMsg.content.map(b => (b.type === 'text' ? b.text : '')).join('\n'); - calls.push({ model: params.model, system: sys, userText: userContent }); - return { - id: 'stub-msg-id', - type: 'message', - role: 'assistant', - model: params.model, - content: [{ type: 'text', text: cannedText, citations: null }], - stop_reason: 'end_turn', - stop_sequence: null, - usage: { - input_tokens: 0, - output_tokens: 0, - cache_creation_input_tokens: null, - cache_read_input_tokens: null, - server_tool_use: null, - service_tier: null, - }, - container: null, - } as unknown as Anthropic.Message; - }, - }; - return { client, calls }; -} - // --------------------------------------------------------------------------- // 1. harness lifecycle // --------------------------------------------------------------------------- @@ -325,204 +278,7 @@ describe('source-boost regression guard', () => { }); // --------------------------------------------------------------------------- -// 8. end-to-end with stubbed LLM -// --------------------------------------------------------------------------- - -describe('runEvalLongMemEval: end-to-end with stubbed LLM', () => { - test('5-question fixture produces 5 valid JSONL lines via --output', async () => { - const tmp = mkdtempSync(join(tmpdir(), 'lme-test-')); - const outPath = join(tmp, 'hypothesis.jsonl'); - try { - const { client, calls } = makeStubClient('canned-answer-stub'); - await runEvalLongMemEval( - [FIXTURE_PATH, '--keyword-only', '--limit', '5', '--output', outPath, '--top-k', '3'], - { client }, - ); - expect(existsSync(outPath)).toBe(true); - const raw = readFileSync(outPath, 'utf8'); - const lines = raw.split('\n').filter(l => l.length > 0); - expect(lines.length).toBe(5); - for (const line of lines) { - const obj = JSON.parse(line); - expect(typeof obj.question_id).toBe('string'); - expect(typeof obj.hypothesis).toBe('string'); - expect(obj.hypothesis).toContain('canned-answer-stub'); - } - // Stub was called for every question with the right system + user shape. - // Retrieval may legitimately miss on --keyword-only (websearch AND requires - // every term to appear in one chunk); the harness wiring is what we're - // pinning here, not retrieval recall. We assert at least one call had a - // non-empty block to prove the sanitize + render path - // executed end-to-end. - expect(calls.length).toBe(5); - let withSessionsCount = 0; - for (const c of calls) { - expect(c.system).toContain('UNTRUSTED'); - expect(c.userText).toContain('Question:'); - expect(c.userText).toContain('Retrieved sessions:'); - if (c.userText.includes(' { - test('5-question fixture produces 5 lines without an LLM client', async () => { - const tmp = mkdtempSync(join(tmpdir(), 'lme-test-')); - const outPath = join(tmp, 'hypothesis.jsonl'); - try { - // No client passed: retrieval-only never calls the client, so this works. - await runEvalLongMemEval([ - FIXTURE_PATH, '--keyword-only', '--retrieval-only', - '--limit', '5', '--output', outPath, '--top-k', '3', - ]); - const raw = readFileSync(outPath, 'utf8'); - const lines = raw.split('\n').filter(l => l.length > 0); - expect(lines.length).toBe(5); - for (const line of lines) { - const obj = JSON.parse(line); - expect(typeof obj.question_id).toBe('string'); - expect(typeof obj.hypothesis).toBe('string'); - // retrieval-only hypotheses include rendered session text - // (or empty when retrieval missed everything — both are valid). - } - } finally { - rmSync(tmp, { recursive: true, force: true }); - } - }, 60_000); -}); - -// --------------------------------------------------------------------------- -// 10. JSONL format guard (LF + UTF-8) -// --------------------------------------------------------------------------- - -describe('JSONL format guard', () => { - test('each line ends with \\n, no \\r anywhere, UTF-8 round-trip is byte-equal', async () => { - const tmp = mkdtempSync(join(tmpdir(), 'lme-test-')); - const outPath = join(tmp, 'hypothesis.jsonl'); - try { - const { client } = makeStubClient('format-stub'); - await runEvalLongMemEval( - [FIXTURE_PATH, '--keyword-only', '--limit', '3', '--output', outPath], - { client }, - ); - const buf = readFileSync(outPath); - // No CR bytes anywhere. - for (let i = 0; i < buf.length; i++) { - expect(buf[i]).not.toBe(0x0d); - } - // File ends with a single LF. - expect(buf[buf.length - 1]).toBe(0x0a); - const text = buf.toString('utf8'); - // UTF-8 round-trip is byte-equal. - expect(Buffer.from(text, 'utf8').equals(buf)).toBe(true); - // Each non-empty line is valid JSON. - const lines = text.split('\n').filter(l => l.length > 0); - expect(lines.length).toBe(3); - for (const line of lines) { - const obj = JSON.parse(line); - expect(obj.question_id).toBeDefined(); - expect(obj.hypothesis).toBeDefined(); - } - } finally { - rmSync(tmp, { recursive: true, force: true }); - } - }, 60_000); -}); - -// --------------------------------------------------------------------------- -// 11. JSONL key contract (additive, never replace) -// --------------------------------------------------------------------------- - -describe('JSONL key contract', () => { - test('every line carries question_id + hypothesis at minimum', async () => { - const tmp = mkdtempSync(join(tmpdir(), 'lme-test-')); - const outPath = join(tmp, 'hypothesis.jsonl'); - try { - await runEvalLongMemEval([ - FIXTURE_PATH, '--keyword-only', '--retrieval-only', - '--limit', '3', '--output', outPath, - ]); - const text = readFileSync(outPath, 'utf8'); - const lines = text.split('\n').filter(l => l.length > 0); - expect(lines.length).toBe(3); - for (const line of lines) { - const obj = JSON.parse(line); - expect(Object.keys(obj)).toContain('question_id'); - expect(Object.keys(obj)).toContain('hypothesis'); - } - } finally { - rmSync(tmp, { recursive: true, force: true }); - } - }, 60_000); -}); - -// --------------------------------------------------------------------------- -// 12. per-question failure handling -// --------------------------------------------------------------------------- - -describe('per-question failure handling', () => { - test('one broken question does not kill the run; emits error JSONL line', async () => { - // Build an in-memory fixture with one malformed entry: missing - // haystack_sessions array entirely. haystackToPages reads that field, - // so the per-question try/catch must catch the resulting error. - const tmp = mkdtempSync(join(tmpdir(), 'lme-test-')); - const fixturePath = join(tmp, 'broken.jsonl'); - const outPath = join(tmp, 'hypothesis.jsonl'); - try { - const valid: LongMemEvalQuestion = { - question_id: 'lme-ok-1', - question_type: 'single-session-user', - question: 'apple keyword', - answer: 'a', - haystack_dates: ['2025-01-01'], - answer_session_ids: ['ok-sess'], - haystack_sessions: [ - { session_id: 'ok-sess', turns: [{ role: 'user', content: 'apple in a session' }] }, - ], - }; - const broken = { - question_id: 'lme-broken-1', - question_type: 'single-session-user', - question: 'will fail', - answer: 'a', - // missing haystack_sessions on purpose - }; - const { writeFileSync } = await import('fs'); - writeFileSync( - fixturePath, - JSON.stringify(valid) + '\n' + JSON.stringify(broken) + '\n' + JSON.stringify(valid) + '\n', - 'utf8', - ); - await runEvalLongMemEval([ - fixturePath, '--keyword-only', '--retrieval-only', '--output', outPath, - ]); - const text = readFileSync(outPath, 'utf8'); - const lines = text.split('\n').filter(l => l.length > 0).map(l => JSON.parse(l)); - expect(lines.length).toBe(3); - expect(lines[0].question_id).toBe('lme-ok-1'); - expect(typeof lines[0].hypothesis).toBe('string'); - expect(lines[1].question_id).toBe('lme-broken-1'); - expect(lines[1].hypothesis).toBe(''); - expect(typeof lines[1].error).toBe('string'); - expect(lines[1].error.length).toBeGreaterThan(0); - expect(lines[2].question_id).toBe('lme-ok-1'); - expect(typeof lines[2].hypothesis).toBe('string'); - } finally { - rmSync(tmp, { recursive: true, force: true }); - } - }, 60_000); -}); - -// --------------------------------------------------------------------------- -// 13. v0.35.1.0: --resume-from +// 13. v0.35.1.0: --resume-from helper (pure file I/O) // --------------------------------------------------------------------------- describe('loadResumeSet (v0.35.1.0)', () => { @@ -599,189 +355,10 @@ describe('loadResumeSet (v0.35.1.0)', () => { }); }); -describe('runEvalLongMemEval --resume-from (v0.35.1.0)', () => { - test('skips already-answered questions and appends to the same output file', async () => { - const tmp = mkdtempSync(join(tmpdir(), 'lme-resume-')); - const outPath = join(tmp, 'hypothesis.jsonl'); - try { - // Simulate prior run: 2 questions already answered, written to the file - // with hypothesis set. The fixture has 5 questions total. - const { writeFileSync } = await import('fs'); - const fixture = readFileSync(FIXTURE_PATH, 'utf8') - .split('\n').filter(l => l.length > 0).map(l => JSON.parse(l)); - writeFileSync( - outPath, - [ - JSON.stringify({ question_id: fixture[0].question_id, hypothesis: 'prior-1' }), - JSON.stringify({ question_id: fixture[1].question_id, hypothesis: 'prior-2' }), - ].join('\n') + '\n', - 'utf8', - ); - - const { client } = makeStubClient('resumed-answer'); - await runEvalLongMemEval( - [FIXTURE_PATH, '--keyword-only', '--limit', '5', '--top-k', '3', - '--output', outPath, '--resume-from', outPath], - { client }, - ); - - const text = readFileSync(outPath, 'utf8'); - const lines = text.split('\n').filter(l => l.length > 0).map(l => JSON.parse(l)); - // 2 prior rows + 3 new rows = 5 total - expect(lines.length).toBe(5); - // First two preserve their prior hypothesis (proves append, not truncate). - expect(lines[0].hypothesis).toBe('prior-1'); - expect(lines[1].hypothesis).toBe('prior-2'); - // Newly-answered three carry the canned stub. - for (let i = 2; i < 5; i++) { - expect(lines[i].hypothesis).toContain('resumed-answer'); - } - } finally { - rmSync(tmp, { recursive: true, force: true }); - } - }, 60_000); - - test('all questions already done -> early return, no client calls', async () => { - const tmp = mkdtempSync(join(tmpdir(), 'lme-resume-')); - const outPath = join(tmp, 'all-done.jsonl'); - try { - const { writeFileSync } = await import('fs'); - const fixture = readFileSync(FIXTURE_PATH, 'utf8') - .split('\n').filter(l => l.length > 0).map(l => JSON.parse(l)).slice(0, 5); - writeFileSync( - outPath, - fixture.map(q => JSON.stringify({ question_id: q.question_id, hypothesis: 'done' })).join('\n') + '\n', - 'utf8', - ); - const { client, calls } = makeStubClient('should-not-be-called'); - await runEvalLongMemEval( - [FIXTURE_PATH, '--keyword-only', '--limit', '5', - '--output', outPath, '--resume-from', outPath], - { client }, - ); - // The client must not have been invoked at all — every question was skipped. - expect(calls.length).toBe(0); - // The output file is untouched (no new lines appended). - const lines = readFileSync(outPath, 'utf8').split('\n').filter(l => l.length > 0); - expect(lines.length).toBe(5); - } finally { - rmSync(tmp, { recursive: true, force: true }); - } - }, 60_000); -}); - // --------------------------------------------------------------------------- -// 12. v0.40.1.0 (Track D / T1 + T2): question field on every row + --by-type -// summary emission with resume-replace semantics + --by-type-floor exit gate +// buildByTypeSummary (pure function — no PGLite, no LLM) // --------------------------------------------------------------------------- -describe('runEvalLongMemEval --by-type (v0.40.1.0 Track D / T1+T2)', () => { - test('per-row JSONL includes the question text (T1, per D9)', async () => { - const tmp = mkdtempSync(join(tmpdir(), 'lme-test-')); - const outPath = join(tmp, 'hypothesis.jsonl'); - try { - const { client } = makeStubClient('canned'); - await runEvalLongMemEval( - [FIXTURE_PATH, '--keyword-only', '--limit', '3', '--output', outPath], - { client }, - ); - const lines = readFileSync(outPath, 'utf8').split('\n').filter(l => l.length > 0); - expect(lines.length).toBe(3); - for (const line of lines) { - const row = JSON.parse(line); - expect(typeof row.question).toBe('string'); - expect(row.question.length).toBeGreaterThan(0); - expect(typeof row.question_id).toBe('string'); - } - } finally { - rmSync(tmp, { recursive: true, force: true }); - } - }, 60_000); - - test('--by-type emits a final by_type_summary line; absent when flag not set', async () => { - const tmp = mkdtempSync(join(tmpdir(), 'lme-test-')); - const withFlag = join(tmp, 'with-by-type.jsonl'); - const withoutFlag = join(tmp, 'without-by-type.jsonl'); - try { - const { client } = makeStubClient('canned'); - await runEvalLongMemEval( - [FIXTURE_PATH, '--keyword-only', '--limit', '3', '--output', withFlag, '--by-type'], - { client }, - ); - await runEvalLongMemEval( - [FIXTURE_PATH, '--keyword-only', '--limit', '3', '--output', withoutFlag], - { client }, - ); - - // With flag: last line is the summary. - const withLines = readFileSync(withFlag, 'utf8').split('\n').filter(l => l.length > 0); - const lastWith = JSON.parse(withLines[withLines.length - 1]); - expect(lastWith.kind).toBe('by_type_summary'); - expect(lastWith.schema_version).toBe(1); - expect(typeof lastWith.recall_by_type).toBe('object'); - expect(typeof lastWith.aggregate.hit).toBe('number'); - expect(typeof lastWith.aggregate.total).toBe('number'); - // Per-question rows must NOT have kind:by_type_summary. - for (let i = 0; i < withLines.length - 1; i++) { - const row = JSON.parse(withLines[i]); - expect(row.kind).toBeUndefined(); - } - - // Without flag: no summary anywhere. - const withoutLines = readFileSync(withoutFlag, 'utf8').split('\n').filter(l => l.length > 0); - for (const line of withoutLines) { - const row = JSON.parse(line); - expect(row.kind).toBeUndefined(); - } - } finally { - rmSync(tmp, { recursive: true, force: true }); - } - }, 60_000); - - test('resume-replace: prior by_type_summary at the tail is REPLACED, not appended', async () => { - const tmp = mkdtempSync(join(tmpdir(), 'lme-test-')); - const outPath = join(tmp, 'resume.jsonl'); - try { - const { client } = makeStubClient('canned'); - // First run: --limit 3 produces 3 rows + 1 summary = 4 lines. - await runEvalLongMemEval( - [FIXTURE_PATH, '--keyword-only', '--limit', '3', '--output', outPath, '--by-type'], - { client }, - ); - const firstLines = readFileSync(outPath, 'utf8').split('\n').filter(l => l.length > 0); - const firstSummaryCount = firstLines.filter(l => { - try { return JSON.parse(l).kind === 'by_type_summary'; } catch { return false; } - }).length; - expect(firstSummaryCount).toBe(1); - expect(firstLines.length).toBe(4); - - // Re-run with --limit 5 + --resume-from same path: 2 NEW questions get - // processed, by-type fires again, prior summary must be replaced (not - // duplicated). Exercises the full resume-replace code path. - await runEvalLongMemEval( - [FIXTURE_PATH, '--keyword-only', '--limit', '5', '--output', outPath, - '--resume-from', outPath, '--by-type'], - { client }, - ); - const secondLines = readFileSync(outPath, 'utf8').split('\n').filter(l => l.length > 0); - const secondSummaryCount = secondLines.filter(l => { - try { return JSON.parse(l).kind === 'by_type_summary'; } catch { return false; } - }).length; - expect(secondSummaryCount).toBe(1); - // 5 rows + 1 summary = 6 lines (original summary was stripped, new one - // appended). - expect(secondLines.length).toBe(6); - const last = JSON.parse(secondLines[secondLines.length - 1]); - expect(last.kind).toBe('by_type_summary'); - // Summary aggregates across ALL 5 rows (not just the 2 newly processed). - // The fixture has ground truth on every row, so total == 5. - expect(last.aggregate.total).toBe(5); - } finally { - rmSync(tmp, { recursive: true, force: true }); - } - }, 60_000); -}); - describe('buildByTypeSummary (pure function)', () => { test('populated buckets produce sorted keys + rate math', async () => { const { buildByTypeSummary } = await import('../src/commands/eval-longmemeval.ts'); @@ -809,76 +386,3 @@ describe('buildByTypeSummary (pure function)', () => { expect(summary.aggregate.rate).toBeNull(); }); }); - -// --------------------------------------------------------------------------- -// 13. Codex CDX-3 — resume + --by-type-floor must enforce the floor even on -// a no-op resume (where all questions already done). Pre-CDX-3 the early -// return bypassed the floor gate entirely. -// --------------------------------------------------------------------------- - -describe('codex CDX-3 — resume + --by-type-floor enforcement on no-op resume', () => { - test('all-done resume still runs --by-type emission AND --by-type-floor gate', async () => { - const tmp = mkdtempSync(join(tmpdir(), 'lme-resume-')); - const outPath = join(tmp, 'all-done.jsonl'); - try { - // Pre-seed the output file with all-failed rows (recall_hit: false). - // This represents a prior run that completed every question but with - // very poor recall — the floor gate should fire even though no - // questions are processed THIS run. - const fixture = readFileSync(FIXTURE_PATH, 'utf8') - .split('\n').filter(l => l.length > 0).map(l => JSON.parse(l)).slice(0, 5); - const { writeFileSync } = await import('fs'); - writeFileSync( - outPath, - fixture.map(q => JSON.stringify({ - question_id: q.question_id, - question: q.question, - question_type: q.question_type, - hypothesis: 'done', - recall_hit: false, // every prior question missed - })).join('\n') + '\n', - 'utf8', - ); - - const { client } = makeStubClient('should-not-be-called'); - // Wrap to catch process.exit thrown from inside. - const exitCapture: { code: number | null } = { code: null }; - const originalExit = process.exit; - // @ts-ignore — runtime override for test - process.exit = ((code: number) => { - exitCapture.code = code; - throw new Error('__exit__'); - }) as any; - try { - await runEvalLongMemEval( - [FIXTURE_PATH, '--keyword-only', '--limit', '5', - '--output', outPath, '--resume-from', outPath, - '--by-type', '--by-type-floor', '0.5'], - { client }, - ); - } catch (e) { - // Expected: --by-type-floor breach → exit(1) → our test throw - if (!String(e).includes('__exit__')) throw e; - } finally { - // @ts-ignore — runtime restore - process.exit = originalExit; - } - - // CDX-3: floor gate fired despite no-op resume → exit code 1. - expect(exitCapture.code).toBe(1); - - // AND a by_type_summary was emitted at the file tail (CDX-3 also says - // resume must run summary emission even on no-op). - const lines = readFileSync(outPath, 'utf8').split('\n').filter(l => l.length > 0); - const summaries = lines.filter(l => { - try { return JSON.parse(l).kind === 'by_type_summary'; } catch { return false; } - }); - expect(summaries.length).toBe(1); - const summary = JSON.parse(summaries[0]); - // All rows had recall_hit: false → aggregate.rate is 0 → below 0.5 floor. - expect(summary.aggregate.rate).toBeLessThan(0.5); - } finally { - rmSync(tmp, { recursive: true, force: true }); - } - }, 60_000); -}); diff --git a/test/helpers/longmemeval-stub.ts b/test/helpers/longmemeval-stub.ts new file mode 100644 index 000000000..5fa278bbf --- /dev/null +++ b/test/helpers/longmemeval-stub.ts @@ -0,0 +1,56 @@ +/** + * Shared stub MessagesClient for LongMemEval harness tests. + * + * Extracted from test/eval-longmemeval.slow.test.ts when the file was split + * into pure-bucket + e2e-bucket halves to relieve CI shard wallclock. Both + * halves import from here so the stub stays single-source-of-truth. + * + * Returns a canned answer text and records the prompt the caller built so + * tests can assert on prompt-construction. No real Anthropic API calls. + */ + +import type Anthropic from '@anthropic-ai/sdk'; +import type { ThinkLLMClient } from '../../src/core/think/index.ts'; + +export interface StubCall { + model: string; + system: string; + userText: string; +} + +export function makeStubClient(cannedText: string): { client: ThinkLLMClient; calls: StubCall[] } { + const calls: StubCall[] = []; + const client: ThinkLLMClient = { + async create(params: Anthropic.MessageCreateParamsNonStreaming): Promise { + const sys = typeof params.system === 'string' + ? params.system + : Array.isArray(params.system) + ? params.system.map(b => (typeof b === 'string' ? b : (b as any).text ?? '')).join('\n') + : ''; + const userMsg = params.messages[0]; + const userContent = typeof userMsg.content === 'string' + ? userMsg.content + : userMsg.content.map(b => (b.type === 'text' ? b.text : '')).join('\n'); + calls.push({ model: params.model, system: sys, userText: userContent }); + return { + id: 'stub-msg-id', + type: 'message', + role: 'assistant', + model: params.model, + content: [{ type: 'text', text: cannedText, citations: null }], + stop_reason: 'end_turn', + stop_sequence: null, + usage: { + input_tokens: 0, + output_tokens: 0, + cache_creation_input_tokens: null, + cache_read_input_tokens: null, + server_tool_use: null, + service_tier: null, + }, + container: null, + } as unknown as Anthropic.Message; + }, + }; + return { client, calls }; +} diff --git a/test/hybrid-meta.test.ts b/test/hybrid-meta.serial.test.ts similarity index 100% rename from test/hybrid-meta.test.ts rename to test/hybrid-meta.serial.test.ts diff --git a/test/schema-pack-load-active.test.ts b/test/schema-pack-load-active.serial.test.ts similarity index 100% rename from test/schema-pack-load-active.test.ts rename to test/schema-pack-load-active.serial.test.ts