From 3a2605e9a0b746a82ab42dd0177fc0f3a666a600 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 25 May 2026 13:15:11 -0700 Subject: [PATCH] =?UTF-8?q?v0.41.6.0=20feat(ci):=20CI=20test=20speedup=20?= =?UTF-8?q?=E2=80=94=2023min=20=E2=86=92=20~9min=20via=20matrix=204?= =?UTF-8?q?=E2=86=926=20+=20weight-aware=20sharding=20+=20auto=20SHA=20cac?= =?UTF-8?q?he=20+=20parallel=20verify=20(#1444)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(ci): scripts/run-verify-parallel.sh — parallel verify dispatcher Fans out the 21 pre-test grep guards via & + wait, captures per-check exit codes in a tempdir, aggregates failures with named check + log tail to stderr on miss. Wallclock 27s sequential → 13s parallel locally (2x). Bigger CI win is shard 1 deload (workflow restructure in a later commit). Pinned by test/scripts/run-verify-parallel.test.ts (6 cases: CLI contract + synthetic dispatcher failure-surfacing). * feat(ci): weight-aware LPT bin-packer + auto SHA cache hash scripts/sharding.ts (NEW) — pure TypeScript LPT bin-packer. Sort weights desc, assign each file to the shard with current minimum total. Worst-case makespan within 4/3 of optimal, O(n log n). Missing weights fall back to corpus median (not 0). New test file → ships immediately without regenerating weights. Pinned by test/scripts/sharding.test.ts (23 cases). scripts/mine-shard-weights.ts (NEW) — scrapes per-file timing from gh run view --log via timestamp delta between ##[group]test/foo.test.ts: headers within a shard. Three input modes: --run , --from-file , stdin. Stable JSON output (sorted keys). Initial weights mined from run 26398061007. Pinned by test/scripts/mine-shard-weights.test.ts (15 cases). scripts/ci-cache-hash.sh (NEW) — deterministic 16-char sha256 over git ls-files -s minus deny-list (CHANGELOG/TODOS/README/LICENSE/ docs/**/*.md). CLAUDE.md, AGENTS.md, skills/**/* deliberately INCLUDED (8+ test files read them; deny-listing would create false-pass holes). ~40ms on 1891 files. Pinned by test/scripts/ci-cache-hash.test.ts (24 cases: 8 CRITICAL false-pass guards + 7 SAFE deny-list invariants + 9 edge cases). scripts/test-weights.json (NEW) — 712 weights. Total 3306s observed runtime; median 30ms; max 6 min outlier. * chore: bump version and changelog (v0.41.6.0) Co-Authored-By: Claude Opus 4.7 --------- Co-authored-by: Claude Opus 4.7 --- .github/workflows/test.yml | 186 +++++- CHANGELOG.md | 49 ++ VERSION | 2 +- package.json | 4 +- scripts/ci-cache-hash.sh | 139 +++++ scripts/mine-shard-weights.ts | 247 ++++++++ scripts/run-verify-parallel.sh | 193 ++++++ scripts/sharding.ts | 256 ++++++++ scripts/test-shard.sh | 100 ++-- scripts/test-weights.json | 713 +++++++++++++++++++++++ test/privacy-script-wired.test.ts | 36 +- test/scripts/ci-cache-hash.test.ts | 422 ++++++++++++++ test/scripts/mine-shard-weights.test.ts | 158 +++++ test/scripts/run-verify-parallel.test.ts | 175 ++++++ test/scripts/sharding.test.ts | 266 +++++++++ test/scripts/test-shard.slow.test.ts | 140 ++++- 16 files changed, 2977 insertions(+), 109 deletions(-) create mode 100755 scripts/ci-cache-hash.sh create mode 100644 scripts/mine-shard-weights.ts create mode 100755 scripts/run-verify-parallel.sh create mode 100644 scripts/sharding.ts create mode 100644 scripts/test-weights.json create mode 100644 test/scripts/ci-cache-hash.test.ts create mode 100644 test/scripts/mine-shard-weights.test.ts create mode 100644 test/scripts/run-verify-parallel.test.ts create mode 100644 test/scripts/sharding.test.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3f7ab2f4a..a80c417bd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,62 @@ permissions: contents: read jobs: + # ────────────────────────────────────────────────────────────────────── + # cache-check: runs first, computes the content hash of every tracked + # file EXCEPT the deny-list (CHANGELOG.md, README.md, docs/**/*.md, etc. + # — see scripts/ci-cache-hash.sh for the full list). Looks up + # `ci-pass-` in actions/cache; if hit, the test matrix + verify + # + serial jobs all skip and test-status reports green immediately. + # If miss, the full suite runs and cache-write seals it on success. + # + # Hit rate covers re-pushes (same SHA twice), branch rebases that + # don't touch tracked code, and any branch update that touches only + # the deny-listed doc files. + # ────────────────────────────────────────────────────────────────────── + cache-check: + runs-on: ubuntu-latest + outputs: + hit: ${{ steps.lookup.outputs.cache-hit }} + hash: ${{ steps.compute.outputs.hash }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Compute content hash + id: compute + run: | + # --verbose writes the "X/Y files in hash" diagnostic to stderr; + # stdout carries the 16-char hash. Capture both. + HASH=$(bash scripts/ci-cache-hash.sh --verbose 2>/tmp/cache-diag) + cat /tmp/cache-diag + echo "Computed cache hash: $HASH" + echo "hash=$HASH" >> "$GITHUB_OUTPUT" + - name: Lookup actions/cache for ci-pass- + id: lookup + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + key: ci-pass-${{ steps.compute.outputs.hash }} + path: .ci-cache-marker + # `lookup-only: true` means we only probe whether the cache + # entry exists — we don't download it (the marker contents + # don't matter, only the key match does). `cache-hit` returns + # true only on EXACT key match (per actions/cache docs); a + # restore-keys prefix fallback would set cache-hit=false, so + # it's deliberately omitted here. Cross-branch scoping works + # naturally: PR branches can read default-branch (master) + # cache entries via exact key match when the content hash + # matches, which happens whenever the tree is doc-only + # different from a green master run. + lookup-only: true + - name: Cache status + run: | + if [ "${{ steps.lookup.outputs.cache-hit }}" = "true" ]; then + echo "✓ cache HIT for hash ${{ steps.compute.outputs.hash }} — test jobs will skip" + else + echo "✗ cache MISS for hash ${{ steps.compute.outputs.hash }} — full suite will run" + fi + gitleaks: + needs: cache-check + if: needs.cache-check.outputs.hit != 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 @@ -20,29 +75,122 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - test: - # ubuntu-latest is free 2-core/7GB. Larger runners (16-cores, etc.) require - # a provisioned runner pool in repo settings. Falling back to default keeps - # the matrix shard speedup (~5-6x via parallelism) at zero cost. + verify: + # Pre-test gates: privacy/jsonb/source-id/etc + typecheck + admin-build. + # Lives in its own runner so the matrix shards aren't carrying ~2-3min + # of verify work in addition to their test files (the old shape stuffed + # this into `test (1)` via `if: matrix.shard == 1`, which made shard 1 + # the slowest matrix worker). scripts/run-verify-parallel.sh fans out + # the 20 checks via & + wait (~5s vs ~15-25s sequential). + needs: cache-check + if: needs.cache-check.outputs.hit != 'true' runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4] steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.3.13 - run: bun install - - name: Pre-test gates (shard 1 only — they're not test files) - if: matrix.shard == 1 - run: bun run verify - - name: Run test shard ${{ matrix.shard }}/4 - run: scripts/test-shard.sh ${{ matrix.shard }} 4 - - name: Run *.serial.test.ts at --max-concurrency=1 (shard 1 only) - # Serial files share file-wide state (top-level mock.module, module - # singletons) that leaks across files in the same bun-test process. - # test-shard.sh excludes them; this step runs them at concurrency=1. - if: matrix.shard == 1 - run: bun run test:serial + - run: bun run verify + + serial-tests: + # *.serial.test.ts at --max-concurrency=1. Lives in its own runner so + # the matrix shards aren't carrying the serial-pass tail (the old shape + # stuffed this into `test (1)` after the matrix work, which compounded + # shard 1's overload). + needs: cache-check + if: needs.cache-check.outputs.hit != 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.13 + - run: bun install + - run: bun run test:serial + + test: + # Pure matrix shard — no verify, no serial. Each shard runs its slice + # of the unit test set under one `bun test` invocation. + # + # 6 shards (not 8) stays under the GitHub free-tier ~20-job concurrency + # budget when multiple PRs land same day: 6 shards + verify + serial + + # gitleaks + cache-check + cache-write + test-status = ~12 jobs × 2 + # concurrent PRs = 24; 8 shards × 2 PRs would queue worse. + # + # Partition policy is weight-aware LPT bin-packing via scripts/sharding.ts + # (replaces FNV-1a path hash). Weights live in scripts/test-weights.json, + # mined from real CI logs via scripts/mine-shard-weights.ts. Missing + # weights fall back to corpus median — new test files work immediately. + needs: cache-check + if: needs.cache-check.outputs.hit != 'true' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4, 5, 6] + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.13 + - run: bun install + - name: Run test shard ${{ matrix.shard }}/6 + run: scripts/test-shard.sh ${{ matrix.shard }} 6 + + # ────────────────────────────────────────────────────────────────────── + # cache-write: ONLY runs when every gated job succeeded. Writes the + # cache entry under `ci-pass-` so future runs at the same hash + # hit cache. Codex's load-bearing correctness point: writing the + # cache before the matrix completes would permanently bless bad states + # (a future run at the same hash would skip tests because of a cache + # entry written when tests hadn't actually passed). + # ────────────────────────────────────────────────────────────────────── + cache-write: + needs: [cache-check, gitleaks, verify, serial-tests, test] + if: success() && needs.cache-check.outputs.hit != 'true' + runs-on: ubuntu-latest + steps: + - name: Create cache marker + run: | + mkdir -p .ci-cache-marker + echo "${{ needs.cache-check.outputs.hash }}" > .ci-cache-marker/hash + echo "$GITHUB_SHA" > .ci-cache-marker/sha + echo "$GITHUB_REF" > .ci-cache-marker/ref + - uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + key: ci-pass-${{ needs.cache-check.outputs.hash }} + path: .ci-cache-marker + + # ────────────────────────────────────────────────────────────────────── + # test-status: the single user-visible "did CI pass?" check. + # Runs always (if: always()), succeeds when EITHER cache-check.hit==true + # OR all gated jobs (gitleaks, verify, serial-tests, test) succeeded. + # Branch protection (when configured) gates on this single job name. + # ────────────────────────────────────────────────────────────────────── + test-status: + needs: [cache-check, gitleaks, verify, serial-tests, test] + if: always() + runs-on: ubuntu-latest + steps: + - name: Aggregate result + run: | + HIT="${{ needs.cache-check.outputs.hit }}" + GITLEAKS="${{ needs.gitleaks.result }}" + VERIFY="${{ needs.verify.result }}" + SERIAL="${{ needs.serial-tests.result }}" + TEST="${{ needs.test.result }}" + echo "cache-check.hit=$HIT" + echo "gitleaks=$GITLEAKS verify=$VERIFY serial-tests=$SERIAL test=$TEST" + if [ "$HIT" = "true" ]; then + echo "✓ cache HIT for hash ${{ needs.cache-check.outputs.hash }} — CI green" + exit 0 + fi + # Cache miss: every gated job must have succeeded. + for r in "$GITLEAKS" "$VERIFY" "$SERIAL" "$TEST"; do + if [ "$r" != "success" ]; then + echo "✗ gated job did not succeed (got $r) — CI fail" + exit 1 + fi + done + echo "✓ all gated jobs succeeded — CI green" diff --git a/CHANGELOG.md b/CHANGELOG.md index 081668a25..4a8ccfb2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,55 @@ All notable changes to GBrain will be documented in this file. +## [0.41.6.0] - 2026-05-25 + +**Your PR CI stops taking 23 minutes.** The Test workflow on every pull request was wallclock-bound by one unlucky shard (shard 3, 22-26 min) while other shards finished in 5-13 min. After this release, every shard finishes in roughly the same time, the whole matrix runs in about 9-10 min on a fresh PR, and PRs that don't actually change test-affecting code (rebases, doc-only commits, retries) skip the test run entirely and finish in under 2 minutes via a content-hash cache. + +To turn it on: `gbrain upgrade` and pull the new `.github/workflows/test.yml`. First run after the merge writes the cache entry; subsequent matching pushes skip the test matrix. + +What you'd see in a concrete example. Pre-this-release: you push a 1-line CHANGELOG typo fix on top of a green PR. CI runs the full 23-minute matrix again. Post-fix: CHANGELOG.md is on the cache-key deny-list, the hash matches your prior green run, every test job skips with the cache-hit signal, the consolidated `Test / test-status` check goes green in under 2 minutes. Same shape for rebases, doc-only PRs, and re-pushes of the same SHA. + +How the speedup actually lands. Five orthogonal levers, all in one PR: + +1. **Restructured `test.yml` jobs.** `verify` (the 20 pre-test grep guards + typecheck + admin-build), `serial-tests`, and the test matrix used to all run on shard 1 via `if: matrix.shard == 1`, which is why shard 1 was the second-slowest. Each now runs in its own GitHub Actions runner in parallel. Shard 1 becomes equal in size to shards 2-6. +2. **Matrix 4 → 6 shards.** Six (not eight) keeps total per-PR job count under the GitHub free-tier ~20-job concurrency budget when multiple PRs land same day (6 shards + verify + serial + gitleaks + cache-check + cache-write + test-status = ~12 jobs × 2 concurrent PRs = 24, tight but workable; 8 shards × 2 PRs would queue worse). +3. **Weight-aware sharding (LPT bin-packer).** `scripts/test-shard.sh` used to partition with `(fnv1a(path) % N) + 1` which is weight-blind. The new `scripts/sharding.ts` does longest-processing-time-first greedy bin-packing over measured per-file weights from `scripts/test-weights.json`. Files absent from the JSON fall back to the corpus median, so adding a new test file works immediately without regenerating weights. Real-weight projection on 712 mined weights: every 6-shard estimated at 534s = 8.9 min wallclock. +4. **Mined CI-log weights, not isolated profiles.** `scripts/mine-shard-weights.ts` scrapes per-file wallclock from `gh run view --log` output (delta between `##[group]test/foo.test.ts:` timestamps within a shard). Free, real-world data, methodologically right (measures CI shard runtime, not per-file cold-start dominantly). Initial `test-weights.json` mined from run `26398061007`. +5. **Auto SHA cache.** `scripts/ci-cache-hash.sh` computes a deterministic 16-char sha256 over every git-tracked file EXCEPT `CHANGELOG.md`, `TODOS.md`, `README.md`, `LICENSE`, `docs/**/*.md`, `docs/**/*.txt`. CLAUDE.md, AGENTS.md, and `skills/**/*` are deliberately INCLUDED in the hash because tests read them; deny-listing those would create false-pass holes. A `cache-check` job runs first via `actions/cache/restore@v4.2.3` in `lookup-only: true` mode; on hit, every gated job skips and the `test-status` aggregator reports green. A `cache-write` job seals the cache key only after every gated job actually succeeded. + +**What we caught and fixed before merging.** Outside-voice (Codex) reviewed the plan and produced four material changes baked into this ship: (a) confirmed `e2e.yml` is fast (3-5 min) and NOT the critical path, validating that targeting `test.yml` is correct; (b) corrected the deny-list to keep CLAUDE.md and AGENTS.md IN the hash (their original deny-listing would have been a real false-pass hole since 8+ test files read them); (c) replaced the original draft's isolated per-file profiling (~57 min run, wrong methodology) with the log-mining approach; (d) added the job restructure (verify/serial split out of shard 1) which was missing from the original plan. + +Coverage. 8 CRITICAL false-pass guards pin the hash invalidation contract: `CLAUDE.md` edit → DIFFERENT hash, `AGENTS.md` edit → DIFFERENT hash, `skills/foo/SKILL.md` edit → DIFFERENT hash, `src/core/db.ts` edit → DIFFERENT hash, `test/foo.test.ts` edit → DIFFERENT hash, `package.json` edit → DIFFERENT hash, `bun.lock` edit → DIFFERENT hash, `.github/workflows/test.yml` edit → DIFFERENT hash. 7 SAFE deny-list invariants pin the cache-hit contract for genuinely test-irrelevant docs. Plus 9 edge cases (untracked-file excluded, rename detection, new-file-type discovery, deny-list-typo guard, symlinks, locale-stable sort, deterministic, usage errors). 24/24 green in `test/scripts/ci-cache-hash.test.ts`. + +### Itemized changes + +**New scripts:** +- `scripts/sharding.ts` (NEW) — Pure TypeScript LPT bin-packer with median fallback. Exports `partition`, `loadWeights`, `computeMedian`, `imbalanceRatio`. Throws `WeightsLoadError` on malformed JSON; runs in O(n log n). Pinned by `test/scripts/sharding.test.ts` (23 cases). +- `scripts/test-shard.sh` (rewrite) — Thin shell wrapper. Same CLI surface (`--dry-run-list` preserved). Streams the file list to `bun run scripts/sharding.ts` via stdin (avoids argv overflow at 676+ files). Slow files (`*.slow.test.ts`) intentionally INCLUDED in CI matrix — local fast loop excludes them; preserves the v0.26.4 CI-vs-local divergence policy. +- `scripts/test-weights.json` (NEW) — 712 mined weights from a green master run. Stats: min=0ms, median=30ms, max=359087ms (~6 min outlier), total 3306.3s observed runtime. +- `scripts/mine-shard-weights.ts` (NEW) — Scrapes `gh run view --log` for per-file timing. Three input modes: `--run `, `--from-file `, or piped from stdin. Stable JSON output (sorted keys) for clean diffs. Pinned by `test/scripts/mine-shard-weights.test.ts` (15 cases). +- `scripts/run-verify-parallel.sh` (NEW) — Fans out 21 fast checks (privacy/jsonb/source-id/admin-build/typecheck/gateway-routed/etc.) via `& wait`, per-check temp dir log, failure aggregation. 27s sequential → 13s parallel (2x) locally; bigger win in CI is shard 1 deload. Pinned by `test/scripts/run-verify-parallel.test.ts` (6 cases including synthetic failure-surfacing). +- `scripts/ci-cache-hash.sh` (NEW) — Deterministic 16-char sha256 over `git ls-files -s` minus deny-list. ~40ms on 1891 files (was ~9s with per-file `git hash-object`). Pinned by `test/scripts/ci-cache-hash.test.ts` (24 cases: 8 CRITICAL false-pass guards + 7 SAFE deny-list invariants + 9 edge cases). + +**Workflow restructure:** +- `.github/workflows/test.yml` — Seven jobs replacing the old 5-shard layout: `cache-check` (runs first via `actions/cache/restore@v4.2.3` `lookup-only`), then `gitleaks` + `verify` + `serial-tests` + `test` (6-shard matrix) all gated on `if: needs.cache-check.outputs.hit != 'true'`, then `cache-write` (post-all-pass via `if: success() && ...`), then `test-status` (the user-visible aggregator, `if: always()` so it reports green on cache-hit OR all-jobs-pass). + +**Coverage extensions:** +- `test/scripts/test-shard.slow.test.ts` (EXTENDED) — New LPT balance contract: 4-shard and 6-shard wallclock imbalance ratio ≤1.5 with real weights from `test-weights.json`. New INCLUDE-slow-files regression guard. New determinism check. Old FNV-1a runtime tests removed (sharding now lives in TS). +- `test/privacy-script-wired.test.ts` (FIXED in same wave) — Updated to follow the verify-script indirection (`package.json` `verify` → `run-verify-parallel.sh` → CHECKS array contains `check:privacy`). The original substring assertion broke when the `&&` chain was replaced with the parallel dispatcher. + +### For contributors + +To regenerate `scripts/test-weights.json` after the corpus drifts significantly: + +```bash +LATEST_RUN=$(gh run list --workflow=test.yml --status success --limit 1 --json databaseId --jq '.[0].databaseId') +bun run scripts/mine-shard-weights.ts --run "$LATEST_RUN" +git add scripts/test-weights.json && git commit -m "chore: refresh test weights from CI run $LATEST_RUN" +``` + +There is no scheduled regen — weights drift gracefully (missing files fall back to median), so this is opt-in when a specific shard starts feeling slow. + ## [0.41.5.0] - 2026-05-24 **Six community bug-fix PRs land + the E2E suite stops lying about itself.** A fix-wave triage swept the 333-PR queue, closed 10 PRs as already-shipped (with credit, naming the commits + files), and bundled 6 real fixes from the community into one collector. Plus three E2E-suite reliability fixes that surfaced while getting the full Docker suite to 100% green. diff --git a/VERSION b/VERSION index d25dbca5c..33a1f1909 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.41.5.0 \ No newline at end of file +0.41.6.0 diff --git a/package.json b/package.json index 5ebabe9d1..ef5598ebf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gbrain", - "version": "0.41.5.0", + "version": "0.41.6.0", "description": "Postgres-native personal knowledge brain with hybrid RAG search", "type": "module", "main": "src/core/index.ts", @@ -40,7 +40,7 @@ "build:pglite-snapshot": "bun run scripts/build-pglite-snapshot.ts", "test": "bash scripts/run-unit-parallel.sh", "test:full": "bun run verify && bash scripts/run-unit-parallel.sh && bun run test:slow && ([ -n \"$DATABASE_URL\" ] && bash scripts/run-e2e.sh || echo '[test:full] skipped E2E (no DATABASE_URL); run docker-compose -f docker-compose.ci.yml up + bun run test:e2e to include' 1>&2)", - "verify": "bun run check:privacy && bun run check:proposal-pii && bun run check:test-names && bun run check:jsonb && bun run check:source-id-projection && bun run check:source-config-leak && bun run check:progress && bun run check:test-isolation && bun run check:wasm && bun run check:admin-build && bun run check:admin-scope-drift && bun run check:cli-exec && bun run check:system-of-record && bun run check:eval-glossary && bun run check:no-pii-agent-voice && bun run check:synthetic-corpus-privacy && bun run check:skill-brain-first && bun run check:fuzz-purity && bun run check:operations-filter-bypass && bun run check:gateway-routed && bun run typecheck", + "verify": "bash scripts/run-verify-parallel.sh", "check:source-config-leak": "scripts/check-source-config-leak.sh", "check:no-pii-agent-voice": "scripts/check-no-pii-in-agent-voice.sh", "check:synthetic-corpus-privacy": "scripts/check-synthetic-corpus-privacy.sh", diff --git a/scripts/ci-cache-hash.sh b/scripts/ci-cache-hash.sh new file mode 100755 index 000000000..ce05465ac --- /dev/null +++ b/scripts/ci-cache-hash.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# scripts/ci-cache-hash.sh — deterministic content hash of all test- +# affecting files for the CI auto-cache. +# +# Outputs a 16-character hex prefix of sha256(sorted list of +# ` ` lines for every tracked file EXCEPT the +# deny-list below). Same tree → same hash. Different code → different +# hash. Doc-only changes → same hash (cache hit). +# +# Used by .github/workflows/test.yml's cache-check job. Cache key is +# `ci-pass-`. When the hash matches a prior green run, the test +# matrix skips and reports green immediately. +# +# DESIGN: deny-list NOT allowlist. New files default to "include in +# hash" — worst case, cache miss (waste 8min). Allowlist would default +# to "exclude", risking false-pass (broken code shipped under green +# check) when someone adds a new file type that tests read. +# +# WHAT'S DENY-LISTED (genuinely test-irrelevant): +# - CHANGELOG.md, TODOS.md pure documentation +# - README.md, LICENSE marketing / metadata, no test reads them +# - docs/**/*.md, *.txt all docs/ subtree is doc-only +# +# WHAT'S DELIBERATELY NOT DENY-LISTED (affects test outcomes): +# - CLAUDE.md 8+ test files reference it (resolver-merge, schema-cli, +# public-exports, eval-cross-modal-batch, etc.) +# - AGENTS.md same — referenced by resolver tests; counterpart to +# CLAUDE.md for OpenClaw hosts +# - skills/** SKILL.md files are read by skill conformance tests +# - everything else under src/, test/, scripts/, .github/, package.json, +# bun.lock, tsconfig*.json, the schema files — obviously test-affecting +# +# Locale-stable: LC_ALL=C on the sort step so byte-order is identical +# across runners (different default locales would re-order the line list +# and change the final hash). +# +# Usage: +# bash scripts/ci-cache-hash.sh # print hash +# bash scripts/ci-cache-hash.sh --verbose # print hash + diagnostics to stderr +# +# Exit codes: +# 0 printed a 16-char hex hash +# 1 internal error (git failure, etc.) +# 2 usage error + +set -euo pipefail + +VERBOSE=0 +if [ "${1:-}" = "--verbose" ]; then + VERBOSE=1 + shift +fi +if [ "$#" -gt 0 ]; then + echo "usage: bash scripts/ci-cache-hash.sh [--verbose]" >&2 + exit 2 +fi + +cd "$(dirname "$0")/.." + +# Deny-list as an extended regex matched against full paths emitted by +# `git ls-files`. -x makes the match anchored (full-line). Each pattern +# is a path predicate, not a glob — `\.` to literal-match dots. +# +# To add a new deny entry: add another `-e ''` line below. To +# REMOVE a deny entry (= include the path back in hash): delete its line. +# Either change invalidates the cache for everyone on next run (different +# hash output), which is the correct behavior. +DENY_PATTERNS=( + -e '^CHANGELOG\.md$' + -e '^TODOS\.md$' + -e '^README\.md$' + -e '^LICENSE$' + -e '^docs/.*\.md$' + -e '^docs/.*\.txt$' +) + +# Use `git ls-files -s` for one-shot enumeration of tracked files + their +# index blob shas. Output shape: ` \t` per line. +# Far faster than per-file `git hash-object` (~30ms vs ~9s on 2000 files). +# +# Trade-off: this reflects the INDEX (committed/staged state), not the +# working tree. CI always works against a committed tree so this matches +# what tests actually run. Local dev with uncommitted edits sees the +# committed-side hash (close enough — the hash is for CI's cache lookup, +# not a tree-state diagnostic). +LS_FILES=$(git ls-files -s) +if [ -z "$LS_FILES" ]; then + echo "error: git ls-files -s returned empty (are we in a git repo?)" >&2 + exit 1 +fi + +# Apply deny-list. Each line ends in `\t` so the deny patterns +# anchor on a tab boundary. Compose the alternation regex from +# DENY_PATTERNS — each entry is `^$`; strip the `^` (since `\t` +# acts as our anchor in `git ls-files -s` output) and the trailing `$` +# stays as-is. +DENY_ALT="" +i=1 +while [ $i -lt ${#DENY_PATTERNS[@]} ]; do + p="${DENY_PATTERNS[$i]}" + p="${p#^}" + if [ -z "$DENY_ALT" ]; then + DENY_ALT="$p" + else + DENY_ALT="$DENY_ALT|$p" + fi + i=$((i + 2)) +done +DENY_RE=$(printf '\t(%s)' "$DENY_ALT") +# Note: each $p already ends in `$` to anchor end-of-line, so the full +# regex is `\t(^CHANGELOG\.md$|^TODOS\.md$|...)`. Wait — we stripped `^` +# from each but kept `$`, so the composed regex is `\t(CHANGELOG\.md$| +# TODOS\.md$|docs/.*\.md$|...)`. Each alternative anchors its own end. +INCLUDED=$(printf '%s\n' "$LS_FILES" | grep -vE "$DENY_RE" || true) + +if [ -z "$INCLUDED" ]; then + echo "error: every tracked file is deny-listed — refusing to hash empty set" >&2 + exit 1 +fi + +# Sort by full line (LC_ALL=C for byte-order stability across locales), +# hash the concatenation. Each line carries (mode, sha, path) so any +# change to content (sha), mode (executable bit), or path (rename) flips +# the final hash. +HASH=$(printf '%s\n' "$INCLUDED" \ + | LC_ALL=C sort \ + | sha256sum \ + | cut -c1-16) + +if [ "$VERBOSE" = "1" ]; then + included_count=$(printf '%s\n' "$INCLUDED" | wc -l | tr -d ' ') + all_count=$(printf '%s\n' "$LS_FILES" | wc -l | tr -d ' ') + denied_count=$((all_count - included_count)) + { + echo "ci-cache-hash: $included_count/$all_count files in hash ($denied_count deny-listed)" + } >&2 +fi + +echo "$HASH" diff --git a/scripts/mine-shard-weights.ts b/scripts/mine-shard-weights.ts new file mode 100644 index 000000000..2b84e3113 --- /dev/null +++ b/scripts/mine-shard-weights.ts @@ -0,0 +1,247 @@ +#!/usr/bin/env bun +/** + * scripts/mine-shard-weights.ts — extract per-file test wallclock from a + * real CI run's logs, write scripts/test-weights.json. + * + * Why this exists: scripts/sharding.ts does LPT bin-packing over per-file + * weights, but the weights have to come from somewhere. The original + * design ran each test file in isolation (`bun test ` per file) + * which (a) takes ~57min to run all 676 files, and (b) measures cold- + * start dominantly because each invocation pays a fresh `bun test` + * startup. CI shards run ~150 files in ONE bun process — cold-start is + * amortized away. Per-file isolated profiles are wrong-by-methodology. + * + * This script scrapes per-file wallclock from a real CI shard's log via + * GitHub's `gh run view --log` output. bun emits an `##[group]test/foo. + * test.ts:` header before each file with an ISO timestamp; the + * difference between consecutive headers = how long the previous file + * took. This is the actual CI shard runtime per file, in the right + * execution mode, for free on every green run. + * + * Usage: + * bun run scripts/mine-shard-weights.ts --run [--out PATH] + * bun run scripts/mine-shard-weights.ts --from-file [--out PATH] + * gh run view --log | bun run scripts/mine-shard-weights.ts [--out PATH] + * + * Default output: scripts/test-weights.json (overwrites). Use --out to + * write elsewhere (useful for diffing before commit). Output is JSON + * with sorted keys for stable diffs. + * + * Regen cadence: there is none. Weights drift continuously but missing + * files fall back to the corpus median in sharding.ts, so stale weights + * degrade gracefully. Run this script when you notice a specific shard + * starts running long, or after a wave that added many heavy tests. + * + * Exit codes: + * 0 wrote weights file (count > 0) + * 1 internal error + * 2 usage error + * 3 no usable timing data found (parsed log but extracted 0 weights) + */ + +import { spawnSync } from "node:child_process"; +import { readFileSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(__dirname, ".."); +const DEFAULT_OUT = resolve(REPO_ROOT, "scripts/test-weights.json"); + +interface Args { + runId?: string; + fromFile?: string; + fromStdin: boolean; + out: string; +} + +function parseArgs(argv: string[]): Args { + const out: Args = { fromStdin: false, out: DEFAULT_OUT }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === "--run" || a === "--run-id") { + out.runId = argv[++i]; + } else if (a === "--from-file") { + out.fromFile = argv[++i]; + } else if (a === "--out") { + out.out = resolve(argv[++i] ?? ""); + } else if (a === "--help" || a === "-h") { + console.log( + "usage: bun run scripts/mine-shard-weights.ts (--run | --from-file | ) [--out ]", + ); + process.exit(0); + } else { + console.error(`error: unknown arg: ${a}`); + process.exit(2); + } + } + if (!out.runId && !out.fromFile) { + out.fromStdin = true; + } + return out; +} + +async function readSource(args: Args): Promise { + if (args.runId) { + const r = spawnSync("gh", ["run", "view", args.runId, "--log"], { + encoding: "utf8", + maxBuffer: 256 * 1024 * 1024, // CI logs can be 50-80MB + }); + if (r.status !== 0) { + throw new Error( + `gh run view ${args.runId} --log failed (exit ${r.status}): ${r.stderr}`, + ); + } + return r.stdout; + } + if (args.fromFile) { + return readFileSync(args.fromFile, "utf8"); + } + // Read stdin + const chunks: Buffer[] = []; + for await (const chunk of process.stdin) { + chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk); + } + return Buffer.concat(chunks).toString("utf8"); +} + +/** + * Parsed timing event from a CI log line. timestamp is ms-since-epoch. + */ +interface TimingEvent { + job: string; + timestampMs: number; + file: string; +} + +/** + * Parse a CI log into a list of `##[group]test/X.test.ts:` events keyed + * by job (so timing deltas don't cross shard boundaries). + * + * GH log line shape: + * \tUNKNOWN STEP\t ##[group]test/foo.test.ts: + * or: + * \t\t ##[group]test/foo.test.ts: + * + * Exported for unit testing. + */ +export function parseLog(raw: string): TimingEvent[] { + const events: TimingEvent[] = []; + const lines = raw.split("\n"); + // Match: TABTAB ##[group]: + const re = /^([^\t]+)\t[^\t]*\t(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)\s+##\[group\](test\/[^\s:]+\.test\.ts):?\s*$/; + for (const line of lines) { + const m = re.exec(line); + if (!m) continue; + const job = m[1]!.trim(); + const ts = Date.parse(m[2]!); + const file = m[3]!; + if (Number.isNaN(ts)) continue; + events.push({ job, timestampMs: ts, file }); + } + return events; +} + +/** + * From a list of file-start events grouped by job, compute per-file + * runtime as (timestamp[i+1] - timestamp[i]) within each job. The last + * file in each job is dropped (we don't know when it ended without + * also parsing the bun summary line; the loss is acceptable since + * sharding.ts's median fallback covers missing files). + * + * When the same file appears in multiple jobs (shouldn't happen, but + * defensive against shard remix during the in-flight CI run that + * generated this log), take the max — heaviest observation wins. + * + * Exported for unit testing. + */ +export function computeWeights(events: TimingEvent[]): Map { + // Group events by job, in stream order. + const byJob = new Map(); + for (const e of events) { + let bucket = byJob.get(e.job); + if (!bucket) { + bucket = []; + byJob.set(e.job, bucket); + } + bucket.push(e); + } + + const weights = new Map(); + for (const [, jobEvents] of byJob) { + for (let i = 0; i + 1 < jobEvents.length; i++) { + const file = jobEvents[i]!.file; + const delta = jobEvents[i + 1]!.timestampMs - jobEvents[i]!.timestampMs; + if (delta < 0) continue; // log out-of-order; defensive + // Round to nearest ms; sub-ms doesn't matter for shard balancing. + const ms = Math.round(delta); + const prev = weights.get(file); + if (prev === undefined || ms > prev) { + weights.set(file, ms); + } + } + // Drop the last event's file (no successor → unknown duration). + } + return weights; +} + +/** + * Serialize a weights map to canonical JSON (keys sorted asc) so the + * committed file produces stable diffs run-to-run. + * + * Exported for unit testing. + */ +export function serializeWeights(weights: Map): string { + const sorted = Array.from(weights.entries()).sort((a, b) => + a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0, + ); + const obj: Record = {}; + for (const [k, v] of sorted) obj[k] = v; + return JSON.stringify(obj, null, 2) + "\n"; +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + console.error( + `[mine-shard-weights] source=${args.runId ?? args.fromFile ?? ""}`, + ); + let raw: string; + try { + raw = await readSource(args); + } catch (e) { + console.error(`error: ${e instanceof Error ? e.message : String(e)}`); + return 1; + } + const events = parseLog(raw); + console.error(`[mine-shard-weights] parsed ${events.length} file-start events`); + if (events.length === 0) { + console.error( + "error: no ##[group]test/*.test.ts: events found in input. Was this a CI test run log?", + ); + return 3; + } + const weights = computeWeights(events); + if (weights.size === 0) { + console.error("error: parsed events but extracted 0 weights (every job had ≤1 file?)"); + return 3; + } + const json = serializeWeights(weights); + writeFileSync(args.out, json); + // Summary: min/median/max/total. Useful for spot-checking the file. + const values = Array.from(weights.values()).sort((a, b) => a - b); + const min = values[0]!; + const max = values[values.length - 1]!; + const median = values[Math.floor(values.length / 2)]!; + const total = values.reduce((a, b) => a + b, 0); + console.error( + `[mine-shard-weights] wrote ${weights.size} weights to ${args.out}`, + ); + console.error( + `[mine-shard-weights] stats: min=${min}ms median=${median}ms max=${max}ms total=${(total / 1000).toFixed(1)}s`, + ); + return 0; +} + +if (import.meta.main) { + main().then((code) => process.exit(code)); +} diff --git a/scripts/run-verify-parallel.sh b/scripts/run-verify-parallel.sh new file mode 100755 index 000000000..45a8dc9e5 --- /dev/null +++ b/scripts/run-verify-parallel.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +# scripts/run-verify-parallel.sh — parallel verify dispatcher. +# +# Runs the 19+ verify checks (privacy, jsonb, source-id, … + typecheck + +# admin-build) as background jobs, waits for all, aggregates exit codes, +# surfaces failed-check name + tail of its log to stderr. +# +# Replaces the sequential `&&`-chain in package.json's `verify` script. +# Wallclock: 19 sequential checks (~15-25s on CI) → parallel (~3-5s). +# +# Usage: +# bash scripts/run-verify-parallel.sh # run every CHECK below +# bash scripts/run-verify-parallel.sh --dry-list # print check list, exit +# +# Env overrides: +# GBRAIN_VERIFY_TIMEOUT per-check wallclock cap, seconds (default 120) +# GBRAIN_VERIFY_LOG_DIR where to write per-check logs (default tempdir) +# +# Exit codes: +# 0 all checks passed +# 1 one or more checks failed (full details in stderr) +# 2 usage error / no checks defined + +set -uo pipefail + +cd "$(dirname "$0")/.." + +# ────────────────────────────────────────────────────────────────────────── +# Checks to run. Order is irrelevant (parallel), but keep stable for log +# determinism + grep-ability. Each entry is a bun-script name (the +# `package.json` "scripts" key), invoked as `bun run `. +# +# To add a check: append to this array. To skip in CI temporarily, comment +# the line — the parallel runner doesn't care about count. +# ────────────────────────────────────────────────────────────────────────── +CHECKS=( + "check:privacy" + "check:proposal-pii" + "check:test-names" + "check:jsonb" + "check:source-id-projection" + "check:source-config-leak" + "check:progress" + "check:test-isolation" + "check:wasm" + "check:admin-build" + "check:admin-scope-drift" + "check:cli-exec" + "check:system-of-record" + "check:eval-glossary" + "check:no-pii-agent-voice" + "check:synthetic-corpus-privacy" + "check:skill-brain-first" + "check:fuzz-purity" + "check:operations-filter-bypass" + "check:gateway-routed" + "typecheck" +) + +if [ "${#CHECKS[@]}" -eq 0 ]; then + echo "ERROR: no checks defined in run-verify-parallel.sh" >&2 + exit 2 +fi + +# Dry-run path: list checks, exit. Used by tests + ops debugging. +if [ "${1:-}" = "--dry-list" ]; then + printf '%s\n' "${CHECKS[@]}" + exit 0 +fi + +if [ "$#" -gt 0 ] && [ "${1:-}" != "" ]; then + echo "ERROR: unknown arg: $1" >&2 + echo "usage: bash scripts/run-verify-parallel.sh [--dry-list]" >&2 + exit 2 +fi + +TIMEOUT="${GBRAIN_VERIFY_TIMEOUT:-120}" + +# Per-check temp dir. Each check gets its own subdir so writes can't race +# on shared scratch state (the checks themselves are read-only — they grep +# the working tree — but defense-in-depth.) +if [ -n "${GBRAIN_VERIFY_LOG_DIR:-}" ]; then + LOG_DIR="$GBRAIN_VERIFY_LOG_DIR" + mkdir -p "$LOG_DIR" || { echo "ERROR: cannot create $LOG_DIR" >&2; exit 2; } +else + LOG_DIR="$(mktemp -d /tmp/gbrain-verify-XXXXXX)" + trap 'rm -rf "$LOG_DIR"' EXIT +fi + +# Resolve `timeout` for per-check wallclock cap. macOS doesn't ship one; +# brew coreutils provides `gtimeout`. If neither is available, fall back to +# bg-pid + sleep-cap (slightly less reliable but still bounded). +TIMEOUT_BIN="" +if command -v gtimeout >/dev/null 2>&1; then TIMEOUT_BIN="gtimeout" +elif command -v timeout >/dev/null 2>&1; then TIMEOUT_BIN="timeout" +fi + +START_TS=$(date +%s) +echo "[verify-parallel] running ${#CHECKS[@]} checks in parallel (timeout=${TIMEOUT}s, logs=$LOG_DIR)" >&2 + +# ────────────────────────────────────────────────────────────────────────── +# Spawn one background process per check. Each child captures its own exit +# code into a sentinel file under $LOG_DIR/.exit; the parent +# never trusts `wait`'s aggregate value because that maps to last-spawned. +# +# safe_name: turn `check:privacy` into `check_privacy` so it fits a filename +# without escaping. +# ────────────────────────────────────────────────────────────────────────── +PIDS=() +SAFE_NAMES=() +for c in "${CHECKS[@]}"; do + safe="${c//:/_}" + SAFE_NAMES+=("$safe") + LOG_FILE="$LOG_DIR/$safe.log" + EXIT_FILE="$LOG_DIR/$safe.exit" + ( + if [ -n "$TIMEOUT_BIN" ]; then + "$TIMEOUT_BIN" "${TIMEOUT}s" bun run "$c" > "$LOG_FILE" 2>&1 + else + bun run "$c" > "$LOG_FILE" 2>&1 & + pid=$! + ( sleep "$TIMEOUT" && kill -TERM "$pid" 2>/dev/null && \ + sleep 5 && kill -KILL "$pid" 2>/dev/null ) & + cap_pid=$! + wait "$pid" 2>/dev/null + kill "$cap_pid" 2>/dev/null + wait "$cap_pid" 2>/dev/null + fi + rc=$? + echo "$rc" > "$EXIT_FILE" + ) & + PIDS+=($!) +done + +# Wait for every background job. Ignore wait's aggregate exit — exit codes +# live in the sentinel files. +for pid in "${PIDS[@]}"; do wait "$pid" 2>/dev/null || true; done + +END_TS=$(date +%s) +ELAPSED=$((END_TS - START_TS)) + +# ────────────────────────────────────────────────────────────────────────── +# Aggregate. For each check, read its exit file; on failure, append a +# labeled block (check name + tail of log) to the failure report. Surface +# one final summary line and the report to stderr if anything failed. +# ────────────────────────────────────────────────────────────────────────── +PASS=0 +FAIL=0 +FAIL_NAMES=() +FAIL_REPORT="" + +for i in "${!CHECKS[@]}"; do + c="${CHECKS[$i]}" + safe="${SAFE_NAMES[$i]}" + EXIT_FILE="$LOG_DIR/$safe.exit" + LOG_FILE="$LOG_DIR/$safe.log" + + rc=1 + [ -f "$EXIT_FILE" ] && rc=$(cat "$EXIT_FILE" 2>/dev/null || echo 1) + + if [ "$rc" = "0" ]; then + PASS=$((PASS + 1)) + else + FAIL=$((FAIL + 1)) + FAIL_NAMES+=("$c") + if [ "$rc" = "124" ]; then + FAIL_REPORT+=$'\n--- '"$c"' (TIMED OUT after '"${TIMEOUT}"'s) ---\n' + else + FAIL_REPORT+=$'\n--- '"$c"' (rc='"$rc"') ---\n' + fi + if [ -f "$LOG_FILE" ]; then + FAIL_REPORT+="$(tail -30 "$LOG_FILE")" + FAIL_REPORT+=$'\n' + fi + fi +done + +if [ "$FAIL" -gt 0 ]; then + { + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "❌ verify failed: $FAIL/${#CHECKS[@]} checks did not pass" + echo "Failed: ${FAIL_NAMES[*]}" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + printf '%s' "$FAIL_REPORT" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "[verify-parallel] elapsed=${ELAPSED}s | pass=$PASS fail=$FAIL" + } >&2 + exit 1 +fi + +echo "[verify-parallel] elapsed=${ELAPSED}s | pass=$PASS fail=0 | all checks green" >&2 +exit 0 diff --git a/scripts/sharding.ts b/scripts/sharding.ts new file mode 100644 index 000000000..53cbca08b --- /dev/null +++ b/scripts/sharding.ts @@ -0,0 +1,256 @@ +#!/usr/bin/env bun +/** + * scripts/sharding.ts — weight-aware test file partitioning. + * + * Replaces FNV-1a path-hash sharding in scripts/test-shard.sh. Uses + * longest-processing-time-first (LPT) greedy bin-packing over measured + * per-file runtimes to balance total wallclock across N shards. + * + * LPT is a textbook approximation algorithm: sort jobs by weight desc, + * assign each to the bin (shard) with the current minimum total. Worst- + * case makespan is within 4/3 of optimal. Runs in O(n log n). + * + * Weights live in scripts/test-weights.json — committed, mined from real + * CI run logs via scripts/mine-shard-weights.ts. Files absent from the + * weights map fall back to the corpus median (not zero — that would + * favor unknown new files into the smallest shard, defeating balance). + * + * CLI: + * bun run scripts/sharding.ts + * Reads test file list from stdin (one path per line). Prints the + * subset assigned to to stdout, one per line. + * + * bun run scripts/sharding.ts --files + * Walks the filesystem for matching files instead of reading stdin. + * + * Exit codes: + * 0 success + * 1 internal error (e.g., malformed weights JSON) + * 2 usage error + * + * Used by: scripts/test-shard.sh (thin wrapper), test/scripts/sharding.test.ts. + */ + +import { readFileSync, existsSync } from "node:fs"; +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(__dirname, ".."); +const DEFAULT_WEIGHTS_PATH = resolve(REPO_ROOT, "scripts/test-weights.json"); + +export type WeightMap = Map; + +export class WeightsLoadError extends Error { + constructor(public readonly path: string, public readonly cause: unknown) { + super(`failed to load weights from ${path}: ${cause}`); + this.name = "WeightsLoadError"; + } +} + +/** + * Read a weights JSON file. Fail-soft on missing file (returns empty map). + * Throws WeightsLoadError on malformed JSON or non-object shape — the + * caller decides whether to fall through to defaults or surface. + */ +export function loadWeights(path: string = DEFAULT_WEIGHTS_PATH): WeightMap { + if (!existsSync(path)) return new Map(); + let raw: string; + try { + raw = readFileSync(path, "utf8"); + } catch (e) { + throw new WeightsLoadError(path, e); + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (e) { + throw new WeightsLoadError(path, `JSON.parse: ${e}`); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + throw new WeightsLoadError(path, "expected top-level object {path: ms}"); + } + const out: WeightMap = new Map(); + for (const [k, v] of Object.entries(parsed as Record)) { + if (typeof v !== "number" || !Number.isFinite(v) || v < 0) { + throw new WeightsLoadError( + path, + `value for "${k}" must be a non-negative finite number, got ${JSON.stringify(v)}`, + ); + } + out.set(k, v); + } + return out; +} + +/** + * Median of a list of numbers. Used as the fallback weight for files + * absent from the weights map. Empty input returns 0 (no signal). + */ +export function computeMedian(values: number[]): number { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? (sorted[mid - 1]! + sorted[mid]!) / 2 + : sorted[mid]!; +} + +export interface PartitionOpts { + /** + * Weight to assign files that are absent from the weights map. Defaults + * to the median of present weights (computed inside `partition`) so + * unknown new files cluster around the typical file's cost. + * + * Override mainly for tests; production callers should use the default. + */ + fallbackWeight?: number; +} + +/** + * Partition `files` into `n` shards using LPT greedy bin-packing. + * + * Returns an array of length `n` where each entry is the (deterministic) + * file list for that shard. Files in each shard are returned in + * assignment order (heaviest first). Same input always produces same + * output — no Math.random, stable sort key (weight desc, then path asc). + * + * Contracts: + * - Every file in `files` appears in exactly one returned shard. + * - If `files` is empty, returns `n` empty arrays. + * - If `n <= 0`, throws RangeError. + * - Files missing from `weights` get `opts.fallbackWeight` (or median). + */ +export function partition( + files: string[], + weights: WeightMap, + n: number, + opts: PartitionOpts = {}, +): string[][] { + if (!Number.isInteger(n) || n <= 0) { + throw new RangeError(`shard count must be a positive integer, got ${n}`); + } + const shards: string[][] = Array.from({ length: n }, () => []); + if (files.length === 0) return shards; + + // Compute fallback weight from the median of present weights, unless + // the caller supplied an explicit override. + let fallback: number; + if (opts.fallbackWeight !== undefined) { + if (!Number.isFinite(opts.fallbackWeight) || opts.fallbackWeight < 0) { + throw new RangeError( + `fallbackWeight must be non-negative finite, got ${opts.fallbackWeight}`, + ); + } + fallback = opts.fallbackWeight; + } else { + fallback = computeMedian(Array.from(weights.values())); + } + // Cold-start guard: if the weights map is empty AND no explicit + // fallback was supplied, every effective weight would be 0 and LPT + // collapses (all ties → lowest-index wins → every file in shard 0). + // Normalize fallback to 1 so LPT degenerates to round-robin, which is + // a strictly better default than "everything in shard 1" until + // test-weights.json gets mined. + if (fallback === 0 && opts.fallbackWeight === undefined) { + fallback = 1; + } + + // Build [weight, path] tuples. Sort by weight desc, then path asc for + // determinism on ties (multiple files with the same weight). + const tuples = files.map((f) => ({ + path: f, + weight: weights.get(f) ?? fallback, + })); + tuples.sort((a, b) => { + if (b.weight !== a.weight) return b.weight - a.weight; + return a.path < b.path ? -1 : a.path > b.path ? 1 : 0; + }); + + // Running per-shard totals. argmin tiebreaker: lowest index (stable). + const totals = new Array(n).fill(0); + for (const t of tuples) { + let minIdx = 0; + for (let i = 1; i < n; i++) { + if (totals[i]! < totals[minIdx]!) minIdx = i; + } + shards[minIdx]!.push(t.path); + totals[minIdx] = totals[minIdx]! + t.weight; + } + return shards; +} + +/** + * Imbalance ratio: max-total / min-total over the partition. 1.0 = perfect. + * Returns Infinity when any shard is empty (degenerate). Use ≤1.5 as a + * loose health gate; LPT typically hits ≤1.1 on real corpora. + * + * Exposed for tests + the slow-test regression that pins corpus health. + */ +export function imbalanceRatio(shards: string[][], weights: WeightMap, fallback: number): number { + if (shards.length === 0) return 1; + const totals = shards.map((s) => + s.reduce((sum, f) => sum + (weights.get(f) ?? fallback), 0), + ); + const max = Math.max(...totals); + const min = Math.min(...totals); + if (min === 0) return max === 0 ? 1 : Infinity; + return max / min; +} + +// ────────────────────────────────────────────────────────────────────────── +// CLI +// ────────────────────────────────────────────────────────────────────────── + +async function readStdinLines(): Promise { + // Bun gives us a readable stream on process.stdin. Read to end. + if (process.stdin.isTTY) return []; // no piped input + const chunks: Buffer[] = []; + for await (const chunk of process.stdin) { + chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk); + } + const text = Buffer.concat(chunks).toString("utf8"); + return text + .split("\n") + .map((l) => l.trim()) + .filter((l) => l.length > 0); +} + +async function main(): Promise { + const argv = process.argv.slice(2); + if (argv.length < 2) { + console.error("usage: bun run scripts/sharding.ts "); + console.error(" (reads file list from stdin, one path per line)"); + return 2; + } + const idx = Number.parseInt(argv[0]!, 10); + const total = Number.parseInt(argv[1]!, 10); + if (!Number.isInteger(idx) || !Number.isInteger(total) || idx < 1 || total < 1 || idx > total) { + console.error( + `error: shard index ${argv[0]} / total ${argv[1]} invalid (need 1 <= index <= total, both ints)`, + ); + return 2; + } + const files = await readStdinLines(); + if (files.length === 0) { + // Caller asked for a shard but piped no files. Exit clean — the wrapper + // will warn or no-op as it sees fit. + return 0; + } + let weights: WeightMap; + try { + weights = loadWeights(); + } catch (e) { + console.error(`error: ${e instanceof Error ? e.message : String(e)}`); + return 1; + } + const shards = partition(files, weights, total); + for (const f of shards[idx - 1]!) { + process.stdout.write(`${f}\n`); + } + return 0; +} + +if (import.meta.main) { + main().then((code) => process.exit(code)); +} diff --git a/scripts/test-shard.sh b/scripts/test-shard.sh index b0065c760..3bc68a52b 100755 --- a/scripts/test-shard.sh +++ b/scripts/test-shard.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Partition unit test files into N shards by stable hash and run one shard. +# Partition unit test files into N shards and run one shard. # # Usage: scripts/test-shard.sh # shard-index: 1-based (1..N) @@ -8,16 +8,27 @@ # Excluded from sharding: # - test/e2e/* — need DATABASE_URL; run via bun run test:e2e # - *.serial.test.ts — concurrency-unsafe (file-wide mock.module / env -# leaks); run via scripts/run-serial-tests.sh on -# shard 1 only. Including these here lets their -# mock.module() calls leak into the rest of the -# shard's bun process and silently break unrelated -# tests. See test/eval-takes-quality-runner.serial.test.ts -# mocking gateway.ts → voyage-multimodal failures. +# leaks); run via bun run test:serial on its +# own runner in CI. Including these here lets +# their mock.module() calls leak into the rest +# of the shard's bun process and silently break +# unrelated tests. # -# Stable partitioning: a file's shard is `(hash(path) % N) + 1`. Same file -# lands in the same shard on every run, regardless of how many other files -# exist, so retries are reproducible. Hash is FNV-1a — pure shell, no jq. +# *.slow.test.ts is deliberately INCLUDED here. CI's matrix is the only +# default place these run; the local fast loop (run-unit-shard.sh) +# excludes them. See CLAUDE.md "CI vs local: intentionally divergent file +# sets" for the rationale. +# +# Partition: weight-aware LPT bin-packing via scripts/sharding.ts. Reads +# per-file runtime weights from scripts/test-weights.json (mined from real +# CI logs by scripts/mine-shard-weights.ts). Files absent from the map +# fall back to the corpus median, so adding a new test file works +# immediately without regenerating weights — worst case it lands in the +# wrong shard until next regen, never silently dropped. +# +# Stable partitioning: same `(files, weights, N)` always produces the +# same assignment, so retries are reproducible. + set -euo pipefail DRY_RUN_LIST=0 @@ -45,60 +56,41 @@ fi cd "$(dirname "$0")/.." -# Find all unit test files, deterministic order. Excludes test/e2e/ and -# *.serial.test.ts. Serial files share file-wide state (top-level -# mock.module, module singletons) that leaks across files in the same -# `bun test` shard process — see scripts/check-test-isolation.sh R2. -# CI runs them via `bun run test:serial` (scripts/run-serial-tests.sh) at -# --max-concurrency=1 in a separate step on shard 1. Local `bun run test` -# already excludes them from the parallel pass and runs them after the -# same way. Portable: avoid `mapfile` (bash 4+) so this runs on macOS -# bash 3.2 too. -FILES=() -while IFS= read -r line; do - FILES+=("$line") -done < <(find test -name '*.test.ts' -not -name '*.serial.test.ts' -not -path 'test/e2e/*' | sort) +# Collect non-E2E, non-serial unit test files. Slow files INCLUDED — see +# header comment. Local run-unit-shard.sh excludes slow files (different +# policy by design). +ALL_FILES=$(find test -name '*.test.ts' \ + -not -name '*.serial.test.ts' \ + -not -path 'test/e2e/*' | sort) -if [ "${#FILES[@]}" -eq 0 ]; then +if [ -z "$ALL_FILES" ]; then echo "no test files found under test/" >&2 exit 1 fi -# FNV-1a 32-bit hash of a string — implemented in pure bash so we don't depend -# on python/openssl/etc on the runner. Output is decimal. -fnv1a() { - local str="$1" - local h=2166136261 # FNV offset basis - local i ord - for (( i=0; i<${#str}; i++ )); do - ord=$(printf '%d' "'${str:$i:1}") - h=$(( (h ^ ord) & 0xFFFFFFFF )) - h=$(( (h * 16777619) & 0xFFFFFFFF )) - done - echo "$h" -} - -SHARD_FILES=() -for f in "${FILES[@]}"; do - hash=$(fnv1a "$f") - bucket=$(( hash % TOTAL_SHARDS + 1 )) - if [ "$bucket" -eq "$SHARD_INDEX" ]; then - SHARD_FILES+=("$f") - fi -done +# Delegate the LPT partition to scripts/sharding.ts. Stream the file list +# via stdin to keep argv small (676+ files would overflow argv in some +# shells / OSes). +SHARD_FILES=$(printf '%s\n' "$ALL_FILES" | bun run scripts/sharding.ts "$SHARD_INDEX" "$TOTAL_SHARDS") if [ "$DRY_RUN_LIST" = "1" ]; then - if [ "${#SHARD_FILES[@]}" -eq 0 ]; then - exit 0 - fi - printf '%s\n' "${SHARD_FILES[@]}" + printf '%s' "$SHARD_FILES" + [ -n "$SHARD_FILES" ] && echo "" # trailing newline if non-empty exit 0 fi -echo "shard $SHARD_INDEX/$TOTAL_SHARDS: ${#SHARD_FILES[@]}/${#FILES[@]} files" -if [ "${#SHARD_FILES[@]}" -eq 0 ]; then - echo "warning: shard $SHARD_INDEX has no files (rehash or reduce shard count)" >&2 +ALL_COUNT=$(printf '%s\n' "$ALL_FILES" | grep -c '^' || true) +SHARD_COUNT=$(printf '%s\n' "$SHARD_FILES" | grep -c '^' || true) +# grep -c on empty input returns 0 even with trailing newline edge cases +[ -z "$SHARD_FILES" ] && SHARD_COUNT=0 + +echo "shard $SHARD_INDEX/$TOTAL_SHARDS: ${SHARD_COUNT}/${ALL_COUNT} files (LPT-balanced)" + +if [ "$SHARD_COUNT" -eq 0 ]; then + echo "warning: shard $SHARD_INDEX has no files (total shards may exceed file count)" >&2 exit 0 fi -exec bun test --timeout=60000 "${SHARD_FILES[@]}" +# Convert newline-separated file list to argv. xargs handles the +# whitespace correctly without word-splitting on spaces in paths. +printf '%s\n' "$SHARD_FILES" | xargs bun test --timeout=60000 diff --git a/scripts/test-weights.json b/scripts/test-weights.json new file mode 100644 index 000000000..bb291b619 --- /dev/null +++ b/scripts/test-weights.json @@ -0,0 +1,713 @@ +{ + "test/active-pack-wiring.test.ts": 3, + "test/admin-agents-spend.test.ts": 4995, + "test/admin-embed-spawn.serial.test.ts": 12792, + "test/agent-cli.test.ts": 2935, + "test/agent-runner.test.ts": 2, + "test/ai/adaptive-embed-batch.test.ts": 34, + "test/ai/build-gateway-config.test.ts": 0, + "test/ai/capabilities.test.ts": 3, + "test/ai/config-no-env-mutation.test.ts": 4, + "test/ai/dims-openai.test.ts": 4, + "test/ai/dims-zeroentropy.test.ts": 4, + "test/ai/embedQuery.test.ts": 8, + "test/ai/gateway-chat.test.ts": 8, + "test/ai/gateway-tool-loop.test.ts": 9, + "test/ai/gateway.test.ts": 444, + "test/ai/header-transport.serial.test.ts": 118, + "test/ai/no-batch-cap-suppression.serial.test.ts": 98, + "test/ai/recipe-azure-openai.test.ts": 7, + "test/ai/recipe-dashscope.test.ts": 5, + "test/ai/recipe-llama-server.test.ts": 0, + "test/ai/recipe-minimax.test.ts": 1, + "test/ai/recipe-openrouter.test.ts": 1, + "test/ai/recipe-zhipu.test.ts": 3, + "test/ai/recipes-existing-regression.test.ts": 5, + "test/ai/rerank.test.ts": 16, + "test/ai/schema-templating.test.ts": 4, + "test/ai/silent-drop-regression.test.ts": 9, + "test/ai/voyage-code-3-recipe.test.ts": 3, + "test/ai/zeroentropy-compat-fetch.test.ts": 16, + "test/ai/zeroentropy-recipe.test.ts": 1, + "test/anomalies.test.ts": 1, + "test/anthropic-model-ids.test.ts": 3, + "test/apply-migrations-pglite-spawn.serial.test.ts": 18874, + "test/apply-migrations.test.ts": 5, + "test/archive-crawler-config.test.ts": 26, + "test/artifact-abstraction.test.ts": 2, + "test/asymmetric-encoding-contract.test.ts": 1, + "test/audit-slug-fallback.serial.test.ts": 98, + "test/audit-synopsis.serial.test.ts": 99, + "test/audit/audit-writer.test.ts": 9, + "test/audit/content-sanity-audit.test.ts": 6, + "test/auto-think-phase.test.ts": 3251, + "test/autopilot-cycle-failure-classification.test.ts": 4, + "test/autopilot-cycle-handler.test.ts": 3039, + "test/autopilot-fanout-wiring.test.ts": 3, + "test/autopilot-fanout.test.ts": 24, + "test/autopilot-install.test.ts": 5, + "test/autopilot-lock-path.test.ts": 13, + "test/autopilot-nightly-probe-wiring.test.ts": 2, + "test/autopilot-reconnect-classifier.test.ts": 5, + "test/autopilot-resolve-cli.test.ts": 11, + "test/autopilot-supervisor-wiring.test.ts": 10, + "test/backfill-base.test.ts": 2, + "test/backfill-concurrency-clamp.serial.test.ts": 103, + "test/backlinks.test.ts": 23, + "test/backoff.test.ts": 8, + "test/balanced-reranker-default.test.ts": 19, + "test/batch-projection.test.ts": 1, + "test/bench-publish.test.ts": 8, + "test/bench/baseline-file.test.ts": 2, + "test/bench/correctness-gate.test.ts": 5, + "test/bench/qrels-file.test.ts": 13, + "test/book-mirror.test.ts": 431, + "test/bootstrap.test.ts": 18481, + "test/brain-allowlist.test.ts": 3257, + "test/brain-registry.serial.test.ts": 104, + "test/brain-resolver.test.ts": 125, + "test/brain-score-breakdown.test.ts": 3087, + "test/brain-score-recommendations.test.ts": 23, + "test/brain-writer-partial-scan.test.ts": 4335, + "test/brain-writer-walk-prune.test.ts": 7, + "test/brain-writer.test.ts": 3949, + "test/brainstorm-timeout.test.ts": 45, + "test/brainstorm/checkpoint.serial.test.ts": 136, + "test/brainstorm/cost-guardrails.test.ts": 13, + "test/brainstorm/distance.test.ts": 26, + "test/brainstorm/eval-brainstorm.test.ts": 13, + "test/brainstorm/lsd-mode-skip.test.ts": 2, + "test/budget-meter.test.ts": 26, + "test/budget-tracker.test.ts": 3265, + "test/build-llms.test.ts": 11, + "test/calibration-cli.test.ts": 13, + "test/calibration-profile.test.ts": 7, + "test/candidate-audit.test.ts": 9, + "test/capture-build-content.test.ts": 18, + "test/capture-runcapture.test.ts": 8, + "test/cathedral-ii-brainbench.test.ts": 9162, + "test/check-resolvable-cli.test.ts": 3600, + "test/check-resolvable.test.ts": 40, + "test/check-system-of-record.test.ts": 70, + "test/check-update.test.ts": 674, + "test/child-worker-supervisor.test.ts": 144, + "test/chunk-grain-fts.test.ts": 70996, + "test/chunker-timeout.test.ts": 27, + "test/chunker-version-gate.test.ts": 6, + "test/chunkers/code.test.ts": 129, + "test/chunkers/recursive.test.ts": 57, + "test/cjk.test.ts": 4, + "test/claw-test-cli.test.ts": 9, + "test/cli-args.test.ts": 15, + "test/cli-dispatch-thin-client.test.ts": 6027, + "test/cli-help-discoverability.test.ts": 1670, + "test/cli-multimodal-integration.test.ts": 26677, + "test/cli-options.test.ts": 1658, + "test/cli-pty-runner.test.ts": 16, + "test/cli-query-image.test.ts": 56, + "test/cli.test.ts": 2633, + "test/code-callers-cli.test.ts": 2, + "test/code-def-refs.test.ts": 28662, + "test/code-edges.test.ts": 2884, + "test/code-intel/edge-densification.test.ts": 33, + "test/code-intel/eval-capture-graph.test.ts": 5, + "test/code-intel/recursive-walk.test.ts": 31107, + "test/code-intel/scope-walker-resolution.test.ts": 19, + "test/code-intel/traversal-cache.test.ts": 42036, + "test/code-retrieval-harness.test.ts": 4, + "test/commands-search.test.ts": 23985, + "test/commands/capture.test.ts": 42431, + "test/config-ensure-gitignore.test.ts": 7, + "test/config-env.test.ts": 6, + "test/config-set.test.ts": 17, + "test/config-unset.test.ts": 23831, + "test/config.test.ts": 6, + "test/connection-manager.serial.test.ts": 117, + "test/connection-resilience.test.ts": 29, + "test/console-prefix.test.ts": 5, + "test/consolidate-valid-until.test.ts": 2471, + "test/content-sanity-literals.test.ts": 2, + "test/content-sanity.test.ts": 5, + "test/context-engine.test.ts": 33, + "test/contextual-retrieval-doctor.serial.test.ts": 2755, + "test/contextual-retrieval-resolver.test.ts": 4, + "test/contextual-retrieval-service-pure.test.ts": 3, + "test/core/audit-week-file.serial.test.ts": 97, + "test/core/base-phase.test.ts": 9, + "test/core/cycle.serial.test.ts": 5563, + "test/core/diarize/payload-fitter-summarize.test.ts": 15, + "test/core/remediation-checkpoint.serial.test.ts": 129, + "test/cosine-rescore-column.test.ts": 3198, + "test/cross-brain-calibration.test.ts": 8, + "test/cross-modal-eval-aggregate.test.ts": 3, + "test/cross-modal-eval-cli.test.ts": 23, + "test/cross-modal-eval-json-repair.test.ts": 31, + "test/cross-modal-hybrid-integration.serial.test.ts": 3312, + "test/cross-modal-phase1.test.ts": 40, + "test/cross-modal-phase2.test.ts": 48, + "test/cycle-abort.test.ts": 108, + "test/cycle-consolidate.test.ts": 23952, + "test/cycle-last-full-cycle-at.test.ts": 3771, + "test/cycle-legacy-phases.test.ts": 4177, + "test/cycle-lock-per-source.test.ts": 9, + "test/cycle-pack-gating.test.ts": 5, + "test/cycle-patterns.test.ts": 2, + "test/cycle-pglite-lock-ordering.serial.test.ts": 2693, + "test/cycle-synthesize-chunker.test.ts": 21, + "test/cycle-synthesize-md-discovery.test.ts": 6, + "test/cycle-synthesize-slug-collection.test.ts": 3140, + "test/cycle-synthesize.test.ts": 17, + "test/cycle/extract-atoms-synthesize-concepts.test.ts": 60181, + "test/cycle/nightly-probe-adapters.test.ts": 6, + "test/data-research.test.ts": 17, + "test/db-lock-election.test.ts": 23346, + "test/db-lock-per-source.test.ts": 33460, + "test/db-lock-refresh.test.ts": 6, + "test/dedup.test.ts": 13, + "test/destructive-guard.test.ts": 6357, + "test/disk-walk.test.ts": 17, + "test/distribution-import-boundary.test.ts": 7, + "test/doctor-behavioral.test.ts": 5729, + "test/doctor-calibration-checks.test.ts": 5, + "test/doctor-child-orphans.test.ts": 4, + "test/doctor-cli-smoke.serial.test.ts": 4239, + "test/doctor-cycle-freshness.test.ts": 33963, + "test/doctor-cycle-phase-scope.test.ts": 7, + "test/doctor-federation-health.test.ts": 4021, + "test/doctor-fix.test.ts": 1301, + "test/doctor-frontmatter-partial.test.ts": 2, + "test/doctor-home-dir-in-worktree.test.ts": 194, + "test/doctor-minions-check.test.ts": 2697, + "test/doctor-remote.serial.test.ts": 210, + "test/doctor-report-remote.serial.test.ts": 2823, + "test/doctor-search-mode.test.ts": 3519, + "test/doctor-subagent-health.test.ts": 2972, + "test/doctor-v0_37_7_checks.test.ts": 2447, + "test/doctor-ze-checks.test.ts": 26042, + "test/doctor.test.ts": 46040, + "test/domain-aggregators.test.ts": 3512, + "test/dream-cli-flags.test.ts": 3, + "test/dream.test.ts": 26718, + "test/drift-watch.test.ts": 255, + "test/dry-fix.test.ts": 442, + "test/edge-extractor.test.ts": 252, + "test/effective-date.test.ts": 6, + "test/embed-backfill-submit.test.ts": 3099, + "test/embed-multimodal-batching.test.ts": 11, + "test/embed-skip.test.ts": 5, + "test/embed-stale.test.ts": 34959, + "test/embed.serial.test.ts": 2312, + "test/embedding-context.test.ts": 6, + "test/embedding-dim-check.test.ts": 4174, + "test/embedding-pricing.test.ts": 3, + "test/emotional-weight.test.ts": 4, + "test/engine-factory.test.ts": 13, + "test/engine-find-trajectory.test.ts": 3351, + "test/engine-parity-event-type.test.ts": 3251, + "test/engine-upsertFile.test.ts": 3801, + "test/engine-weight-rounding-integration.test.ts": 2367, + "test/enrichable-pack.test.ts": 8, + "test/enrichment-service.test.ts": 4, + "test/enrichment.test.ts": 3498, + "test/entity-resolve-perf.slow.test.ts": 158667, + "test/entity-resolve.test.ts": 24029, + "test/error-classify.test.ts": 3, + "test/errors.test.ts": 18, + "test/eval-candidates.test.ts": 18453, + "test/eval-capture-scrub.test.ts": 1, + "test/eval-capture.test.ts": 6, + "test/eval-compare.test.ts": 36, + "test/eval-contradictions-auto-supersession.test.ts": 3, + "test/eval-contradictions-cache.test.ts": 4601, + "test/eval-contradictions-calibration-join.test.ts": 2, + "test/eval-contradictions-calibration.test.ts": 12, + "test/eval-contradictions-cost-prompt.test.ts": 4223, + "test/eval-contradictions-cost.test.ts": 16, + "test/eval-contradictions-cross-source.test.ts": 5, + "test/eval-contradictions-date-filter.test.ts": 5, + "test/eval-contradictions-engine.test.ts": 5226, + "test/eval-contradictions-fixture-redact.test.ts": 5, + "test/eval-contradictions-integrations.test.ts": 4067, + "test/eval-contradictions-judge-errors.test.ts": 3, + "test/eval-contradictions-judge.test.ts": 32, + "test/eval-contradictions-runner.test.ts": 5998, + "test/eval-contradictions-severity.test.ts": 5, + "test/eval-contradictions-trends.test.ts": 36220, + "test/eval-contradictions/no-valid-until-write.test.ts": 33, + "test/eval-cross-modal-batch.test.ts": 141, + "test/eval-export.test.ts": 2399, + "test/eval-gate.test.ts": 3884, + "test/eval-longmemeval.slow.test.ts": 359087, + "test/eval-prune.test.ts": 18396, + "test/eval-replay-gate.test.ts": 34958, + "test/eval-replay-metadata-skip.test.ts": 3430, + "test/eval-replay.test.ts": 16, + "test/eval-run-all.test.ts": 17, + "test/eval-schema-authoring.test.ts": 4, + "test/eval-shared-json-repair-shim.test.ts": 6, + "test/eval-takes-quality-aggregate.test.ts": 32, + "test/eval-takes-quality-boundaries.test.ts": 3193, + "test/eval-takes-quality-cli.test.ts": 13, + "test/eval-takes-quality-pricing.test.ts": 2, + "test/eval-takes-quality-receipt-name.test.ts": 4, + "test/eval-takes-quality-receipt-write.test.ts": 3213, + "test/eval-takes-quality-regress.test.ts": 14, + "test/eval-takes-quality-replay.test.ts": 2890, + "test/eval-takes-quality-rubric.test.ts": 3, + "test/eval-takes-quality-runner.serial.test.ts": 2642, + "test/eval-takes-quality-trend.test.ts": 23553, + "test/eval-trajectory.test.ts": 3424, + "test/eval-v041_2-scaffolds.test.ts": 23, + "test/eval-whoknows.test.ts": 8, + "test/eval.test.ts": 2, + "test/exit-classification.test.ts": 4, + "test/expert-types-pack.test.ts": 60, + "test/extract-db.test.ts": 3200, + "test/extract-facts-phase.test.ts": 24617, + "test/extract-from-fence.test.ts": 6, + "test/extract-fs.test.ts": 2618, + "test/extract-incremental.test.ts": 4523, + "test/extract-source-aware.test.ts": 23970, + "test/extract-takes-holder-producer-seam.test.ts": 2469, + "test/extract-takes.test.ts": 3153, + "test/extract.test.ts": 24, + "test/extractable-pack.test.ts": 76, + "test/facts-absorb-log.test.ts": 3161, + "test/facts-anti-loop.test.ts": 2478, + "test/facts-backstop-gating.test.ts": 3278, + "test/facts-backstop-integration.test.ts": 23696, + "test/facts-backstop.test.ts": 3118, + "test/facts-canonicality.test.ts": 3218, + "test/facts-classify.test.ts": 3, + "test/facts-context-injection.serial.test.ts": 2703, + "test/facts-decay.test.ts": 19, + "test/facts-doctor-shape.test.ts": 3109, + "test/facts-eligibility.test.ts": 5, + "test/facts-engine.test.ts": 12954, + "test/facts-extract-silent-no-op.test.ts": 4, + "test/facts-extract-smoke.test.ts": 6, + "test/facts-extract.test.ts": 1, + "test/facts-fence-typed.test.ts": 4, + "test/facts-fence.test.ts": 5, + "test/facts-mcp-allowlist.serial.test.ts": 2633, + "test/facts-meta-cache.test.ts": 3117, + "test/facts-migration-dim.test.ts": 3329, + "test/facts-multi-tenant.test.ts": 23425, + "test/facts-queue.test.ts": 604, + "test/facts-recall-render.test.ts": 2968, + "test/facts-separation-pglite.test.ts": 3260, + "test/facts-visibility.test.ts": 2917, + "test/fail-improve.test.ts": 132, + "test/feature-flags.test.ts": 3864, + "test/features.test.ts": 5, + "test/fence-extraction.test.ts": 3427, + "test/fence-write.test.ts": 2335, + "test/file-migration.test.ts": 4, + "test/file-resolver.test.ts": 24, + "test/file-upload-security.test.ts": 3, + "test/files.test.ts": 6, + "test/filing-audit.test.ts": 9, + "test/find-experts-op.test.ts": 3125, + "test/fix-wave-structural.test.ts": 6, + "test/founder-scorecard.test.ts": 9, + "test/friction-cli.test.ts": 21, + "test/friction.test.ts": 29, + "test/frontmatter-cli.test.ts": 2936, + "test/frontmatter-inference.test.ts": 10, + "test/frontmatter-install-hook.test.ts": 56, + "test/fuzz/filesystem-validators.test.ts": 87, + "test/fuzz/mixed-validators.test.ts": 279, + "test/fuzz/pure-validators.test.ts": 220, + "test/gateway-embed-model-override.test.ts": 1, + "test/gbrain-home-isolation.test.ts": 7, + "test/get-brain-identity.test.ts": 36332, + "test/git-remote.test.ts": 365, + "test/grade-takes-ensemble.test.ts": 30, + "test/grade-takes.test.ts": 6, + "test/graph-query.test.ts": 3163, + "test/gstack-learnings-coupling.test.ts": 15, + "test/handlers-embed-backfill.test.ts": 2978, + "test/handlers.test.ts": 25084, + "test/helpers/schema-diff-indexes.test.ts": 4, + "test/helpers/schema-diff.test.ts": 2, + "test/helpers/with-env.test.ts": 19, + "test/http-transport.test.ts": 100, + "test/hybrid-meta.test.ts": 3314, + "test/hybrid-search-lite.serial.test.ts": 2849, + "test/import-checkpoint.test.ts": 5, + "test/import-file-content-sanity.test.ts": 3363, + "test/import-file.test.ts": 87, + "test/import-image-file.test.ts": 3539, + "test/import-resume.test.ts": 2811, + "test/import-source-id.test.ts": 3149, + "test/import-walker.test.ts": 1, + "test/incremental-chunking.test.ts": 3212, + "test/infer-type-pack.test.ts": 16, + "test/ingestion/daemon.test.ts": 39, + "test/ingestion/dedup.test.ts": 5, + "test/ingestion/gstack-learnings.test.ts": 8, + "test/ingestion/ingest-capture.test.ts": 53213, + "test/ingestion/markdown-greenfield.test.ts": 13, + "test/ingestion/migration-mode.test.ts": 8, + "test/ingestion/put-page-write-through.test.ts": 38741, + "test/ingestion/skillpack-load.test.ts": 15, + "test/ingestion/sources/file-watcher.test.ts": 1007, + "test/ingestion/sources/inbox-folder.test.ts": 518, + "test/ingestion/test-harness.test.ts": 68, + "test/ingestion/types.test.ts": 9, + "test/init-env-detection.test.ts": 11, + "test/init-mcp-only.test.ts": 4380, + "test/init-migrate-only.test.ts": 7098, + "test/init-mode-picker.test.ts": 2440, + "test/init-provider-picker.test.ts": 2, + "test/insert-facts-batch.test.ts": 2976, + "test/integrations-install.test.ts": 183, + "test/integrations.test.ts": 24, + "test/integrity.test.ts": 2678, + "test/intent-weights.test.ts": 4, + "test/jobs-watch-snapshot.test.ts": 12, + "test/language-manifest.test.ts": 17, + "test/lease-cap-controller.test.ts": 3161, + "test/lens-pack-manifests.test.ts": 13, + "test/levenshtein.test.ts": 3, + "test/link-extraction-code-refs.test.ts": 4, + "test/link-extraction.test.ts": 25, + "test/link-inference-pack.test.ts": 46, + "test/lint-content-sanity.test.ts": 36, + "test/lint-frontmatter.test.ts": 15, + "test/lint.test.ts": 1, + "test/list-all-sources.test.ts": 39599, + "test/llm-intent-escalation.test.ts": 6, + "test/llm-intent-hybrid-integration.serial.test.ts": 3246, + "test/loadConfig-merge.test.ts": 4, + "test/longmemeval-extract.test.ts": 2991, + "test/longmemeval-intent.test.ts": 3, + "test/longmemeval-sanitize.test.ts": 5, + "test/longmemeval-trajectory-routing.test.ts": 12926, + "test/markdown-serializer.test.ts": 8, + "test/markdown-validation.test.ts": 3, + "test/markdown.test.ts": 27, + "test/mcp-client-hardening.test.ts": 305, + "test/mcp-client.test.ts": 65, + "test/mcp-dispatch-summarize.test.ts": 6, + "test/mcp-eval-capture.test.ts": 25246, + "test/mcp-tool-defs.test.ts": 15, + "test/metric-glossary.test.ts": 16, + "test/migrate-extensions.test.ts": 3, + "test/migrate.test.ts": 102260, + "test/migration-orchestrator-v0_21_0.test.ts": 7, + "test/migration-orchestrator-v0_31_0.test.ts": 12960, + "test/migration-resume.test.ts": 9, + "test/migration-v0-29-1.serial.test.ts": 3273, + "test/migrations-cjk-wave.test.ts": 3280, + "test/migrations-registry.test.ts": 3, + "test/migrations-v0_11_0.test.ts": 14, + "test/migrations-v0_12_0.test.ts": 6, + "test/migrations-v0_12_2.test.ts": 3, + "test/migrations-v0_13_0.test.ts": 1, + "test/migrations-v0_13_1.test.ts": 23, + "test/migrations-v0_14_0.test.ts": 20, + "test/migrations-v0_16_0.test.ts": 1, + "test/migrations-v0_19_0.test.ts": 20826, + "test/migrations-v0_21_0.test.ts": 2, + "test/migrations-v0_22_4.test.ts": 1, + "test/migrations-v0_27_1.test.ts": 3248, + "test/migrations-v0_32_2.test.ts": 2510, + "test/migrations-v48-takes-weight-backfill.test.ts": 2501, + "test/migrations-v94.test.ts": 32374, + "test/minions-lease-full-retry.test.ts": 3031, + "test/minions-quiet-hours.test.ts": 3341, + "test/minions-shell-inherit.test.ts": 24, + "test/minions-shell-redact.test.ts": 19, + "test/minions-shell-validate.test.ts": 7, + "test/minions-shell.test.ts": 3268, + "test/minions.test.ts": 15231, + "test/minions/agent-audit.test.ts": 9, + "test/minions/budget-meter.test.ts": 44657, + "test/mode-switch-ux.test.ts": 3698, + "test/model-config.serial.test.ts": 101, + "test/mounts-cache.test.ts": 21, + "test/mounts-cli.test.ts": 20, + "test/multi-source-drift.test.ts": 2479, + "test/multi-source-integration.test.ts": 3120, + "test/nightly-quality-probe.test.ts": 50, + "test/notability-eval.test.ts": 36, + "test/nudge.test.ts": 16, + "test/oauth-confidential-client.test.ts": 3221, + "test/oauth-scope-probe.test.ts": 112, + "test/oauth.test.ts": 2151, + "test/op-checkpoint.test.ts": 5333, + "test/openai-compat-multimodal.test.ts": 6, + "test/operation-context-sourceid-required.test.ts": 10, + "test/operations-allow-list.test.ts": 4, + "test/operations-descriptions.test.ts": 11, + "test/operations-embedding-column.test.ts": 30, + "test/operations-find-trajectory.test.ts": 3419, + "test/operations-schema-pack.test.ts": 5842, + "test/operations-trust-boundary.test.ts": 43020, + "test/orphans.test.ts": 16921, + "test/page-lock.test.ts": 266, + "test/page-type-exhaustive.test.ts": 17, + "test/pages-soft-delete.test.ts": 15271, + "test/parallel.test.ts": 108, + "test/parent-scope.test.ts": 23673, + "test/parity.test.ts": 16, + "test/performfullsync-source-id.test.ts": 2869, + "test/pglite-engine.test.ts": 33476, + "test/pglite-lock.test.ts": 1007, + "test/phantom-redirect-engine-parity.test.ts": 3605, + "test/phantom-redirect-per-source-lock.test.ts": 3, + "test/phantom-redirect.test.ts": 7303, + "test/phase-scope-coverage.test.ts": 3, + "test/plugin-loader.test.ts": 12, + "test/post-install-advisory.test.ts": 15, + "test/post-write-lint.test.ts": 4180, + "test/postgres-engine.test.ts": 32, + "test/preferences.test.ts": 14, + "test/privacy-script-wired.test.ts": 3, + "test/privacy-strip-and-forget.test.ts": 2762, + "test/progress-tail.test.ts": 3, + "test/progress.test.ts": 215, + "test/propose-takes.test.ts": 15, + "test/providers.test.ts": 17, + "test/public-exports.test.ts": 35, + "test/publish.test.ts": 128, + "test/put-page-namespace.test.ts": 4, + "test/put-page-provenance.test.ts": 24839, + "test/qualified-names.test.ts": 4, + "test/query-cache-gate.test.ts": 31003, + "test/query-cache-knobs-hash.test.ts": 3122, + "test/query-cache.test.ts": 2565, + "test/query-image-flag.serial.test.ts": 2965, + "test/query-intent-legacy.test.ts": 8, + "test/query-intent.test.ts": 6, + "test/query-sanitization.test.ts": 4, + "test/queue-child-done.test.ts": 2548, + "test/rate-leases-uncapped.test.ts": 2884, + "test/rate-leases.test.ts": 2959, + "test/readme-hero-anchors.test.ts": 0, + "test/recall-extensions.test.ts": 4668, + "test/recall-footer.test.ts": 17, + "test/recall-rollup.test.ts": 84, + "test/recency-decay.test.ts": 5, + "test/recompute-emotional-weight.test.ts": 2, + "test/reconcile-links.serial.test.ts": 2702, + "test/regression-strict-source-id.test.ts": 4, + "test/regression-v0_16_4.test.ts": 28, + "test/regressions/gbrain-base-equivalence.test.ts": 12, + "test/regressions/v0.36.1.0-iron-rule.test.ts": 3, + "test/regressions/v0_36_frontier_cap.test.ts": 27151, + "test/regressions/v0_40_2_0-trajectory-backcompat.test.ts": 27576, + "test/reindex-code-max-cost.serial.test.ts": 2636, + "test/reindex-code-model-source.serial.test.ts": 2782, + "test/reindex-code-nudge.serial.test.ts": 2710, + "test/reindex-code.test.ts": 38123, + "test/reindex-frontmatter-connect.test.ts": 18383, + "test/reindex.test.ts": 19003, + "test/remediation-step.test.ts": 25, + "test/repair-jsonb.test.ts": 3, + "test/repo-root.test.ts": 14, + "test/report.test.ts": 2, + "test/repos-alias.test.ts": 3163, + "test/rerank-audit.test.ts": 24, + "test/resolve-prepare.test.ts": 11, + "test/resolver-merge.test.ts": 5, + "test/resolver.test.ts": 58, + "test/resolvers.test.ts": 4130, + "test/restart-sweep.test.ts": 42, + "test/retrieval-upgrade-planner.test.ts": 8375, + "test/retry-matcher.test.ts": 2, + "test/routing-eval-cli.test.ts": 1118, + "test/routing-eval.test.ts": 34, + "test/salience.test.ts": 2, + "test/scenarios.test.ts": 29, + "test/schema-bootstrap-coverage.test.ts": 6449, + "test/schema-cli-contract.test.ts": 4, + "test/schema-cli.test.ts": 3375, + "test/schema-pack-best-effort.test.ts": 9, + "test/schema-pack-lint-rules.test.ts": 11, + "test/schema-pack-load-active.test.ts": 43, + "test/schema-pack-loader.test.ts": 10, + "test/schema-pack-manifest-v041_2.test.ts": 17, + "test/schema-pack-mutate-audit.test.ts": 8, + "test/schema-pack-mutate.test.ts": 80, + "test/schema-pack-pack-lock.test.ts": 65, + "test/schema-pack-query-cache-invalidator.test.ts": 22822, + "test/schema-pack-registry-reload.test.ts": 40, + "test/schema-pack-registry.test.ts": 52, + "test/schema-pack-stats.test.ts": 43032, + "test/schema-pack-sync.test.ts": 5580, + "test/schema-pack-trust-boundary.test.ts": 8, + "test/schema-verify.test.ts": 7, + "test/scope-agent-isolation.test.ts": 30, + "test/scope-normalize.test.ts": 24, + "test/scope.test.ts": 5, + "test/scripts/check-proposal-pii.test.ts": 511, + "test/scripts/check-test-isolation.test.ts": 223, + "test/scripts/run-unit-parallel.test.ts": 427, + "test/scripts/run-unit-shard.test.ts": 131, + "test/scripts/serial-files.test.ts": 41, + "test/scripts/test-shard.slow.test.ts": 32161, + "test/search-by-image-op.test.ts": 3869, + "test/search-image-column.test.ts": 30089, + "test/search-lang-symbol-kind.test.ts": 2556, + "test/search-limit.test.ts": 2909, + "test/search-mode.test.ts": 8, + "test/search-telemetry.test.ts": 25699, + "test/search-types-filter.test.ts": 3023, + "test/search.test.ts": 9, + "test/search/attribution-stamping.test.ts": 5, + "test/search/embedding-column.serial.test.ts": 109, + "test/search/explain-formatter.test.ts": 13, + "test/search/graph-signals-wire-integration.test.ts": 28358, + "test/search/graph-signals.test.ts": 14, + "test/search/hybrid-reranker-integration.serial.test.ts": 2905, + "test/search/knobs-hash-reranker.test.ts": 5, + "test/search/rerank.test.ts": 23, + "test/search/search-stats-graph-signals.test.ts": 23476, + "test/seed-pglite.test.ts": 12966, + "test/select-e2e.test.ts": 11, + "test/self-fix.test.ts": 28842, + "test/serve-http-bootstrap-token.test.ts": 2, + "test/serve-http-health.test.ts": 301, + "test/serve-stdio-lifecycle.test.ts": 1252, + "test/setup-branching.test.ts": 15, + "test/skill-brain-first.test.ts": 9, + "test/skill-manifest.test.ts": 25, + "test/skillify-check.test.ts": 554, + "test/skillify-scaffold.test.ts": 31, + "test/skillpack-apply-hunks.test.ts": 19, + "test/skillpack-bootstrap-display.test.ts": 4, + "test/skillpack-changed-since-version.test.ts": 361, + "test/skillpack-check.test.ts": 4691, + "test/skillpack-copy.test.ts": 11, + "test/skillpack-endorse.test.ts": 114, + "test/skillpack-frontmatter-sources.test.ts": 10, + "test/skillpack-harvest-lint.test.ts": 5, + "test/skillpack-harvest.test.ts": 35, + "test/skillpack-init-pack.test.ts": 50, + "test/skillpack-install.test.ts": 51, + "test/skillpack-manifest-v1.test.ts": 4, + "test/skillpack-migrate-fence.test.ts": 10, + "test/skillpack-reference-apply.test.ts": 13, + "test/skillpack-reference-pack-is-ten.test.ts": 14, + "test/skillpack-reference.test.ts": 29, + "test/skillpack-registry-client.test.ts": 27, + "test/skillpack-registry-schema.test.ts": 13, + "test/skillpack-remote-source.test.ts": 65, + "test/skillpack-rubric-doctor.test.ts": 45, + "test/skillpack-scaffold-third-party.test.ts": 17, + "test/skillpack-scaffold.test.ts": 44, + "test/skillpack-scrub-legacy.test.ts": 51, + "test/skillpack-state.test.ts": 30, + "test/skillpack-tarball.test.ts": 132, + "test/skillpack-trust-prompt.test.ts": 9, + "test/skills-conformance.test.ts": 25, + "test/slug-validation.test.ts": 12, + "test/sort-newest-first.test.ts": 120, + "test/source-config-redact.test.ts": 1, + "test/source-health.test.ts": 2940, + "test/source-id-routing.test.ts": 3687, + "test/source-id-tx-regression.test.ts": 3200, + "test/source-id.test.ts": 4, + "test/source-resolver-silent-fallback.test.ts": 40514, + "test/source-resolver-with-tier.test.ts": 1, + "test/source-resolver.test.ts": 27, + "test/sources-load.test.ts": 4120, + "test/sources-mcp.test.ts": 41488, + "test/sources-ops.test.ts": 4235, + "test/sources-resync-recovery.test.ts": 40765, + "test/sources-set-cr-mode.test.ts": 37772, + "test/sources-webhook.test.ts": 20, + "test/sources.test.ts": 11, + "test/spawn-helpers.test.ts": 18, + "test/sql-query.test.ts": 2327, + "test/sql-ranking.test.ts": 17, + "test/ssrf-validate.test.ts": 6, + "test/storage-backfill.test.ts": 4, + "test/storage-config.test.ts": 8, + "test/storage-export.test.ts": 3170, + "test/storage-pglite.test.ts": 23978, + "test/storage-status.test.ts": 12, + "test/storage-sync.test.ts": 27, + "test/storage.test.ts": 146, + "test/stub-guard-audit.test.ts": 27, + "test/subagent-aggregator.test.ts": 5, + "test/subagent-audit.test.ts": 29, + "test/subagent-handler.test.ts": 3178, + "test/subagent-prompt-too-long.test.ts": 17, + "test/subagent-transcript.test.ts": 3276, + "test/subagent-v1-v2-shim.test.ts": 4830, + "test/submit-agent.test.ts": 4992, + "test/supabase-admin.test.ts": 0, + "test/supervisor-audit.test.ts": 8, + "test/supervisor-tini.test.ts": 1, + "test/supervisor.test.ts": 326, + "test/svg-renderer.test.ts": 7, + "test/sync-all-parallel.test.ts": 6, + "test/sync-classifier-widening.test.ts": 4, + "test/sync-concurrency.test.ts": 6, + "test/sync-cost-preview.test.ts": 115, + "test/sync-failures.test.ts": 18, + "test/sync-parallel.test.ts": 22640, + "test/sync-strategy.test.ts": 15, + "test/sync-trigger-cli.test.ts": 3100, + "test/sync-walker-submodule.test.ts": 5, + "test/sync-walker-symlink.test.ts": 12, + "test/sync.test.ts": 5841, + "test/synth-enabled-default.test.ts": 3, + "test/system-prompt.test.ts": 8, + "test/take-forecast.test.ts": 7, + "test/takes-engine.test.ts": 3302, + "test/takes-fence-read-ops.serial.test.ts": 2720, + "test/takes-fence.test.ts": 132, + "test/takes-holder-semantics.test.ts": 5, + "test/takes-holder-validation.test.ts": 13, + "test/takes-mcp-allowlist.serial.test.ts": 2721, + "test/takes-resolution.test.ts": 5, + "test/takes-weight-rounding.test.ts": 6, + "test/thin-client-routing-audit.test.ts": 5, + "test/thin-client-upgrade-prompt.test.ts": 23, + "test/think-ab.test.ts": 18, + "test/think-entity-extract.test.ts": 4, + "test/think-gateway-adapter.test.ts": 33, + "test/think-intent.test.ts": 8, + "test/think-pipeline.serial.test.ts": 2723, + "test/think-sanitize-trajectory.test.ts": 7, + "test/think-trajectory-injection.test.ts": 3346, + "test/think-with-calibration.test.ts": 3, + "test/timing-safe.test.ts": 17, + "test/token-budget.test.ts": 3, + "test/trajectory-format.test.ts": 4, + "test/transcript-capture.test.ts": 246, + "test/transcription.test.ts": 20, + "test/transcripts.test.ts": 20, + "test/traverse-graph-dedup.test.ts": 2933, + "test/trust-boundary-contract.test.ts": 24, + "test/two-pass.test.ts": 3137, + "test/undo-wave.test.ts": 5, + "test/unified-multimodal.serial.test.ts": 3286, + "test/upgrade-checkpoint.serial.test.ts": 109, + "test/upgrade-reembed-prompt.test.ts": 3219, + "test/upgrade-reference-sweep.test.ts": 56, + "test/upgrade.serial.test.ts": 723, + "test/url-redact.test.ts": 2, + "test/utils.test.ts": 5, + "test/v0_29-tool-surfaces.test.ts": 4, + "test/v0_37_fix_wave.serial.test.ts": 211, + "test/v81-v82-smoke.test.ts": 3160, + "test/vector-index-lifecycle.test.ts": 5, + "test/voice-gate.test.ts": 9, + "test/voyage-multimodal.test.ts": 22, + "test/voyage-response-cap.test.ts": 1, + "test/wait-for-completion.test.ts": 3626, + "test/whoami.test.ts": 3, + "test/whoknows-doctor.test.ts": 70, + "test/whoknows.test.ts": 3, + "test/worker-rss.test.ts": 3, + "test/worker-shutdown-disconnect.test.ts": 3260, + "test/writer.test.ts": 37365, + "test/yaml-lite.test.ts": 9, + "test/ze-switch-cli.test.ts": 4227, + "test/zombie-reap.test.ts": 3 +} diff --git a/test/privacy-script-wired.test.ts b/test/privacy-script-wired.test.ts index 993c00f26..f2fdb9547 100644 --- a/test/privacy-script-wired.test.ts +++ b/test/privacy-script-wired.test.ts @@ -8,19 +8,26 @@ * * v0.26.4 split: `bun run test` is now the fast parallel loop and does * NOT chain pre-checks; the privacy gate moved to `bun run verify`, - * which CI's test.yml runs on shard 1 before the matrix fans out. - * Regression guard now asserts both: (1) verify chains check:privacy, - * (2) CI workflow's pre-test gate calls `bun run verify`. Together those - * guarantee the privacy check runs before any merge. + * which CI's test.yml runs as its own job before the matrix fans out. + * + * v0.41.4+ wave: `bun run verify` now delegates to + * scripts/run-verify-parallel.sh which fans out all 20 checks in + * parallel via & + wait. The privacy check is one entry in that + * script's CHECKS[] array. Regression guard updated to follow the + * indirection: (1) verify points at the parallel dispatcher, + * (2) the dispatcher's CHECKS array contains check:privacy, + * (3) CI workflow's verify job calls `bun run verify`. */ import { describe, it, expect } from 'bun:test'; import { readFileSync, existsSync } from 'fs'; import { resolve } from 'path'; +import { spawnSync } from 'child_process'; const REPO_ROOT = resolve(import.meta.dir, '..'); const PACKAGE_JSON = resolve(REPO_ROOT, 'package.json'); const PRIVACY_SCRIPT = resolve(REPO_ROOT, 'scripts/check-privacy.sh'); +const VERIFY_DISPATCHER = resolve(REPO_ROOT, 'scripts/run-verify-parallel.sh'); const TEST_WORKFLOW = resolve(REPO_ROOT, '.github/workflows/test.yml'); describe('check-privacy.sh CI wiring', () => { @@ -31,10 +38,27 @@ describe('check-privacy.sh CI wiring', () => { expect((stat.mode & 0o100) !== 0).toBe(true); }); - it('package.json "verify" script chains check:privacy', () => { + it('package.json "verify" script delegates to run-verify-parallel.sh', () => { const pkg = JSON.parse(readFileSync(PACKAGE_JSON, 'utf-8')); expect(typeof pkg.scripts?.verify).toBe('string'); - expect(pkg.scripts.verify).toContain('check:privacy'); + // verify body is now `bash scripts/run-verify-parallel.sh`. The + // direct check:privacy substring assertion broke when the && chain + // was replaced with the parallel dispatcher. Follow the indirection. + expect(pkg.scripts.verify).toContain('run-verify-parallel.sh'); + }); + + it('run-verify-parallel.sh dispatches check:privacy', () => { + expect(existsSync(VERIFY_DISPATCHER)).toBe(true); + // The dispatcher exposes --dry-list which prints one check name per + // line. Authoritative check than substring-grepping the script body + // (which could pass on a commented-out entry). + const r = spawnSync('bash', [VERIFY_DISPATCHER, '--dry-list'], { + cwd: REPO_ROOT, + encoding: 'utf-8', + }); + expect(r.status).toBe(0); + const checks = r.stdout.trim().split('\n'); + expect(checks).toContain('check:privacy'); }); it('package.json "check:privacy" alias points at the script', () => { diff --git a/test/scripts/ci-cache-hash.test.ts b/test/scripts/ci-cache-hash.test.ts new file mode 100644 index 000000000..fb55f1fa5 --- /dev/null +++ b/test/scripts/ci-cache-hash.test.ts @@ -0,0 +1,422 @@ +/** + * ci-cache-hash.test.ts — adversarial coverage of the auto-cache hash. + * + * The cache key controls false-pass risk. If a test-affecting file is + * accidentally deny-listed (or a file change fails to invalidate the + * hash), broken code ships under a green CI check. This suite is the + * primary safety net. + * + * Strategy: each test spins up a synthetic git repo, copies the + * ci-cache-hash.sh script in, populates a small file set covering the + * deny-list edges, runs the script, modifies one file, runs again, + * asserts hash behavior. + * + * Critical invariants (the false-pass guards): + * - CLAUDE.md edit → DIFFERENT hash + * - AGENTS.md edit → DIFFERENT hash + * - skills/foo/SKILL.md edit → DIFFERENT hash + * - src/core/db.ts edit → DIFFERENT hash + * - test/foo.test.ts edit → DIFFERENT hash + * + * Safe deny-list invariants: + * - CHANGELOG.md edit → same hash + * - README.md edit → same hash + * - docs/guide.md edit → same hash + * - TODOS.md edit → same hash + * - LICENSE edit → same hash + */ + +import { afterAll, beforeAll, describe, expect, it } from "bun:test"; +import { execFileSync, spawnSync } from "node:child_process"; +import { + cpSync, + mkdirSync, + mkdtempSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +const REPO_ROOT = resolve(import.meta.dir, "..", ".."); +const SCRIPT_SRC = resolve(REPO_ROOT, "scripts/ci-cache-hash.sh"); + +interface Sandbox { + dir: string; + scriptPath: string; +} + +function makeSandbox(files: Record): Sandbox { + const dir = mkdtempSync(join(tmpdir(), "ci-cache-hash-")); + // Init git repo + execFileSync("git", ["init", "--quiet"], { cwd: dir }); + execFileSync("git", ["config", "user.email", "test@example.com"], { cwd: dir }); + execFileSync("git", ["config", "user.name", "Test"], { cwd: dir }); + + // Copy script under scripts/ so cd "$(dirname "$0")/.." lands at sandbox root + mkdirSync(join(dir, "scripts"), { recursive: true }); + cpSync(SCRIPT_SRC, join(dir, "scripts/ci-cache-hash.sh")); + + // Write each fixture file under its declared path + for (const [path, content] of Object.entries(files)) { + const abs = join(dir, path); + mkdirSync(resolve(abs, ".."), { recursive: true }); + writeFileSync(abs, content); + } + + // Stage + commit so git ls-files returns them + execFileSync("git", ["add", "-A"], { cwd: dir }); + execFileSync("git", ["commit", "--quiet", "-m", "fixture"], { cwd: dir }); + + return { dir, scriptPath: join(dir, "scripts/ci-cache-hash.sh") }; +} + +function hash(s: Sandbox): string { + const r = spawnSync("bash", [s.scriptPath], { + cwd: s.dir, + encoding: "utf8", + }); + if (r.status !== 0) { + throw new Error(`ci-cache-hash exit ${r.status}: ${r.stderr}`); + } + return r.stdout.trim(); +} + +function modify(s: Sandbox, path: string, content: string) { + writeFileSync(join(s.dir, path), content); + // The script reads `git ls-files -s` which reflects the INDEX, not + // the working tree. Stage + commit so the change is visible to the + // hash. This matches what CI sees on a checked-out PR (committed + // tree, not working tree). + execFileSync("git", ["add", path], { cwd: s.dir }); + execFileSync("git", ["commit", "--quiet", "-m", `modify ${path}`], { + cwd: s.dir, + }); +} + +// ────────────────────────────────────────────────────────────────────── +// Baseline fixtures used across most cases. +// ────────────────────────────────────────────────────────────────────── +const BASELINE_FILES: Record = { + "CHANGELOG.md": "# Changelog\n\n## [1.0.0]\n", + "TODOS.md": "- [ ] thing\n", + "README.md": "# Project\n", + "LICENSE": "MIT\n", + "CLAUDE.md": "# CLAUDE.md\n\nproject instructions\n", + "AGENTS.md": "# AGENTS.md\n\nopenclaw entry\n", + "package.json": '{"name":"test","version":"0.0.0"}\n', + "bun.lock": "lockfile-v1\n", + "tsconfig.json": '{"compilerOptions":{"strict":true}}\n', + "src/core/db.ts": "export const db = 1;\n", + "src/cli.ts": "console.log('hi');\n", + "test/foo.test.ts": "import {test} from 'bun:test';\ntest('x', () => {});\n", + "scripts/check-x.sh": "#!/bin/bash\necho ok\n", + ".github/workflows/test.yml": "name: Test\n", + "skills/foo/SKILL.md": "---\nname: foo\n---\nFoo skill\n", + "docs/guide.md": "# Guide\n", + "docs/sub/notes.md": "Notes\n", + "docs/raw.txt": "raw text\n", +}; + +describe("ci-cache-hash.sh — determinism", () => { + let sb: Sandbox; + beforeAll(() => { + sb = makeSandbox(BASELINE_FILES); + }); + afterAll(() => { + if (sb) rmSync(sb.dir, { recursive: true, force: true }); + }); + + it("same tree → same hash (run twice)", () => { + const h1 = hash(sb); + const h2 = hash(sb); + expect(h1).toBe(h2); + expect(h1).toMatch(/^[0-9a-f]{16}$/); + }); + + it("hash is 16-char lowercase hex", () => { + const h = hash(sb); + expect(h).toMatch(/^[0-9a-f]{16}$/); + }); +}); + +describe("ci-cache-hash.sh — CRITICAL false-pass guards (must invalidate)", () => { + // Each test gets its own sandbox so modifications don't leak across cases. + + function withSandbox(test: (sb: Sandbox) => void) { + const sb = makeSandbox(BASELINE_FILES); + try { + test(sb); + } finally { + rmSync(sb.dir, { recursive: true, force: true }); + } + } + + it("CLAUDE.md edit MUST change hash (8+ test files reference it)", () => { + withSandbox((sb) => { + const before = hash(sb); + modify(sb, "CLAUDE.md", "# CLAUDE.md\n\nNEW INSTRUCTIONS\n"); + const after = hash(sb); + expect(after).not.toBe(before); + }); + }); + + it("AGENTS.md edit MUST change hash (resolver tests read it)", () => { + withSandbox((sb) => { + const before = hash(sb); + modify(sb, "AGENTS.md", "# AGENTS.md\n\nNEW DISPATCH\n"); + const after = hash(sb); + expect(after).not.toBe(before); + }); + }); + + it("skills/foo/SKILL.md edit MUST change hash (skill conformance tests)", () => { + withSandbox((sb) => { + const before = hash(sb); + modify(sb, "skills/foo/SKILL.md", "---\nname: foo\n---\nNEW SKILL BODY\n"); + const after = hash(sb); + expect(after).not.toBe(before); + }); + }); + + it("src/core/db.ts edit MUST change hash", () => { + withSandbox((sb) => { + const before = hash(sb); + modify(sb, "src/core/db.ts", "export const db = 2;\n"); + const after = hash(sb); + expect(after).not.toBe(before); + }); + }); + + it("test/foo.test.ts edit MUST change hash", () => { + withSandbox((sb) => { + const before = hash(sb); + modify( + sb, + "test/foo.test.ts", + "import {test} from 'bun:test';\ntest('y', () => {});\n", + ); + const after = hash(sb); + expect(after).not.toBe(before); + }); + }); + + it("package.json edit MUST change hash", () => { + withSandbox((sb) => { + const before = hash(sb); + modify(sb, "package.json", '{"name":"test","version":"0.0.1"}\n'); + const after = hash(sb); + expect(after).not.toBe(before); + }); + }); + + it("bun.lock edit MUST change hash (dependency drift catches false-pass)", () => { + withSandbox((sb) => { + const before = hash(sb); + modify(sb, "bun.lock", "lockfile-v2\n"); + const after = hash(sb); + expect(after).not.toBe(before); + }); + }); + + it(".github/workflows/test.yml edit MUST change hash (CI shape change is test-affecting)", () => { + withSandbox((sb) => { + const before = hash(sb); + modify(sb, ".github/workflows/test.yml", "name: Test\n# changed\n"); + const after = hash(sb); + expect(after).not.toBe(before); + }); + }); +}); + +describe("ci-cache-hash.sh — SAFE deny-list invariants (must NOT invalidate)", () => { + function withSandbox(test: (sb: Sandbox) => void) { + const sb = makeSandbox(BASELINE_FILES); + try { + test(sb); + } finally { + rmSync(sb.dir, { recursive: true, force: true }); + } + } + + it("CHANGELOG.md edit produces SAME hash (deny-listed)", () => { + withSandbox((sb) => { + const before = hash(sb); + modify(sb, "CHANGELOG.md", "# Changelog\n\n## [1.1.0]\nnew release\n"); + const after = hash(sb); + expect(after).toBe(before); + }); + }); + + it("README.md edit produces SAME hash (deny-listed)", () => { + withSandbox((sb) => { + const before = hash(sb); + modify(sb, "README.md", "# Project\n\nupdated tagline\n"); + const after = hash(sb); + expect(after).toBe(before); + }); + }); + + it("TODOS.md edit produces SAME hash (deny-listed)", () => { + withSandbox((sb) => { + const before = hash(sb); + modify(sb, "TODOS.md", "- [x] thing\n"); + const after = hash(sb); + expect(after).toBe(before); + }); + }); + + it("LICENSE edit produces SAME hash (deny-listed)", () => { + withSandbox((sb) => { + const before = hash(sb); + modify(sb, "LICENSE", "Apache-2.0\n"); + const after = hash(sb); + expect(after).toBe(before); + }); + }); + + it("docs/guide.md edit produces SAME hash (deny-listed via docs/**/*.md)", () => { + withSandbox((sb) => { + const before = hash(sb); + modify(sb, "docs/guide.md", "# Guide v2\n"); + const after = hash(sb); + expect(after).toBe(before); + }); + }); + + it("docs/sub/notes.md edit produces SAME hash (nested docs match)", () => { + withSandbox((sb) => { + const before = hash(sb); + modify(sb, "docs/sub/notes.md", "Updated notes\n"); + const after = hash(sb); + expect(after).toBe(before); + }); + }); + + it("docs/raw.txt edit produces SAME hash (deny-listed via docs/**/*.txt)", () => { + withSandbox((sb) => { + const before = hash(sb); + modify(sb, "docs/raw.txt", "new raw text\n"); + const after = hash(sb); + expect(after).toBe(before); + }); + }); +}); + +describe("ci-cache-hash.sh — edge cases", () => { + function withSandbox( + files: Record, + test: (sb: Sandbox) => void, + ) { + const sb = makeSandbox(files); + try { + test(sb); + } finally { + rmSync(sb.dir, { recursive: true, force: true }); + } + } + + it("untracked file does NOT affect hash (git ls-files only)", () => { + withSandbox(BASELINE_FILES, (sb) => { + const before = hash(sb); + // Write a new file but don't `git add` it. + writeFileSync(join(sb.dir, "src/untracked.ts"), "const x = 1;\n"); + const after = hash(sb); + expect(after).toBe(before); + }); + }); + + it("rename detection: same content, new path → DIFFERENT hash", () => { + withSandbox(BASELINE_FILES, (sb) => { + const before = hash(sb); + execFileSync("git", ["mv", "src/cli.ts", "src/main.ts"], { cwd: sb.dir }); + const after = hash(sb); + // Same content, new path. Our hash includes the path, so it changes. + // This is correct: a rename can absolutely affect test outcomes + // (imports change, file resolution changes, etc.). + expect(after).not.toBe(before); + }); + }); + + it("new file under unknown path (e.g. prompts/x.md) DOES affect hash (deny-list default)", () => { + withSandbox(BASELINE_FILES, (sb) => { + const before = hash(sb); + // Add a NEW file in a path that isn't deny-listed and didn't exist + // before. Default behavior: include in hash. Closes the "missed file + // type" false-pass class (e.g. someone adds `prompts/*.md` that + // tests read). + mkdirSync(join(sb.dir, "prompts"), { recursive: true }); + writeFileSync(join(sb.dir, "prompts/extract-takes.md"), "system prompt\n"); + execFileSync("git", ["add", "prompts/extract-takes.md"], { cwd: sb.dir }); + execFileSync("git", ["commit", "--quiet", "-m", "add prompts"], { + cwd: sb.dir, + }); + const after = hash(sb); + expect(after).not.toBe(before); + }); + }); + + it("symlink target change affects hash (git hash-object follows symlink contents)", () => { + // Note: git stores symlinks as the link target string, so changing + // the LINK TARGET changes the git blob sha. Our hash will catch + // that. We can't easily test "change of target's content" without + // creating an out-of-tree file, so this case pins the link-target + // path change. + withSandbox(BASELINE_FILES, (sb) => { + // Create a symlink and commit. + symlinkSync("./db.ts", join(sb.dir, "src/core/db-link.ts")); + execFileSync("git", ["add", "src/core/db-link.ts"], { cwd: sb.dir }); + execFileSync("git", ["commit", "--quiet", "-m", "add link"], { + cwd: sb.dir, + }); + const before = hash(sb); + // Point the symlink elsewhere — and commit so it lands in the + // index that `git ls-files -s` reads. + rmSync(join(sb.dir, "src/core/db-link.ts")); + symlinkSync("./cli.ts", join(sb.dir, "src/core/db-link.ts")); + execFileSync("git", ["add", "src/core/db-link.ts"], { cwd: sb.dir }); + execFileSync("git", ["commit", "--quiet", "-m", "repoint link"], { + cwd: sb.dir, + }); + const after = hash(sb); + expect(after).not.toBe(before); + }); + }); + + it("locale-stable: LC_ALL=de_DE.UTF-8 produces the same hash as default", () => { + withSandbox(BASELINE_FILES, (sb) => { + const defaultHash = spawnSync("bash", [sb.scriptPath], { + cwd: sb.dir, + encoding: "utf8", + }).stdout.trim(); + const altLocale = spawnSync("bash", [sb.scriptPath], { + cwd: sb.dir, + encoding: "utf8", + env: { ...process.env, LC_ALL: "de_DE.UTF-8", LANG: "de_DE.UTF-8" }, + }).stdout.trim(); + expect(altLocale).toBe(defaultHash); + }); + }); +}); + +describe("ci-cache-hash.sh — usage errors", () => { + it("--bogus arg exits 2", () => { + const r = spawnSync("bash", [SCRIPT_SRC, "--bogus"], { + cwd: REPO_ROOT, + encoding: "utf8", + }); + expect(r.status).toBe(2); + expect(r.stderr).toContain("usage:"); + }); + + it("--verbose flag works (writes diagnostics to stderr, hash to stdout)", () => { + const r = spawnSync("bash", [SCRIPT_SRC, "--verbose"], { + cwd: REPO_ROOT, + encoding: "utf8", + }); + expect(r.status).toBe(0); + expect(r.stdout.trim()).toMatch(/^[0-9a-f]{16}$/); + expect(r.stderr).toContain("files in hash"); + }); +}); diff --git a/test/scripts/mine-shard-weights.test.ts b/test/scripts/mine-shard-weights.test.ts new file mode 100644 index 000000000..1f037d06c --- /dev/null +++ b/test/scripts/mine-shard-weights.test.ts @@ -0,0 +1,158 @@ +// mine-shard-weights.test.ts — pure-function tests for the log parser +// and weight extraction. The full pipeline (gh run view → write JSON) +// is integration-tested by actually running it once during T4. + +import { describe, expect, it } from "bun:test"; +import { + computeWeights, + parseLog, + serializeWeights, +} from "../../scripts/mine-shard-weights.ts"; + +const SAMPLE = `test (1)\tUNKNOWN STEP\t2026-05-25T11:26:40.000000Z ##[group]test/alpha.test.ts: +test (1)\tUNKNOWN STEP\t2026-05-25T11:26:41.000000Z (pass) some test [1.00ms] +test (1)\tUNKNOWN STEP\t2026-05-25T11:26:43.500000Z ##[group]test/beta.test.ts: +test (1)\tUNKNOWN STEP\t2026-05-25T11:26:45.000000Z (pass) another test +test (1)\tUNKNOWN STEP\t2026-05-25T11:26:48.000000Z ##[group]test/gamma.test.ts: +test (1)\tUNKNOWN STEP\t2026-05-25T11:26:50.000000Z ##[group]test/delta.test.ts: +test (2)\tUNKNOWN STEP\t2026-05-25T11:26:42.000000Z ##[group]test/epsilon.test.ts: +test (2)\tUNKNOWN STEP\t2026-05-25T11:26:55.000000Z ##[group]test/zeta.test.ts: +`; + +describe("parseLog", () => { + it("extracts file-start events with timestamps and jobs", () => { + const events = parseLog(SAMPLE); + expect(events.length).toBe(6); + expect(events[0]).toMatchObject({ + job: "test (1)", + file: "test/alpha.test.ts", + }); + expect(events[5]).toMatchObject({ + job: "test (2)", + file: "test/zeta.test.ts", + }); + }); + + it("preserves stream order", () => { + const events = parseLog(SAMPLE); + const files = events.map((e) => e.file); + expect(files).toEqual([ + "test/alpha.test.ts", + "test/beta.test.ts", + "test/gamma.test.ts", + "test/delta.test.ts", + "test/epsilon.test.ts", + "test/zeta.test.ts", + ]); + }); + + it("ignores non-group lines", () => { + // The sample has (pass) lines mixed in. They should not appear in the + // event list; parseLog only emits ##[group]test/X.test.ts: matches. + const events = parseLog(SAMPLE); + expect(events.every((e) => e.file.startsWith("test/"))).toBe(true); + expect(events.every((e) => e.file.endsWith(".test.ts"))).toBe(true); + }); + + it("handles empty input", () => { + expect(parseLog("")).toEqual([]); + }); + + it("ignores malformed timestamps", () => { + const bad = "test (1)\tUNKNOWN\tBAD-TS ##[group]test/x.test.ts:\n"; + expect(parseLog(bad)).toEqual([]); + }); + + it("rejects non-test-file groups (e.g. setup-bun action groups)", () => { + const noise = + "test (1)\tUNKNOWN\t2026-05-25T11:26:40.000Z ##[group]Run actions/checkout\n"; + expect(parseLog(noise)).toEqual([]); + }); +}); + +describe("computeWeights", () => { + it("computes delta between consecutive file headers within a job", () => { + const events = parseLog(SAMPLE); + const weights = computeWeights(events); + // job test (1): alpha → 3500ms (40s → 43.5s), beta → 4500ms (43.5s → 48s), + // gamma → 2000ms (48s → 50s), delta dropped (no successor). + expect(weights.get("test/alpha.test.ts")).toBe(3500); + expect(weights.get("test/beta.test.ts")).toBe(4500); + expect(weights.get("test/gamma.test.ts")).toBe(2000); + expect(weights.get("test/delta.test.ts")).toBeUndefined(); + // job test (2): epsilon → 13000ms (42s → 55s), zeta dropped. + expect(weights.get("test/epsilon.test.ts")).toBe(13000); + expect(weights.get("test/zeta.test.ts")).toBeUndefined(); + }); + + it("does not cross job boundaries", () => { + // If we naively diffed across jobs we'd get bogus values when + // alpha (test 1) was followed by epsilon (test 2) by clock skew. + const events = parseLog(SAMPLE); + const weights = computeWeights(events); + // alpha → beta within same job: 3500ms. Not 2000ms (which would be + // alpha's stamp to epsilon's stamp across jobs). + expect(weights.get("test/alpha.test.ts")).toBe(3500); + }); + + it("takes the max when a file appears with multiple deltas", () => { + const events = [ + { job: "test (1)", timestampMs: 1000, file: "test/x.test.ts" }, + { job: "test (1)", timestampMs: 2000, file: "test/y.test.ts" }, + { job: "test (2)", timestampMs: 3000, file: "test/x.test.ts" }, + { job: "test (2)", timestampMs: 8000, file: "test/y.test.ts" }, + ]; + const w = computeWeights(events); + // x: 1000→2000 (1000ms in job 1), 3000→8000 (5000ms in job 2). Max = 5000. + expect(w.get("test/x.test.ts")).toBe(5000); + }); + + it("skips out-of-order events defensively (negative delta)", () => { + const events = [ + { job: "test (1)", timestampMs: 5000, file: "test/x.test.ts" }, + { job: "test (1)", timestampMs: 3000, file: "test/y.test.ts" }, + ]; + const w = computeWeights(events); + // x → negative delta → skipped. y has no successor → not recorded. + expect(w.size).toBe(0); + }); + + it("empty input → empty map", () => { + expect(computeWeights([]).size).toBe(0); + }); +}); + +describe("serializeWeights", () => { + it("sorts keys alphabetically for stable diffs", () => { + const w = new Map([ + ["test/z.test.ts", 100], + ["test/a.test.ts", 200], + ["test/m.test.ts", 50], + ]); + const json = serializeWeights(w); + const lines = json.split("\n").filter((l) => l.includes('"test/')); + expect(lines[0]).toContain("a.test.ts"); + expect(lines[1]).toContain("m.test.ts"); + expect(lines[2]).toContain("z.test.ts"); + }); + + it("ends with a trailing newline (POSIX-friendly diff)", () => { + const w = new Map([["test/x.test.ts", 1]]); + expect(serializeWeights(w).endsWith("\n")).toBe(true); + }); + + it("empty map → empty object JSON", () => { + expect(serializeWeights(new Map())).toBe("{}\n"); + }); + + it("round-trips: parse our own output", () => { + const w = new Map([ + ["test/alpha.test.ts", 100], + ["test/beta.test.ts", 250], + ]); + const json = serializeWeights(w); + const parsed = JSON.parse(json); + expect(parsed["test/alpha.test.ts"]).toBe(100); + expect(parsed["test/beta.test.ts"]).toBe(250); + }); +}); diff --git a/test/scripts/run-verify-parallel.test.ts b/test/scripts/run-verify-parallel.test.ts new file mode 100644 index 000000000..949fc3d83 --- /dev/null +++ b/test/scripts/run-verify-parallel.test.ts @@ -0,0 +1,175 @@ +// run-verify-parallel.test.ts — pin the dispatcher's contract. +// +// We can't easily fake the 20-check list inside the script (it's a static +// array), but we CAN verify: +// 1. --dry-list emits one line per check +// 2. Unknown args exit 2 +// 3. The fast-path measurement claim — running it on a clean tree +// completes faster than the sequential `&&`-chain would have +// 4. A made-up failing check propagates exit 1 with the named check +// in the failure report (covered by overriding the CHECKS array +// via a sibling temp script) +// +// (3) is a soft regression guard, not a hard timing assertion (CI runners +// vary). (4) is the load-bearing test: failure surfaces with name + log. + +import { describe, expect, it } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { writeFileSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const SCRIPT = "scripts/run-verify-parallel.sh"; + +describe("run-verify-parallel.sh — CLI contract", () => { + it("--dry-list emits one line per check, exit 0", () => { + const r = spawnSync("bash", [SCRIPT, "--dry-list"], { encoding: "utf8" }); + expect(r.status).toBe(0); + const lines = r.stdout.trim().split("\n").filter(Boolean); + // The exact count is allowed to grow; assert > 10 and that every line + // looks like a script name (no whitespace, no quotes). + expect(lines.length).toBeGreaterThan(10); + for (const l of lines) { + expect(l).toMatch(/^[a-z][a-z0-9:_-]+$/); + } + }); + + it("includes the load-bearing privacy + jsonb + typecheck checks", () => { + const r = spawnSync("bash", [SCRIPT, "--dry-list"], { encoding: "utf8" }); + expect(r.status).toBe(0); + const set = new Set(r.stdout.trim().split("\n")); + expect(set.has("check:privacy")).toBe(true); + expect(set.has("check:jsonb")).toBe(true); + expect(set.has("typecheck")).toBe(true); + expect(set.has("check:operations-filter-bypass")).toBe(true); + }); + + it("unknown arg exits 2 with usage error", () => { + const r = spawnSync("bash", [SCRIPT, "--bogus"], { encoding: "utf8" }); + expect(r.status).toBe(2); + expect(r.stderr).toContain("unknown arg"); + expect(r.stderr).toContain("usage:"); + }); +}); + +describe("run-verify-parallel.sh — failure surfacing (synthetic dispatcher)", () => { + // We can't inject a fake check into the real script without touching the + // CHECKS array. Instead, we write a SMALLER synthetic dispatcher that + // shares the same shape (background jobs, exit-file aggregation, log + // tail on failure) and assert the surfacing contract on it. This proves + // the design, leaving the real script tested via CLI contract above. + // + // Each "check" is written as its own executable file so bash array + // expansion doesn't try to word-split shell strings (the natural + // failure mode of expressing checks as bash command strings is that + // spaces and quotes don't survive `"${CHECKS[@]}"`). + + function writeSynth(checkBodies: { name: string; body: string }[]): string { + const dir = mkdtempSync(join(tmpdir(), "verify-synth-")); + const checkPaths: string[] = []; + for (const c of checkBodies) { + const p = `${dir}/${c.name}.sh`; + writeFileSync(p, `#!/usr/bin/env bash\n${c.body}\n`, { mode: 0o755 }); + checkPaths.push(p); + } + const dispatcher = `${dir}/dispatch.sh`; + // CHECKS is an array of absolute paths to executable scripts. + const arrLit = checkPaths.map((p) => `"${p}"`).join(" "); + writeFileSync( + dispatcher, + `#!/usr/bin/env bash +set -uo pipefail +LOG_DIR=$(mktemp -d /tmp/verify-synth-log-XXXXXX) +trap 'rm -rf "$LOG_DIR"' EXIT +CHECKS=(${arrLit}) +PIDS=() +NAMES=() +for c in "\${CHECKS[@]}"; do + base=$(basename "$c" .sh) + NAMES+=("$base") + ( + "$c" > "$LOG_DIR/$base.log" 2>&1 + echo $? > "$LOG_DIR/$base.exit" + ) & + PIDS+=($!) +done +for p in "\${PIDS[@]}"; do wait "$p" 2>/dev/null || true; done +FAIL=0 +FAILED_NAMES="" +for i in "\${!NAMES[@]}"; do + n="\${NAMES[$i]}" + rc=$(cat "$LOG_DIR/$n.exit") + if [ "$rc" != "0" ]; then + FAIL=$((FAIL+1)) + FAILED_NAMES="$FAILED_NAMES $n" + echo "--- $n (rc=$rc) ---" >&2 + tail -10 "$LOG_DIR/$n.log" >&2 + fi +done +if [ "$FAIL" -gt 0 ]; then + echo "Failed:$FAILED_NAMES" >&2 + exit 1 +fi +exit 0 +`, + { mode: 0o755 }, + ); + return dispatcher; + } + + function cleanup(dispatcher: string) { + rmSync(dispatcher.replace(/\/dispatch\.sh$/, ""), { recursive: true, force: true }); + } + + it("all-pass: exit 0, no failure block", () => { + const d = writeSynth([ + { name: "alpha", body: "exit 0" }, + { name: "beta", body: "exit 0" }, + { name: "gamma", body: "exit 0" }, + ]); + try { + const r = spawnSync("bash", [d], { encoding: "utf8" }); + expect(r.status).toBe(0); + expect(r.stderr).not.toContain("Failed:"); + } finally { + cleanup(d); + } + }); + + it("one-fails: exit 1, failure block names the check + shows log tail", () => { + const d = writeSynth([ + { name: "alpha", body: "exit 0" }, + { + name: "beta", + body: 'echo "synthetic failure detail line" >&2\nexit 7', + }, + { name: "gamma", body: "exit 0" }, + ]); + try { + const r = spawnSync("bash", [d], { encoding: "utf8" }); + expect(r.status).toBe(1); + expect(r.stderr).toContain("Failed: beta"); + expect(r.stderr).toContain("synthetic failure detail line"); + expect(r.stderr).toMatch(/--- beta \(rc=7\) ---/); + } finally { + cleanup(d); + } + }); + + it("two-fail: both names appear in the Failed: list", () => { + const d = writeSynth([ + { name: "alpha", body: "exit 0" }, + { name: "beta", body: "echo err-A >&2\nexit 1" }, + { name: "gamma", body: "echo err-B >&2\nexit 2" }, + ]); + try { + const r = spawnSync("bash", [d], { encoding: "utf8" }); + expect(r.status).toBe(1); + expect(r.stderr).toContain("err-A"); + expect(r.stderr).toContain("err-B"); + expect(r.stderr).toMatch(/Failed:\s+beta\s+gamma/); + } finally { + cleanup(d); + } + }); +}); diff --git a/test/scripts/sharding.test.ts b/test/scripts/sharding.test.ts new file mode 100644 index 000000000..bd42f169e --- /dev/null +++ b/test/scripts/sharding.test.ts @@ -0,0 +1,266 @@ +// sharding.test.ts — pure unit tests for the LPT bin-packer. +// +// Covers: happy path, fallback semantics, full coverage, determinism, +// balance ratio, N=1 trivial, weight-equal-to-zero handling, malformed +// weights, missing weights file fail-soft. + +import { describe, expect, it } from "bun:test"; +import { + computeMedian, + imbalanceRatio, + loadWeights, + partition, + WeightsLoadError, + type WeightMap, +} from "../../scripts/sharding.ts"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +function tempJson(content: string): string { + const dir = mkdtempSync(join(tmpdir(), "weights-test-")); + const path = join(dir, "weights.json"); + writeFileSync(path, content, "utf8"); + return path; +} + +describe("computeMedian", () => { + it("returns 0 on empty input", () => { + expect(computeMedian([])).toBe(0); + }); + it("single value is itself", () => { + expect(computeMedian([42])).toBe(42); + }); + it("odd count: middle value", () => { + expect(computeMedian([1, 5, 3])).toBe(3); + }); + it("even count: mean of middle two", () => { + expect(computeMedian([1, 2, 3, 4])).toBe(2.5); + }); +}); + +describe("partition — happy path", () => { + it("balances 4 known-weight files across 2 shards", () => { + const weights: WeightMap = new Map([ + ["a.test.ts", 100], + ["b.test.ts", 100], + ["c.test.ts", 50], + ["d.test.ts", 50], + ]); + const out = partition( + ["a.test.ts", "b.test.ts", "c.test.ts", "d.test.ts"], + weights, + 2, + ); + expect(out.length).toBe(2); + // LPT assigns 100→s0, 100→s1, 50→s0 (now 150), 50→s1 (now 150). + // Wait: 100→s0 (s0=100), 100→s1 (s1=100), tie → s0 gets 50 (s0=150), + // then s1 gets 50 (s1=150). Balanced. + const totals = out.map((s) => + s.reduce((acc, f) => acc + (weights.get(f) ?? 0), 0), + ); + expect(totals[0]).toBe(150); + expect(totals[1]).toBe(150); + }); + + it("LPT prefers heavy-first assignment", () => { + // 3 files: 100, 10, 10. 2 shards. Naive assignment (alpha) would put + // 100 + 10 = 110 in s0 and 10 in s1 (imbalance 11x). LPT puts 100 in + // s0, then 10 + 10 in s1 (totals 100 + 20 = imbalance 5x). + const weights: WeightMap = new Map([ + ["big.test.ts", 100], + ["small1.test.ts", 10], + ["small2.test.ts", 10], + ]); + const out = partition( + ["big.test.ts", "small1.test.ts", "small2.test.ts"], + weights, + 2, + ); + expect(out[0]).toEqual(["big.test.ts"]); + expect(out[1]?.sort()).toEqual(["small1.test.ts", "small2.test.ts"]); + }); +}); + +describe("partition — fallback semantics", () => { + it("missing weights default to corpus median", () => { + // weights = {a:100, b:50}, median = 75. Files c + d are unknown → 75 each. + const weights: WeightMap = new Map([ + ["a", 100], + ["b", 50], + ]); + const out = partition(["a", "b", "c", "d"], weights, 2); + // Effective weights: a=100, b=50, c=75, d=75. LPT: 100→s0, 75→s1 (c), + // 75→s1 (d, ties broken alpha)... actually: 100→s0=100, 75→s1=75, + // 75→s1 vs s0 → s1=150, 50→s0=150. Balanced 150/150. + const totalsEffective = out.map((s) => + s.reduce((acc, f) => acc + (weights.get(f) ?? 75), 0), + ); + expect(totalsEffective[0]).toBe(totalsEffective[1]); + }); + + it("explicit fallback override beats median", () => { + const weights: WeightMap = new Map([["a", 100]]); + const out = partition(["a", "unknown"], weights, 2, { fallbackWeight: 500 }); + // a=100 to s0, unknown=500 to s1 (heavier goes first actually — sort + // desc puts unknown=500 first). Wait, LPT sorts desc, so unknown + // (500) goes to s0 first, then a (100) goes to s1. + expect(out[0]).toEqual(["unknown"]); + expect(out[1]).toEqual(["a"]); + }); + + it("zero-weight files are valid (just always go to current min shard)", () => { + const weights: WeightMap = new Map([ + ["zero1", 0], + ["zero2", 0], + ["big", 100], + ]); + const out = partition(["zero1", "zero2", "big"], weights, 2); + // 100→s0=100, 0→s1=0, 0→s1=0. Both zeroes go to s1. + expect(out[0]).toEqual(["big"]); + expect(out[1]?.sort()).toEqual(["zero1", "zero2"]); + }); +}); + +describe("partition — invariants", () => { + it("every input file lands in exactly one shard (full coverage)", () => { + const files = Array.from({ length: 25 }, (_, i) => `f${i}.test.ts`); + const weights: WeightMap = new Map( + files.map((f, i) => [f, (i % 5) * 10 + 5]), + ); + const out = partition(files, weights, 4); + const seen: string[] = []; + for (const shard of out) seen.push(...shard); + expect(seen.sort()).toEqual([...files].sort()); + expect(new Set(seen).size).toBe(files.length); + }); + + it("deterministic — same input always produces same output", () => { + const files = ["d.test.ts", "a.test.ts", "c.test.ts", "b.test.ts"]; + const weights: WeightMap = new Map([ + ["a.test.ts", 30], + ["b.test.ts", 30], + ["c.test.ts", 30], + ["d.test.ts", 30], + ]); + const r1 = partition(files, weights, 2); + const r2 = partition([...files].reverse(), weights, 2); + const r3 = partition(files, weights, 2); + // Same input → identical output + expect(r3).toEqual(r1); + // Input order shuffled but contents identical → identical output + // (ties broken by path asc so order is canonical regardless of input) + expect(r2).toEqual(r1); + }); + + it("N=1 trivial: everything in one shard", () => { + const out = partition(["x.test.ts", "y.test.ts"], new Map(), 1); + expect(out.length).toBe(1); + expect(out[0]?.sort()).toEqual(["x.test.ts", "y.test.ts"]); + }); + + it("empty file list returns N empty shards", () => { + const out = partition([], new Map(), 3); + expect(out).toEqual([[], [], []]); + }); + + it("invalid shard count throws RangeError", () => { + expect(() => partition(["a"], new Map(), 0)).toThrow(RangeError); + expect(() => partition(["a"], new Map(), -1)).toThrow(RangeError); + expect(() => partition(["a"], new Map(), 1.5)).toThrow(RangeError); + }); + + it("invalid fallbackWeight throws RangeError", () => { + expect(() => + partition(["a"], new Map(), 2, { fallbackWeight: -1 }), + ).toThrow(RangeError); + expect(() => + partition(["a"], new Map(), 2, { fallbackWeight: NaN }), + ).toThrow(RangeError); + }); +}); + +describe("partition — balance quality", () => { + it("synthetic skewed corpus produces imbalance ratio ≤ 1.5", () => { + // 100 files, weights drawn from a Zipf-ish distribution (a few heavy, + // many light). This is the shape that broke FNV-1a hash sharding. + const files = Array.from({ length: 100 }, (_, i) => `f${i}.test.ts`); + const weights: WeightMap = new Map(); + for (let i = 0; i < files.length; i++) { + // Heavy tail: file 0 = 1000ms, file 1 = 500, file 2 = 333, ... + // Average about 50ms; max about 1000ms. + const w = Math.max(10, Math.floor(1000 / (i + 1))); + weights.set(files[i]!, w); + } + const out = partition(files, weights, 6); + const ratio = imbalanceRatio(out, weights, 0); + expect(ratio).toBeLessThanOrEqual(1.5); + }); + + it("imbalance ratio of 1.0 when weights are perfectly divisible", () => { + const files = ["a", "b", "c", "d"]; + const weights: WeightMap = new Map([ + ["a", 50], + ["b", 50], + ["c", 50], + ["d", 50], + ]); + const out = partition(files, weights, 2); + expect(imbalanceRatio(out, weights, 0)).toBe(1.0); + }); +}); + +describe("loadWeights", () => { + it("missing file → empty map (fail-soft)", () => { + const w = loadWeights("/tmp/definitely-does-not-exist-weights.json"); + expect(w.size).toBe(0); + }); + + it("malformed JSON → throws WeightsLoadError", () => { + const p = tempJson("not json {"); + try { + expect(() => loadWeights(p)).toThrow(WeightsLoadError); + } finally { + rmSync(p.replace(/\/weights\.json$/, ""), { recursive: true, force: true }); + } + }); + + it("array JSON → throws (wrong shape)", () => { + const p = tempJson("[1,2,3]"); + try { + expect(() => loadWeights(p)).toThrow(/expected top-level object/); + } finally { + rmSync(p.replace(/\/weights\.json$/, ""), { recursive: true, force: true }); + } + }); + + it("negative weight → throws (semantic invalid)", () => { + const p = tempJson('{"foo.test.ts": -5}'); + try { + expect(() => loadWeights(p)).toThrow(/non-negative finite/); + } finally { + rmSync(p.replace(/\/weights\.json$/, ""), { recursive: true, force: true }); + } + }); + + it("non-number weight → throws (semantic invalid)", () => { + const p = tempJson('{"foo.test.ts": "100ms"}'); + try { + expect(() => loadWeights(p)).toThrow(/non-negative finite/); + } finally { + rmSync(p.replace(/\/weights\.json$/, ""), { recursive: true, force: true }); + } + }); + + it("valid weights JSON round-trips", () => { + const p = tempJson('{"a.test.ts": 100, "b.test.ts": 250}'); + try { + const w = loadWeights(p); + expect(w.size).toBe(2); + expect(w.get("a.test.ts")).toBe(100); + expect(w.get("b.test.ts")).toBe(250); + } finally { + rmSync(p.replace(/\/weights\.json$/, ""), { recursive: true, force: true }); + } + }); +}); diff --git a/test/scripts/test-shard.slow.test.ts b/test/scripts/test-shard.slow.test.ts index 21afee02d..e3e4c18cd 100644 --- a/test/scripts/test-shard.slow.test.ts +++ b/test/scripts/test-shard.slow.test.ts @@ -1,18 +1,30 @@ /** - * Regression test: scripts/test-shard.sh exclusion symmetry. + * Regression test: scripts/test-shard.sh exclusion + balance contract. * - * Pins the contract that CI's hash-bucketed shard script EXCLUDES - * *.serial.test.ts and test/e2e/ from every shard. Serial files share - * file-wide state (top-level mock.module, module singletons) that leaks - * across files in the same `bun test` shard process. Before v0.31.4.1 - * they were hashed into the same buckets as parallel files, which broke - * the quarantine — `eval-takes-quality-runner.serial.test.ts` stubbed - * `gateway.ts` and broke every `gateway.embedMultimodal` test in - * `voyage-multimodal.test.ts` on shard 2. + * Pins two invariants of the CI matrix shard script: * - * Without this guard, a future refactor that drops the `-not -name - * '*.serial.test.ts'` clause from test-shard.sh would silently undo the - * fix and re-introduce the contention flake. + * 1. EXCLUDE *.serial.test.ts and test/e2e/ from every shard. Serial + * files share file-wide state (top-level mock.module, module + * singletons) that leaks across files in the same `bun test` shard + * process. Before v0.31.4.1 they were hashed into the same buckets + * as parallel files, which broke the quarantine — + * `eval-takes-quality-runner.serial.test.ts` stubbed `gateway.ts` + * and broke every `gateway.embedMultimodal` test in + * `voyage-multimodal.test.ts` on shard 2. + * + * 2. INCLUDE *.slow.test.ts. CI is the only default place slow files + * run; the local fast loop excludes them via run-unit-shard.sh. See + * CLAUDE.md "CI vs local: intentionally divergent file sets". + * + * 3. LPT balance: file counts across shards are within 5% of each other + * under the default (no-weights / cold-start) configuration. Once + * test-weights.json is populated, weighted balance also lands the + * imbalance ratio ≤ 1.5. + * + * Without (1), the v0.31.4.1 mock.module leak comes back. Without (2), + * slow tests stop running in CI (silent regression — they don't fail, + * they just never execute). Without (3), shard 3 will drift back to + * 26-min p100 wallclock. */ import { describe, it, expect, beforeAll } from 'bun:test'; @@ -22,30 +34,27 @@ import { resolve } from 'path'; const REPO_ROOT = resolve(import.meta.dir, '..', '..'); const SHARD_SH = resolve(REPO_ROOT, 'scripts/test-shard.sh'); -// Pure-bash FNV-1a per shard takes ~4s on a M-series Mac because the script -// computes a hash for every test file. Compute all 4 shards once in beforeAll -// and reuse across cases so the suite finishes in one shell-out per shard. -const shardCache: Record = {}; +// LPT partition via sharding.ts is ~30ms per shard (much faster than the +// pure-bash FNV-1a it replaced). Cache results so repeated assertions +// don't shell out per case. +const shardCache: Record = {}; function dryRunList(shard: number, total: number): string[] { - if (shardCache[shard]) return shardCache[shard]; + const key = `${shard}/${total}`; + if (shardCache[key]) return shardCache[key]; const out = execFileSync('bash', [SHARD_SH, '--dry-run-list', String(shard), String(total)], { cwd: REPO_ROOT, encoding: 'utf-8', }); - shardCache[shard] = out.split('\n').map(s => s.trim()).filter(Boolean); - return shardCache[shard]; + shardCache[key] = out.split('\n').map(s => s.trim()).filter(Boolean); + return shardCache[key]; } -describe('test-shard.sh exclusion symmetry', () => { +describe('test-shard.sh — exclusion contract', () => { beforeAll(() => { for (const shard of [1, 2, 3, 4]) dryRunList(shard, 4); - // v0.40.10 flake-hardening: bump beforeAll budget 60s → 180s. Each - // FNV-1a dry-run shells out and computes a hash for every test file - // (~4s solo). Under slow-shard concurrency (longmemeval E2E at ~50s - // hogging CPU), the 4 sequential shell-outs slip past 60s and time - // out even though they'd complete fine solo. - }, 180_000); + }, 60_000); + it('includes plain *.test.ts files in at least one shard', () => { const allFiles = [1, 2, 3, 4].flatMap(s => dryRunList(s, 4)); expect(allFiles.length).toBeGreaterThan(0); @@ -68,7 +77,17 @@ describe('test-shard.sh exclusion symmetry', () => { } }); - it('partitions plain files across shards without overlap', () => { + it('INCLUDES *.slow.test.ts files (CI matrix is where slow files run)', () => { + const allFiles = [1, 2, 3, 4].flatMap(s => dryRunList(s, 4)); + const slowFiles = allFiles.filter(f => /\.slow\.test\.ts$/.test(f)); + // The repo currently has 3 slow files — pin >= 1 so the test stays + // green if the count changes but fails loud if slow files vanish + // entirely (regression: someone re-added -not -name '*.slow.test.ts' + // to the find clause). + expect(slowFiles.length).toBeGreaterThan(0); + }); + + it('partitions every file across shards without overlap', () => { const seen = new Map(); for (const shard of [1, 2, 3, 4]) { for (const f of dryRunList(shard, 4)) { @@ -81,3 +100,70 @@ describe('test-shard.sh exclusion symmetry', () => { expect(seen.size).toBeGreaterThan(0); }); }); + +describe('test-shard.sh — LPT balance contract', () => { + // LPT balances WALLCLOCK (sum of weights), not file count. When real + // weights are loaded from scripts/test-weights.json, shards with + // heavier files have FEWER files — that's the optimization working. + // We assert wallclock balance via the imbalance ratio (≤1.5). + + // Read weights from the committed JSON. If it doesn't exist yet (early + // in the wave) or is empty, we degrade to cold-start round-robin and + // assert file-count balance instead. + let weightsMap = new Map(); + let weightsLoaded = false; + beforeAll(() => { + try { + const fs = require('fs'); + const path = require('path'); + const p = path.resolve(REPO_ROOT, 'scripts/test-weights.json'); + if (fs.existsSync(p)) { + const raw = JSON.parse(fs.readFileSync(p, 'utf8')); + for (const [k, v] of Object.entries(raw)) { + if (typeof v === 'number' && Number.isFinite(v)) weightsMap.set(k, v); + } + weightsLoaded = weightsMap.size > 0; + } + } catch { + // fall through to cold-start mode + } + }); + + function totalsFor(shards: string[][]): number[] { + // Use 30ms as the cold-start fallback (matches mine-shard-weights + // median observation). When weights are loaded, missing files get + // the corpus median anyway via sharding.ts. + const fallback = weightsLoaded + ? Array.from(weightsMap.values()).sort((a, b) => a - b)[ + Math.floor(weightsMap.size / 2) + ] ?? 30 + : 1; + return shards.map((s) => + s.reduce((acc, f) => acc + (weightsMap.get(f) ?? fallback), 0), + ); + } + + it('4-shard wallclock imbalance ratio ≤ 1.5', () => { + const shards = [1, 2, 3, 4].map(s => dryRunList(s, 4)); + for (const s of shards) expect(s.length).toBeGreaterThan(0); + const totals = totalsFor(shards); + const ratio = Math.max(...totals) / Math.min(...totals); + expect(ratio).toBeLessThanOrEqual(1.5); + }); + + it('6-shard wallclock imbalance ratio ≤ 1.5', () => { + const shards = [1, 2, 3, 4, 5, 6].map(s => dryRunList(s, 6)); + for (const s of shards) expect(s.length).toBeGreaterThan(0); + const totals = totalsFor(shards); + const ratio = Math.max(...totals) / Math.min(...totals); + expect(ratio).toBeLessThanOrEqual(1.5); + }); + + it('6-shard partition is deterministic across runs', () => { + // Clear cache + re-run for one shard, compare to the cached result. + const cached = [...dryRunList(1, 6)]; + delete shardCache['1/6']; + const fresh = dryRunList(1, 6); + expect(fresh).toEqual(cached); + }); +});