diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e958392bd..f4a134102 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,3 +40,22 @@ jobs: run: bun run verify - name: Run test shard ${{ matrix.shard }}/4 run: scripts/test-shard.sh ${{ matrix.shard }} 4 + + # Serial tests run in their own job because *.serial.test.ts files use + # mock.module() which leaks across files in the same bun-test process. + # Quarantining them here mirrors the local fast-loop convention (`bun run + # test:serial` is invoked separately from the parallel shard runs). Without + # this split, mock.module substitutions from one serial file break tests + # in other files in the same shard — exactly the regression that surfaced + # when eval-takes-quality-runner.serial.test.ts's mock leaked into + # voyage-multimodal.test.ts and broke 18 of its 22 tests. + test-serial: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: latest + - run: bun install + - name: Run *.serial.test.ts files (max-concurrency=1) + run: bun run test:serial diff --git a/scripts/test-shard.sh b/scripts/test-shard.sh index 5f836c268..08cc19870 100755 --- a/scripts/test-shard.sh +++ b/scripts/test-shard.sh @@ -32,12 +32,21 @@ fi cd "$(dirname "$0")/.." -# Find all unit test files, deterministic order. Excludes test/e2e/. +# Find all unit test files, deterministic order. Excludes: +# - test/e2e/* (runs separately via scripts/run-e2e.sh) +# - *.serial.test.ts (concurrency-unsafe; runs via scripts/run-serial-tests.sh) +# - *.slow.test.ts (cold-path correctness; runs via scripts/run-slow-tests.sh) +# +# Mirrors the exclusion in scripts/run-unit-shard.sh (the local fast-loop +# helper). Without these exclusions, mock.module() in *.serial.test.ts files +# leaks across the shard's bun process and breaks downstream tests — exactly +# the failure mode CLAUDE.md test isolation rules call out for the .serial +# quarantine. Honors the same quarantine in CI as locally. # Portable: avoid `mapfile` (bash 4+) so this runs on macOS bash 3.2 too. FILES=() while IFS= read -r line; do FILES+=("$line") -done < <(find test -name '*.test.ts' -not -path 'test/e2e/*' | sort) +done < <(find test -name '*.test.ts' -not -path 'test/e2e/*' -not -name '*.serial.test.ts' -not -name '*.slow.test.ts' | sort) if [ "${#FILES[@]}" -eq 0 ]; then echo "no test files found under test/" >&2 diff --git a/test/scripts/test-shard.test.ts b/test/scripts/test-shard.test.ts new file mode 100644 index 000000000..16165e807 --- /dev/null +++ b/test/scripts/test-shard.test.ts @@ -0,0 +1,40 @@ +/** + * Regression test: scripts/test-shard.sh (the CI test-sharding script) + * MUST exclude *.serial.test.ts and *.slow.test.ts files, mirroring the + * exclusion in scripts/run-unit-shard.sh (the local fast-loop equivalent). + * + * Why this regression test exists: serial files use `mock.module()` which + * leaks across files in the same `bun test` process. When test-shard.sh + * included serial files alongside other tests in the same shard, mocks from + * eval-takes-quality-runner.serial.test.ts leaked into voyage-multimodal.test.ts + * and broke 18 of its 22 tests in CI shard 2 — even though local runs (which + * already excluded serial files via run-unit-shard.sh) were green. + * + * Without this guard, a future refactor that drops the `-not -name '*.serial.test.ts'` + * clause from test-shard.sh would silently re-introduce the cross-file mock + * leak in CI without any local repro. + */ + +import { describe, it, expect } from 'bun:test'; +import { readFileSync } from 'fs'; +import { resolve } from 'path'; + +const REPO_ROOT = resolve(import.meta.dir, '..', '..'); +const TEST_SHARD_SH = resolve(REPO_ROOT, 'scripts/test-shard.sh'); + +describe('scripts/test-shard.sh exclusion contract', () => { + it('excludes *.serial.test.ts files from the shard', () => { + const source = readFileSync(TEST_SHARD_SH, 'utf-8'); + expect(source).toMatch(/-not -name ['"]\*\.serial\.test\.ts['"]/); + }); + + it('excludes *.slow.test.ts files from the shard', () => { + const source = readFileSync(TEST_SHARD_SH, 'utf-8'); + expect(source).toMatch(/-not -name ['"]\*\.slow\.test\.ts['"]/); + }); + + it('excludes test/e2e/* (always-on contract)', () => { + const source = readFileSync(TEST_SHARD_SH, 'utf-8'); + expect(source).toMatch(/-not -path ['"]test\/e2e\/\*['"]/); + }); +});