mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-28 06:23:01 +00:00
d69f211629
The signal-handler test asserted an absolute liveReporters===0 on a module-global set, so any other test file in the shard holding a live reporter flaked it — it red-flagged ~12 unrelated PR runs and one master push in two days, purely as a function of shard composition. The delta form pins the same claim (50 reporter lifecycles leak nothing). The 15-minute shard timeout cancelled 13 fully-passing runs under parallel PR load (PGLite WASM cold-starts stretch shards); the test-status gate then reported the cancellations as failures. Co-authored-by: Garry Tan <garrytan@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
291 lines
14 KiB
YAML
291 lines
14 KiB
YAML
name: Test
|
||
|
||
on:
|
||
push:
|
||
branches: [master]
|
||
pull_request:
|
||
branches: [master]
|
||
# Manual dispatch lets a local dev/agent offload the suite to GitHub's
|
||
# on-demand runners from ANY branch (see scripts/ship-remote-tests.sh).
|
||
# Frees a load-saturated local machine (e.g. many Conductor agents running
|
||
# their own bun-test suites at once — load avg 120 on 16 cores).
|
||
workflow_dispatch:
|
||
|
||
permissions:
|
||
contents: read
|
||
|
||
# Cancel a superseded run when a newer commit lands on the same PR/branch.
|
||
# Keyed on the PR number for pull_request events (unique per PR, so two PRs
|
||
# from forks sharing a branch name don't cancel each other) and falls back to
|
||
# github.ref for push/scheduled runs. Mirrors heavy-tests.yml; frees runners
|
||
# and stops a stale-SHA run from reporting a flaky failure on an obsolete commit.
|
||
concurrency:
|
||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||
cancel-in-progress: true
|
||
|
||
jobs:
|
||
# ──────────────────────────────────────────────────────────────────────
|
||
# cache-check: runs first, computes the content hash of every tracked
|
||
# file EXCEPT the deny-list (CHANGELOG.md, README.md, docs/**/*.md, etc.
|
||
# — see scripts/ci-cache-hash.sh for the full list). Looks up
|
||
# `ci-pass-<hash>` in actions/cache; if hit, the test matrix + verify
|
||
# + serial jobs all skip and test-status reports green immediately.
|
||
# If miss, the full suite runs and cache-write seals it on success.
|
||
#
|
||
# Hit rate covers re-pushes (same SHA twice), branch rebases that
|
||
# don't touch tracked code, and any branch update that touches only
|
||
# the deny-listed doc files.
|
||
# ──────────────────────────────────────────────────────────────────────
|
||
cache-check:
|
||
runs-on: ubuntu-latest
|
||
timeout-minutes: 10
|
||
outputs:
|
||
hit: ${{ steps.lookup.outputs.cache-hit }}
|
||
hash: ${{ steps.compute.outputs.hash }}
|
||
steps:
|
||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||
- name: Compute content hash
|
||
id: compute
|
||
run: |
|
||
# --verbose writes the "X/Y files in hash" diagnostic to stderr;
|
||
# stdout carries the 16-char hash. Capture both.
|
||
HASH=$(bash scripts/ci-cache-hash.sh --verbose 2>/tmp/cache-diag)
|
||
cat /tmp/cache-diag
|
||
echo "Computed cache hash: $HASH"
|
||
echo "hash=$HASH" >> "$GITHUB_OUTPUT"
|
||
- name: Lookup actions/cache for ci-pass-<hash>
|
||
id: lookup
|
||
uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
|
||
with:
|
||
key: ci-pass-${{ steps.compute.outputs.hash }}
|
||
path: .ci-cache-marker
|
||
# `lookup-only: true` means we only probe whether the cache
|
||
# entry exists — we don't download it (the marker contents
|
||
# don't matter, only the key match does). `cache-hit` returns
|
||
# true only on EXACT key match (per actions/cache docs); a
|
||
# restore-keys prefix fallback would set cache-hit=false, so
|
||
# it's deliberately omitted here. Cross-branch scoping works
|
||
# naturally: PR branches can read default-branch (master)
|
||
# cache entries via exact key match when the content hash
|
||
# matches, which happens whenever the tree is doc-only
|
||
# different from a green master run.
|
||
lookup-only: true
|
||
- name: Cache status
|
||
run: |
|
||
if [ "${{ steps.lookup.outputs.cache-hit }}" = "true" ]; then
|
||
echo "✓ cache HIT for hash ${{ steps.compute.outputs.hash }} — test jobs will skip"
|
||
else
|
||
echo "✗ cache MISS for hash ${{ steps.compute.outputs.hash }} — full suite will run"
|
||
fi
|
||
|
||
gitleaks:
|
||
needs: cache-check
|
||
if: needs.cache-check.outputs.hit != 'true'
|
||
runs-on: ubuntu-latest
|
||
timeout-minutes: 10
|
||
steps:
|
||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||
with:
|
||
fetch-depth: 0
|
||
- uses: gitleaks/gitleaks-action@dcedce43c6f43de0b836d1fe38946645c9c638dc # v2
|
||
env:
|
||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||
|
||
verify:
|
||
# Pre-test gates: privacy/jsonb/source-id/etc + typecheck + admin-build.
|
||
# Lives in its own runner so the matrix shards aren't carrying ~2-3min
|
||
# of verify work in addition to their test files (the old shape stuffed
|
||
# this into `test (1)` via `if: matrix.shard == 1`, which made shard 1
|
||
# the slowest matrix worker). scripts/run-verify-parallel.sh fans out
|
||
# the 20 checks via & + wait (~5s vs ~15-25s sequential).
|
||
needs: cache-check
|
||
if: needs.cache-check.outputs.hit != 'true'
|
||
runs-on: ubuntu-latest
|
||
timeout-minutes: 12
|
||
steps:
|
||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||
with:
|
||
bun-version: 1.3.13
|
||
- uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
|
||
with:
|
||
path: ~/.bun/install/cache
|
||
key: bun-cache-${{ runner.os }}-${{ hashFiles('bun.lock') }}
|
||
- run: bun install
|
||
- run: bun run verify
|
||
|
||
serial-tests:
|
||
# *.serial.test.ts at --max-concurrency=1. Lives in its own runner so
|
||
# the matrix shards aren't carrying the serial-pass tail (the old shape
|
||
# stuffed this into `test (1)` after the matrix work, which compounded
|
||
# shard 1's overload).
|
||
needs: cache-check
|
||
if: needs.cache-check.outputs.hit != 'true'
|
||
runs-on: ubuntu-latest
|
||
timeout-minutes: 15
|
||
steps:
|
||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||
with:
|
||
bun-version: 1.3.13
|
||
- uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
|
||
with:
|
||
path: ~/.bun/install/cache
|
||
key: bun-cache-${{ runner.os }}-${{ hashFiles('bun.lock') }}
|
||
- run: bun install
|
||
- run: bun run test:serial
|
||
|
||
slow-eval-longmemeval:
|
||
# Dedicated runner for the LongMemEval end-to-end test file. The file
|
||
# was originally 359s. TODO #1 (engine-sharing in runEvalLongMemEval
|
||
# via RunOpts.engine) cut it to ~200s by amortizing PGLite cold-create
|
||
# across all 13 runEvalLongMemEval calls in one beforeAll-shared brain.
|
||
# Pulled out of the matrix (see scripts/test-shard.sh) so a single 200s
|
||
# atom doesn't dominate a shard's wallclock. Companion file
|
||
# test/eval-longmemeval.slow.test.ts (the pure-bucket half) stays in
|
||
# the matrix because it's light (~42s).
|
||
needs: cache-check
|
||
if: needs.cache-check.outputs.hit != 'true'
|
||
runs-on: ubuntu-latest
|
||
timeout-minutes: 12
|
||
steps:
|
||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||
with:
|
||
bun-version: 1.3.13
|
||
- uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
|
||
with:
|
||
path: ~/.bun/install/cache
|
||
key: bun-cache-${{ runner.os }}-${{ hashFiles('bun.lock') }}
|
||
- run: bun install
|
||
- run: bun test test/eval-longmemeval-e2e.slow.test.ts --timeout=60000
|
||
|
||
slow-entity-resolve-perf:
|
||
# Dedicated runner for the entity-resolve perf test (~159s, single perf
|
||
# describe with one test that builds 5000+ pages and asserts the NEW
|
||
# tryPrefixExpansion shape is 5x faster than the OLD shape — not
|
||
# subdivisible without weakening the perf guarantee). Pulled out of the
|
||
# matrix (see scripts/test-shard.sh) so a single 159s atom doesn't
|
||
# dominate a shard's wallclock. Runs in parallel with the matrix.
|
||
needs: cache-check
|
||
if: needs.cache-check.outputs.hit != 'true'
|
||
runs-on: ubuntu-latest
|
||
timeout-minutes: 12
|
||
steps:
|
||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||
with:
|
||
bun-version: 1.3.13
|
||
- uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
|
||
with:
|
||
path: ~/.bun/install/cache
|
||
key: bun-cache-${{ runner.os }}-${{ hashFiles('bun.lock') }}
|
||
- run: bun install
|
||
- run: bun test test/entity-resolve-perf.slow.test.ts --timeout=300000
|
||
|
||
test:
|
||
# Pure matrix shard — no verify, no serial. Each shard runs its slice
|
||
# of the unit test set under one `bun test` invocation.
|
||
#
|
||
# 10 shards (was 6) drops per-shard total from 532s → 287s. With the two
|
||
# dedicated jobs (slow-eval-longmemeval, slow-entity-resolve-perf) also
|
||
# pulled out, the matrix is bounded by ~287s ≈ 4.8 min. Total CI ≈ max
|
||
# of matrix + slow-eval (~3.3 min after engine-sharing in TODO #1) +
|
||
# slow-entity-resolve-perf (~2.6 min) ≈ 4.8 min.
|
||
#
|
||
# Concurrency budget: 10 shards + verify + serial + slow-eval +
|
||
# slow-entity-resolve-perf + gitleaks + cache-check + cache-write +
|
||
# test-status = ~18 jobs × 2 concurrent PRs = 36. GitHub free-tier
|
||
# caps at ~20 concurrent jobs, so multi-PR days will see some queue
|
||
# pressure. Single-PR runs are unaffected.
|
||
#
|
||
# Partition policy is weight-aware LPT bin-packing via scripts/sharding.ts
|
||
# (replaces FNV-1a path hash). Weights live in scripts/test-weights.json,
|
||
# mined from real CI logs via scripts/mine-shard-weights.ts. Missing
|
||
# weights fall back to corpus median — new test files work immediately.
|
||
needs: cache-check
|
||
if: needs.cache-check.outputs.hit != 'true'
|
||
runs-on: ubuntu-latest
|
||
# 22, not 15: under parallel PR load the PGLite WASM cold-starts stretch a
|
||
# shard past 15 min while every test is still passing — the timeout then
|
||
# cancels the job and the test-status gate reads it as a failure. 13 runs
|
||
# died this way on 2026-07-21/22 alone.
|
||
timeout-minutes: 22
|
||
strategy:
|
||
fail-fast: false
|
||
matrix:
|
||
shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
||
steps:
|
||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||
with:
|
||
bun-version: 1.3.13
|
||
- uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
|
||
with:
|
||
path: ~/.bun/install/cache
|
||
key: bun-cache-${{ runner.os }}-${{ hashFiles('bun.lock') }}
|
||
- run: bun install
|
||
- name: Run test shard ${{ matrix.shard }}/10
|
||
run: scripts/test-shard.sh ${{ matrix.shard }} 10
|
||
|
||
# ──────────────────────────────────────────────────────────────────────
|
||
# cache-write: ONLY runs when every gated job succeeded. Writes the
|
||
# cache entry under `ci-pass-<hash>` so future runs at the same hash
|
||
# hit cache. Codex's load-bearing correctness point: writing the
|
||
# cache before the matrix completes would permanently bless bad states
|
||
# (a future run at the same hash would skip tests because of a cache
|
||
# entry written when tests hadn't actually passed).
|
||
# ──────────────────────────────────────────────────────────────────────
|
||
cache-write:
|
||
needs: [cache-check, gitleaks, verify, serial-tests, slow-eval-longmemeval, slow-entity-resolve-perf, test]
|
||
if: success() && needs.cache-check.outputs.hit != 'true'
|
||
runs-on: ubuntu-latest
|
||
timeout-minutes: 5
|
||
steps:
|
||
- name: Create cache marker
|
||
run: |
|
||
mkdir -p .ci-cache-marker
|
||
echo "${{ needs.cache-check.outputs.hash }}" > .ci-cache-marker/hash
|
||
echo "$GITHUB_SHA" > .ci-cache-marker/sha
|
||
echo "$GITHUB_REF" > .ci-cache-marker/ref
|
||
- uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
|
||
with:
|
||
key: ci-pass-${{ needs.cache-check.outputs.hash }}
|
||
path: .ci-cache-marker
|
||
|
||
# ──────────────────────────────────────────────────────────────────────
|
||
# test-status: the single user-visible "did CI pass?" check.
|
||
# Runs always (if: always()), succeeds when EITHER cache-check.hit==true
|
||
# OR all gated jobs (gitleaks, verify, serial-tests, test) succeeded.
|
||
# Branch protection (when configured) gates on this single job name.
|
||
# ──────────────────────────────────────────────────────────────────────
|
||
test-status:
|
||
needs: [cache-check, gitleaks, verify, serial-tests, slow-eval-longmemeval, slow-entity-resolve-perf, test]
|
||
if: always()
|
||
runs-on: ubuntu-latest
|
||
timeout-minutes: 5
|
||
steps:
|
||
- name: Aggregate result
|
||
run: |
|
||
HIT="${{ needs.cache-check.outputs.hit }}"
|
||
GITLEAKS="${{ needs.gitleaks.result }}"
|
||
VERIFY="${{ needs.verify.result }}"
|
||
SERIAL="${{ needs.serial-tests.result }}"
|
||
SLOW_EVAL="${{ needs.slow-eval-longmemeval.result }}"
|
||
SLOW_PERF="${{ needs.slow-entity-resolve-perf.result }}"
|
||
TEST="${{ needs.test.result }}"
|
||
echo "cache-check.hit=$HIT"
|
||
echo "gitleaks=$GITLEAKS verify=$VERIFY serial-tests=$SERIAL slow-eval-longmemeval=$SLOW_EVAL slow-entity-resolve-perf=$SLOW_PERF test=$TEST"
|
||
if [ "$HIT" = "true" ]; then
|
||
echo "✓ cache HIT for hash ${{ needs.cache-check.outputs.hash }} — CI green"
|
||
exit 0
|
||
fi
|
||
# Cache miss: every gated job must have succeeded.
|
||
for r in "$GITLEAKS" "$VERIFY" "$SERIAL" "$SLOW_EVAL" "$SLOW_PERF" "$TEST"; do
|
||
if [ "$r" != "success" ]; then
|
||
echo "✗ gated job did not succeed (got $r) — CI fail"
|
||
exit 1
|
||
fi
|
||
done
|
||
echo "✓ all gated jobs succeeded — CI green"
|