From 70d5f36db60d435b40f83031473f1911f6bc2f9a Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Wed, 17 Jun 2026 07:44:08 -0700 Subject: [PATCH] =?UTF-8?q?v0.42.50.0=20ci:=20reliability=20hardening=20?= =?UTF-8?q?=E2=80=94=20cancel-superseded=20+=20per-job=20timeouts=20+=20ac?= =?UTF-8?q?tionlint=20+=20hermetic=20E2E=20env=20(#2254)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: cancel superseded runs + per-job timeouts on test.yml & e2e.yml Ports the GH-Actions hygiene gbrain already uses in heavy-tests.yml to the two hot-path workflows. concurrency cancels a superseded run (keyed on PR number for pull_request events — fork-safe — with github.ref fallback for push/scheduled); frees runners and stops a stale-SHA run reporting a flaky failure on an obsolete commit. Per-job timeout-minutes (test matrix 15, verify 12, serial 15, slow-* 12, e2e tier1 20 / tier2 30, trivial jobs 5-10) convert a wedged job from a 6-hour zombie (GitHub's default) into a fast legible fail. fail-fast:false already set. Co-Authored-By: Claude Opus 4.8 (1M context) * ci: add actionlint workflow (rhysd/actionlint v1.7.11, SHA-pinned) Lints workflow YAML on .github/workflows/** changes so a malformed workflow / bad action ref / missing-permission bug is caught before it ships a broken pipeline. gbrain edits these workflows often; cheap preventive guard. Mirrors GStack's actionlint job, SHA-pinned to gbrain's convention. Co-Authored-By: Claude Opus 4.8 (1M context) * ci(e2e): scrub operator/agent env before E2E (hermetic runner) A dev or Conductor shell exports CONDUCTOR_*/MCP_*/OPENCLAW_*/GBRAIN_* overrides that silently change test behavior, making hermetic E2E non-hermetic and its failures unreproducible across machines. run-e2e.sh drops those prefixes before bun starts (denylist — PATH/HOME/TMPDIR/DATABASE_URL survive; GBRAIN_HOME kept for the existing HOME isolation). Adapts GStack's buildHermeticEnv to gbrain's shell runner. Verified: a 78-test e2e file passes with DATABASE_URL surviving + a planted GBRAIN_BRAIN_ID scrubbed. Co-Authored-By: Claude Opus 4.8 (1M context) * v0.42.50.0 ci: reliability hardening — cancel-superseded + per-job timeouts + actionlint + hermetic E2E env Ports GStack's CI-reliability hygiene to gbrain's hot-path workflows: concurrency cancel-in-progress (PR-number keyed), per-job timeout-minutes (no more 6-hour zombie jobs), an actionlint workflow, and an operator-env scrub in run-e2e.sh. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .github/workflows/actionlint.yml | 32 ++++++++++++++++++++++++++++++++ .github/workflows/e2e.yml | 9 +++++++++ .github/workflows/test.yml | 18 ++++++++++++++++++ CHANGELOG.md | 15 +++++++++++++++ VERSION | 2 +- package.json | 2 +- scripts/run-e2e.sh | 18 ++++++++++++++++++ 7 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/actionlint.yml diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml new file mode 100644 index 000000000..b4b6b8272 --- /dev/null +++ b/.github/workflows/actionlint.yml @@ -0,0 +1,32 @@ +name: Actionlint + +# Lints the GitHub Actions workflow YAML on every change so a malformed +# workflow / bad action ref / missing-permission bug is caught before it ships +# a broken pipeline. gbrain edits .github/workflows/* often (sharding, cache, +# timeouts); this is the cheap guard that keeps those edits honest. + +on: + push: + branches: [master] + paths: + - '.github/workflows/**' + pull_request: + branches: [master] + paths: + - '.github/workflows/**' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + actionlint: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: rhysd/actionlint@393031adb9afb225ee52ae2ccd7a5af5525e03e8 # v1.7.11 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 556afcf42..3857043ba 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -12,10 +12,18 @@ on: permissions: contents: read +# Cancel a superseded run when a newer commit lands on the same PR/branch. +# PR number for pull_request events (fork-safe), github.ref fallback for +# push/scheduled runs. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: tier1: name: Tier 1 (Mechanical) runs-on: ubuntu-latest + timeout-minutes: 20 services: postgres: image: pgvector/pgvector:pg16 @@ -49,6 +57,7 @@ jobs: # from repo/org secrets. Nightly + manual triggers still supported via # the workflow-level `on:` list. needs: tier1 + timeout-minutes: 30 services: postgres: image: pgvector/pgvector:pg16 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f548af4c2..6a80e4b77 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,15 @@ on: permissions: contents: read +# Cancel a superseded run when a newer commit lands on the same PR/branch. +# Keyed on the PR number for pull_request events (unique per PR, so two PRs +# from forks sharing a branch name don't cancel each other) and falls back to +# github.ref for push/scheduled runs. Mirrors heavy-tests.yml; frees runners +# and stops a stale-SHA run from reporting a flaky failure on an obsolete commit. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: # ────────────────────────────────────────────────────────────────────── # cache-check: runs first, computes the content hash of every tracked @@ -29,6 +38,7 @@ jobs: # ────────────────────────────────────────────────────────────────────── cache-check: runs-on: ubuntu-latest + timeout-minutes: 10 outputs: hit: ${{ steps.lookup.outputs.cache-hit }} hash: ${{ steps.compute.outputs.hash }} @@ -72,6 +82,7 @@ jobs: needs: cache-check if: needs.cache-check.outputs.hit != 'true' runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: @@ -90,6 +101,7 @@ jobs: needs: cache-check if: needs.cache-check.outputs.hit != 'true' runs-on: ubuntu-latest + timeout-minutes: 12 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 @@ -110,6 +122,7 @@ jobs: needs: cache-check if: needs.cache-check.outputs.hit != 'true' runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 @@ -134,6 +147,7 @@ jobs: needs: cache-check if: needs.cache-check.outputs.hit != 'true' runs-on: ubuntu-latest + timeout-minutes: 12 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 @@ -156,6 +170,7 @@ jobs: needs: cache-check if: needs.cache-check.outputs.hit != 'true' runs-on: ubuntu-latest + timeout-minutes: 12 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 @@ -191,6 +206,7 @@ jobs: needs: cache-check if: needs.cache-check.outputs.hit != 'true' runs-on: ubuntu-latest + timeout-minutes: 15 strategy: fail-fast: false matrix: @@ -220,6 +236,7 @@ jobs: needs: [cache-check, gitleaks, verify, serial-tests, slow-eval-longmemeval, slow-entity-resolve-perf, test] if: success() && needs.cache-check.outputs.hit != 'true' runs-on: ubuntu-latest + timeout-minutes: 5 steps: - name: Create cache marker run: | @@ -242,6 +259,7 @@ jobs: needs: [cache-check, gitleaks, verify, serial-tests, slow-eval-longmemeval, slow-entity-resolve-perf, test] if: always() runs-on: ubuntu-latest + timeout-minutes: 5 steps: - name: Aggregate result run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index af8687491..380ae0a75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ All notable changes to GBrain will be documented in this file. +## [0.42.50.0] - 2026-06-17 + +**CI reliability hardening — a wedged job can no longer run for six hours, a superseded run no longer reports a stale flaky failure, and broken workflow YAML is caught before it ships.** gbrain's CI already had the deep machinery (content-hash run-skip cache, weight-aware shard balancing, test-isolation guards, hermetic E2E). What it lacked was the cheap GitHub-Actions hygiene that was already wired into `heavy-tests.yml` but never into the two hot-path workflows. This pass closes that gap, porting the patterns from the sibling GStack project's CI-reliability work. + +### Changed +- **`test.yml` and `e2e.yml` cancel a superseded run** when a newer commit lands on the same PR (`concurrency` keyed on the PR number — fork-safe — with a `github.ref` fallback so push and scheduled runs always complete). Frees runners and stops a run against an obsolete commit from reporting a flaky failure. +- **Every job in `test.yml`/`e2e.yml` now has a `timeout-minutes` bound** (test matrix 15, verify 12, serial 15, slow jobs 12, E2E tier 1 20 / tier 2 30, trivial jobs 5-10). A wedged job is converted from a six-hour zombie (GitHub's default) into a fast, legible failure. +- **`scripts/run-e2e.sh` scrubs operator/agent environment before E2E.** A dev or Conductor shell exporting `CONDUCTOR_*` / `MCP_*` / `GBRAIN_*` config overrides no longer bleeds into E2E child processes (which made "hermetic" E2E non-hermetic and its failures unreproducible across machines). Denylist scrub — `PATH`/`HOME`/`TMPDIR`/`DATABASE_URL` survive; `GBRAIN_HOME` is preserved for the existing HOME isolation. + +### Added +- **`actionlint` workflow** (`rhysd/actionlint`, SHA-pinned) lints all workflow YAML on `.github/workflows/**` changes, catching a malformed workflow / bad action ref / missing-permission bug before it ships a broken pipeline. + +### To take advantage of v0.42.50.0 +Nothing to do — these are CI/test-infra changes that take effect automatically on the next push. Contributors running the suite locally get the same hermetic-E2E env scrub via `bash scripts/run-e2e.sh` (or `bun run ci:local`). + ## [0.42.49.0] - 2026-06-16 **Big embed backfills and syncs now throttle themselves when the database gets busy, so clearing a backlog can't starve the job queue — no more external babysitter scripts.** A naive `gbrain embed --stale` or large `gbrain sync` against a PgBouncer transaction-mode pooler could saturate it and starve the minion supervisor's lock renewals, cascading `lock-renewal-failed` into dead jobs. The field workaround was an external wrapper that SIGSTOP/SIGCONT'd the process off a side-pool latency probe. That approach was blind (the side pool read low latency while the pool that mattered starved), unsafe (SIGSTOP can freeze a process mid-transaction holding locks), and couldn't touch peak pressure. gbrain now does this natively, and better. diff --git a/VERSION b/VERSION index bf317619b..0578db35f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.42.49.0 \ No newline at end of file +0.42.50.0 diff --git a/package.json b/package.json index 4c08fb119..4dd05db81 100644 --- a/package.json +++ b/package.json @@ -143,5 +143,5 @@ "bun": ">=1.3.10" }, "license": "MIT", - "version": "0.42.49.0" + "version": "0.42.50.0" } diff --git a/scripts/run-e2e.sh b/scripts/run-e2e.sh index d945ea2e2..4cb075084 100755 --- a/scripts/run-e2e.sh +++ b/scripts/run-e2e.sh @@ -66,6 +66,24 @@ export HOME="$E2E_TMP_HOME" export GBRAIN_HOME="$E2E_TMP_HOME" mkdir -p "$E2E_TMP_HOME/.gbrain" +# --- Hermetic env scrub: operator/agent context must not bleed into E2E --- +# A dev shell or a Conductor workspace exports CONDUCTOR_*, MCP_*, OPENCLAW_*, +# and GBRAIN_* config overrides (e.g. a stray GBRAIN_BRAIN_ID, GBRAIN_SOURCE, +# GBRAIN_*_THRESHOLD, GBRAIN_SUPERVISOR_PID_FILE) that would silently change +# test behavior — making "hermetic" E2E non-hermetic and its failures +# unreproducible across machines. Drop them before bun starts. This is a +# DENYLIST of operator-context prefixes (not an allowlist rebuild), so PATH, +# HOME, TMPDIR, CI, DATABASE_URL, and bun internals survive untouched. We keep +# GBRAIN_HOME (just set above for HOME isolation); everything else GBRAIN_* is +# an operator override the suite must not inherit. Adapts GStack's +# buildHermeticEnv() allowlist to gbrain's shell E2E runner. +for _e2e_var in $(env | grep -oE '^(CONDUCTOR_|MCP_|OPENCLAW_|GBRAIN_)[A-Za-z0-9_]*' | sort -u); do + case "$_e2e_var" in + GBRAIN_HOME) ;; # required for HOME isolation (set above) — keep + *) unset "$_e2e_var" || true ;; + esac +done + # --dry-run-list: print the resolved file list (one per line) and exit. Used # by scripts/ci-local.sh to smoke-test the argv branching at startup. DRY_RUN_LIST=0