mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
v0.41.6.0 feat(ci): CI test speedup — 23min → ~9min via matrix 4→6 + weight-aware sharding + auto SHA cache + parallel verify (#1444)
* feat(ci): scripts/run-verify-parallel.sh — parallel verify dispatcher Fans out the 21 pre-test grep guards via & + wait, captures per-check exit codes in a tempdir, aggregates failures with named check + log tail to stderr on miss. Wallclock 27s sequential → 13s parallel locally (2x). Bigger CI win is shard 1 deload (workflow restructure in a later commit). Pinned by test/scripts/run-verify-parallel.test.ts (6 cases: CLI contract + synthetic dispatcher failure-surfacing). * feat(ci): weight-aware LPT bin-packer + auto SHA cache hash scripts/sharding.ts (NEW) — pure TypeScript LPT bin-packer. Sort weights desc, assign each file to the shard with current minimum total. Worst-case makespan within 4/3 of optimal, O(n log n). Missing weights fall back to corpus median (not 0). New test file → ships immediately without regenerating weights. Pinned by test/scripts/sharding.test.ts (23 cases). scripts/mine-shard-weights.ts (NEW) — scrapes per-file timing from gh run view --log via timestamp delta between ##[group]test/foo.test.ts: headers within a shard. Three input modes: --run <ID>, --from-file <PATH>, stdin. Stable JSON output (sorted keys). Initial weights mined from run 26398061007. Pinned by test/scripts/mine-shard-weights.test.ts (15 cases). scripts/ci-cache-hash.sh (NEW) — deterministic 16-char sha256 over git ls-files -s minus deny-list (CHANGELOG/TODOS/README/LICENSE/ docs/**/*.md). CLAUDE.md, AGENTS.md, skills/**/* deliberately INCLUDED (8+ test files read them; deny-listing would create false-pass holes). ~40ms on 1891 files. Pinned by test/scripts/ci-cache-hash.test.ts (24 cases: 8 CRITICAL false-pass guards + 7 SAFE deny-list invariants + 9 edge cases). scripts/test-weights.json (NEW) — 712 weights. Total 3306s observed runtime; median 30ms; max 6 min outlier. * chore: bump version and changelog (v0.41.6.0) Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
e084457c2d
commit
3a2605e9a0
@@ -0,0 +1,422 @@
|
||||
/**
|
||||
* ci-cache-hash.test.ts — adversarial coverage of the auto-cache hash.
|
||||
*
|
||||
* The cache key controls false-pass risk. If a test-affecting file is
|
||||
* accidentally deny-listed (or a file change fails to invalidate the
|
||||
* hash), broken code ships under a green CI check. This suite is the
|
||||
* primary safety net.
|
||||
*
|
||||
* Strategy: each test spins up a synthetic git repo, copies the
|
||||
* ci-cache-hash.sh script in, populates a small file set covering the
|
||||
* deny-list edges, runs the script, modifies one file, runs again,
|
||||
* asserts hash behavior.
|
||||
*
|
||||
* Critical invariants (the false-pass guards):
|
||||
* - CLAUDE.md edit → DIFFERENT hash
|
||||
* - AGENTS.md edit → DIFFERENT hash
|
||||
* - skills/foo/SKILL.md edit → DIFFERENT hash
|
||||
* - src/core/db.ts edit → DIFFERENT hash
|
||||
* - test/foo.test.ts edit → DIFFERENT hash
|
||||
*
|
||||
* Safe deny-list invariants:
|
||||
* - CHANGELOG.md edit → same hash
|
||||
* - README.md edit → same hash
|
||||
* - docs/guide.md edit → same hash
|
||||
* - TODOS.md edit → same hash
|
||||
* - LICENSE edit → same hash
|
||||
*/
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import {
|
||||
cpSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
rmSync,
|
||||
symlinkSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
const REPO_ROOT = resolve(import.meta.dir, "..", "..");
|
||||
const SCRIPT_SRC = resolve(REPO_ROOT, "scripts/ci-cache-hash.sh");
|
||||
|
||||
interface Sandbox {
|
||||
dir: string;
|
||||
scriptPath: string;
|
||||
}
|
||||
|
||||
function makeSandbox(files: Record<string, string>): Sandbox {
|
||||
const dir = mkdtempSync(join(tmpdir(), "ci-cache-hash-"));
|
||||
// Init git repo
|
||||
execFileSync("git", ["init", "--quiet"], { cwd: dir });
|
||||
execFileSync("git", ["config", "user.email", "test@example.com"], { cwd: dir });
|
||||
execFileSync("git", ["config", "user.name", "Test"], { cwd: dir });
|
||||
|
||||
// Copy script under scripts/ so cd "$(dirname "$0")/.." lands at sandbox root
|
||||
mkdirSync(join(dir, "scripts"), { recursive: true });
|
||||
cpSync(SCRIPT_SRC, join(dir, "scripts/ci-cache-hash.sh"));
|
||||
|
||||
// Write each fixture file under its declared path
|
||||
for (const [path, content] of Object.entries(files)) {
|
||||
const abs = join(dir, path);
|
||||
mkdirSync(resolve(abs, ".."), { recursive: true });
|
||||
writeFileSync(abs, content);
|
||||
}
|
||||
|
||||
// Stage + commit so git ls-files returns them
|
||||
execFileSync("git", ["add", "-A"], { cwd: dir });
|
||||
execFileSync("git", ["commit", "--quiet", "-m", "fixture"], { cwd: dir });
|
||||
|
||||
return { dir, scriptPath: join(dir, "scripts/ci-cache-hash.sh") };
|
||||
}
|
||||
|
||||
function hash(s: Sandbox): string {
|
||||
const r = spawnSync("bash", [s.scriptPath], {
|
||||
cwd: s.dir,
|
||||
encoding: "utf8",
|
||||
});
|
||||
if (r.status !== 0) {
|
||||
throw new Error(`ci-cache-hash exit ${r.status}: ${r.stderr}`);
|
||||
}
|
||||
return r.stdout.trim();
|
||||
}
|
||||
|
||||
function modify(s: Sandbox, path: string, content: string) {
|
||||
writeFileSync(join(s.dir, path), content);
|
||||
// The script reads `git ls-files -s` which reflects the INDEX, not
|
||||
// the working tree. Stage + commit so the change is visible to the
|
||||
// hash. This matches what CI sees on a checked-out PR (committed
|
||||
// tree, not working tree).
|
||||
execFileSync("git", ["add", path], { cwd: s.dir });
|
||||
execFileSync("git", ["commit", "--quiet", "-m", `modify ${path}`], {
|
||||
cwd: s.dir,
|
||||
});
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Baseline fixtures used across most cases.
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
const BASELINE_FILES: Record<string, string> = {
|
||||
"CHANGELOG.md": "# Changelog\n\n## [1.0.0]\n",
|
||||
"TODOS.md": "- [ ] thing\n",
|
||||
"README.md": "# Project\n",
|
||||
"LICENSE": "MIT\n",
|
||||
"CLAUDE.md": "# CLAUDE.md\n\nproject instructions\n",
|
||||
"AGENTS.md": "# AGENTS.md\n\nopenclaw entry\n",
|
||||
"package.json": '{"name":"test","version":"0.0.0"}\n',
|
||||
"bun.lock": "lockfile-v1\n",
|
||||
"tsconfig.json": '{"compilerOptions":{"strict":true}}\n',
|
||||
"src/core/db.ts": "export const db = 1;\n",
|
||||
"src/cli.ts": "console.log('hi');\n",
|
||||
"test/foo.test.ts": "import {test} from 'bun:test';\ntest('x', () => {});\n",
|
||||
"scripts/check-x.sh": "#!/bin/bash\necho ok\n",
|
||||
".github/workflows/test.yml": "name: Test\n",
|
||||
"skills/foo/SKILL.md": "---\nname: foo\n---\nFoo skill\n",
|
||||
"docs/guide.md": "# Guide\n",
|
||||
"docs/sub/notes.md": "Notes\n",
|
||||
"docs/raw.txt": "raw text\n",
|
||||
};
|
||||
|
||||
describe("ci-cache-hash.sh — determinism", () => {
|
||||
let sb: Sandbox;
|
||||
beforeAll(() => {
|
||||
sb = makeSandbox(BASELINE_FILES);
|
||||
});
|
||||
afterAll(() => {
|
||||
if (sb) rmSync(sb.dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("same tree → same hash (run twice)", () => {
|
||||
const h1 = hash(sb);
|
||||
const h2 = hash(sb);
|
||||
expect(h1).toBe(h2);
|
||||
expect(h1).toMatch(/^[0-9a-f]{16}$/);
|
||||
});
|
||||
|
||||
it("hash is 16-char lowercase hex", () => {
|
||||
const h = hash(sb);
|
||||
expect(h).toMatch(/^[0-9a-f]{16}$/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ci-cache-hash.sh — CRITICAL false-pass guards (must invalidate)", () => {
|
||||
// Each test gets its own sandbox so modifications don't leak across cases.
|
||||
|
||||
function withSandbox(test: (sb: Sandbox) => void) {
|
||||
const sb = makeSandbox(BASELINE_FILES);
|
||||
try {
|
||||
test(sb);
|
||||
} finally {
|
||||
rmSync(sb.dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
it("CLAUDE.md edit MUST change hash (8+ test files reference it)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "CLAUDE.md", "# CLAUDE.md\n\nNEW INSTRUCTIONS\n");
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("AGENTS.md edit MUST change hash (resolver tests read it)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "AGENTS.md", "# AGENTS.md\n\nNEW DISPATCH\n");
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("skills/foo/SKILL.md edit MUST change hash (skill conformance tests)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "skills/foo/SKILL.md", "---\nname: foo\n---\nNEW SKILL BODY\n");
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("src/core/db.ts edit MUST change hash", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "src/core/db.ts", "export const db = 2;\n");
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("test/foo.test.ts edit MUST change hash", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(
|
||||
sb,
|
||||
"test/foo.test.ts",
|
||||
"import {test} from 'bun:test';\ntest('y', () => {});\n",
|
||||
);
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("package.json edit MUST change hash", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "package.json", '{"name":"test","version":"0.0.1"}\n');
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("bun.lock edit MUST change hash (dependency drift catches false-pass)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "bun.lock", "lockfile-v2\n");
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it(".github/workflows/test.yml edit MUST change hash (CI shape change is test-affecting)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, ".github/workflows/test.yml", "name: Test\n# changed\n");
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("ci-cache-hash.sh — SAFE deny-list invariants (must NOT invalidate)", () => {
|
||||
function withSandbox(test: (sb: Sandbox) => void) {
|
||||
const sb = makeSandbox(BASELINE_FILES);
|
||||
try {
|
||||
test(sb);
|
||||
} finally {
|
||||
rmSync(sb.dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
it("CHANGELOG.md edit produces SAME hash (deny-listed)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "CHANGELOG.md", "# Changelog\n\n## [1.1.0]\nnew release\n");
|
||||
const after = hash(sb);
|
||||
expect(after).toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("README.md edit produces SAME hash (deny-listed)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "README.md", "# Project\n\nupdated tagline\n");
|
||||
const after = hash(sb);
|
||||
expect(after).toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("TODOS.md edit produces SAME hash (deny-listed)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "TODOS.md", "- [x] thing\n");
|
||||
const after = hash(sb);
|
||||
expect(after).toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("LICENSE edit produces SAME hash (deny-listed)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "LICENSE", "Apache-2.0\n");
|
||||
const after = hash(sb);
|
||||
expect(after).toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("docs/guide.md edit produces SAME hash (deny-listed via docs/**/*.md)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "docs/guide.md", "# Guide v2\n");
|
||||
const after = hash(sb);
|
||||
expect(after).toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("docs/sub/notes.md edit produces SAME hash (nested docs match)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "docs/sub/notes.md", "Updated notes\n");
|
||||
const after = hash(sb);
|
||||
expect(after).toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("docs/raw.txt edit produces SAME hash (deny-listed via docs/**/*.txt)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "docs/raw.txt", "new raw text\n");
|
||||
const after = hash(sb);
|
||||
expect(after).toBe(before);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("ci-cache-hash.sh — edge cases", () => {
|
||||
function withSandbox(
|
||||
files: Record<string, string>,
|
||||
test: (sb: Sandbox) => void,
|
||||
) {
|
||||
const sb = makeSandbox(files);
|
||||
try {
|
||||
test(sb);
|
||||
} finally {
|
||||
rmSync(sb.dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
it("untracked file does NOT affect hash (git ls-files only)", () => {
|
||||
withSandbox(BASELINE_FILES, (sb) => {
|
||||
const before = hash(sb);
|
||||
// Write a new file but don't `git add` it.
|
||||
writeFileSync(join(sb.dir, "src/untracked.ts"), "const x = 1;\n");
|
||||
const after = hash(sb);
|
||||
expect(after).toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("rename detection: same content, new path → DIFFERENT hash", () => {
|
||||
withSandbox(BASELINE_FILES, (sb) => {
|
||||
const before = hash(sb);
|
||||
execFileSync("git", ["mv", "src/cli.ts", "src/main.ts"], { cwd: sb.dir });
|
||||
const after = hash(sb);
|
||||
// Same content, new path. Our hash includes the path, so it changes.
|
||||
// This is correct: a rename can absolutely affect test outcomes
|
||||
// (imports change, file resolution changes, etc.).
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("new file under unknown path (e.g. prompts/x.md) DOES affect hash (deny-list default)", () => {
|
||||
withSandbox(BASELINE_FILES, (sb) => {
|
||||
const before = hash(sb);
|
||||
// Add a NEW file in a path that isn't deny-listed and didn't exist
|
||||
// before. Default behavior: include in hash. Closes the "missed file
|
||||
// type" false-pass class (e.g. someone adds `prompts/*.md` that
|
||||
// tests read).
|
||||
mkdirSync(join(sb.dir, "prompts"), { recursive: true });
|
||||
writeFileSync(join(sb.dir, "prompts/extract-takes.md"), "system prompt\n");
|
||||
execFileSync("git", ["add", "prompts/extract-takes.md"], { cwd: sb.dir });
|
||||
execFileSync("git", ["commit", "--quiet", "-m", "add prompts"], {
|
||||
cwd: sb.dir,
|
||||
});
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("symlink target change affects hash (git hash-object follows symlink contents)", () => {
|
||||
// Note: git stores symlinks as the link target string, so changing
|
||||
// the LINK TARGET changes the git blob sha. Our hash will catch
|
||||
// that. We can't easily test "change of target's content" without
|
||||
// creating an out-of-tree file, so this case pins the link-target
|
||||
// path change.
|
||||
withSandbox(BASELINE_FILES, (sb) => {
|
||||
// Create a symlink and commit.
|
||||
symlinkSync("./db.ts", join(sb.dir, "src/core/db-link.ts"));
|
||||
execFileSync("git", ["add", "src/core/db-link.ts"], { cwd: sb.dir });
|
||||
execFileSync("git", ["commit", "--quiet", "-m", "add link"], {
|
||||
cwd: sb.dir,
|
||||
});
|
||||
const before = hash(sb);
|
||||
// Point the symlink elsewhere — and commit so it lands in the
|
||||
// index that `git ls-files -s` reads.
|
||||
rmSync(join(sb.dir, "src/core/db-link.ts"));
|
||||
symlinkSync("./cli.ts", join(sb.dir, "src/core/db-link.ts"));
|
||||
execFileSync("git", ["add", "src/core/db-link.ts"], { cwd: sb.dir });
|
||||
execFileSync("git", ["commit", "--quiet", "-m", "repoint link"], {
|
||||
cwd: sb.dir,
|
||||
});
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("locale-stable: LC_ALL=de_DE.UTF-8 produces the same hash as default", () => {
|
||||
withSandbox(BASELINE_FILES, (sb) => {
|
||||
const defaultHash = spawnSync("bash", [sb.scriptPath], {
|
||||
cwd: sb.dir,
|
||||
encoding: "utf8",
|
||||
}).stdout.trim();
|
||||
const altLocale = spawnSync("bash", [sb.scriptPath], {
|
||||
cwd: sb.dir,
|
||||
encoding: "utf8",
|
||||
env: { ...process.env, LC_ALL: "de_DE.UTF-8", LANG: "de_DE.UTF-8" },
|
||||
}).stdout.trim();
|
||||
expect(altLocale).toBe(defaultHash);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("ci-cache-hash.sh — usage errors", () => {
|
||||
it("--bogus arg exits 2", () => {
|
||||
const r = spawnSync("bash", [SCRIPT_SRC, "--bogus"], {
|
||||
cwd: REPO_ROOT,
|
||||
encoding: "utf8",
|
||||
});
|
||||
expect(r.status).toBe(2);
|
||||
expect(r.stderr).toContain("usage:");
|
||||
});
|
||||
|
||||
it("--verbose flag works (writes diagnostics to stderr, hash to stdout)", () => {
|
||||
const r = spawnSync("bash", [SCRIPT_SRC, "--verbose"], {
|
||||
cwd: REPO_ROOT,
|
||||
encoding: "utf8",
|
||||
});
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stdout.trim()).toMatch(/^[0-9a-f]{16}$/);
|
||||
expect(r.stderr).toContain("files in hash");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,158 @@
|
||||
// mine-shard-weights.test.ts — pure-function tests for the log parser
|
||||
// and weight extraction. The full pipeline (gh run view → write JSON)
|
||||
// is integration-tested by actually running it once during T4.
|
||||
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import {
|
||||
computeWeights,
|
||||
parseLog,
|
||||
serializeWeights,
|
||||
} from "../../scripts/mine-shard-weights.ts";
|
||||
|
||||
const SAMPLE = `test (1)\tUNKNOWN STEP\t2026-05-25T11:26:40.000000Z ##[group]test/alpha.test.ts:
|
||||
test (1)\tUNKNOWN STEP\t2026-05-25T11:26:41.000000Z (pass) some test [1.00ms]
|
||||
test (1)\tUNKNOWN STEP\t2026-05-25T11:26:43.500000Z ##[group]test/beta.test.ts:
|
||||
test (1)\tUNKNOWN STEP\t2026-05-25T11:26:45.000000Z (pass) another test
|
||||
test (1)\tUNKNOWN STEP\t2026-05-25T11:26:48.000000Z ##[group]test/gamma.test.ts:
|
||||
test (1)\tUNKNOWN STEP\t2026-05-25T11:26:50.000000Z ##[group]test/delta.test.ts:
|
||||
test (2)\tUNKNOWN STEP\t2026-05-25T11:26:42.000000Z ##[group]test/epsilon.test.ts:
|
||||
test (2)\tUNKNOWN STEP\t2026-05-25T11:26:55.000000Z ##[group]test/zeta.test.ts:
|
||||
`;
|
||||
|
||||
describe("parseLog", () => {
|
||||
it("extracts file-start events with timestamps and jobs", () => {
|
||||
const events = parseLog(SAMPLE);
|
||||
expect(events.length).toBe(6);
|
||||
expect(events[0]).toMatchObject({
|
||||
job: "test (1)",
|
||||
file: "test/alpha.test.ts",
|
||||
});
|
||||
expect(events[5]).toMatchObject({
|
||||
job: "test (2)",
|
||||
file: "test/zeta.test.ts",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves stream order", () => {
|
||||
const events = parseLog(SAMPLE);
|
||||
const files = events.map((e) => e.file);
|
||||
expect(files).toEqual([
|
||||
"test/alpha.test.ts",
|
||||
"test/beta.test.ts",
|
||||
"test/gamma.test.ts",
|
||||
"test/delta.test.ts",
|
||||
"test/epsilon.test.ts",
|
||||
"test/zeta.test.ts",
|
||||
]);
|
||||
});
|
||||
|
||||
it("ignores non-group lines", () => {
|
||||
// The sample has (pass) lines mixed in. They should not appear in the
|
||||
// event list; parseLog only emits ##[group]test/X.test.ts: matches.
|
||||
const events = parseLog(SAMPLE);
|
||||
expect(events.every((e) => e.file.startsWith("test/"))).toBe(true);
|
||||
expect(events.every((e) => e.file.endsWith(".test.ts"))).toBe(true);
|
||||
});
|
||||
|
||||
it("handles empty input", () => {
|
||||
expect(parseLog("")).toEqual([]);
|
||||
});
|
||||
|
||||
it("ignores malformed timestamps", () => {
|
||||
const bad = "test (1)\tUNKNOWN\tBAD-TS ##[group]test/x.test.ts:\n";
|
||||
expect(parseLog(bad)).toEqual([]);
|
||||
});
|
||||
|
||||
it("rejects non-test-file groups (e.g. setup-bun action groups)", () => {
|
||||
const noise =
|
||||
"test (1)\tUNKNOWN\t2026-05-25T11:26:40.000Z ##[group]Run actions/checkout\n";
|
||||
expect(parseLog(noise)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("computeWeights", () => {
|
||||
it("computes delta between consecutive file headers within a job", () => {
|
||||
const events = parseLog(SAMPLE);
|
||||
const weights = computeWeights(events);
|
||||
// job test (1): alpha → 3500ms (40s → 43.5s), beta → 4500ms (43.5s → 48s),
|
||||
// gamma → 2000ms (48s → 50s), delta dropped (no successor).
|
||||
expect(weights.get("test/alpha.test.ts")).toBe(3500);
|
||||
expect(weights.get("test/beta.test.ts")).toBe(4500);
|
||||
expect(weights.get("test/gamma.test.ts")).toBe(2000);
|
||||
expect(weights.get("test/delta.test.ts")).toBeUndefined();
|
||||
// job test (2): epsilon → 13000ms (42s → 55s), zeta dropped.
|
||||
expect(weights.get("test/epsilon.test.ts")).toBe(13000);
|
||||
expect(weights.get("test/zeta.test.ts")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not cross job boundaries", () => {
|
||||
// If we naively diffed across jobs we'd get bogus values when
|
||||
// alpha (test 1) was followed by epsilon (test 2) by clock skew.
|
||||
const events = parseLog(SAMPLE);
|
||||
const weights = computeWeights(events);
|
||||
// alpha → beta within same job: 3500ms. Not 2000ms (which would be
|
||||
// alpha's stamp to epsilon's stamp across jobs).
|
||||
expect(weights.get("test/alpha.test.ts")).toBe(3500);
|
||||
});
|
||||
|
||||
it("takes the max when a file appears with multiple deltas", () => {
|
||||
const events = [
|
||||
{ job: "test (1)", timestampMs: 1000, file: "test/x.test.ts" },
|
||||
{ job: "test (1)", timestampMs: 2000, file: "test/y.test.ts" },
|
||||
{ job: "test (2)", timestampMs: 3000, file: "test/x.test.ts" },
|
||||
{ job: "test (2)", timestampMs: 8000, file: "test/y.test.ts" },
|
||||
];
|
||||
const w = computeWeights(events);
|
||||
// x: 1000→2000 (1000ms in job 1), 3000→8000 (5000ms in job 2). Max = 5000.
|
||||
expect(w.get("test/x.test.ts")).toBe(5000);
|
||||
});
|
||||
|
||||
it("skips out-of-order events defensively (negative delta)", () => {
|
||||
const events = [
|
||||
{ job: "test (1)", timestampMs: 5000, file: "test/x.test.ts" },
|
||||
{ job: "test (1)", timestampMs: 3000, file: "test/y.test.ts" },
|
||||
];
|
||||
const w = computeWeights(events);
|
||||
// x → negative delta → skipped. y has no successor → not recorded.
|
||||
expect(w.size).toBe(0);
|
||||
});
|
||||
|
||||
it("empty input → empty map", () => {
|
||||
expect(computeWeights([]).size).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeWeights", () => {
|
||||
it("sorts keys alphabetically for stable diffs", () => {
|
||||
const w = new Map([
|
||||
["test/z.test.ts", 100],
|
||||
["test/a.test.ts", 200],
|
||||
["test/m.test.ts", 50],
|
||||
]);
|
||||
const json = serializeWeights(w);
|
||||
const lines = json.split("\n").filter((l) => l.includes('"test/'));
|
||||
expect(lines[0]).toContain("a.test.ts");
|
||||
expect(lines[1]).toContain("m.test.ts");
|
||||
expect(lines[2]).toContain("z.test.ts");
|
||||
});
|
||||
|
||||
it("ends with a trailing newline (POSIX-friendly diff)", () => {
|
||||
const w = new Map([["test/x.test.ts", 1]]);
|
||||
expect(serializeWeights(w).endsWith("\n")).toBe(true);
|
||||
});
|
||||
|
||||
it("empty map → empty object JSON", () => {
|
||||
expect(serializeWeights(new Map())).toBe("{}\n");
|
||||
});
|
||||
|
||||
it("round-trips: parse our own output", () => {
|
||||
const w = new Map([
|
||||
["test/alpha.test.ts", 100],
|
||||
["test/beta.test.ts", 250],
|
||||
]);
|
||||
const json = serializeWeights(w);
|
||||
const parsed = JSON.parse(json);
|
||||
expect(parsed["test/alpha.test.ts"]).toBe(100);
|
||||
expect(parsed["test/beta.test.ts"]).toBe(250);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,175 @@
|
||||
// run-verify-parallel.test.ts — pin the dispatcher's contract.
|
||||
//
|
||||
// We can't easily fake the 20-check list inside the script (it's a static
|
||||
// array), but we CAN verify:
|
||||
// 1. --dry-list emits one line per check
|
||||
// 2. Unknown args exit 2
|
||||
// 3. The fast-path measurement claim — running it on a clean tree
|
||||
// completes faster than the sequential `&&`-chain would have
|
||||
// 4. A made-up failing check propagates exit 1 with the named check
|
||||
// in the failure report (covered by overriding the CHECKS array
|
||||
// via a sibling temp script)
|
||||
//
|
||||
// (3) is a soft regression guard, not a hard timing assertion (CI runners
|
||||
// vary). (4) is the load-bearing test: failure surfaces with name + log.
|
||||
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { writeFileSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
const SCRIPT = "scripts/run-verify-parallel.sh";
|
||||
|
||||
describe("run-verify-parallel.sh — CLI contract", () => {
|
||||
it("--dry-list emits one line per check, exit 0", () => {
|
||||
const r = spawnSync("bash", [SCRIPT, "--dry-list"], { encoding: "utf8" });
|
||||
expect(r.status).toBe(0);
|
||||
const lines = r.stdout.trim().split("\n").filter(Boolean);
|
||||
// The exact count is allowed to grow; assert > 10 and that every line
|
||||
// looks like a script name (no whitespace, no quotes).
|
||||
expect(lines.length).toBeGreaterThan(10);
|
||||
for (const l of lines) {
|
||||
expect(l).toMatch(/^[a-z][a-z0-9:_-]+$/);
|
||||
}
|
||||
});
|
||||
|
||||
it("includes the load-bearing privacy + jsonb + typecheck checks", () => {
|
||||
const r = spawnSync("bash", [SCRIPT, "--dry-list"], { encoding: "utf8" });
|
||||
expect(r.status).toBe(0);
|
||||
const set = new Set(r.stdout.trim().split("\n"));
|
||||
expect(set.has("check:privacy")).toBe(true);
|
||||
expect(set.has("check:jsonb")).toBe(true);
|
||||
expect(set.has("typecheck")).toBe(true);
|
||||
expect(set.has("check:operations-filter-bypass")).toBe(true);
|
||||
});
|
||||
|
||||
it("unknown arg exits 2 with usage error", () => {
|
||||
const r = spawnSync("bash", [SCRIPT, "--bogus"], { encoding: "utf8" });
|
||||
expect(r.status).toBe(2);
|
||||
expect(r.stderr).toContain("unknown arg");
|
||||
expect(r.stderr).toContain("usage:");
|
||||
});
|
||||
});
|
||||
|
||||
describe("run-verify-parallel.sh — failure surfacing (synthetic dispatcher)", () => {
|
||||
// We can't inject a fake check into the real script without touching the
|
||||
// CHECKS array. Instead, we write a SMALLER synthetic dispatcher that
|
||||
// shares the same shape (background jobs, exit-file aggregation, log
|
||||
// tail on failure) and assert the surfacing contract on it. This proves
|
||||
// the design, leaving the real script tested via CLI contract above.
|
||||
//
|
||||
// Each "check" is written as its own executable file so bash array
|
||||
// expansion doesn't try to word-split shell strings (the natural
|
||||
// failure mode of expressing checks as bash command strings is that
|
||||
// spaces and quotes don't survive `"${CHECKS[@]}"`).
|
||||
|
||||
function writeSynth(checkBodies: { name: string; body: string }[]): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "verify-synth-"));
|
||||
const checkPaths: string[] = [];
|
||||
for (const c of checkBodies) {
|
||||
const p = `${dir}/${c.name}.sh`;
|
||||
writeFileSync(p, `#!/usr/bin/env bash\n${c.body}\n`, { mode: 0o755 });
|
||||
checkPaths.push(p);
|
||||
}
|
||||
const dispatcher = `${dir}/dispatch.sh`;
|
||||
// CHECKS is an array of absolute paths to executable scripts.
|
||||
const arrLit = checkPaths.map((p) => `"${p}"`).join(" ");
|
||||
writeFileSync(
|
||||
dispatcher,
|
||||
`#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
LOG_DIR=$(mktemp -d /tmp/verify-synth-log-XXXXXX)
|
||||
trap 'rm -rf "$LOG_DIR"' EXIT
|
||||
CHECKS=(${arrLit})
|
||||
PIDS=()
|
||||
NAMES=()
|
||||
for c in "\${CHECKS[@]}"; do
|
||||
base=$(basename "$c" .sh)
|
||||
NAMES+=("$base")
|
||||
(
|
||||
"$c" > "$LOG_DIR/$base.log" 2>&1
|
||||
echo $? > "$LOG_DIR/$base.exit"
|
||||
) &
|
||||
PIDS+=($!)
|
||||
done
|
||||
for p in "\${PIDS[@]}"; do wait "$p" 2>/dev/null || true; done
|
||||
FAIL=0
|
||||
FAILED_NAMES=""
|
||||
for i in "\${!NAMES[@]}"; do
|
||||
n="\${NAMES[$i]}"
|
||||
rc=$(cat "$LOG_DIR/$n.exit")
|
||||
if [ "$rc" != "0" ]; then
|
||||
FAIL=$((FAIL+1))
|
||||
FAILED_NAMES="$FAILED_NAMES $n"
|
||||
echo "--- $n (rc=$rc) ---" >&2
|
||||
tail -10 "$LOG_DIR/$n.log" >&2
|
||||
fi
|
||||
done
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
echo "Failed:$FAILED_NAMES" >&2
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
return dispatcher;
|
||||
}
|
||||
|
||||
function cleanup(dispatcher: string) {
|
||||
rmSync(dispatcher.replace(/\/dispatch\.sh$/, ""), { recursive: true, force: true });
|
||||
}
|
||||
|
||||
it("all-pass: exit 0, no failure block", () => {
|
||||
const d = writeSynth([
|
||||
{ name: "alpha", body: "exit 0" },
|
||||
{ name: "beta", body: "exit 0" },
|
||||
{ name: "gamma", body: "exit 0" },
|
||||
]);
|
||||
try {
|
||||
const r = spawnSync("bash", [d], { encoding: "utf8" });
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stderr).not.toContain("Failed:");
|
||||
} finally {
|
||||
cleanup(d);
|
||||
}
|
||||
});
|
||||
|
||||
it("one-fails: exit 1, failure block names the check + shows log tail", () => {
|
||||
const d = writeSynth([
|
||||
{ name: "alpha", body: "exit 0" },
|
||||
{
|
||||
name: "beta",
|
||||
body: 'echo "synthetic failure detail line" >&2\nexit 7',
|
||||
},
|
||||
{ name: "gamma", body: "exit 0" },
|
||||
]);
|
||||
try {
|
||||
const r = spawnSync("bash", [d], { encoding: "utf8" });
|
||||
expect(r.status).toBe(1);
|
||||
expect(r.stderr).toContain("Failed: beta");
|
||||
expect(r.stderr).toContain("synthetic failure detail line");
|
||||
expect(r.stderr).toMatch(/--- beta \(rc=7\) ---/);
|
||||
} finally {
|
||||
cleanup(d);
|
||||
}
|
||||
});
|
||||
|
||||
it("two-fail: both names appear in the Failed: list", () => {
|
||||
const d = writeSynth([
|
||||
{ name: "alpha", body: "exit 0" },
|
||||
{ name: "beta", body: "echo err-A >&2\nexit 1" },
|
||||
{ name: "gamma", body: "echo err-B >&2\nexit 2" },
|
||||
]);
|
||||
try {
|
||||
const r = spawnSync("bash", [d], { encoding: "utf8" });
|
||||
expect(r.status).toBe(1);
|
||||
expect(r.stderr).toContain("err-A");
|
||||
expect(r.stderr).toContain("err-B");
|
||||
expect(r.stderr).toMatch(/Failed:\s+beta\s+gamma/);
|
||||
} finally {
|
||||
cleanup(d);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,266 @@
|
||||
// sharding.test.ts — pure unit tests for the LPT bin-packer.
|
||||
//
|
||||
// Covers: happy path, fallback semantics, full coverage, determinism,
|
||||
// balance ratio, N=1 trivial, weight-equal-to-zero handling, malformed
|
||||
// weights, missing weights file fail-soft.
|
||||
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import {
|
||||
computeMedian,
|
||||
imbalanceRatio,
|
||||
loadWeights,
|
||||
partition,
|
||||
WeightsLoadError,
|
||||
type WeightMap,
|
||||
} from "../../scripts/sharding.ts";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
function tempJson(content: string): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "weights-test-"));
|
||||
const path = join(dir, "weights.json");
|
||||
writeFileSync(path, content, "utf8");
|
||||
return path;
|
||||
}
|
||||
|
||||
describe("computeMedian", () => {
|
||||
it("returns 0 on empty input", () => {
|
||||
expect(computeMedian([])).toBe(0);
|
||||
});
|
||||
it("single value is itself", () => {
|
||||
expect(computeMedian([42])).toBe(42);
|
||||
});
|
||||
it("odd count: middle value", () => {
|
||||
expect(computeMedian([1, 5, 3])).toBe(3);
|
||||
});
|
||||
it("even count: mean of middle two", () => {
|
||||
expect(computeMedian([1, 2, 3, 4])).toBe(2.5);
|
||||
});
|
||||
});
|
||||
|
||||
describe("partition — happy path", () => {
|
||||
it("balances 4 known-weight files across 2 shards", () => {
|
||||
const weights: WeightMap = new Map([
|
||||
["a.test.ts", 100],
|
||||
["b.test.ts", 100],
|
||||
["c.test.ts", 50],
|
||||
["d.test.ts", 50],
|
||||
]);
|
||||
const out = partition(
|
||||
["a.test.ts", "b.test.ts", "c.test.ts", "d.test.ts"],
|
||||
weights,
|
||||
2,
|
||||
);
|
||||
expect(out.length).toBe(2);
|
||||
// LPT assigns 100→s0, 100→s1, 50→s0 (now 150), 50→s1 (now 150).
|
||||
// Wait: 100→s0 (s0=100), 100→s1 (s1=100), tie → s0 gets 50 (s0=150),
|
||||
// then s1 gets 50 (s1=150). Balanced.
|
||||
const totals = out.map((s) =>
|
||||
s.reduce((acc, f) => acc + (weights.get(f) ?? 0), 0),
|
||||
);
|
||||
expect(totals[0]).toBe(150);
|
||||
expect(totals[1]).toBe(150);
|
||||
});
|
||||
|
||||
it("LPT prefers heavy-first assignment", () => {
|
||||
// 3 files: 100, 10, 10. 2 shards. Naive assignment (alpha) would put
|
||||
// 100 + 10 = 110 in s0 and 10 in s1 (imbalance 11x). LPT puts 100 in
|
||||
// s0, then 10 + 10 in s1 (totals 100 + 20 = imbalance 5x).
|
||||
const weights: WeightMap = new Map([
|
||||
["big.test.ts", 100],
|
||||
["small1.test.ts", 10],
|
||||
["small2.test.ts", 10],
|
||||
]);
|
||||
const out = partition(
|
||||
["big.test.ts", "small1.test.ts", "small2.test.ts"],
|
||||
weights,
|
||||
2,
|
||||
);
|
||||
expect(out[0]).toEqual(["big.test.ts"]);
|
||||
expect(out[1]?.sort()).toEqual(["small1.test.ts", "small2.test.ts"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("partition — fallback semantics", () => {
|
||||
it("missing weights default to corpus median", () => {
|
||||
// weights = {a:100, b:50}, median = 75. Files c + d are unknown → 75 each.
|
||||
const weights: WeightMap = new Map([
|
||||
["a", 100],
|
||||
["b", 50],
|
||||
]);
|
||||
const out = partition(["a", "b", "c", "d"], weights, 2);
|
||||
// Effective weights: a=100, b=50, c=75, d=75. LPT: 100→s0, 75→s1 (c),
|
||||
// 75→s1 (d, ties broken alpha)... actually: 100→s0=100, 75→s1=75,
|
||||
// 75→s1 vs s0 → s1=150, 50→s0=150. Balanced 150/150.
|
||||
const totalsEffective = out.map((s) =>
|
||||
s.reduce((acc, f) => acc + (weights.get(f) ?? 75), 0),
|
||||
);
|
||||
expect(totalsEffective[0]).toBe(totalsEffective[1]);
|
||||
});
|
||||
|
||||
it("explicit fallback override beats median", () => {
|
||||
const weights: WeightMap = new Map([["a", 100]]);
|
||||
const out = partition(["a", "unknown"], weights, 2, { fallbackWeight: 500 });
|
||||
// a=100 to s0, unknown=500 to s1 (heavier goes first actually — sort
|
||||
// desc puts unknown=500 first). Wait, LPT sorts desc, so unknown
|
||||
// (500) goes to s0 first, then a (100) goes to s1.
|
||||
expect(out[0]).toEqual(["unknown"]);
|
||||
expect(out[1]).toEqual(["a"]);
|
||||
});
|
||||
|
||||
it("zero-weight files are valid (just always go to current min shard)", () => {
|
||||
const weights: WeightMap = new Map([
|
||||
["zero1", 0],
|
||||
["zero2", 0],
|
||||
["big", 100],
|
||||
]);
|
||||
const out = partition(["zero1", "zero2", "big"], weights, 2);
|
||||
// 100→s0=100, 0→s1=0, 0→s1=0. Both zeroes go to s1.
|
||||
expect(out[0]).toEqual(["big"]);
|
||||
expect(out[1]?.sort()).toEqual(["zero1", "zero2"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("partition — invariants", () => {
|
||||
it("every input file lands in exactly one shard (full coverage)", () => {
|
||||
const files = Array.from({ length: 25 }, (_, i) => `f${i}.test.ts`);
|
||||
const weights: WeightMap = new Map(
|
||||
files.map((f, i) => [f, (i % 5) * 10 + 5]),
|
||||
);
|
||||
const out = partition(files, weights, 4);
|
||||
const seen: string[] = [];
|
||||
for (const shard of out) seen.push(...shard);
|
||||
expect(seen.sort()).toEqual([...files].sort());
|
||||
expect(new Set(seen).size).toBe(files.length);
|
||||
});
|
||||
|
||||
it("deterministic — same input always produces same output", () => {
|
||||
const files = ["d.test.ts", "a.test.ts", "c.test.ts", "b.test.ts"];
|
||||
const weights: WeightMap = new Map([
|
||||
["a.test.ts", 30],
|
||||
["b.test.ts", 30],
|
||||
["c.test.ts", 30],
|
||||
["d.test.ts", 30],
|
||||
]);
|
||||
const r1 = partition(files, weights, 2);
|
||||
const r2 = partition([...files].reverse(), weights, 2);
|
||||
const r3 = partition(files, weights, 2);
|
||||
// Same input → identical output
|
||||
expect(r3).toEqual(r1);
|
||||
// Input order shuffled but contents identical → identical output
|
||||
// (ties broken by path asc so order is canonical regardless of input)
|
||||
expect(r2).toEqual(r1);
|
||||
});
|
||||
|
||||
it("N=1 trivial: everything in one shard", () => {
|
||||
const out = partition(["x.test.ts", "y.test.ts"], new Map(), 1);
|
||||
expect(out.length).toBe(1);
|
||||
expect(out[0]?.sort()).toEqual(["x.test.ts", "y.test.ts"]);
|
||||
});
|
||||
|
||||
it("empty file list returns N empty shards", () => {
|
||||
const out = partition([], new Map(), 3);
|
||||
expect(out).toEqual([[], [], []]);
|
||||
});
|
||||
|
||||
it("invalid shard count throws RangeError", () => {
|
||||
expect(() => partition(["a"], new Map(), 0)).toThrow(RangeError);
|
||||
expect(() => partition(["a"], new Map(), -1)).toThrow(RangeError);
|
||||
expect(() => partition(["a"], new Map(), 1.5)).toThrow(RangeError);
|
||||
});
|
||||
|
||||
it("invalid fallbackWeight throws RangeError", () => {
|
||||
expect(() =>
|
||||
partition(["a"], new Map(), 2, { fallbackWeight: -1 }),
|
||||
).toThrow(RangeError);
|
||||
expect(() =>
|
||||
partition(["a"], new Map(), 2, { fallbackWeight: NaN }),
|
||||
).toThrow(RangeError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("partition — balance quality", () => {
|
||||
it("synthetic skewed corpus produces imbalance ratio ≤ 1.5", () => {
|
||||
// 100 files, weights drawn from a Zipf-ish distribution (a few heavy,
|
||||
// many light). This is the shape that broke FNV-1a hash sharding.
|
||||
const files = Array.from({ length: 100 }, (_, i) => `f${i}.test.ts`);
|
||||
const weights: WeightMap = new Map();
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
// Heavy tail: file 0 = 1000ms, file 1 = 500, file 2 = 333, ...
|
||||
// Average about 50ms; max about 1000ms.
|
||||
const w = Math.max(10, Math.floor(1000 / (i + 1)));
|
||||
weights.set(files[i]!, w);
|
||||
}
|
||||
const out = partition(files, weights, 6);
|
||||
const ratio = imbalanceRatio(out, weights, 0);
|
||||
expect(ratio).toBeLessThanOrEqual(1.5);
|
||||
});
|
||||
|
||||
it("imbalance ratio of 1.0 when weights are perfectly divisible", () => {
|
||||
const files = ["a", "b", "c", "d"];
|
||||
const weights: WeightMap = new Map([
|
||||
["a", 50],
|
||||
["b", 50],
|
||||
["c", 50],
|
||||
["d", 50],
|
||||
]);
|
||||
const out = partition(files, weights, 2);
|
||||
expect(imbalanceRatio(out, weights, 0)).toBe(1.0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("loadWeights", () => {
|
||||
it("missing file → empty map (fail-soft)", () => {
|
||||
const w = loadWeights("/tmp/definitely-does-not-exist-weights.json");
|
||||
expect(w.size).toBe(0);
|
||||
});
|
||||
|
||||
it("malformed JSON → throws WeightsLoadError", () => {
|
||||
const p = tempJson("not json {");
|
||||
try {
|
||||
expect(() => loadWeights(p)).toThrow(WeightsLoadError);
|
||||
} finally {
|
||||
rmSync(p.replace(/\/weights\.json$/, ""), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("array JSON → throws (wrong shape)", () => {
|
||||
const p = tempJson("[1,2,3]");
|
||||
try {
|
||||
expect(() => loadWeights(p)).toThrow(/expected top-level object/);
|
||||
} finally {
|
||||
rmSync(p.replace(/\/weights\.json$/, ""), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("negative weight → throws (semantic invalid)", () => {
|
||||
const p = tempJson('{"foo.test.ts": -5}');
|
||||
try {
|
||||
expect(() => loadWeights(p)).toThrow(/non-negative finite/);
|
||||
} finally {
|
||||
rmSync(p.replace(/\/weights\.json$/, ""), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("non-number weight → throws (semantic invalid)", () => {
|
||||
const p = tempJson('{"foo.test.ts": "100ms"}');
|
||||
try {
|
||||
expect(() => loadWeights(p)).toThrow(/non-negative finite/);
|
||||
} finally {
|
||||
rmSync(p.replace(/\/weights\.json$/, ""), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("valid weights JSON round-trips", () => {
|
||||
const p = tempJson('{"a.test.ts": 100, "b.test.ts": 250}');
|
||||
try {
|
||||
const w = loadWeights(p);
|
||||
expect(w.size).toBe(2);
|
||||
expect(w.get("a.test.ts")).toBe(100);
|
||||
expect(w.get("b.test.ts")).toBe(250);
|
||||
} finally {
|
||||
rmSync(p.replace(/\/weights\.json$/, ""), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,18 +1,30 @@
|
||||
/**
|
||||
* Regression test: scripts/test-shard.sh exclusion symmetry.
|
||||
* Regression test: scripts/test-shard.sh exclusion + balance contract.
|
||||
*
|
||||
* Pins the contract that CI's hash-bucketed shard script EXCLUDES
|
||||
* *.serial.test.ts and test/e2e/ from every shard. Serial files share
|
||||
* file-wide state (top-level mock.module, module singletons) that leaks
|
||||
* across files in the same `bun test` shard process. Before v0.31.4.1
|
||||
* they were hashed into the same buckets as parallel files, which broke
|
||||
* the quarantine — `eval-takes-quality-runner.serial.test.ts` stubbed
|
||||
* `gateway.ts` and broke every `gateway.embedMultimodal` test in
|
||||
* `voyage-multimodal.test.ts` on shard 2.
|
||||
* Pins two invariants of the CI matrix shard script:
|
||||
*
|
||||
* Without this guard, a future refactor that drops the `-not -name
|
||||
* '*.serial.test.ts'` clause from test-shard.sh would silently undo the
|
||||
* fix and re-introduce the contention flake.
|
||||
* 1. EXCLUDE *.serial.test.ts and test/e2e/ from every shard. Serial
|
||||
* files share file-wide state (top-level mock.module, module
|
||||
* singletons) that leaks across files in the same `bun test` shard
|
||||
* process. Before v0.31.4.1 they were hashed into the same buckets
|
||||
* as parallel files, which broke the quarantine —
|
||||
* `eval-takes-quality-runner.serial.test.ts` stubbed `gateway.ts`
|
||||
* and broke every `gateway.embedMultimodal` test in
|
||||
* `voyage-multimodal.test.ts` on shard 2.
|
||||
*
|
||||
* 2. INCLUDE *.slow.test.ts. CI is the only default place slow files
|
||||
* run; the local fast loop excludes them via run-unit-shard.sh. See
|
||||
* CLAUDE.md "CI vs local: intentionally divergent file sets".
|
||||
*
|
||||
* 3. LPT balance: file counts across shards are within 5% of each other
|
||||
* under the default (no-weights / cold-start) configuration. Once
|
||||
* test-weights.json is populated, weighted balance also lands the
|
||||
* imbalance ratio ≤ 1.5.
|
||||
*
|
||||
* Without (1), the v0.31.4.1 mock.module leak comes back. Without (2),
|
||||
* slow tests stop running in CI (silent regression — they don't fail,
|
||||
* they just never execute). Without (3), shard 3 will drift back to
|
||||
* 26-min p100 wallclock.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll } from 'bun:test';
|
||||
@@ -22,30 +34,27 @@ import { resolve } from 'path';
|
||||
const REPO_ROOT = resolve(import.meta.dir, '..', '..');
|
||||
const SHARD_SH = resolve(REPO_ROOT, 'scripts/test-shard.sh');
|
||||
|
||||
// Pure-bash FNV-1a per shard takes ~4s on a M-series Mac because the script
|
||||
// computes a hash for every test file. Compute all 4 shards once in beforeAll
|
||||
// and reuse across cases so the suite finishes in one shell-out per shard.
|
||||
const shardCache: Record<number, string[]> = {};
|
||||
// LPT partition via sharding.ts is ~30ms per shard (much faster than the
|
||||
// pure-bash FNV-1a it replaced). Cache results so repeated assertions
|
||||
// don't shell out per case.
|
||||
const shardCache: Record<string, string[]> = {};
|
||||
|
||||
function dryRunList(shard: number, total: number): string[] {
|
||||
if (shardCache[shard]) return shardCache[shard];
|
||||
const key = `${shard}/${total}`;
|
||||
if (shardCache[key]) return shardCache[key];
|
||||
const out = execFileSync('bash', [SHARD_SH, '--dry-run-list', String(shard), String(total)], {
|
||||
cwd: REPO_ROOT,
|
||||
encoding: 'utf-8',
|
||||
});
|
||||
shardCache[shard] = out.split('\n').map(s => s.trim()).filter(Boolean);
|
||||
return shardCache[shard];
|
||||
shardCache[key] = out.split('\n').map(s => s.trim()).filter(Boolean);
|
||||
return shardCache[key];
|
||||
}
|
||||
|
||||
describe('test-shard.sh exclusion symmetry', () => {
|
||||
describe('test-shard.sh — exclusion contract', () => {
|
||||
beforeAll(() => {
|
||||
for (const shard of [1, 2, 3, 4]) dryRunList(shard, 4);
|
||||
// v0.40.10 flake-hardening: bump beforeAll budget 60s → 180s. Each
|
||||
// FNV-1a dry-run shells out and computes a hash for every test file
|
||||
// (~4s solo). Under slow-shard concurrency (longmemeval E2E at ~50s
|
||||
// hogging CPU), the 4 sequential shell-outs slip past 60s and time
|
||||
// out even though they'd complete fine solo.
|
||||
}, 180_000);
|
||||
}, 60_000);
|
||||
|
||||
it('includes plain *.test.ts files in at least one shard', () => {
|
||||
const allFiles = [1, 2, 3, 4].flatMap(s => dryRunList(s, 4));
|
||||
expect(allFiles.length).toBeGreaterThan(0);
|
||||
@@ -68,7 +77,17 @@ describe('test-shard.sh exclusion symmetry', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('partitions plain files across shards without overlap', () => {
|
||||
it('INCLUDES *.slow.test.ts files (CI matrix is where slow files run)', () => {
|
||||
const allFiles = [1, 2, 3, 4].flatMap(s => dryRunList(s, 4));
|
||||
const slowFiles = allFiles.filter(f => /\.slow\.test\.ts$/.test(f));
|
||||
// The repo currently has 3 slow files — pin >= 1 so the test stays
|
||||
// green if the count changes but fails loud if slow files vanish
|
||||
// entirely (regression: someone re-added -not -name '*.slow.test.ts'
|
||||
// to the find clause).
|
||||
expect(slowFiles.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('partitions every file across shards without overlap', () => {
|
||||
const seen = new Map<string, number>();
|
||||
for (const shard of [1, 2, 3, 4]) {
|
||||
for (const f of dryRunList(shard, 4)) {
|
||||
@@ -81,3 +100,70 @@ describe('test-shard.sh exclusion symmetry', () => {
|
||||
expect(seen.size).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('test-shard.sh — LPT balance contract', () => {
|
||||
// LPT balances WALLCLOCK (sum of weights), not file count. When real
|
||||
// weights are loaded from scripts/test-weights.json, shards with
|
||||
// heavier files have FEWER files — that's the optimization working.
|
||||
// We assert wallclock balance via the imbalance ratio (≤1.5).
|
||||
|
||||
// Read weights from the committed JSON. If it doesn't exist yet (early
|
||||
// in the wave) or is empty, we degrade to cold-start round-robin and
|
||||
// assert file-count balance instead.
|
||||
let weightsMap = new Map<string, number>();
|
||||
let weightsLoaded = false;
|
||||
beforeAll(() => {
|
||||
try {
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const p = path.resolve(REPO_ROOT, 'scripts/test-weights.json');
|
||||
if (fs.existsSync(p)) {
|
||||
const raw = JSON.parse(fs.readFileSync(p, 'utf8'));
|
||||
for (const [k, v] of Object.entries(raw)) {
|
||||
if (typeof v === 'number' && Number.isFinite(v)) weightsMap.set(k, v);
|
||||
}
|
||||
weightsLoaded = weightsMap.size > 0;
|
||||
}
|
||||
} catch {
|
||||
// fall through to cold-start mode
|
||||
}
|
||||
});
|
||||
|
||||
function totalsFor(shards: string[][]): number[] {
|
||||
// Use 30ms as the cold-start fallback (matches mine-shard-weights
|
||||
// median observation). When weights are loaded, missing files get
|
||||
// the corpus median anyway via sharding.ts.
|
||||
const fallback = weightsLoaded
|
||||
? Array.from(weightsMap.values()).sort((a, b) => a - b)[
|
||||
Math.floor(weightsMap.size / 2)
|
||||
] ?? 30
|
||||
: 1;
|
||||
return shards.map((s) =>
|
||||
s.reduce((acc, f) => acc + (weightsMap.get(f) ?? fallback), 0),
|
||||
);
|
||||
}
|
||||
|
||||
it('4-shard wallclock imbalance ratio ≤ 1.5', () => {
|
||||
const shards = [1, 2, 3, 4].map(s => dryRunList(s, 4));
|
||||
for (const s of shards) expect(s.length).toBeGreaterThan(0);
|
||||
const totals = totalsFor(shards);
|
||||
const ratio = Math.max(...totals) / Math.min(...totals);
|
||||
expect(ratio).toBeLessThanOrEqual(1.5);
|
||||
});
|
||||
|
||||
it('6-shard wallclock imbalance ratio ≤ 1.5', () => {
|
||||
const shards = [1, 2, 3, 4, 5, 6].map(s => dryRunList(s, 6));
|
||||
for (const s of shards) expect(s.length).toBeGreaterThan(0);
|
||||
const totals = totalsFor(shards);
|
||||
const ratio = Math.max(...totals) / Math.min(...totals);
|
||||
expect(ratio).toBeLessThanOrEqual(1.5);
|
||||
});
|
||||
|
||||
it('6-shard partition is deterministic across runs', () => {
|
||||
// Clear cache + re-run for one shard, compare to the cached result.
|
||||
const cached = [...dryRunList(1, 6)];
|
||||
delete shardCache['1/6'];
|
||||
const fresh = dryRunList(1, 6);
|
||||
expect(fresh).toEqual(cached);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user