mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
Merge remote-tracking branch 'origin/master' into garrytan/fix-wave-warm-narwhal
# Conflicts: # test/e2e/autopilot-fanout-postgres.test.ts # test/e2e/dream-cycle-phase-order-pglite.test.ts # test/e2e/fresh-install-pglite.test.ts # test/e2e/voyage-multimodal.test.ts
This commit is contained in:
Garry Tan
@@ -20,7 +20,19 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
# Resolution order for the scan root:
|
||||
# 1. $GBRAIN_SCAN_ROOT explicit override — tests pass this so they
|
||||
# don't depend on `git rev-parse` walking up to an unrelated parent
|
||||
# .git/ on filesystems where `git init` silently fails under
|
||||
# shard-concurrency load (v0.40.10 flake-hardening fix).
|
||||
# 2. `git rev-parse --show-toplevel` — production callers from inside
|
||||
# the gbrain repo.
|
||||
# 3. $PWD — last-resort fallback for callers without git.
|
||||
if [ -n "${GBRAIN_SCAN_ROOT:-}" ]; then
|
||||
ROOT="$GBRAIN_SCAN_ROOT"
|
||||
else
|
||||
ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
fi
|
||||
cd "$ROOT"
|
||||
|
||||
# Banned direct-call patterns. Each is a method on BrainEngine that
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* v0.41 E5 A/B harness (D11 + codex pass-2 #7 spec).
|
||||
*
|
||||
* Manually-runnable script that proves the auto-adaptive lease-cap
|
||||
* controller beats fixed-cap on a real upstream. Writes a structured
|
||||
* receipt to test/fixtures/e5-lease-cap-ab/{timestamp}.json — that file
|
||||
* is committed as the baseline. Future controller changes ship with
|
||||
* their own receipt + diff against prior.
|
||||
*
|
||||
* **Spec (D11):**
|
||||
* Workload: 500 subagent jobs, log-normal prompt distribution
|
||||
* (mean 2k tokens, p99 16k tokens). Synthesized via fixture file.
|
||||
* Provider: Anthropic (real API) via gateway.
|
||||
* Cost cap: --budget-usd 8 per arm (D5 enforced).
|
||||
* Failure injection: synthetic 429 burst at minute 15 (10s window).
|
||||
* Statistical threshold: controller arm must beat fixed-cap on
|
||||
* (completed_jobs / wall_clock_time) by ≥5% AND match
|
||||
* (completed_jobs / dollars_spent) within ±2%.
|
||||
*
|
||||
* **Usage:**
|
||||
* ANTHROPIC_API_KEY=sk-... bun scripts/e5-lease-cap-ab.ts
|
||||
* ANTHROPIC_API_KEY=sk-... bun scripts/e5-lease-cap-ab.ts --dry-run
|
||||
*
|
||||
* **Cost:** ~$16 per full run ($8/arm × 2 arms). Approximate; depends on
|
||||
* actual prompt lengths sampled from the fixture.
|
||||
*
|
||||
* **Not in CI:** This script requires a real API key + ~30min wall-clock
|
||||
* + real Anthropic budget. Intended to be run BEFORE landing a controller
|
||||
* change; receipt is the durable artifact. CI gating happens via the
|
||||
* unit-test suite (`lease-cap-controller.test.ts` covers the pure
|
||||
* decision function exhaustively).
|
||||
*/
|
||||
|
||||
import { writeFileSync, mkdirSync, existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { PostgresEngine } from '../src/core/postgres-engine.ts';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { loadConfig } from '../src/core/config.ts';
|
||||
import type { BrainEngine } from '../src/core/engine.ts';
|
||||
|
||||
interface ArmStats {
|
||||
arm: 'fixed' | 'adaptive';
|
||||
jobs_submitted: number;
|
||||
jobs_completed: number;
|
||||
jobs_dead: number;
|
||||
wall_clock_ms: number;
|
||||
total_cost_usd: number;
|
||||
lease_cap_history: number[];
|
||||
bounces: number;
|
||||
upstream_429s: number;
|
||||
}
|
||||
|
||||
interface ABReceipt {
|
||||
schema_version: 1;
|
||||
timestamp: string;
|
||||
spec: {
|
||||
job_count: number;
|
||||
budget_per_arm_usd: number;
|
||||
injection_at_min: number;
|
||||
};
|
||||
arms: ArmStats[];
|
||||
verdict: {
|
||||
throughput_advantage_pct: number;
|
||||
cost_efficiency_delta_pct: number;
|
||||
pr_gate_pass: boolean;
|
||||
note: string;
|
||||
};
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]) {
|
||||
const args = {
|
||||
dryRun: argv.includes('--dry-run'),
|
||||
jobs: 500,
|
||||
budgetUsd: 8,
|
||||
};
|
||||
for (const arg of argv) {
|
||||
if (arg.startsWith('--jobs=')) args.jobs = parseInt(arg.split('=')[1] ?? '500', 10);
|
||||
if (arg.startsWith('--budget-usd=')) args.budgetUsd = parseFloat(arg.split('=')[1] ?? '8');
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
async function openEngine(): Promise<BrainEngine> {
|
||||
const cfg = loadConfig();
|
||||
if (cfg?.database_url) {
|
||||
const engine = new PostgresEngine();
|
||||
await engine.connect({ database_url: cfg.database_url });
|
||||
return engine;
|
||||
}
|
||||
// Fallback: PGLite ephemeral. Real A/B runs should use Postgres so the
|
||||
// lease-cap controller's elected-mutator pattern is exercised cross-process.
|
||||
process.stderr.write('[e5-ab] WARN: using PGLite ephemeral (no DATABASE_URL); cross-worker tests will not run\n');
|
||||
const engine = new PGLiteEngine();
|
||||
await engine.connect({ database_url: '' });
|
||||
await engine.initSchema();
|
||||
return engine;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a synthetic prompt with the spec'd token distribution.
|
||||
* Approximate — uses character counts as a stand-in for tokens (1 token
|
||||
* ≈ 4 chars on English).
|
||||
*/
|
||||
function syntheticPrompt(index: number): string {
|
||||
// Log-normal mean=2k tokens, σ such that p99 = 16k tokens.
|
||||
// log(p99/p50) = z_99 * σ → σ ≈ ln(8) / 2.33 ≈ 0.89
|
||||
const mu = Math.log(2000);
|
||||
const sigma = 0.89;
|
||||
// Box-Muller for deterministic-per-index pseudo-Normal sample.
|
||||
const u1 = ((index * 9301 + 49297) % 233280) / 233280;
|
||||
const u2 = ((index * 13849 + 65521) % 233280) / 233280;
|
||||
const z = Math.sqrt(-2 * Math.log(Math.max(u1, 1e-9))) * Math.cos(2 * Math.PI * u2);
|
||||
const tokens = Math.exp(mu + sigma * z);
|
||||
const chars = Math.max(40, Math.min(64000, Math.floor(tokens * 4)));
|
||||
return `Synthetic A/B test prompt #${index}. Body: ` + 'X'.repeat(chars - 40);
|
||||
}
|
||||
|
||||
async function runArm(
|
||||
engine: BrainEngine,
|
||||
arm: 'fixed' | 'adaptive',
|
||||
opts: { jobs: number; budgetUsd: number; dryRun: boolean },
|
||||
): Promise<ArmStats> {
|
||||
const start = Date.now();
|
||||
const lease_cap_history: number[] = [];
|
||||
let jobs_submitted = 0;
|
||||
let jobs_completed = 0;
|
||||
let jobs_dead = 0;
|
||||
let total_cost_usd = 0;
|
||||
let bounces = 0;
|
||||
let upstream_429s = 0;
|
||||
|
||||
process.stderr.write(`[e5-ab] === arm=${arm} starting ===\n`);
|
||||
|
||||
// Configure the cap policy for this arm.
|
||||
if (arm === 'fixed') {
|
||||
await engine.setConfig('minions.auto_lease_cap', 'false');
|
||||
await engine.setConfig('minions.lease_cap_current', '8');
|
||||
} else {
|
||||
await engine.setConfig('minions.auto_lease_cap', 'true');
|
||||
await engine.setConfig('minions.lease_cap_current', '8');
|
||||
}
|
||||
|
||||
if (opts.dryRun) {
|
||||
process.stderr.write(`[e5-ab] --dry-run: skipping real submission. Would submit ${opts.jobs} jobs.\n`);
|
||||
return {
|
||||
arm,
|
||||
jobs_submitted: opts.jobs,
|
||||
jobs_completed: 0,
|
||||
jobs_dead: 0,
|
||||
wall_clock_ms: Date.now() - start,
|
||||
total_cost_usd: 0,
|
||||
lease_cap_history: [8],
|
||||
bounces: 0,
|
||||
upstream_429s: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Real-run scaffolding lives here. v0.41 ships the spec; the full
|
||||
// dispatcher (queue submit + worker spin-up + 15-min 429 injector +
|
||||
// tick loop) lands in the follow-up wave when the controller has been
|
||||
// exercised manually first. Receipt fixture is committed as a baseline
|
||||
// shape for future runs to diff against.
|
||||
process.stderr.write(`[e5-ab] arm=${arm}: real-run implementation deferred to v0.41.1 follow-up.\n`);
|
||||
process.stderr.write(`[e5-ab] See CHANGELOG.md "v0.41.0.0 → v0.41.1.0 follow-up" for details.\n`);
|
||||
|
||||
return {
|
||||
arm,
|
||||
jobs_submitted,
|
||||
jobs_completed,
|
||||
jobs_dead,
|
||||
wall_clock_ms: Date.now() - start,
|
||||
total_cost_usd,
|
||||
lease_cap_history,
|
||||
bounces,
|
||||
upstream_429s,
|
||||
};
|
||||
}
|
||||
|
||||
function computeVerdict(fixed: ArmStats, adaptive: ArmStats): ABReceipt['verdict'] {
|
||||
// Throughput ratio: completed_jobs / wall_clock_ms. Higher is better.
|
||||
const tputFixed = fixed.wall_clock_ms > 0 ? fixed.jobs_completed / fixed.wall_clock_ms : 0;
|
||||
const tputAdaptive = adaptive.wall_clock_ms > 0 ? adaptive.jobs_completed / adaptive.wall_clock_ms : 0;
|
||||
const throughputAdvantage = tputFixed > 0 ? ((tputAdaptive - tputFixed) / tputFixed) * 100 : 0;
|
||||
|
||||
// Cost efficiency ratio: completed_jobs / dollars. Higher is better.
|
||||
const effFixed = fixed.total_cost_usd > 0 ? fixed.jobs_completed / fixed.total_cost_usd : 0;
|
||||
const effAdaptive = adaptive.total_cost_usd > 0 ? adaptive.jobs_completed / adaptive.total_cost_usd : 0;
|
||||
const costEfficiencyDelta = effFixed > 0 ? ((effAdaptive - effFixed) / effFixed) * 100 : 0;
|
||||
|
||||
// PR gate: adaptive must beat fixed by ≥5% on throughput AND match
|
||||
// within ±2% on cost efficiency.
|
||||
const pr_gate_pass = throughputAdvantage >= 5 && Math.abs(costEfficiencyDelta) <= 2;
|
||||
|
||||
return {
|
||||
throughput_advantage_pct: Math.round(throughputAdvantage * 100) / 100,
|
||||
cost_efficiency_delta_pct: Math.round(costEfficiencyDelta * 100) / 100,
|
||||
pr_gate_pass,
|
||||
note: pr_gate_pass
|
||||
? 'controller beats fixed-cap; safe to default ON'
|
||||
: 'controller does NOT meet PR gate; defaults stay OFF',
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const argv = process.argv.slice(2);
|
||||
const opts = parseArgs(argv);
|
||||
|
||||
const engine = await openEngine();
|
||||
try {
|
||||
const fixed = await runArm(engine, 'fixed', opts);
|
||||
const adaptive = await runArm(engine, 'adaptive', opts);
|
||||
const verdict = computeVerdict(fixed, adaptive);
|
||||
|
||||
const receipt: ABReceipt = {
|
||||
schema_version: 1,
|
||||
timestamp: new Date().toISOString(),
|
||||
spec: {
|
||||
job_count: opts.jobs,
|
||||
budget_per_arm_usd: opts.budgetUsd,
|
||||
injection_at_min: 15,
|
||||
},
|
||||
arms: [fixed, adaptive],
|
||||
verdict,
|
||||
};
|
||||
|
||||
const fixtureDir = join(process.cwd(), 'test/fixtures/e5-lease-cap-ab');
|
||||
if (!existsSync(fixtureDir)) mkdirSync(fixtureDir, { recursive: true });
|
||||
const receiptPath = join(
|
||||
fixtureDir,
|
||||
`${new Date().toISOString().replace(/[:.]/g, '-')}${opts.dryRun ? '-dry-run' : ''}.json`,
|
||||
);
|
||||
writeFileSync(receiptPath, JSON.stringify(receipt, null, 2));
|
||||
process.stderr.write(`[e5-ab] receipt written: ${receiptPath}\n`);
|
||||
process.stderr.write(`[e5-ab] verdict: ${verdict.note}\n`);
|
||||
process.exit(verdict.pr_gate_pass || opts.dryRun ? 0 : 1);
|
||||
} finally {
|
||||
await engine.disconnect().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
main().catch(err => {
|
||||
process.stderr.write(`[e5-ab] FATAL: ${err instanceof Error ? err.message : String(err)}\n`);
|
||||
process.exit(2);
|
||||
});
|
||||
}
|
||||
@@ -17,4 +17,9 @@ if [ "${#slow_files[@]}" -eq 0 ]; then
|
||||
fi
|
||||
|
||||
echo "[run-slow-tests] running ${#slow_files[@]} slow files (CI runs these as part of bun run test)"
|
||||
exec bun test --timeout=60000 "${slow_files[@]}"
|
||||
# v0.40.10 flake-hardening: bump per-test timeout 60s → 120s. Slow tests
|
||||
# legitimately approach 60s in isolation (longmemeval E2E suite is ~50s);
|
||||
# when bun runs slow files in parallel, CPU contention pushes them past
|
||||
# 60s and individual tests timeout even though they'd pass solo. Slow
|
||||
# tests are explicit by-name — generous per-test budget is correct.
|
||||
exec bun test --timeout=120000 "${slow_files[@]}"
|
||||
|
||||
@@ -58,10 +58,27 @@ N="${SHARDS_OVERRIDE:-${SHARDS:-$(detect_cpus)}}"
|
||||
if ! printf '%s' "$N" | grep -qE '^[0-9]+$' || [ "$N" -lt 1 ]; then
|
||||
echo "ERROR: invalid shard count: $N" >&2; exit 2
|
||||
fi
|
||||
# v0.40.10 flake-hardening: clamp default to 4 (was 8) to match CI's
|
||||
# test-shard.sh fan-out. At 8-shard parallel on Apple Silicon we observed
|
||||
# shard 5 SIGKILL during source-health.test.ts's PGLite migration replay —
|
||||
# 8 parallel PGLite WASM inits contend severely on the lockfile, and the
|
||||
# 92-migration replay × 8 simultaneous can wedge past even 900s. CI uses
|
||||
# 4 and is stable. Trade ~2x wallclock for reliability + parity with CI's
|
||||
# fan-out. Override via --shards N or SHARDS=N (still capped at 8).
|
||||
[ "$N" -gt 8 ] && N=8
|
||||
if [ -z "${SHARDS_OVERRIDE:-}" ] && [ -z "${SHARDS:-}" ] && [ "$N" -gt 4 ]; then
|
||||
N=4
|
||||
fi
|
||||
|
||||
INTRA_CONC="${MAX_CONCURRENCY_OVERRIDE:-${GBRAIN_TEST_MAX_CONCURRENCY:-4}}"
|
||||
SHARD_TIMEOUT="${GBRAIN_TEST_SHARD_TIMEOUT:-600}"
|
||||
# v0.40.10 flake-hardening: bump per-shard cap 600 → 1500 (was 900). At
|
||||
# 4-shard default each shard runs 159 files / ~2420 tests with internal
|
||||
# wallclock 960-1020s. The 900s value (sized for 8-shard's ~80 files /
|
||||
# 1100 tests at 620-770s) false-killed shard 1 at 900s even though it
|
||||
# had completed in 968s. 1500s cap gives ~55% headroom over observed
|
||||
# 4-shard wallclock; real hangs still hit it. Override via
|
||||
# GBRAIN_TEST_SHARD_TIMEOUT=N.
|
||||
SHARD_TIMEOUT="${GBRAIN_TEST_SHARD_TIMEOUT:-1500}"
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Output directories. Prefer workspace-local .context/, fall back to /tmp.
|
||||
|
||||
Reference in New Issue
Block a user