From 0887dcc958ba6c143f39765e9eafd59d07c2df37 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 24 May 2026 00:32:31 -0700 Subject: [PATCH] v0.41: doctor subagent_health + jobs stats lease_pressure line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator visibility for the v0.41 Bug 2 audit data. src/commands/doctor.ts checkSubagentHealth(engine) — new exported check function. Reads the last 24h of minion_lease_pressure_log and classifies by bounce volume + forward progress: 0 bounces → ok 1-99 bounces → ok ("transient") 100+ bounces + subagent jobs completing → ok ("healthy backpressure") 100+ bounces + NO completed subagent jobs → warn (paste-ready hint) 1000+ bounces → fail (blocking) Warn/fail messages embed `export GBRAIN_ANTHROPIC_MAX_INFLIGHT=64` for copy-paste. Pre-v93 brains (no table) silently skip with OK. Works on both Postgres + PGLite. src/commands/jobs.ts (case 'stats') Adds `Lease pressure (1h)` line to the stats output. When >0 bounces, cross-checks completed subagent count and surfaces the same binding-but-healthy vs cap-too-tight distinction inline so operators don't have to run `gbrain doctor` to see it. Pre-v93 silent skip. test/doctor-subagent-health.test.ts (NEW) 4 cases pinning all threshold bands. Uses `allowProtectedSubmit: true` on the queue.add for `subagent`-named owner jobs. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/commands/doctor.ts | 74 ++++++++++++++++ src/commands/jobs.ts | 31 +++++++ test/doctor-subagent-health.test.ts | 125 ++++++++++++++++++++++++++++ 3 files changed, 230 insertions(+) create mode 100644 test/doctor-subagent-health.test.ts diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 2594935ed..caadfd77d 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -563,6 +563,9 @@ export async function doctorReportRemote(engine: BrainEngine): Promise { + try { + const bounceRows = await engine.executeRaw<{ count: string }>( + `SELECT count(*)::text AS count FROM minion_lease_pressure_log + WHERE bounced_at > now() - interval '24 hours'`, + ); + const bounces = parseInt(bounceRows[0]?.count ?? '0', 10); + if (bounces === 0) { + return { + name: 'subagent_health', + status: 'ok', + message: 'No rate-lease pressure in last 24h', + }; + } + if (bounces >= 1000) { + return { + name: 'subagent_health', + status: 'fail', + message: `${bounces} lease-pressure bounces in last 24h — this is blocking real work. Raise the cap: \`export GBRAIN_ANTHROPIC_MAX_INFLIGHT=64\` (or \`unlimited\` for Azure / Bedrock / self-hosted upstreams with no provider-side rate limit). After raising, restart \`gbrain jobs work\`.`, + }; + } + // 1-999 bounces: cross-check forward progress. + const completedRows = await engine.executeRaw<{ count: string }>( + `SELECT count(*)::text AS count FROM minion_jobs + WHERE finished_at > now() - interval '24 hours' + AND status = 'completed' + AND name = 'subagent'`, + ).catch(() => [{ count: '0' }]); + const completed = parseInt(completedRows[0]?.count ?? '0', 10); + if (bounces >= 100 && completed === 0) { + return { + name: 'subagent_health', + status: 'warn', + message: `${bounces} lease-pressure bounces in last 24h with no completed subagent jobs — cap is too tight. Raise via \`export GBRAIN_ANTHROPIC_MAX_INFLIGHT=64\` (or \`unlimited\` for upstreams with no provider-side cap).`, + }; + } + return { + name: 'subagent_health', + status: 'ok', + message: `Lease pressure: ${bounces} bounces in last 24h, ${completed} subagent jobs completed — backpressure is binding but throughput is healthy`, + }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + if (process.env.GBRAIN_DEBUG === '1') { + process.stderr.write(`[doctor] subagent_health skipped: ${msg}\n`); + } + return { + name: 'subagent_health', + status: 'ok', + message: 'Skipped (minion_lease_pressure_log unavailable — pre-v0.41 brain)', + }; + } +} + export async function checkVoiceGateHealth(engine: BrainEngine): Promise { try { const rows = await engine.executeRaw<{ total: number; failures: number }>( diff --git a/src/commands/jobs.ts b/src/commands/jobs.ts index 880ed0fb5..e6d938605 100644 --- a/src/commands/jobs.ts +++ b/src/commands/jobs.ts @@ -545,6 +545,37 @@ HANDLER TYPES (built in) console.log(' No jobs in the last 24 hours.'); } console.log(`\n Queue health: ${stats.queue_health.waiting} waiting, ${stats.queue_health.active} active, ${stats.queue_health.stalled} stalled`); + + // v0.41 Bug 2 / Eng D8 — surface lease pressure to the operator. + // Reads minion_lease_pressure_log windowed at 1h. Best-effort: pre-v93 + // brains (no table) silently skip; the queue_health line above is the + // operator's primary signal in that case. + try { + const lpRows = await engine.executeRaw<{ count: string }>( + `SELECT count(*)::text AS count FROM minion_lease_pressure_log + WHERE bounced_at > now() - interval '1 hour'`, + ); + const lpCount = parseInt(lpRows[0]?.count ?? '0', 10); + if (lpCount > 0) { + // Also surface whether any of those bounces stalled forward progress. + // Bounces with rising completed counts = healthy backpressure; bounces + // with zero completes = real blocker (matches doctor's subagent_health). + const completedRows = await engine.executeRaw<{ count: string }>( + `SELECT count(*)::text AS count FROM minion_jobs + WHERE finished_at > now() - interval '1 hour' + AND status = 'completed' AND name = 'subagent'`, + ).catch(() => [{ count: '0' }]); + const completed = parseInt(completedRows[0]?.count ?? '0', 10); + const tag = completed > 0 + ? `(${completed} subagent job${completed === 1 ? '' : 's'} completed, throughput healthy)` + : `(no subagent jobs completed — cap may be too tight; \`export GBRAIN_ANTHROPIC_MAX_INFLIGHT=64\`)`; + console.log(` Lease pressure (1h): ${lpCount} bounce${lpCount === 1 ? '' : 's'} ${tag}`); + } else { + console.log(` Lease pressure (1h): 0 bounces`); + } + } catch { + // Pre-v93 brain — no table. Silent skip. + } break; } diff --git a/test/doctor-subagent-health.test.ts b/test/doctor-subagent-health.test.ts new file mode 100644 index 000000000..247837760 --- /dev/null +++ b/test/doctor-subagent-health.test.ts @@ -0,0 +1,125 @@ +/** + * v0.41 Bug 2 / Eng D8 — `gbrain doctor` `subagent_health` check coverage. + * + * Reads the last 24h of `minion_lease_pressure_log` (populated by the + * Bug 2 worker bypass path) and classifies pressure into ok / warn / fail + * thresholds. The doctor check is the operator's primary forensic signal + * for "is the lease cap too tight" — without it, the v0.41 bypass would + * be invisible (no dead-letter, but also no operator visibility). + * + * Four cases pinned per the plan: + * 1. 0 bounces → ok ("no pressure") + * 2. 100+ bounces with subagent jobs completing → ok ("healthy backpressure") + * 3. 100+ bounces with NO subagent jobs completing → warn (paste-ready hint) + * 4. 1000+ bounces → fail (blocking) + */ + +import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { MinionQueue } from '../src/core/minions/queue.ts'; +import { checkSubagentHealth } from '../src/commands/doctor.ts'; +import { logLeasePressure } from '../src/core/minions/lease-pressure-audit.ts'; + +let engine: PGLiteEngine; +let queue: MinionQueue; + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({ database_url: '' }); + await engine.initSchema(); + queue = new MinionQueue(engine); +}); + +afterAll(async () => { + await engine.disconnect(); +}); + +beforeEach(async () => { + await engine.executeRaw('DELETE FROM minion_lease_pressure_log'); + await engine.executeRaw('DELETE FROM minion_jobs'); +}); + +/** Wrapper for terse test bodies. */ +async function getSubagentHealthCheck() { + return checkSubagentHealth(engine); +} + +describe('doctor subagent_health (v0.41 Bug 2 / Eng D8)', () => { + test('0 bounces → ok ("no pressure")', async () => { + const check = await getSubagentHealthCheck(); + expect(check.status).toBe('ok'); + expect(check.message).toContain('No rate-lease pressure'); + }); + + test('healthy backpressure (1-99 bounces with completed jobs) → ok', async () => { + // 5 bounces + 3 completed subagent jobs in the last hour. + const owner = await queue.add('subagent', {}, {}, { allowProtectedSubmit: true }); + for (let i = 0; i < 5; i++) { + await logLeasePressure(engine, { + job_id: owner.id, + lease_key: 'anthropic:messages', + active_at_bounce: 8, + max_concurrent: 8, + }); + } + // Insert 3 completed subagent jobs. + for (let i = 0; i < 3; i++) { + await engine.executeRaw( + `INSERT INTO minion_jobs + (name, queue, status, attempts_made, attempts_started, + finished_at, started_at, max_attempts) + VALUES ('subagent', 'default', 'completed', 1, 1, + now(), now() - interval '1 second', 3)`, + ); + } + + const check = await getSubagentHealthCheck(); + expect(check.status).toBe('ok'); + expect(check.message).toContain('bounces'); + // Sub-100 bounces routes through the second-tier OK message + // (not the "no pressure" message; falls into "healthy backpressure"). + expect(check.message).not.toContain('No rate-lease pressure'); + }); + + test('100+ bounces with NO completed subagent jobs → warn with paste-ready hint', async () => { + const owner = await queue.add('subagent', {}, {}, { allowProtectedSubmit: true }); + for (let i = 0; i < 100; i++) { + await logLeasePressure(engine, { + job_id: owner.id, + lease_key: 'anthropic:messages', + active_at_bounce: 8, + max_concurrent: 8, + }); + } + // NO completed subagent jobs — pressure is BLOCKING real work. + + const check = await getSubagentHealthCheck(); + expect(check.status).toBe('warn'); + expect(check.message).toContain('100'); + // Paste-ready hint with the canonical env-var name. + expect(check.message).toContain('GBRAIN_ANTHROPIC_MAX_INFLIGHT'); + }); + + test('1000+ bounces → fail (blocking real work)', async () => { + const owner = await queue.add('subagent', {}, {}, { allowProtectedSubmit: true }); + // Batch insert via VALUES rather than per-row logLeasePressure (faster). + const valuesList: string[] = []; + const params: Array = []; + for (let i = 0; i < 1000; i++) { + valuesList.push(`($${params.length + 1}, $${params.length + 2}, $${params.length + 3}, $${params.length + 4})`); + params.push(owner.id, 'anthropic:messages', 8, 8); + } + await engine.executeRaw( + `INSERT INTO minion_lease_pressure_log + (job_id, lease_key, active_at_bounce, max_concurrent) + VALUES ${valuesList.join(', ')}`, + params, + ); + + const check = await getSubagentHealthCheck(); + expect(check.status).toBe('fail'); + expect(check.message).toContain('1000'); + expect(check.message).toContain('blocking real work'); + expect(check.message).toContain('GBRAIN_ANTHROPIC_MAX_INFLIGHT'); + }); +});