v0.41: doctor subagent_health + jobs stats lease_pressure line

Operator visibility for the v0.41 Bug 2 audit data.

src/commands/doctor.ts
  checkSubagentHealth(engine) — new exported check function. Reads the
  last 24h of minion_lease_pressure_log and classifies by bounce volume
  + forward progress:
    0 bounces                                            → ok
    1-99 bounces                                         → ok ("transient")
    100+ bounces + subagent jobs completing             → ok ("healthy backpressure")
    100+ bounces + NO completed subagent jobs           → warn (paste-ready hint)
    1000+ bounces                                       → fail (blocking)
  Warn/fail messages embed `export GBRAIN_ANTHROPIC_MAX_INFLIGHT=64` for
  copy-paste. Pre-v93 brains (no table) silently skip with OK. Works on
  both Postgres + PGLite.

src/commands/jobs.ts (case 'stats')
  Adds `Lease pressure (1h)` line to the stats output. When >0 bounces,
  cross-checks completed subagent count and surfaces the same
  binding-but-healthy vs cap-too-tight distinction inline so operators
  don't have to run `gbrain doctor` to see it. Pre-v93 silent skip.

test/doctor-subagent-health.test.ts (NEW)
  4 cases pinning all threshold bands. Uses `allowProtectedSubmit: true`
  on the queue.add for `subagent`-named owner jobs.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-05-24 00:32:31 -07:00
co-authored by Claude Opus 4.7
parent bd751f6f99
commit 0887dcc958
3 changed files with 230 additions and 0 deletions
+74
View File
@@ -563,6 +563,9 @@ export async function doctorReportRemote(engine: BrainEngine): Promise<DoctorRep
checks.push({ name: 'queue_health', status: 'ok', message: 'PGLite — no queue to check' });
}
// v0.41 Bug 2 / Eng D8 — subagent_health surfaces rate-lease pressure to the operator.
checks.push(await checkSubagentHealth(engine));
// v0.31.12 subagent runtime enforcement (Layer 3 of 3 — Codex F13).
// The subagent loop is Anthropic-only. If models.tier.subagent or
// models.default is explicitly set to a non-Anthropic provider, warn here
@@ -852,6 +855,77 @@ export async function checkGradeConfidenceDrift(engine: BrainEngine): Promise<Ch
* isolation (template fallback is fine), but a sustained high rate signals
* the rubric needs tuning.
*/
/**
* v0.41 Bug 2 / Eng D8 — surfaces rate-lease pressure from
* `minion_lease_pressure_log` (populated by the worker's lease-full bypass
* path). The operator's primary forensic signal for "is the lease cap too
* tight" — without this check, the v0.41 bypass would be invisible (no
* dead-letter, but also no operator awareness).
*
* Thresholds (windowed at 24h):
* 0 bounces → ok ("no pressure")
* 1-99 bounces → ok ("transient")
* 100+ bounces + subagent jobs completed in same window → ok ("healthy backpressure")
* 100+ bounces + ZERO completed subagent jobs → warn (paste-ready cap-raise hint)
* 1000+ bounces → fail ("blocking real work")
*
* Works on both Postgres + PGLite (migration v93 creates the table on both).
* Pre-v93 brains (no table) silently skip with an OK message.
*/
export async function checkSubagentHealth(engine: BrainEngine): Promise<Check> {
try {
const bounceRows = await engine.executeRaw<{ count: string }>(
`SELECT count(*)::text AS count FROM minion_lease_pressure_log
WHERE bounced_at > now() - interval '24 hours'`,
);
const bounces = parseInt(bounceRows[0]?.count ?? '0', 10);
if (bounces === 0) {
return {
name: 'subagent_health',
status: 'ok',
message: 'No rate-lease pressure in last 24h',
};
}
if (bounces >= 1000) {
return {
name: 'subagent_health',
status: 'fail',
message: `${bounces} lease-pressure bounces in last 24h — this is blocking real work. Raise the cap: \`export GBRAIN_ANTHROPIC_MAX_INFLIGHT=64\` (or \`unlimited\` for Azure / Bedrock / self-hosted upstreams with no provider-side rate limit). After raising, restart \`gbrain jobs work\`.`,
};
}
// 1-999 bounces: cross-check forward progress.
const completedRows = await engine.executeRaw<{ count: string }>(
`SELECT count(*)::text AS count FROM minion_jobs
WHERE finished_at > now() - interval '24 hours'
AND status = 'completed'
AND name = 'subagent'`,
).catch(() => [{ count: '0' }]);
const completed = parseInt(completedRows[0]?.count ?? '0', 10);
if (bounces >= 100 && completed === 0) {
return {
name: 'subagent_health',
status: 'warn',
message: `${bounces} lease-pressure bounces in last 24h with no completed subagent jobs — cap is too tight. Raise via \`export GBRAIN_ANTHROPIC_MAX_INFLIGHT=64\` (or \`unlimited\` for upstreams with no provider-side cap).`,
};
}
return {
name: 'subagent_health',
status: 'ok',
message: `Lease pressure: ${bounces} bounces in last 24h, ${completed} subagent jobs completed — backpressure is binding but throughput is healthy`,
};
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
if (process.env.GBRAIN_DEBUG === '1') {
process.stderr.write(`[doctor] subagent_health skipped: ${msg}\n`);
}
return {
name: 'subagent_health',
status: 'ok',
message: 'Skipped (minion_lease_pressure_log unavailable — pre-v0.41 brain)',
};
}
}
export async function checkVoiceGateHealth(engine: BrainEngine): Promise<Check> {
try {
const rows = await engine.executeRaw<{ total: number; failures: number }>(
+31
View File
@@ -545,6 +545,37 @@ HANDLER TYPES (built in)
console.log(' No jobs in the last 24 hours.');
}
console.log(`\n Queue health: ${stats.queue_health.waiting} waiting, ${stats.queue_health.active} active, ${stats.queue_health.stalled} stalled`);
// v0.41 Bug 2 / Eng D8 — surface lease pressure to the operator.
// Reads minion_lease_pressure_log windowed at 1h. Best-effort: pre-v93
// brains (no table) silently skip; the queue_health line above is the
// operator's primary signal in that case.
try {
const lpRows = await engine.executeRaw<{ count: string }>(
`SELECT count(*)::text AS count FROM minion_lease_pressure_log
WHERE bounced_at > now() - interval '1 hour'`,
);
const lpCount = parseInt(lpRows[0]?.count ?? '0', 10);
if (lpCount > 0) {
// Also surface whether any of those bounces stalled forward progress.
// Bounces with rising completed counts = healthy backpressure; bounces
// with zero completes = real blocker (matches doctor's subagent_health).
const completedRows = await engine.executeRaw<{ count: string }>(
`SELECT count(*)::text AS count FROM minion_jobs
WHERE finished_at > now() - interval '1 hour'
AND status = 'completed' AND name = 'subagent'`,
).catch(() => [{ count: '0' }]);
const completed = parseInt(completedRows[0]?.count ?? '0', 10);
const tag = completed > 0
? `(${completed} subagent job${completed === 1 ? '' : 's'} completed, throughput healthy)`
: `(no subagent jobs completed — cap may be too tight; \`export GBRAIN_ANTHROPIC_MAX_INFLIGHT=64\`)`;
console.log(` Lease pressure (1h): ${lpCount} bounce${lpCount === 1 ? '' : 's'} ${tag}`);
} else {
console.log(` Lease pressure (1h): 0 bounces`);
}
} catch {
// Pre-v93 brain — no table. Silent skip.
}
break;
}
+125
View File
@@ -0,0 +1,125 @@
/**
* v0.41 Bug 2 / Eng D8 — `gbrain doctor` `subagent_health` check coverage.
*
* Reads the last 24h of `minion_lease_pressure_log` (populated by the
* Bug 2 worker bypass path) and classifies pressure into ok / warn / fail
* thresholds. The doctor check is the operator's primary forensic signal
* for "is the lease cap too tight" — without it, the v0.41 bypass would
* be invisible (no dead-letter, but also no operator visibility).
*
* Four cases pinned per the plan:
* 1. 0 bounces → ok ("no pressure")
* 2. 100+ bounces with subagent jobs completing → ok ("healthy backpressure")
* 3. 100+ bounces with NO subagent jobs completing → warn (paste-ready hint)
* 4. 1000+ bounces → fail (blocking)
*/
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { MinionQueue } from '../src/core/minions/queue.ts';
import { checkSubagentHealth } from '../src/commands/doctor.ts';
import { logLeasePressure } from '../src/core/minions/lease-pressure-audit.ts';
let engine: PGLiteEngine;
let queue: MinionQueue;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({ database_url: '' });
await engine.initSchema();
queue = new MinionQueue(engine);
});
afterAll(async () => {
await engine.disconnect();
});
beforeEach(async () => {
await engine.executeRaw('DELETE FROM minion_lease_pressure_log');
await engine.executeRaw('DELETE FROM minion_jobs');
});
/** Wrapper for terse test bodies. */
async function getSubagentHealthCheck() {
return checkSubagentHealth(engine);
}
describe('doctor subagent_health (v0.41 Bug 2 / Eng D8)', () => {
test('0 bounces → ok ("no pressure")', async () => {
const check = await getSubagentHealthCheck();
expect(check.status).toBe('ok');
expect(check.message).toContain('No rate-lease pressure');
});
test('healthy backpressure (1-99 bounces with completed jobs) → ok', async () => {
// 5 bounces + 3 completed subagent jobs in the last hour.
const owner = await queue.add('subagent', {}, {}, { allowProtectedSubmit: true });
for (let i = 0; i < 5; i++) {
await logLeasePressure(engine, {
job_id: owner.id,
lease_key: 'anthropic:messages',
active_at_bounce: 8,
max_concurrent: 8,
});
}
// Insert 3 completed subagent jobs.
for (let i = 0; i < 3; i++) {
await engine.executeRaw(
`INSERT INTO minion_jobs
(name, queue, status, attempts_made, attempts_started,
finished_at, started_at, max_attempts)
VALUES ('subagent', 'default', 'completed', 1, 1,
now(), now() - interval '1 second', 3)`,
);
}
const check = await getSubagentHealthCheck();
expect(check.status).toBe('ok');
expect(check.message).toContain('bounces');
// Sub-100 bounces routes through the second-tier OK message
// (not the "no pressure" message; falls into "healthy backpressure").
expect(check.message).not.toContain('No rate-lease pressure');
});
test('100+ bounces with NO completed subagent jobs → warn with paste-ready hint', async () => {
const owner = await queue.add('subagent', {}, {}, { allowProtectedSubmit: true });
for (let i = 0; i < 100; i++) {
await logLeasePressure(engine, {
job_id: owner.id,
lease_key: 'anthropic:messages',
active_at_bounce: 8,
max_concurrent: 8,
});
}
// NO completed subagent jobs — pressure is BLOCKING real work.
const check = await getSubagentHealthCheck();
expect(check.status).toBe('warn');
expect(check.message).toContain('100');
// Paste-ready hint with the canonical env-var name.
expect(check.message).toContain('GBRAIN_ANTHROPIC_MAX_INFLIGHT');
});
test('1000+ bounces → fail (blocking real work)', async () => {
const owner = await queue.add('subagent', {}, {}, { allowProtectedSubmit: true });
// Batch insert via VALUES rather than per-row logLeasePressure (faster).
const valuesList: string[] = [];
const params: Array<string | number> = [];
for (let i = 0; i < 1000; i++) {
valuesList.push(`($${params.length + 1}, $${params.length + 2}, $${params.length + 3}, $${params.length + 4})`);
params.push(owner.id, 'anthropic:messages', 8, 8);
}
await engine.executeRaw(
`INSERT INTO minion_lease_pressure_log
(job_id, lease_key, active_at_bounce, max_concurrent)
VALUES ${valuesList.join(', ')}`,
params,
);
const check = await getSubagentHealthCheck();
expect(check.status).toBe('fail');
expect(check.message).toContain('1000');
expect(check.message).toContain('blocking real work');
expect(check.message).toContain('GBRAIN_ANTHROPIC_MAX_INFLIGHT');
});
});