mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
v0.41: doctor subagent_health + jobs stats lease_pressure line
Operator visibility for the v0.41 Bug 2 audit data.
src/commands/doctor.ts
checkSubagentHealth(engine) — new exported check function. Reads the
last 24h of minion_lease_pressure_log and classifies by bounce volume
+ forward progress:
0 bounces → ok
1-99 bounces → ok ("transient")
100+ bounces + subagent jobs completing → ok ("healthy backpressure")
100+ bounces + NO completed subagent jobs → warn (paste-ready hint)
1000+ bounces → fail (blocking)
Warn/fail messages embed `export GBRAIN_ANTHROPIC_MAX_INFLIGHT=64` for
copy-paste. Pre-v93 brains (no table) silently skip with OK. Works on
both Postgres + PGLite.
src/commands/jobs.ts (case 'stats')
Adds `Lease pressure (1h)` line to the stats output. When >0 bounces,
cross-checks completed subagent count and surfaces the same
binding-but-healthy vs cap-too-tight distinction inline so operators
don't have to run `gbrain doctor` to see it. Pre-v93 silent skip.
test/doctor-subagent-health.test.ts (NEW)
4 cases pinning all threshold bands. Uses `allowProtectedSubmit: true`
on the queue.add for `subagent`-named owner jobs.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
bd751f6f99
commit
0887dcc958
@@ -563,6 +563,9 @@ export async function doctorReportRemote(engine: BrainEngine): Promise<DoctorRep
|
||||
checks.push({ name: 'queue_health', status: 'ok', message: 'PGLite — no queue to check' });
|
||||
}
|
||||
|
||||
// v0.41 Bug 2 / Eng D8 — subagent_health surfaces rate-lease pressure to the operator.
|
||||
checks.push(await checkSubagentHealth(engine));
|
||||
|
||||
// v0.31.12 subagent runtime enforcement (Layer 3 of 3 — Codex F13).
|
||||
// The subagent loop is Anthropic-only. If models.tier.subagent or
|
||||
// models.default is explicitly set to a non-Anthropic provider, warn here
|
||||
@@ -852,6 +855,77 @@ export async function checkGradeConfidenceDrift(engine: BrainEngine): Promise<Ch
|
||||
* isolation (template fallback is fine), but a sustained high rate signals
|
||||
* the rubric needs tuning.
|
||||
*/
|
||||
/**
|
||||
* v0.41 Bug 2 / Eng D8 — surfaces rate-lease pressure from
|
||||
* `minion_lease_pressure_log` (populated by the worker's lease-full bypass
|
||||
* path). The operator's primary forensic signal for "is the lease cap too
|
||||
* tight" — without this check, the v0.41 bypass would be invisible (no
|
||||
* dead-letter, but also no operator awareness).
|
||||
*
|
||||
* Thresholds (windowed at 24h):
|
||||
* 0 bounces → ok ("no pressure")
|
||||
* 1-99 bounces → ok ("transient")
|
||||
* 100+ bounces + subagent jobs completed in same window → ok ("healthy backpressure")
|
||||
* 100+ bounces + ZERO completed subagent jobs → warn (paste-ready cap-raise hint)
|
||||
* 1000+ bounces → fail ("blocking real work")
|
||||
*
|
||||
* Works on both Postgres + PGLite (migration v93 creates the table on both).
|
||||
* Pre-v93 brains (no table) silently skip with an OK message.
|
||||
*/
|
||||
export async function checkSubagentHealth(engine: BrainEngine): Promise<Check> {
|
||||
try {
|
||||
const bounceRows = await engine.executeRaw<{ count: string }>(
|
||||
`SELECT count(*)::text AS count FROM minion_lease_pressure_log
|
||||
WHERE bounced_at > now() - interval '24 hours'`,
|
||||
);
|
||||
const bounces = parseInt(bounceRows[0]?.count ?? '0', 10);
|
||||
if (bounces === 0) {
|
||||
return {
|
||||
name: 'subagent_health',
|
||||
status: 'ok',
|
||||
message: 'No rate-lease pressure in last 24h',
|
||||
};
|
||||
}
|
||||
if (bounces >= 1000) {
|
||||
return {
|
||||
name: 'subagent_health',
|
||||
status: 'fail',
|
||||
message: `${bounces} lease-pressure bounces in last 24h — this is blocking real work. Raise the cap: \`export GBRAIN_ANTHROPIC_MAX_INFLIGHT=64\` (or \`unlimited\` for Azure / Bedrock / self-hosted upstreams with no provider-side rate limit). After raising, restart \`gbrain jobs work\`.`,
|
||||
};
|
||||
}
|
||||
// 1-999 bounces: cross-check forward progress.
|
||||
const completedRows = await engine.executeRaw<{ count: string }>(
|
||||
`SELECT count(*)::text AS count FROM minion_jobs
|
||||
WHERE finished_at > now() - interval '24 hours'
|
||||
AND status = 'completed'
|
||||
AND name = 'subagent'`,
|
||||
).catch(() => [{ count: '0' }]);
|
||||
const completed = parseInt(completedRows[0]?.count ?? '0', 10);
|
||||
if (bounces >= 100 && completed === 0) {
|
||||
return {
|
||||
name: 'subagent_health',
|
||||
status: 'warn',
|
||||
message: `${bounces} lease-pressure bounces in last 24h with no completed subagent jobs — cap is too tight. Raise via \`export GBRAIN_ANTHROPIC_MAX_INFLIGHT=64\` (or \`unlimited\` for upstreams with no provider-side cap).`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
name: 'subagent_health',
|
||||
status: 'ok',
|
||||
message: `Lease pressure: ${bounces} bounces in last 24h, ${completed} subagent jobs completed — backpressure is binding but throughput is healthy`,
|
||||
};
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
if (process.env.GBRAIN_DEBUG === '1') {
|
||||
process.stderr.write(`[doctor] subagent_health skipped: ${msg}\n`);
|
||||
}
|
||||
return {
|
||||
name: 'subagent_health',
|
||||
status: 'ok',
|
||||
message: 'Skipped (minion_lease_pressure_log unavailable — pre-v0.41 brain)',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function checkVoiceGateHealth(engine: BrainEngine): Promise<Check> {
|
||||
try {
|
||||
const rows = await engine.executeRaw<{ total: number; failures: number }>(
|
||||
|
||||
@@ -545,6 +545,37 @@ HANDLER TYPES (built in)
|
||||
console.log(' No jobs in the last 24 hours.');
|
||||
}
|
||||
console.log(`\n Queue health: ${stats.queue_health.waiting} waiting, ${stats.queue_health.active} active, ${stats.queue_health.stalled} stalled`);
|
||||
|
||||
// v0.41 Bug 2 / Eng D8 — surface lease pressure to the operator.
|
||||
// Reads minion_lease_pressure_log windowed at 1h. Best-effort: pre-v93
|
||||
// brains (no table) silently skip; the queue_health line above is the
|
||||
// operator's primary signal in that case.
|
||||
try {
|
||||
const lpRows = await engine.executeRaw<{ count: string }>(
|
||||
`SELECT count(*)::text AS count FROM minion_lease_pressure_log
|
||||
WHERE bounced_at > now() - interval '1 hour'`,
|
||||
);
|
||||
const lpCount = parseInt(lpRows[0]?.count ?? '0', 10);
|
||||
if (lpCount > 0) {
|
||||
// Also surface whether any of those bounces stalled forward progress.
|
||||
// Bounces with rising completed counts = healthy backpressure; bounces
|
||||
// with zero completes = real blocker (matches doctor's subagent_health).
|
||||
const completedRows = await engine.executeRaw<{ count: string }>(
|
||||
`SELECT count(*)::text AS count FROM minion_jobs
|
||||
WHERE finished_at > now() - interval '1 hour'
|
||||
AND status = 'completed' AND name = 'subagent'`,
|
||||
).catch(() => [{ count: '0' }]);
|
||||
const completed = parseInt(completedRows[0]?.count ?? '0', 10);
|
||||
const tag = completed > 0
|
||||
? `(${completed} subagent job${completed === 1 ? '' : 's'} completed, throughput healthy)`
|
||||
: `(no subagent jobs completed — cap may be too tight; \`export GBRAIN_ANTHROPIC_MAX_INFLIGHT=64\`)`;
|
||||
console.log(` Lease pressure (1h): ${lpCount} bounce${lpCount === 1 ? '' : 's'} ${tag}`);
|
||||
} else {
|
||||
console.log(` Lease pressure (1h): 0 bounces`);
|
||||
}
|
||||
} catch {
|
||||
// Pre-v93 brain — no table. Silent skip.
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* v0.41 Bug 2 / Eng D8 — `gbrain doctor` `subagent_health` check coverage.
|
||||
*
|
||||
* Reads the last 24h of `minion_lease_pressure_log` (populated by the
|
||||
* Bug 2 worker bypass path) and classifies pressure into ok / warn / fail
|
||||
* thresholds. The doctor check is the operator's primary forensic signal
|
||||
* for "is the lease cap too tight" — without it, the v0.41 bypass would
|
||||
* be invisible (no dead-letter, but also no operator visibility).
|
||||
*
|
||||
* Four cases pinned per the plan:
|
||||
* 1. 0 bounces → ok ("no pressure")
|
||||
* 2. 100+ bounces with subagent jobs completing → ok ("healthy backpressure")
|
||||
* 3. 100+ bounces with NO subagent jobs completing → warn (paste-ready hint)
|
||||
* 4. 1000+ bounces → fail (blocking)
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { MinionQueue } from '../src/core/minions/queue.ts';
|
||||
import { checkSubagentHealth } from '../src/commands/doctor.ts';
|
||||
import { logLeasePressure } from '../src/core/minions/lease-pressure-audit.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
let queue: MinionQueue;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({ database_url: '' });
|
||||
await engine.initSchema();
|
||||
queue = new MinionQueue(engine);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await engine.executeRaw('DELETE FROM minion_lease_pressure_log');
|
||||
await engine.executeRaw('DELETE FROM minion_jobs');
|
||||
});
|
||||
|
||||
/** Wrapper for terse test bodies. */
|
||||
async function getSubagentHealthCheck() {
|
||||
return checkSubagentHealth(engine);
|
||||
}
|
||||
|
||||
describe('doctor subagent_health (v0.41 Bug 2 / Eng D8)', () => {
|
||||
test('0 bounces → ok ("no pressure")', async () => {
|
||||
const check = await getSubagentHealthCheck();
|
||||
expect(check.status).toBe('ok');
|
||||
expect(check.message).toContain('No rate-lease pressure');
|
||||
});
|
||||
|
||||
test('healthy backpressure (1-99 bounces with completed jobs) → ok', async () => {
|
||||
// 5 bounces + 3 completed subagent jobs in the last hour.
|
||||
const owner = await queue.add('subagent', {}, {}, { allowProtectedSubmit: true });
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await logLeasePressure(engine, {
|
||||
job_id: owner.id,
|
||||
lease_key: 'anthropic:messages',
|
||||
active_at_bounce: 8,
|
||||
max_concurrent: 8,
|
||||
});
|
||||
}
|
||||
// Insert 3 completed subagent jobs.
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO minion_jobs
|
||||
(name, queue, status, attempts_made, attempts_started,
|
||||
finished_at, started_at, max_attempts)
|
||||
VALUES ('subagent', 'default', 'completed', 1, 1,
|
||||
now(), now() - interval '1 second', 3)`,
|
||||
);
|
||||
}
|
||||
|
||||
const check = await getSubagentHealthCheck();
|
||||
expect(check.status).toBe('ok');
|
||||
expect(check.message).toContain('bounces');
|
||||
// Sub-100 bounces routes through the second-tier OK message
|
||||
// (not the "no pressure" message; falls into "healthy backpressure").
|
||||
expect(check.message).not.toContain('No rate-lease pressure');
|
||||
});
|
||||
|
||||
test('100+ bounces with NO completed subagent jobs → warn with paste-ready hint', async () => {
|
||||
const owner = await queue.add('subagent', {}, {}, { allowProtectedSubmit: true });
|
||||
for (let i = 0; i < 100; i++) {
|
||||
await logLeasePressure(engine, {
|
||||
job_id: owner.id,
|
||||
lease_key: 'anthropic:messages',
|
||||
active_at_bounce: 8,
|
||||
max_concurrent: 8,
|
||||
});
|
||||
}
|
||||
// NO completed subagent jobs — pressure is BLOCKING real work.
|
||||
|
||||
const check = await getSubagentHealthCheck();
|
||||
expect(check.status).toBe('warn');
|
||||
expect(check.message).toContain('100');
|
||||
// Paste-ready hint with the canonical env-var name.
|
||||
expect(check.message).toContain('GBRAIN_ANTHROPIC_MAX_INFLIGHT');
|
||||
});
|
||||
|
||||
test('1000+ bounces → fail (blocking real work)', async () => {
|
||||
const owner = await queue.add('subagent', {}, {}, { allowProtectedSubmit: true });
|
||||
// Batch insert via VALUES rather than per-row logLeasePressure (faster).
|
||||
const valuesList: string[] = [];
|
||||
const params: Array<string | number> = [];
|
||||
for (let i = 0; i < 1000; i++) {
|
||||
valuesList.push(`($${params.length + 1}, $${params.length + 2}, $${params.length + 3}, $${params.length + 4})`);
|
||||
params.push(owner.id, 'anthropic:messages', 8, 8);
|
||||
}
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO minion_lease_pressure_log
|
||||
(job_id, lease_key, active_at_bounce, max_concurrent)
|
||||
VALUES ${valuesList.join(', ')}`,
|
||||
params,
|
||||
);
|
||||
|
||||
const check = await getSubagentHealthCheck();
|
||||
expect(check.status).toBe('fail');
|
||||
expect(check.message).toContain('1000');
|
||||
expect(check.message).toContain('blocking real work');
|
||||
expect(check.message).toContain('GBRAIN_ANTHROPIC_MAX_INFLIGHT');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user