mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
fix(cycle): synthesize budgets its sequential child waits from remaining job time (#2781 follow-up)
Rebase of #2960 onto master, reconciled with the landed #2959 (23e0541d):
master shipped the patterns-phase deadline clamp inline in patterns.ts;
this extracts those helpers into a shared src/core/cycle/deadline-budget.ts
(MIN_PATTERNS_SUBAGENT_BUDGET_MS -> MIN_SUBAGENT_START_BUDGET_MS, new
remainingChildBudgetMs primitive) so synthesize can reuse them without a
patterns<->synthesize import cycle. patterns behavior is byte-identical —
pinned by the existing test suite, repointed to the shared module.
synthesize (#2781 follow-up, the half #2959 left open):
- pre-fanout gate: skips honestly (insufficient_cycle_budget) before
submitting any Sonnet work when the remaining budget is under the
start minimum.
- per-transcript + per-chunk submit guards: stop submitting below the
start minimum; each child's timeout_ms is clamped to the remaining
time at ITS submit. Idempotency keys resume the rest next cycle.
- SEQUENTIAL wait loop recomputes the remaining budget every iteration,
so N children can't accumulate N x subagent_wait_timeout_ms past the
parent budget. On a clamped-wait timeout the child is cancelled
(a queued child could outlive the clamp).
- exhausted budget settles unwaited children cancel-first; an
already-terminal sibling keeps its real status.
- budget-truncated runs (including a timeout on a deadline-CLAMPED
wait) do NOT stamp dream.synthesize.last_completion_ts — the 12h
cooldown would contradict "next cycle retries with a fresh budget".
Co-authored-by: Masashi-Ono0611 <masashi.ono.0611@gmail.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Masashi-Ono0611
Claude Fable 5
parent
9f7244a77f
commit
e453687a60
@@ -1695,6 +1695,7 @@ export async function runCycle(
|
||||
// #1586: scope synthesized writes to the cycle's resolved source
|
||||
// (explicit --source wins, else derived from the checkout dir).
|
||||
sourceId: cycleSourceId,
|
||||
deadlineAtMs: opts.deadlineAtMs ?? null,
|
||||
}));
|
||||
result.duration_ms = duration_ms;
|
||||
phaseResults.push(result);
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
/**
|
||||
* #2781 — shared deadline-budget helpers for cycle phases that spawn
|
||||
* subagent children (patterns, synthesize).
|
||||
*
|
||||
* The enclosing minion job carries an absolute deadline
|
||||
* (`MinionJobContext.deadlineAtMs`, from the claim-time `timeout_at`
|
||||
* stamp). Phases clamp their child-job timeouts and wait timeouts to the
|
||||
* REMAINING time so one phase's fixed worst-case can't blow past the job
|
||||
* budget and dead-letter the whole cycle mid-phase.
|
||||
*
|
||||
* Lives in its own module because patterns.ts imports from synthesize.ts
|
||||
* (allow-list helpers) — synthesize importing these back from patterns
|
||||
* would create an import cycle.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Stop-margin reserved under the parent deadline when clamping subagent
|
||||
* budgets. NOT a promise that tail phases complete — the cycle is allowed
|
||||
* to go partial and resume next tick. This only guarantees the phase's
|
||||
* wait returns and the handler unwinds cleanly before the worker's abort
|
||||
* fires: wait poll interval (5s) + worker force-evict grace (30s) + lock
|
||||
* and DB cleanup headroom.
|
||||
*/
|
||||
export const CYCLE_DEADLINE_RESERVE_MS = 60 * 1000;
|
||||
|
||||
/**
|
||||
* Smallest remaining budget worth submitting a subagent for. Below this,
|
||||
* the LLM call is near-certain to be killed mid-flight — wasted spend and
|
||||
* a guaranteed-timeout child — so the phase skips honestly instead
|
||||
* (`insufficient_cycle_budget`) and the next cycle retries with a fresh
|
||||
* budget.
|
||||
*/
|
||||
export const MIN_SUBAGENT_START_BUDGET_MS = 2 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Remaining child budget under the parent deadline, or null when the
|
||||
* caller has no deadline (direct `gbrain dream`). May be <= 0 when the
|
||||
* reserve is already breached — callers treat that as "no budget left".
|
||||
*/
|
||||
export function remainingChildBudgetMs(
|
||||
deadlineAtMs: number | null | undefined,
|
||||
nowMs: number,
|
||||
): number | null {
|
||||
if (deadlineAtMs == null) return null;
|
||||
return deadlineAtMs - CYCLE_DEADLINE_RESERVE_MS - nowMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clamp the configured subagent budgets to the remaining parent-job time.
|
||||
* Both timeouts derive from the SAME absolute child deadline
|
||||
* (`deadlineAtMs - reserve`) so the child job's kill switch and our wait
|
||||
* agree. Returns null when the remaining budget is below the minimum —
|
||||
* caller should skip the phase without submitting.
|
||||
*/
|
||||
export function clampSubagentBudgets(
|
||||
config: { subagentTimeoutMs: number; subagentWaitTimeoutMs: number },
|
||||
deadlineAtMs: number | null | undefined,
|
||||
nowMs: number,
|
||||
): { timeoutMs: number; waitTimeoutMs: number } | null {
|
||||
const childBudgetMs = remainingChildBudgetMs(deadlineAtMs, nowMs);
|
||||
if (childBudgetMs === null) {
|
||||
return { timeoutMs: config.subagentTimeoutMs, waitTimeoutMs: config.subagentWaitTimeoutMs };
|
||||
}
|
||||
if (childBudgetMs < MIN_SUBAGENT_START_BUDGET_MS) return null;
|
||||
return {
|
||||
timeoutMs: Math.min(config.subagentTimeoutMs, childBudgetMs),
|
||||
waitTimeoutMs: Math.min(config.subagentWaitTimeoutMs, childBudgetMs),
|
||||
};
|
||||
}
|
||||
@@ -30,6 +30,11 @@ import type { Page, PageType } from '../types.ts';
|
||||
// #2415: allow-list + output-root resolution shared with the synthesize
|
||||
// phase — both phases must agree on the configured namespace.
|
||||
import { loadAllowedSlugPrefixes, loadOutputRoot } from './synthesize.ts';
|
||||
import {
|
||||
clampSubagentBudgets,
|
||||
CYCLE_DEADLINE_RESERVE_MS,
|
||||
MIN_SUBAGENT_START_BUDGET_MS,
|
||||
} from './deadline-budget.ts';
|
||||
import { probeChatModel } from '../ai/gateway.ts';
|
||||
import { normalizeModelId } from '../model-id.ts';
|
||||
|
||||
@@ -48,48 +53,6 @@ export interface PatternsPhaseOpts {
|
||||
deadlineAtMs?: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop-margin reserved under the parent deadline when clamping subagent
|
||||
* budgets. NOT a promise that tail phases complete — the cycle is allowed
|
||||
* to go partial and resume next tick. This only guarantees the phase's
|
||||
* wait returns and the handler unwinds cleanly before the worker's abort
|
||||
* fires: wait poll interval (5s) + worker force-evict grace (30s) + lock
|
||||
* and DB cleanup headroom.
|
||||
*/
|
||||
export const CYCLE_DEADLINE_RESERVE_MS = 60 * 1000;
|
||||
|
||||
/**
|
||||
* Smallest remaining budget worth submitting a subagent for. Below this,
|
||||
* the LLM call is near-certain to be killed mid-flight — wasted spend and
|
||||
* a guaranteed-timeout child — so the phase skips honestly instead
|
||||
* (`insufficient_cycle_budget`) and the next cycle retries with a fresh
|
||||
* budget.
|
||||
*/
|
||||
export const MIN_PATTERNS_SUBAGENT_BUDGET_MS = 2 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Clamp the configured subagent budgets to the remaining parent-job time.
|
||||
* Both timeouts derive from the SAME absolute child deadline
|
||||
* (`deadlineAtMs - reserve`) so the child job's kill switch and our wait
|
||||
* agree. Returns null when the remaining budget is below the minimum —
|
||||
* caller should skip the phase without submitting.
|
||||
*/
|
||||
export function clampSubagentBudgets(
|
||||
config: { subagentTimeoutMs: number; subagentWaitTimeoutMs: number },
|
||||
deadlineAtMs: number | null | undefined,
|
||||
nowMs: number,
|
||||
): { timeoutMs: number; waitTimeoutMs: number } | null {
|
||||
if (deadlineAtMs == null) {
|
||||
return { timeoutMs: config.subagentTimeoutMs, waitTimeoutMs: config.subagentWaitTimeoutMs };
|
||||
}
|
||||
const childBudgetMs = deadlineAtMs - CYCLE_DEADLINE_RESERVE_MS - nowMs;
|
||||
if (childBudgetMs < MIN_PATTERNS_SUBAGENT_BUDGET_MS) return null;
|
||||
return {
|
||||
timeoutMs: Math.min(config.subagentTimeoutMs, childBudgetMs),
|
||||
waitTimeoutMs: Math.min(config.subagentWaitTimeoutMs, childBudgetMs),
|
||||
};
|
||||
}
|
||||
|
||||
export async function runPhasePatterns(
|
||||
engine: BrainEngine,
|
||||
opts: PatternsPhaseOpts,
|
||||
@@ -149,7 +112,7 @@ export async function runPhasePatterns(
|
||||
if (budgets === null) {
|
||||
return skipped(
|
||||
'insufficient_cycle_budget',
|
||||
`remaining cycle budget under ${Math.round(MIN_PATTERNS_SUBAGENT_BUDGET_MS / 1000)}s ` +
|
||||
`remaining cycle budget under ${Math.round(MIN_SUBAGENT_START_BUDGET_MS / 1000)}s ` +
|
||||
`(reserve ${Math.round(CYCLE_DEADLINE_RESERVE_MS / 1000)}s); next cycle retries with a fresh budget`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -39,6 +39,11 @@ import { MinionQueue } from '../minions/queue.ts';
|
||||
import { waitForCompletion, TimeoutError } from '../minions/wait-for-completion.ts';
|
||||
import type { MinionJobInput, SubagentHandlerData } from '../minions/types.ts';
|
||||
import { discoverTranscripts, type DiscoveredTranscript } from './transcript-discovery.ts';
|
||||
import {
|
||||
remainingChildBudgetMs,
|
||||
CYCLE_DEADLINE_RESERVE_MS,
|
||||
MIN_SUBAGENT_START_BUDGET_MS,
|
||||
} from './deadline-budget.ts';
|
||||
import { serializeMarkdown, serializePageToMarkdown } from '../markdown.ts';
|
||||
import type { Page, PageType } from '../types.ts';
|
||||
import { validateSourceId } from '../utils.ts';
|
||||
@@ -252,6 +257,15 @@ export interface SynthesizePhaseOpts {
|
||||
* correct (source_id, slug) row. Unset → legacy 'default'.
|
||||
*/
|
||||
sourceId?: string;
|
||||
/**
|
||||
* Absolute deadline (epoch ms) of the enclosing minion job, or null for
|
||||
* direct callers (`gbrain dream`). When set, every child's job timeout
|
||||
* is clamped to the remaining time at submit and — unlike patterns'
|
||||
* single wait — the SEQUENTIAL per-child wait loop recomputes the
|
||||
* remaining budget before each wait, so N children can't accumulate
|
||||
* N × subagent_wait_timeout_ms past the parent budget (#2781).
|
||||
*/
|
||||
deadlineAtMs?: number | null;
|
||||
}
|
||||
|
||||
export async function runPhaseSynthesize(
|
||||
@@ -405,6 +419,21 @@ export async function runPhaseSynthesize(
|
||||
});
|
||||
}
|
||||
|
||||
// #2781: budget the fan-out from the REMAINING parent-job time.
|
||||
// Checked after the cheap gates (not_configured / cooldown / verdicts /
|
||||
// dry-run) so a budget skip only fires when the phase would otherwise
|
||||
// have submitted Sonnet work. Note the verdict loop above may itself
|
||||
// have consumed wall-clock — that's why this reads Date.now() here,
|
||||
// not at phase entry.
|
||||
const preFanoutBudgetMs = remainingChildBudgetMs(opts.deadlineAtMs, Date.now());
|
||||
if (preFanoutBudgetMs !== null && preFanoutBudgetMs < MIN_SUBAGENT_START_BUDGET_MS) {
|
||||
return skipped(
|
||||
'insufficient_cycle_budget',
|
||||
`remaining cycle budget under ${Math.round(MIN_SUBAGENT_START_BUDGET_MS / 1000)}s ` +
|
||||
`(reserve ${Math.round(CYCLE_DEADLINE_RESERVE_MS / 1000)}s); next cycle retries with a fresh budget`,
|
||||
);
|
||||
}
|
||||
|
||||
// Fan-out: submit one subagent per worth-processing transcript (or one
|
||||
// per chunk for transcripts that exceed the model's per-prompt budget).
|
||||
const allowedSlugPrefixes = await loadAllowedSlugPrefixes(config.outputRoot);
|
||||
@@ -422,7 +451,25 @@ export async function runPhaseSynthesize(
|
||||
|
||||
const maxCharsPerChunk = computeChunkCharBudget(config.model, config.maxPromptTokens);
|
||||
|
||||
// Set when the budget runs out anywhere past the pre-fanout gate. A
|
||||
// budget-truncated run must NOT stamp last_completion_ts — the default
|
||||
// 12h cooldown would contradict "next cycle retries with a fresh
|
||||
// budget" and pin the skipped work for half a day.
|
||||
let budgetExhausted = false;
|
||||
|
||||
for (const t of worthProcessing) {
|
||||
// #2781: the submit loop itself is fast (DB inserts, no LLM), but the
|
||||
// budget can still decay across a long worthProcessing list. Below
|
||||
// the start minimum, stop submitting — remaining transcripts
|
||||
// re-attempt next cycle (idempotency keys make the partial fan-out
|
||||
// safe to resume).
|
||||
const remainingMs = remainingChildBudgetMs(opts.deadlineAtMs, Date.now());
|
||||
if (remainingMs !== null && remainingMs < MIN_SUBAGENT_START_BUDGET_MS) {
|
||||
budgetExhausted = true;
|
||||
skipReports.push({ filePath: t.filePath, reason: 'insufficient_cycle_budget' });
|
||||
continue;
|
||||
}
|
||||
|
||||
const hash16 = t.contentHash.slice(0, 16);
|
||||
const hash6 = t.contentHash.slice(0, 6);
|
||||
|
||||
@@ -487,11 +534,27 @@ export async function runPhaseSynthesize(
|
||||
const idempotency_key = isChunked
|
||||
? `dream:synth:${t.filePath}:${hash16}:c${i}of${chunks.length}`
|
||||
: `dream:synth:${t.filePath}:${hash16}`;
|
||||
// #2781: clamp each child's own kill switch to the remaining time
|
||||
// at submit. The child's timeout_ms clock starts at ITS claim, so
|
||||
// this alone can't bound a child that sits queued — the wait loop's
|
||||
// cancel below closes that — but it bounds the common promptly-
|
||||
// claimed case without waiting for the cancel path. Below the
|
||||
// start minimum, stop submitting this transcript's remaining
|
||||
// chunks (per-chunk idempotency keys resume the rest next cycle).
|
||||
const remainingAtSubmitMs = remainingChildBudgetMs(opts.deadlineAtMs, Date.now());
|
||||
if (remainingAtSubmitMs !== null && remainingAtSubmitMs < MIN_SUBAGENT_START_BUDGET_MS) {
|
||||
budgetExhausted = true;
|
||||
skipReports.push({ filePath: t.filePath, reason: 'insufficient_cycle_budget' });
|
||||
break;
|
||||
}
|
||||
const childTimeoutMs = remainingAtSubmitMs === null
|
||||
? config.subagentTimeoutMs
|
||||
: Math.min(config.subagentTimeoutMs, remainingAtSubmitMs);
|
||||
const submitOpts: Partial<MinionJobInput> = {
|
||||
max_stalled: 3,
|
||||
on_child_fail: 'continue',
|
||||
idempotency_key,
|
||||
timeout_ms: config.subagentTimeoutMs,
|
||||
timeout_ms: childTimeoutMs,
|
||||
};
|
||||
const child = await queue.add(
|
||||
'subagent',
|
||||
@@ -508,17 +571,45 @@ export async function runPhaseSynthesize(
|
||||
|
||||
// Wait for every child to reach a terminal state. Tick yieldDuringPhase
|
||||
// every 5 min so the cycle lock TTL refreshes.
|
||||
//
|
||||
// #2781: the waits are SEQUENTIAL, so a fixed per-wait timeout
|
||||
// accumulates N × subagent_wait_timeout_ms across N children — past any
|
||||
// parent budget. Each iteration recomputes the remaining budget against
|
||||
// the same absolute deadline; once it's gone, the rest of the children
|
||||
// are cancelled instead of waited on (their timeout_ms clock starts at
|
||||
// THEIR claim, so an unwaited child could otherwise outlive the parent).
|
||||
// A child that already reached a terminal state on its own keeps its
|
||||
// real status — earlier siblings' waits gave it time to finish, and its
|
||||
// pages still collect normally.
|
||||
const childOutcomes: Array<{ jobId: number; status: string }> = [];
|
||||
for (const jobId of childIds) {
|
||||
let waitIdx = 0;
|
||||
for (; waitIdx < childIds.length; waitIdx++) {
|
||||
const jobId = childIds[waitIdx];
|
||||
const remainingWaitMs = remainingChildBudgetMs(opts.deadlineAtMs, Date.now());
|
||||
if (remainingWaitMs !== null && remainingWaitMs <= 0) break;
|
||||
const waitTimeoutMs = remainingWaitMs === null
|
||||
? config.subagentWaitTimeoutMs
|
||||
: Math.min(config.subagentWaitTimeoutMs, remainingWaitMs);
|
||||
try {
|
||||
const job = await waitForCompletion(queue, jobId, {
|
||||
timeoutMs: config.subagentWaitTimeoutMs,
|
||||
timeoutMs: waitTimeoutMs,
|
||||
pollMs: 5 * 1000,
|
||||
});
|
||||
childOutcomes.push({ jobId, status: job.status });
|
||||
} catch (e) {
|
||||
if (e instanceof TimeoutError) {
|
||||
childOutcomes.push({ jobId, status: 'timeout' });
|
||||
// Same rationale as patterns: a child that sat queued can outlive
|
||||
// the deadline this wait was clamped to. Strip it.
|
||||
try { await queue.cancelJob(jobId); } catch { /* best-effort */ }
|
||||
// A timeout on a DEADLINE-CLAMPED wait is budget truncation, not
|
||||
// a genuine child overrun — the child never got its full
|
||||
// configured wait. Suppress the cooldown stamp so the next cycle
|
||||
// retries. A timeout at the full configured wait keeps the
|
||||
// pre-existing stamping behavior.
|
||||
if (remainingWaitMs !== null && waitTimeoutMs < config.subagentWaitTimeoutMs) {
|
||||
budgetExhausted = true;
|
||||
}
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
@@ -528,6 +619,30 @@ export async function runPhaseSynthesize(
|
||||
try { await opts.yieldDuringPhase(); } catch { /* best-effort */ }
|
||||
}
|
||||
}
|
||||
// Budget ran out before every child was waited on: settle the rest
|
||||
// without waiting. Cancel-first (one write per child): a non-null
|
||||
// return means we cancelled it; null means it was already terminal —
|
||||
// read its real status so a completed sibling isn't misreported as
|
||||
// timeout. Deliberately sequential per-child rather than one bulk SQL:
|
||||
// cancelJob carries per-job semantics (child_done emission, aggregator
|
||||
// unblocking) a bulk UPDATE would skip, each call is a millisecond-
|
||||
// scale write with NO waits, and realistic fan-outs (children ≤
|
||||
// max_chunks × transcripts per cycle) settle well inside the reserve.
|
||||
if (waitIdx < childIds.length) {
|
||||
budgetExhausted = true;
|
||||
for (; waitIdx < childIds.length; waitIdx++) {
|
||||
const jobId = childIds[waitIdx];
|
||||
let status = 'timeout';
|
||||
try {
|
||||
const cancelled = await queue.cancelJob(jobId);
|
||||
if (cancelled === null) {
|
||||
const current = await queue.getJob(jobId);
|
||||
if (current) status = current.status;
|
||||
}
|
||||
} catch { /* best-effort */ }
|
||||
childOutcomes.push({ jobId, status });
|
||||
}
|
||||
}
|
||||
|
||||
// Collect slugs from put_page tool executions across the children
|
||||
// (codex finding #2: deterministic provenance, NOT pages.updated_at).
|
||||
@@ -560,8 +675,13 @@ export async function runPhaseSynthesize(
|
||||
await writeSummaryPage(engine, opts.brainDir, summarySlug, summaryDate, writtenSlugs, childOutcomes, cycleSourceId);
|
||||
}
|
||||
|
||||
// Write completion timestamp ON SUCCESS only.
|
||||
await engine.setConfig('dream.synthesize.last_completion_ts', new Date().toISOString());
|
||||
// Write completion timestamp ON SUCCESS only — and NOT on a
|
||||
// budget-truncated run: stamping would arm the cooldown (default 12h)
|
||||
// over transcripts/children this run never gave a real chance,
|
||||
// contradicting "next cycle retries with a fresh budget" (#2781).
|
||||
if (!budgetExhausted) {
|
||||
await engine.setConfig('dream.synthesize.last_completion_ts', new Date().toISOString());
|
||||
}
|
||||
|
||||
const ms = Date.now() - start;
|
||||
const submittedTranscripts = worthProcessing.length - skipReports.length;
|
||||
|
||||
@@ -20,8 +20,8 @@ import { MinionQueue } from '../src/core/minions/queue.ts';
|
||||
import {
|
||||
clampSubagentBudgets,
|
||||
CYCLE_DEADLINE_RESERVE_MS,
|
||||
MIN_PATTERNS_SUBAGENT_BUDGET_MS,
|
||||
} from '../src/core/cycle/patterns.ts';
|
||||
MIN_SUBAGENT_START_BUDGET_MS,
|
||||
} from '../src/core/cycle/deadline-budget.ts';
|
||||
|
||||
const CONFIG = {
|
||||
subagentTimeoutMs: 30 * 60 * 1000,
|
||||
@@ -60,15 +60,15 @@ describe('clampSubagentBudgets', () => {
|
||||
});
|
||||
|
||||
test('remaining budget below minimum → null (caller skips, no submit)', () => {
|
||||
const deadline = now + CYCLE_DEADLINE_RESERVE_MS + MIN_PATTERNS_SUBAGENT_BUDGET_MS - 1;
|
||||
const deadline = now + CYCLE_DEADLINE_RESERVE_MS + MIN_SUBAGENT_START_BUDGET_MS - 1;
|
||||
expect(clampSubagentBudgets(CONFIG, deadline, now)).toBeNull();
|
||||
});
|
||||
|
||||
test('boundary: exactly the minimum budget → submit allowed', () => {
|
||||
const deadline = now + CYCLE_DEADLINE_RESERVE_MS + MIN_PATTERNS_SUBAGENT_BUDGET_MS;
|
||||
const deadline = now + CYCLE_DEADLINE_RESERVE_MS + MIN_SUBAGENT_START_BUDGET_MS;
|
||||
expect(clampSubagentBudgets(CONFIG, deadline, now)).toEqual({
|
||||
timeoutMs: MIN_PATTERNS_SUBAGENT_BUDGET_MS,
|
||||
waitTimeoutMs: MIN_PATTERNS_SUBAGENT_BUDGET_MS,
|
||||
timeoutMs: MIN_SUBAGENT_START_BUDGET_MS,
|
||||
waitTimeoutMs: MIN_SUBAGENT_START_BUDGET_MS,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -159,7 +159,7 @@ describe('deadline plumbing wiring (structural)', () => {
|
||||
// Budget gate sits AFTER the provider probe so a no-provider brain
|
||||
// still reports no_provider (cheaper, more actionable reason).
|
||||
const probeIdx = patternsSrc.indexOf("skipped('no_provider'");
|
||||
// lastIndexOf: the doc comment on MIN_PATTERNS_SUBAGENT_BUDGET_MS
|
||||
// lastIndexOf: the doc comment on MIN_SUBAGENT_START_BUDGET_MS
|
||||
// mentions the reason string too; the CALL SITE is the later hit.
|
||||
const budgetIdx = patternsSrc.lastIndexOf('insufficient_cycle_budget');
|
||||
expect(probeIdx).toBeGreaterThan(0);
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
/**
|
||||
* #2781 follow-up — synthesize budgets its fan-out and its SEQUENTIAL
|
||||
* per-child waits from the remaining parent-job time. Without the
|
||||
* per-wait recompute, N children accumulate N × subagent_wait_timeout_ms
|
||||
* past any parent budget (patterns' single-wait clamp can't help here).
|
||||
*
|
||||
* Layers mirror test/cycle-patterns-deadline-budget.test.ts:
|
||||
* 1. Unit tests on `remainingChildBudgetMs` (the per-iteration primitive).
|
||||
* 2. Structural pins on the synthesize wiring.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { readFileSync } from 'fs';
|
||||
import {
|
||||
remainingChildBudgetMs,
|
||||
CYCLE_DEADLINE_RESERVE_MS,
|
||||
} from '../src/core/cycle/deadline-budget.ts';
|
||||
|
||||
describe('remainingChildBudgetMs', () => {
|
||||
const now = 1_000_000_000_000;
|
||||
|
||||
test('null deadline → null (direct `gbrain dream` back-compat)', () => {
|
||||
expect(remainingChildBudgetMs(null, now)).toBeNull();
|
||||
expect(remainingChildBudgetMs(undefined, now)).toBeNull();
|
||||
});
|
||||
|
||||
test('subtracts the reserve from the remaining time', () => {
|
||||
const deadline = now + 10 * 60 * 1000;
|
||||
expect(remainingChildBudgetMs(deadline, now)).toBe(10 * 60 * 1000 - CYCLE_DEADLINE_RESERVE_MS);
|
||||
});
|
||||
|
||||
test('goes non-positive once the reserve is breached (callers stop waiting)', () => {
|
||||
expect(remainingChildBudgetMs(now + CYCLE_DEADLINE_RESERVE_MS, now)).toBe(0);
|
||||
expect(remainingChildBudgetMs(now - 1000, now)).toBeLessThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('synthesize deadline wiring (structural)', () => {
|
||||
const synthSrc = readFileSync(new URL('../src/core/cycle/synthesize.ts', import.meta.url), 'utf-8');
|
||||
const cycleSrc = readFileSync(new URL('../src/core/cycle.ts', import.meta.url), 'utf-8');
|
||||
const patternsSrc = readFileSync(new URL('../src/core/cycle/patterns.ts', import.meta.url), 'utf-8');
|
||||
|
||||
test('runCycle forwards deadlineAtMs to the synthesize phase', () => {
|
||||
// Both subagent-spawning phases receive the deadline (patterns pinned
|
||||
// in its own test file; this pins the synthesize call site).
|
||||
const callSite = cycleSrc.indexOf('runPhaseSynthesize(engine, {');
|
||||
expect(callSite).toBeGreaterThan(0);
|
||||
const callSlice = cycleSrc.slice(callSite, callSite + 600);
|
||||
expect(callSlice).toContain('deadlineAtMs: opts.deadlineAtMs ?? null');
|
||||
});
|
||||
|
||||
test('pre-fanout gate skips honestly before submitting any Sonnet work', () => {
|
||||
expect(synthSrc).toContain('insufficient_cycle_budget');
|
||||
// Gate sits after the verdict loop (which consumes wall-clock) and
|
||||
// before the fan-out submit loop.
|
||||
const gateIdx = synthSrc.indexOf('preFanoutBudgetMs');
|
||||
const fanoutIdx = synthSrc.indexOf('const queue = new MinionQueue(engine)');
|
||||
expect(gateIdx).toBeGreaterThan(0);
|
||||
expect(fanoutIdx).toBeGreaterThan(gateIdx);
|
||||
});
|
||||
|
||||
test('per-child submit clamps timeout_ms to the remaining budget', () => {
|
||||
expect(synthSrc).toContain('timeout_ms: childTimeoutMs');
|
||||
expect(synthSrc).not.toContain('timeout_ms: config.subagentTimeoutMs');
|
||||
});
|
||||
|
||||
test('sequential wait loop recomputes the remaining budget EVERY iteration', () => {
|
||||
// The recompute call must be INSIDE the wait-loop body — a single
|
||||
// pre-loop computation is exactly the accumulation bug.
|
||||
const loopIdx = synthSrc.indexOf('for (; waitIdx < childIds.length; waitIdx++)');
|
||||
expect(loopIdx).toBeGreaterThan(0);
|
||||
const loopSlice = synthSrc.slice(loopIdx, loopIdx + 800);
|
||||
expect(loopSlice).toContain('remainingChildBudgetMs(opts.deadlineAtMs, Date.now())');
|
||||
expect(synthSrc).toContain('timeoutMs: waitTimeoutMs');
|
||||
expect(synthSrc).not.toContain('timeoutMs: config.subagentWaitTimeoutMs');
|
||||
});
|
||||
|
||||
test('exhausted budget settles remaining children: cancel-first, real status when already terminal', () => {
|
||||
// Cancel returns null for an already-terminal row → refetch so a
|
||||
// completed sibling isn't misreported as timeout.
|
||||
expect(synthSrc).toContain('cancelled === null');
|
||||
expect(synthSrc).toContain('queue.getJob(jobId)');
|
||||
});
|
||||
|
||||
test('wait-timeout path cancels the child (queued child can outlive the clamp)', () => {
|
||||
// Two cancel sites: TimeoutError in the wait loop + the settle loop.
|
||||
const matches = synthSrc.match(/queue\.cancelJob\(jobId\)/g) ?? [];
|
||||
expect(matches.length).toBe(2);
|
||||
});
|
||||
|
||||
test('budget-truncated runs do not arm the synthesize cooldown', () => {
|
||||
expect(synthSrc).toContain('if (!budgetExhausted) {');
|
||||
const stampIdx = synthSrc.indexOf("setConfig('dream.synthesize.last_completion_ts'");
|
||||
const guardIdx = synthSrc.indexOf('if (!budgetExhausted) {');
|
||||
expect(stampIdx).toBeGreaterThan(guardIdx);
|
||||
});
|
||||
|
||||
test('a timeout on a deadline-CLAMPED wait counts as budget truncation', () => {
|
||||
// The child never got its full configured wait — suppressing the
|
||||
// cooldown stamp lets the next cycle retry. A timeout at the full
|
||||
// configured wait keeps the pre-existing stamping behavior.
|
||||
expect(synthSrc).toContain('waitTimeoutMs < config.subagentWaitTimeoutMs');
|
||||
});
|
||||
|
||||
test('mid-fanout exhaustion stops submitting below the start minimum, not merely at zero', () => {
|
||||
// Both the per-transcript guard and the per-chunk guard compare against
|
||||
// MIN_SUBAGENT_START_BUDGET_MS (a <= 0 guard would still submit
|
||||
// guaranteed-kill children with second-scale timeouts).
|
||||
const matches = synthSrc.match(/< MIN_SUBAGENT_START_BUDGET_MS/g) ?? [];
|
||||
expect(matches.length).toBeGreaterThanOrEqual(3); // pre-fanout + transcript + chunk guards
|
||||
});
|
||||
|
||||
test('shared helpers live in deadline-budget.ts (no patterns↔synthesize cycle)', () => {
|
||||
expect(synthSrc).toContain("from './deadline-budget.ts'");
|
||||
expect(patternsSrc).toContain("from './deadline-budget.ts'");
|
||||
// patterns must NOT re-define the helpers it now imports.
|
||||
expect(patternsSrc).not.toContain('export function clampSubagentBudgets');
|
||||
expect(patternsSrc).not.toContain('export const CYCLE_DEADLINE_RESERVE_MS');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user