Revert "fix(jobs/autopilot): --install interval persistence, --lock-duration flag, dead-jobs doctor check, deployment-shape docs (#3129)"

This reverts commit 8345abce42.
This commit is contained in:
Garry Tan
2026-07-23 15:26:33 -07:00
parent 8345abce42
commit 1392243d3b
9 changed files with 11 additions and 376 deletions
+5 -33
View File
@@ -361,7 +361,7 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
if (args.includes('--help') || args.includes('-h')) {
console.log(
'Usage: gbrain autopilot [--repo <path>] [--interval N] [--json] [--no-worker]\n' +
' gbrain autopilot --install [--repo <path>] [--interval N]\n' +
' gbrain autopilot --install [--repo <path>]\n' +
' gbrain autopilot --uninstall\n' +
' gbrain autopilot --status [--json]\n\n' +
'Self-maintaining brain daemon. Runs the full maintenance cycle\n' +
@@ -385,12 +385,7 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
}
const repoPath = parseArg(args, '--repo') || await engine.getConfig('sync.repo_path');
// Flag → persisted config (written by --install --interval, #2794) → 300s.
const intervalRaw = parseArg(args, '--interval')
|| await engine.getConfig('autopilot.interval')
|| '300';
const intervalParsed = parseInt(intervalRaw, 10);
const baseInterval = Number.isFinite(intervalParsed) && intervalParsed >= 1 ? intervalParsed : 300;
const baseInterval = parseInt(parseArg(args, '--interval') || '300', 10);
const jsonMode = args.includes('--json');
const forceInline = args.includes('--inline');
const noWorker = !shouldSpawnAutopilotWorker(args);
@@ -1303,8 +1298,7 @@ function detectOpenClaw(): { detected: boolean; bootstrapCandidates: string[] }
return { detected: signal, bootstrapCandidates: existing };
}
// Exported for tests (issue #2794: the wrapper must carry --interval).
export function writeWrapperScript(repoPath: string, intervalSeconds?: number): string {
function writeWrapperScript(repoPath: string): string {
const home = process.env.HOME || '';
const gbrainDir = join(home, '.gbrain');
mkdirSync(gbrainDir, { recursive: true });
@@ -1324,7 +1318,7 @@ export function writeWrapperScript(repoPath: string, intervalSeconds?: number):
# OPENAI/ANTHROPIC keys exported in zshenv reach autopilot.
[ -f ~/.zshenv ] && source ~/.zshenv 2>/dev/null
source ~/.zshrc 2>/dev/null || source ~/.bashrc 2>/dev/null || true
exec '${safeGbrainPath}' autopilot --repo '${safeRepoPath}'${intervalSeconds !== undefined ? ` --interval '${intervalSeconds}'` : ''}
exec '${safeGbrainPath}' autopilot --repo '${safeRepoPath}'
`;
writeFileSync(wrapperPath, wrapper, { mode: 0o755 });
return wrapperPath;
@@ -1343,29 +1337,7 @@ async function installDaemon(engine: BrainEngine, args: string[]) {
const injectBootstrap = args.includes('--inject-bootstrap');
const noInject = args.includes('--no-inject');
// issue #2794: --install used to silently drop --interval — the wrapper
// always exec'd bare `autopilot --repo ...` (default 300s). Parse it here,
// persist to config so a later flag-less --install regenerates the wrapper
// with the same tuning, and thread it into the exec line.
const intervalRaw = parseArg(args, '--interval');
let intervalSeconds: number | undefined;
if (intervalRaw !== undefined) {
const parsed = Number(intervalRaw);
if (!Number.isInteger(parsed) || parsed < 1) {
console.error(`Error: --interval must be a positive integer (seconds), got "${intervalRaw}"`);
process.exit(2);
}
intervalSeconds = parsed;
await engine.setConfig('autopilot.interval', String(parsed));
} else {
const persisted = await engine.getConfig('autopilot.interval');
if (persisted) {
const parsed = Number(persisted);
if (Number.isInteger(parsed) && parsed >= 1) intervalSeconds = parsed;
}
}
const wrapperPath = writeWrapperScript(repoPath, intervalSeconds);
const wrapperPath = writeWrapperScript(repoPath);
const home = process.env.HOME || '';
switch (target) {
-83
View File
@@ -707,14 +707,6 @@ async function cmdDoctor(args: string[]): Promise<void> {
}
}
// Cross-cutting check (PR #1185, @ethanbeard): dead jobs in the minions
// queue. Shell jobs submitted by collectors die silently from the
// per-integration perspective — the collector's drain sees the submit
// succeed and stays green while the worker job dead-letters on the queue
// side, so a whole pipeline can be broken for days before anyone notices.
const deadJobsCheck = checkDeadJobs();
if (deadJobsCheck) results.push(deadJobsCheck);
if (jsonMode) {
const fails = results.filter(r => r.status !== 'ok');
console.log(JSON.stringify({
@@ -734,85 +726,10 @@ async function cmdDoctor(args: string[]): Promise<void> {
}
}
// Surface cross-cutting [queue] checks below the per-integration sections.
const queueChecks = results.filter(r => r.integration === '[queue]');
if (queueChecks.length > 0) {
console.log(` [queue]: ${queueChecks.every(c => c.status === 'ok') ? 'OK' : 'ISSUES'}`);
for (const c of queueChecks) {
const icon = c.status === 'ok' ? ' ✓' : c.status === 'timeout' ? ' ⏱' : ' ✗';
console.log(`${icon} ${c.output}`);
}
}
const totalFails = results.filter(r => r.status !== 'ok').length;
console.log(`\n OVERALL: ${totalFails === 0 ? 'All checks passed' : `${totalFails} issue(s) found`}`);
}
/**
* Pure aggregation for the dead-jobs queue check (PR #1185 rework): given
* parsed `jobs list --status dead --json` output, count jobs whose
* finished_at falls inside the last 24 hours — parity with the main
* `gbrain doctor` queue checks, so one ancient dead job can't flag
* `[queue]: ISSUES` forever — grouped by job name, biggest first.
* Exported for unit tests.
*/
export function summarizeDeadJobs(
jobs: Array<{ name?: unknown; finished_at?: unknown }>,
now: Date = new Date(),
): { total: number; breakdown: string } {
const cutoff = now.getTime() - 24 * 60 * 60 * 1000;
const byName: Record<string, number> = {};
let total = 0;
for (const job of jobs) {
if (typeof job.finished_at !== 'string' && !(job.finished_at instanceof Date)) continue;
const finished = new Date(job.finished_at as string | Date);
if (!Number.isFinite(finished.getTime()) || finished.getTime() < cutoff) continue;
const name = typeof job.name === 'string' ? job.name : 'unknown';
byName[name] = (byName[name] ?? 0) + 1;
total++;
}
const breakdown = Object.entries(byName)
.sort((a, b) => b[1] - a[1])
.map(([n, c]) => `${n}:${c}`)
.join(', ');
return { total, breakdown };
}
/**
* Dead minion jobs in the last 24h. Returns a CheckResult only when dead
* jobs exist (silent when healthy, per doctor's ISSUES-only model).
*
* Shells out to `gbrain jobs list --json` instead of opening a DB
* connection — preserves this file's standalone-CLI design. The JSON
* surface exists for exactly this: the human table's column widths shift
* with long job names, so screen-scraping it is not an option.
*/
function checkDeadJobs(): CheckResult | null {
try {
// --limit 500 caps memory while comfortably covering a day of failures;
// getJobs orders by created_at DESC so the most recent dead jobs come first.
const out = execSync('gbrain jobs list --status dead --limit 500 --json', {
encoding: 'utf8',
timeout: 10_000,
stdio: ['ignore', 'pipe', 'ignore'],
});
const jobs: unknown = JSON.parse(out);
if (!Array.isArray(jobs)) return null;
const { total, breakdown } = summarizeDeadJobs(jobs);
if (total === 0) return null;
return {
integration: '[queue]',
check: 'dead_jobs',
status: 'fail',
output: `${total} dead job(s) in the last 24h (${breakdown}). Inspect: gbrain jobs list --status dead; details: gbrain jobs get <id>`,
};
} catch {
// DB unreachable / gbrain not on PATH / non-JSON output — skip silently.
// Doctor still surfaces the per-recipe checks; this one is best-effort.
return null;
}
}
function cmdStats(args: string[]): void {
const jsonMode = args.includes('--json');
const recipes = loadAllRecipes();
+4 -61
View File
@@ -122,27 +122,6 @@ export function parseNiceFlag(args: string[], env: NodeJS.ProcessEnv = process.e
}
}
/** Parse `--lock-duration MS` (then `GBRAIN_LOCK_DURATION` env; flag wins).
* Tunes the worker's stall-lock window — the wall-clock dead-letter cap is
* lockDuration × max_stalled, so the default 30s lock caps long jobs at
* ~180s with the schema defaults (issue #1014). Returns undefined when
* absent (worker default 30000ms applies). Throws on non-integer values or
* values < 1000 — sub-1000 is almost always a seconds-vs-ms unit-confusion
* typo (parity with --health-interval). Exported for unit tests; CLI call
* sites wrap with console.error + process.exit(1). */
export function parseLockDurationFlag(args: string[], env: NodeJS.ProcessEnv = process.env): number | undefined {
const raw = parseFlag(args, '--lock-duration') ?? env.GBRAIN_LOCK_DURATION;
if (raw === undefined || raw === '') return undefined;
const parsed = Number(raw);
if (!Number.isInteger(parsed) || parsed < 1000) {
throw new Error(
`--lock-duration must be an integer >= 1000 (milliseconds), got "${raw}". ` +
`The flag takes milliseconds; for a 60-second lock pass 60000.`,
);
}
return parsed;
}
export function resolveWorkerConcurrency(args: string[], env: NodeJS.ProcessEnv = process.env): number {
const raw = parseFlag(args, '--concurrency') ?? env.GBRAIN_WORKER_CONCURRENCY ?? '1';
const parsed = parseInt(raw, 10);
@@ -225,7 +204,7 @@ USAGE
[--idempotency-key K] [--queue Q] [--dry-run]
[--redact-secrets] (shell only; scrubs inherit
values from stdout/stderr)
gbrain jobs list [--status S] [--queue Q] [--limit N] [--json]
gbrain jobs list [--status S] [--queue Q] [--limit N]
gbrain jobs get <id>
gbrain jobs cancel <id>
gbrain jobs retry <id>
@@ -234,17 +213,12 @@ USAGE
gbrain jobs stats
gbrain jobs smoke
gbrain jobs work [--queue Q] [--concurrency N] [--max-rss MB]
[--health-interval MS] [--nice N] [--lock-duration MS]
[--health-interval MS] [--nice N]
gbrain jobs supervisor [start] [--detach] [--json]
[--concurrency N] [--queue Q] [--pid-file PATH]
[--max-crashes N] [--health-interval N]
[--allow-shell-jobs] [--cli-path PATH]
[--max-rss MB] [--nice N] [--lock-duration MS]
--lock-duration MS Worker stall-lock window in milliseconds (default
30000). The wall-clock dead-letter cap for a running job is
lock-duration × max_stalled, so raise this for long-running
handlers. Minimum 1000. Env: GBRAIN_LOCK_DURATION (flag wins).
[--max-rss MB] [--nice N]
--nice N OS scheduling priority, -20 (highest) to 19 (nicest). Lowers CPU
priority without cutting concurrency — full throughput when the
@@ -510,7 +484,6 @@ HANDLER TYPES (built in)
const status = parseFlag(args, '--status') as MinionJobStatus | undefined;
const queueName = parseFlag(args, '--queue');
const limit = parseInt(parseFlag(args, '--limit') ?? '20', 10);
const jsonMode = hasFlag(args, '--json');
// v0.32: thin-client routing. The `list_jobs` MCP op is admin-scoped
// but not localOnly, so a thin-client install with admin access can
@@ -530,14 +503,6 @@ HANDLER TYPES (built in)
jobs = await queue.getJobs({ status, queue: queueName, limit });
}
// --json: machine-readable output (one JSON array on stdout) so
// downstream tooling (e.g. `gbrain integrations doctor`) never
// screen-scrapes the human table, whose column widths shift.
if (jsonMode) {
console.log(JSON.stringify(jobs));
return;
}
if (jobs.length === 0) {
console.log('No jobs found.');
return;
@@ -980,16 +945,6 @@ HANDLER TYPES (built in)
healthCheckInterval = parsed;
}
// --lock-duration MS (issue #1014): tune the stall-lock window (and so
// the lockDuration × max_stalled wall-clock dead-letter cap).
let lockDuration: number | undefined;
try {
lockDuration = parseLockDurationFlag(args);
} catch (e) {
console.error(`Error: ${e instanceof Error ? e.message : String(e)}`);
process.exit(1);
}
// --nice N (issue #1815): renice this worker process so background work
// yields CPU to foreground tasks without sacrificing concurrency. Applied
// at the CLI layer (worker.ts stays embeddable). Niceness inherits to the
@@ -1011,7 +966,6 @@ HANDLER TYPES (built in)
const worker = new MinionWorker(engine, {
queue: queueName, concurrency, maxRssMb, healthCheckInterval,
...(lockDuration !== undefined ? { lockDuration } : {}),
});
await registerBuiltinHandlers(worker, engine);
@@ -1052,8 +1006,7 @@ HANDLER TYPES (built in)
: `, health-check: ${Math.round(healthCheckInterval / 1000)}s`)
: '';
const niceNote = niceResult ? `, nice: ${formatNice(niceResult.effective ?? niceVal!)}` : '';
const lockNote = lockDuration !== undefined ? `, lock: ${lockDuration}ms` : '';
console.log(`Minion worker started (queue: ${queueName}, concurrency: ${concurrency}${watchdogNote}${healthNote}${niceNote}${lockNote})`);
console.log(`Minion worker started (queue: ${queueName}, concurrency: ${concurrency}${watchdogNote}${healthNote}${niceNote})`);
console.log(`Registered handlers: ${worker.registeredNames.join(', ')}`);
// Register in the live worker registry (issue #1815) so jobs stats / doctor
@@ -1305,15 +1258,6 @@ HANDLER TYPES (built in)
}
healthInterval = parsed;
}
// --lock-duration (supervisor): validated here (fail-fast even for
// --detach), passed down to the spawned worker via buildWorkerArgs.
let supLockDuration: number | undefined;
try {
supLockDuration = parseLockDurationFlag(args);
} catch (e) {
console.error(`Error: ${e instanceof Error ? e.message : String(e)}`);
process.exit(1);
}
const allowShellJobs = hasFlag(args, '--allow-shell-jobs') ||
!!process.env.GBRAIN_ALLOW_SHELL_JOBS;
const detach = hasFlag(args, '--detach');
@@ -1381,7 +1325,6 @@ HANDLER TYPES (built in)
allowShellJobs,
json: jsonMode,
maxRssMb,
...(supLockDuration !== undefined ? { lockDuration: supLockDuration } : {}),
...(supNice !== undefined ? { nice_requested: supNice } : {}),
...(supNiceResult?.effective != null ? { nice_effective: supNiceResult.effective } : {}),
...(supNiceResult?.error ? { nice_error: supNiceResult.error } : {}),
+1 -8
View File
@@ -87,10 +87,6 @@ export interface SupervisorOpts {
* resolveDefaultMaxRssMb() (issue #1678) instead of a flat default.
* Set to 0 to spawn the worker without a watchdog. */
maxRssMb: number;
/** Worker stall-lock window in ms (issue #1014), passed to the spawned
* worker as `--lock-duration N`. The wall-clock dead-letter cap is
* lockDuration × max_stalled. Undefined → worker default (30000ms). */
lockDuration?: number;
/** Niceness (issue #1815) the operator requested via `--nice` / `GBRAIN_NICE`,
* or undefined to inherit. When set, the worker is spawned with `--nice N` so
* it re-applies the value (the supervisor itself is reniced by the CLI layer,
@@ -176,7 +172,7 @@ const DEFAULTS: Omit<SupervisorOpts, 'cliPath'> = {
* niceness also inherits to the worker's own children automatically.
*/
export function buildWorkerArgs(
opts: Pick<SupervisorOpts, 'concurrency' | 'queue' | 'maxRssMb' | 'nice_requested' | 'lockDuration'>,
opts: Pick<SupervisorOpts, 'concurrency' | 'queue' | 'maxRssMb' | 'nice_requested'>,
): string[] {
const args = [
'jobs', 'work',
@@ -189,9 +185,6 @@ export function buildWorkerArgs(
if (opts.nice_requested !== undefined) {
args.push('--nice', String(opts.nice_requested));
}
if (opts.lockDuration !== undefined) {
args.push('--lock-duration', String(opts.lockDuration));
}
return args;
}