fix(onboard): stop repeating the same auto-remediation within a run (#2854)

When the recommendation list is refreshed between remediation steps, a
remediation that doesn't clear its own health signal is reintroduced
under its stable id and attempted again, indefinitely on long runs.
Track attempted recommendation ids for the run and skip re-attempts.

Includes a behavioral regression test: a persistently-stuck signal is
attempted once, the loop terminates, and other remediations still run.
This commit is contained in:
Sanchal Ranjan
2026-07-23 11:38:10 -07:00
committed by GitHub
parent b98fae9b61
commit 054badbe60
2 changed files with 91 additions and 5 deletions
+12 -5
View File
@@ -182,6 +182,7 @@ export async function runRemediation(
// Real submission path
const submitted: StepResult[] = [];
const abortedIds = new Set<string>();
const attemptedIds = new Set<string>();
const doctorRunId = crypto.randomUUID();
const { MinionQueue } = await import('../minions/queue.ts');
@@ -231,6 +232,7 @@ export async function runRemediation(
if (completedFromCheckpoint.has(step.id)) {
const result: StepResult = { step: stepCount, id: step.id, job_id: null, status: 'completed' };
submitted.push(result);
attemptedIds.add(step.id);
hooks.onStepEnd?.(result);
recs.shift();
continue;
@@ -241,6 +243,7 @@ export async function runRemediation(
const result: StepResult = { step: stepCount, id: step.id, job_id: null, status: 'skipped_dep_aborted' };
submitted.push(result);
abortedIds.add(step.id);
attemptedIds.add(step.id);
hooks.onStepEnd?.(result);
recs.shift();
continue;
@@ -299,13 +302,17 @@ export async function runRemediation(
hooks.onStepEnd?.(errResult);
}
attemptedIds.add(step.id);
recs.shift();
// D7: scoped recheck — re-compute plan from fresh health snapshot.
// The next plan may drop completed steps and re-introduce failed
// steps with bumped retry suffix (D1).
// Queue-level max_attempts handles retries within a submitted attempt.
// A stuck health signal regenerates the same stable id, so keep ids this
// run already attempted out of the refreshed list to avoid re-enqueueing
// them forever.
if (recs.length === 0 || stepCount >= maxJobs) break;
const freshHealth = await engine.getHealth();
recs = computeRecommendations(freshHealth, ctx).filter((r) => r.status === 'remediable');
recs = computeRecommendations(freshHealth, ctx)
.filter((r) => r.status === 'remediable' && !attemptedIds.has(r.id));
}
};
@@ -322,8 +329,8 @@ export async function runRemediation(
}
// Clear checkpoint on a clean run (no budget abort). Failed steps in the
// submitted set don't disqualify the cleanup — they re-surface on the
// next plan with bumped suffixes.
// submitted set don't disqualify cleanup; an uncleared health signal can
// produce the same stable id again in a later run.
if (!budgetAbort) {
clearRemediationCheckpoint(planHash);
}
+79
View File
@@ -0,0 +1,79 @@
import { describe, expect, mock, test } from 'bun:test';
import type { BrainEngine } from '../src/core/engine.ts';
import type { BrainHealth } from '../src/core/types.ts';
const attemptedJobs: string[] = [];
mock.module('../src/core/minions/queue.ts', () => ({
MinionQueue: class {
async add(name: string) {
attemptedJobs.push(name);
return { id: attemptedJobs.length };
}
},
}));
mock.module('../src/core/minions/wait-for-completion.ts', () => ({
waitForCompletion: async (_queue: unknown, jobId: number) => ({
id: jobId,
status: 'completed',
}),
}));
mock.module('../src/core/remediation-checkpoint.ts', () => ({
computePlanHash: (ids: string[]) => [...ids].sort().join('|'),
saveRemediationCheckpoint: () => undefined,
loadRemediationCheckpoint: () => null,
listRemediationCheckpoints: () => [],
clearRemediationCheckpoint: () => undefined,
}));
mock.module('../src/core/ai/gateway.ts', () => ({
getEmbeddingModel: () => 'ollama:nomic-embed-text',
getEmbeddingDimensions: () => 768,
withBudgetTracker: async (_tracker: unknown, fn: () => Promise<void>) => fn(),
}));
const { runRemediation } = await import('../src/core/remediation/run.ts');
function makeHealth(): BrainHealth {
return {
page_count: 100,
embed_coverage: 1,
stale_pages: 1,
orphan_pages: 0,
missing_embeddings: 0,
brain_score: 80,
dead_links: 1,
link_coverage: 1,
timeline_coverage: 1,
most_connected: [],
embed_coverage_score: 35,
link_density_score: 25,
timeline_coverage_score: 15,
no_orphans_score: 15,
no_dead_links_score: 0,
};
}
describe('runRemediation recheck loop guard', () => {
test('attempts a stable stuck remediation once and continues to later work', async () => {
attemptedJobs.length = 0;
const health = makeHealth();
const engine = {
kind: 'postgres',
getHealth: async () => health,
getConfig: async (key: string) => key === 'sync.repo_path' ? '/brain' : null,
} as BrainEngine;
const result = await runRemediation(engine, { maxJobs: 4 });
expect(attemptedJobs.filter((name) => name === 'backlinks')).toHaveLength(1);
expect(attemptedJobs).toEqual(['backlinks', 'sync', 'extract']);
expect(result.submitted.map((step) => step.id)).toEqual([
'backlinks.fix',
'sync.repo',
'extract.all',
]);
});
});