mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
dream.drift.enabled has gated an unwired scaffold since v0.28: runPhaseDrift had zero call sites, the resolved model + BudgetMeter were discarded (void modelId; void meter), and no operator-readable output existed. - Wire 'drift' as a CyclePhase (default OFF via dream.drift.enabled), ordered after the calibration trio and before embed so the report page gets embedded same-cycle. PHASE_SCOPE=global, cycle-lock coordinated, --once (--phase drift --once) bypasses the gate for one run. - Implement the LLM judge: soft-band candidates (weight 0.3-0.85, active, unresolved, fresh timeline evidence) are judged against their page's recent timeline entries via gateway chat; BudgetMeter-gated (dream.drift.budget, default $1), capped by dream.drift.max_per_cycle (default 20). Judge model resolves models.drift -> reasoning tier -> sonnet fallback (unchanged from scaffold). - Report-only v1: judged candidates land on a reports/drift-<date> page. dream.drift.auto_update mutates NOTHING; the flag state is recorded in the report for operators. Co-authored-by: Garry Tan <garrytan@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Time Attakc
co-authored by
Garry Tan
Claude Fable 5
Garry Tan
Claude Fable 5
parent
69bc37f745
commit
40d9b83d5c
@@ -86,7 +86,7 @@ interface DreamArgs {
|
||||
* `--phase <name>`; bare `--once` is a usage error (there'd be no single
|
||||
* phase to target). Applies only to phases with a config `.enabled` gate
|
||||
* (patterns, synthesize, conversation_facts_backfill, enrich_thin,
|
||||
* skillopt) — a no-op for phases that always run when named directly.
|
||||
* skillopt, drift) — a no-op for phases that always run when named directly.
|
||||
*/
|
||||
once: boolean;
|
||||
}
|
||||
@@ -367,7 +367,7 @@ Options:
|
||||
unlike toggling the flag on/off around the run, a
|
||||
crash mid-invocation can't leave it stuck. Applies to
|
||||
patterns, synthesize, conversation_facts_backfill,
|
||||
enrich_thin, skillopt; no-op on phases with no such
|
||||
enrich_thin, skillopt, drift; no-op on phases with no such
|
||||
gate. Requires an EXPLICIT --phase <name> — a phase
|
||||
implied by --input or --drain does not count (bare
|
||||
--once, or --once with --input/--drain and no
|
||||
|
||||
@@ -66,6 +66,10 @@ export type CyclePhase =
|
||||
// - calibration_profile: aggregates the resolved subset into 2-4
|
||||
// narrative pattern statements + active bias tags. Voice-gated.
|
||||
| 'propose_takes' | 'grade_takes' | 'calibration_profile'
|
||||
// #2653 — drift detection (default OFF; dream.drift.enabled). LLM-judges
|
||||
// soft-band takes against recent timeline evidence; report-only in v1
|
||||
// (writes reports/drift-<date>; auto_update mutates nothing).
|
||||
| 'drift'
|
||||
| 'embed' | 'orphans' | 'purge'
|
||||
// v0.39 T12: schema-suggest passive trigger (D3 + D4 plan-eng-review).
|
||||
// Wraps runSuggest() — same library the CLI verb + EIIRP call.
|
||||
@@ -151,6 +155,10 @@ export const ALL_PHASES: CyclePhase[] = [
|
||||
'propose_takes',
|
||||
'grade_takes',
|
||||
'calibration_profile',
|
||||
// #2653 — drift detection. Default OFF (dream.drift.enabled). Runs AFTER
|
||||
// the calibration trio (fresh take resolutions) and BEFORE embed so the
|
||||
// drift report page gets embedded same-cycle. Report-only in v1.
|
||||
'drift',
|
||||
// v0.41.11.0 — opt-in conversation-facts backfill. Default OFF; reads
|
||||
// cycle.conversation_facts_backfill.enabled gate inside the wrapper.
|
||||
// Ordered AFTER calibration_profile (matches the runCycle dispatch
|
||||
@@ -221,6 +229,9 @@ export const PHASE_SCOPE: Record<CyclePhase, PhaseScope> = {
|
||||
propose_takes: 'source',
|
||||
grade_takes: 'global',
|
||||
calibration_profile: 'global',
|
||||
// #2653 — drift walks takes brain-wide (same posture as grade_takes) and
|
||||
// writes one brain-global report page.
|
||||
drift: 'global',
|
||||
embed: 'global',
|
||||
orphans: 'global',
|
||||
purge: 'global',
|
||||
@@ -289,6 +300,8 @@ const NEEDS_LOCK_PHASES: ReadonlySet<CyclePhase> = new Set([
|
||||
'propose_takes',
|
||||
'grade_takes',
|
||||
'calibration_profile',
|
||||
// #2653 — writes the reports/drift-<date> page.
|
||||
'drift',
|
||||
// v0.41 T9 — extract_atoms writes atom-typed pages via put_page;
|
||||
// synthesize_concepts writes concept-typed pages + tier updates. Both
|
||||
// mutate DB state and need the lock.
|
||||
@@ -2151,6 +2164,49 @@ export async function runCycle(
|
||||
}
|
||||
}
|
||||
|
||||
// ── #2653: drift detection ──────────────────────────────────
|
||||
// Default OFF (dream.drift.enabled). LLM-judges soft-band takes
|
||||
// against recent timeline evidence; report-only in v1 — writes one
|
||||
// reports/drift-<date> page, mutates no takes regardless of
|
||||
// dream.drift.auto_update.
|
||||
if (phases.includes('drift')) {
|
||||
checkAborted(opts.signal);
|
||||
if (!engine) {
|
||||
phaseResults.push({
|
||||
phase: 'drift',
|
||||
status: 'skipped',
|
||||
duration_ms: 0,
|
||||
summary: 'no database connected',
|
||||
details: { reason: 'no_database' },
|
||||
});
|
||||
} else {
|
||||
progress.start('cycle.drift');
|
||||
const { runPhaseDrift } = await import('./cycle/drift.ts');
|
||||
const { result, duration_ms } = await timePhase(async (): Promise<PhaseResult> => {
|
||||
const r = await runPhaseDrift(engine, {
|
||||
dryRun,
|
||||
brainDir: brainDir ?? undefined,
|
||||
forceEnabled: opts.onceForPhase === 'drift',
|
||||
});
|
||||
const status: PhaseStatus =
|
||||
r.status === 'complete' ? 'ok' :
|
||||
r.status === 'partial' ? 'warn' :
|
||||
r.status === 'failed' ? 'fail' : 'skipped';
|
||||
return {
|
||||
phase: 'drift',
|
||||
status,
|
||||
duration_ms: 0,
|
||||
summary: r.detail,
|
||||
details: { ...(r.totals ?? {}) },
|
||||
};
|
||||
});
|
||||
result.duration_ms = duration_ms;
|
||||
phaseResults.push(result);
|
||||
progress.finish();
|
||||
}
|
||||
await safeYield(opts.yieldBetweenPhases);
|
||||
}
|
||||
|
||||
// ── v0.41.11.0: conversation_facts_backfill ─────────────────
|
||||
// Opt-in (default OFF). Walks long-form conversation/meeting/slack/
|
||||
// email pages, segments by 30-min gap, runs facts extractor with a
|
||||
|
||||
+250
-35
@@ -1,18 +1,25 @@
|
||||
/**
|
||||
* v0.28: drift dream phase.
|
||||
* Drift dream phase (#2653 — wired v0.42.x; scaffold shipped v0.28).
|
||||
*
|
||||
* Detects takes where the underlying evidence has shifted since the take
|
||||
* was made. v0.28 ships the SCAFFOLD: the phase iterates active takes,
|
||||
* runs a lightweight check against recent timeline entries on the same
|
||||
* page, and writes a drift-report-<date>.md if any takes look stale.
|
||||
* Detects takes whose underlying evidence has shifted since the take was
|
||||
* made. Two stages:
|
||||
* 1. Cheap SQL heuristic (findDriftCandidates): soft-band takes
|
||||
* (weight 0.3–0.85, active, unresolved) on pages with fresh
|
||||
* timeline_entries evidence inside the lookback window.
|
||||
* 2. LLM judge: each candidate's claim is compared against the recent
|
||||
* timeline evidence; the judge returns {drifted, confidence,
|
||||
* reasoning, suggested_weight}. BudgetMeter-gated.
|
||||
*
|
||||
* The full LLM-driven drift detection (compare each take's claim to recent
|
||||
* page evidence and propose a weight adjustment) is the v0.29 follow-up.
|
||||
* v0.28 lays the phase orchestration so the contract is stable.
|
||||
* Output is REPORT-ONLY (v1 conservative posture): judged candidates land
|
||||
* on a `reports/drift-<date>` page. `dream.drift.auto_update` mutates
|
||||
* NOTHING in v1 — it is recorded in the report so operators can see the
|
||||
* flag state, and a future wave may wire weight adjustment behind it.
|
||||
*
|
||||
* Default-disabled. Operator opts in:
|
||||
* gbrain config set dream.drift.enabled true
|
||||
* gbrain config set dream.drift.lookback_days 30
|
||||
* gbrain config set dream.drift.max_per_cycle 20
|
||||
* gbrain config set dream.drift.budget 1.0
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from '../engine.ts';
|
||||
@@ -25,6 +32,10 @@ export interface DriftPhaseOpts {
|
||||
dryRun: boolean;
|
||||
/** Override the audit ledger path (tests). */
|
||||
auditPath?: string;
|
||||
/** issue #2860 --once: bypass the dream.drift.enabled gate for this run only. */
|
||||
forceEnabled?: boolean;
|
||||
/** Inject the judge model call (tests). Defaults to gateway chat. */
|
||||
judge?: DriftJudgeFn;
|
||||
}
|
||||
|
||||
export interface DriftConfig {
|
||||
@@ -32,6 +43,7 @@ export interface DriftConfig {
|
||||
lookbackDays: number;
|
||||
budgetUsd: number;
|
||||
autoUpdate: boolean;
|
||||
maxPerCycle: number;
|
||||
}
|
||||
|
||||
async function loadDriftConfig(engine: BrainEngine): Promise<DriftConfig> {
|
||||
@@ -39,16 +51,19 @@ async function loadDriftConfig(engine: BrainEngine): Promise<DriftConfig> {
|
||||
const lookbackStr = await engine.getConfig('dream.drift.lookback_days');
|
||||
const budgetStr = await engine.getConfig('dream.drift.budget');
|
||||
const autoStr = await engine.getConfig('dream.drift.auto_update');
|
||||
const maxPerStr = await engine.getConfig('dream.drift.max_per_cycle');
|
||||
return {
|
||||
enabled: enabledStr === 'true',
|
||||
lookbackDays: lookbackStr ? Math.max(1, parseInt(lookbackStr, 10) || 30) : 30,
|
||||
budgetUsd: budgetStr ? Math.max(0, parseFloat(budgetStr) || 1.0) : 1.0,
|
||||
autoUpdate: autoStr === 'true',
|
||||
maxPerCycle: maxPerStr ? Math.max(1, parseInt(maxPerStr, 10) || 20) : 20,
|
||||
};
|
||||
}
|
||||
|
||||
interface DriftCandidate {
|
||||
export interface DriftCandidate {
|
||||
takeId: number;
|
||||
pageId: number;
|
||||
pageSlug: string;
|
||||
rowNum: number;
|
||||
claim: string;
|
||||
@@ -57,24 +72,121 @@ interface DriftCandidate {
|
||||
recentEvidenceCount: number;
|
||||
}
|
||||
|
||||
/** Judge verdict: has the claim drifted relative to the evidence? */
|
||||
export interface DriftVerdict {
|
||||
drifted: boolean;
|
||||
confidence: number;
|
||||
reasoning: string;
|
||||
/** Judge's suggested new weight (advisory only — v1 never applies it). */
|
||||
suggested_weight?: number;
|
||||
}
|
||||
|
||||
/** Judge function signature — injected for tests. */
|
||||
export type DriftJudgeFn = (input: {
|
||||
candidate: DriftCandidate;
|
||||
evidence: string;
|
||||
modelHint?: string;
|
||||
}) => Promise<DriftVerdict>;
|
||||
|
||||
export const DRIFT_JUDGE_PROMPT = `You are auditing a knowledge-base "take" (a weighted claim) for drift:
|
||||
has newer evidence shifted the ground under this claim since it was made?
|
||||
|
||||
Output ONLY one JSON object with these fields:
|
||||
- drifted (boolean) — true when the evidence meaningfully contradicts,
|
||||
supersedes, or reframes the claim; false when it is
|
||||
consistent or merely adjacent.
|
||||
- confidence (number in [0,1]) — your confidence in the drifted verdict.
|
||||
- reasoning (string, <=300 chars) — what in the evidence drove the verdict.
|
||||
- suggested_weight (number in [0,1], optional) — where the take's weight should
|
||||
move if drifted. Advisory only.
|
||||
|
||||
If the evidence is sparse or unrelated to the claim, return drifted=false with
|
||||
low confidence.
|
||||
|
||||
TAKE:
|
||||
Claim: {CLAIM}
|
||||
Weight: {WEIGHT}
|
||||
Page: {PAGE}
|
||||
|
||||
RECENT EVIDENCE (timeline entries on the same page):
|
||||
{EVIDENCE_BLOCK}
|
||||
`;
|
||||
|
||||
/**
|
||||
* Parse the judge model's JSON output. Tolerant of fence wrapping and
|
||||
* leading prose; returns null on unrecoverable parse failure.
|
||||
*/
|
||||
export function parseDriftOutput(raw: string): DriftVerdict | null {
|
||||
if (!raw || raw.trim().length === 0) return null;
|
||||
let text = raw.trim();
|
||||
const fenced = text.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```$/);
|
||||
if (fenced) text = (fenced[1] ?? '').trim();
|
||||
const firstObj = text.indexOf('{');
|
||||
if (firstObj === -1) return null;
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(text.slice(firstObj));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (typeof parsed !== 'object' || parsed === null) return null;
|
||||
const r = parsed as Record<string, unknown>;
|
||||
if (typeof r.drifted !== 'boolean') return null;
|
||||
const confRaw = typeof r.confidence === 'number' ? r.confidence : Number.parseFloat(String(r.confidence ?? ''));
|
||||
if (!Number.isFinite(confRaw)) return null;
|
||||
const verdict: DriftVerdict = {
|
||||
drifted: r.drifted,
|
||||
confidence: Math.max(0, Math.min(1, confRaw)),
|
||||
reasoning: typeof r.reasoning === 'string' ? r.reasoning.slice(0, 300) : '',
|
||||
};
|
||||
const sw = typeof r.suggested_weight === 'number' ? r.suggested_weight : NaN;
|
||||
if (Number.isFinite(sw)) verdict.suggested_weight = Math.max(0, Math.min(1, sw));
|
||||
return verdict;
|
||||
}
|
||||
|
||||
/** Production judge — calls gateway.chat with the DRIFT_JUDGE_PROMPT. */
|
||||
export async function defaultDriftJudge(input: {
|
||||
candidate: DriftCandidate;
|
||||
evidence: string;
|
||||
modelHint?: string;
|
||||
}): Promise<DriftVerdict> {
|
||||
const { chat } = await import('../ai/gateway.ts');
|
||||
const prompt = DRIFT_JUDGE_PROMPT
|
||||
.replace('{CLAIM}', input.candidate.claim)
|
||||
.replace('{WEIGHT}', String(input.candidate.weight))
|
||||
.replace('{PAGE}', input.candidate.pageSlug)
|
||||
.replace('{EVIDENCE_BLOCK}', input.evidence);
|
||||
const result = await chat({
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
...(input.modelHint ? { model: input.modelHint } : {}),
|
||||
maxTokens: 400,
|
||||
});
|
||||
const parsed = parseDriftOutput(result.text);
|
||||
if (!parsed) {
|
||||
// Failed parse — conservative no-drift at zero confidence so the row
|
||||
// still surfaces in the report instead of disappearing silently.
|
||||
return { drifted: false, confidence: 0, reasoning: 'judge_output_parse_failed' };
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cheap pre-LLM heuristic: takes that have substantial recent timeline
|
||||
* evidence on the same page MAY have drifted. Surface them; the v0.29
|
||||
* LLM judge will decide if the weight should move.
|
||||
* evidence on the same page MAY have drifted. Surface them; the LLM judge
|
||||
* decides.
|
||||
*/
|
||||
async function findDriftCandidates(
|
||||
engine: BrainEngine,
|
||||
lookbackDays: number,
|
||||
): Promise<DriftCandidate[]> {
|
||||
const cutoffMs = Date.now() - lookbackDays * 86_400_000;
|
||||
const cutoffIso = new Date(cutoffMs).toISOString().slice(0, 10);
|
||||
const cutoffIso = lookbackCutoffIso(lookbackDays);
|
||||
// Only consider takes with weight in the "soft" middle band (0.3..0.85)
|
||||
// — facts (1.0) don't drift, very-low hunches (<0.3) aren't actionable yet.
|
||||
const rows = await engine.executeRaw<{
|
||||
take_id: number; page_slug: string; row_num: number;
|
||||
take_id: number; page_id: number; page_slug: string; row_num: number;
|
||||
claim: string; weight: number; recent_evidence: number;
|
||||
}>(`
|
||||
SELECT t.id AS take_id, p.slug AS page_slug, t.row_num,
|
||||
SELECT t.id AS take_id, p.id AS page_id, p.slug AS page_slug, t.row_num,
|
||||
t.claim, t.weight,
|
||||
(SELECT count(*)::int FROM timeline_entries te
|
||||
WHERE te.page_id = p.id
|
||||
@@ -92,6 +204,7 @@ async function findDriftCandidates(
|
||||
.filter(r => Number(r.recent_evidence) >= 1)
|
||||
.map(r => ({
|
||||
takeId: Number(r.take_id),
|
||||
pageId: Number(r.page_id),
|
||||
pageSlug: String(r.page_slug),
|
||||
rowNum: Number(r.row_num),
|
||||
claim: String(r.claim),
|
||||
@@ -100,6 +213,58 @@ async function findDriftCandidates(
|
||||
}));
|
||||
}
|
||||
|
||||
function lookbackCutoffIso(lookbackDays: number): string {
|
||||
return new Date(Date.now() - lookbackDays * 86_400_000).toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
/** Format the candidate page's recent timeline entries as judge evidence. */
|
||||
async function loadEvidence(
|
||||
engine: BrainEngine,
|
||||
pageId: number,
|
||||
cutoffIso: string,
|
||||
): Promise<string> {
|
||||
const rows = await engine.executeRaw<{ date: string; source: string; summary: string }>(
|
||||
`SELECT date::text AS date, source, summary FROM timeline_entries
|
||||
WHERE page_id = $1 AND date >= $2::date
|
||||
ORDER BY date DESC LIMIT 12`,
|
||||
[pageId, cutoffIso],
|
||||
);
|
||||
if (rows.length === 0) return '(no timeline entries in window)';
|
||||
return rows.map(r => `- ${r.date} [${r.source}] ${r.summary}`).join('\n');
|
||||
}
|
||||
|
||||
interface JudgedCandidate {
|
||||
candidate: DriftCandidate;
|
||||
verdict: DriftVerdict;
|
||||
}
|
||||
|
||||
function buildReportBody(
|
||||
judged: JudgedCandidate[],
|
||||
cfg: DriftConfig,
|
||||
modelId: string,
|
||||
): string {
|
||||
const drifted = judged.filter(j => j.verdict.drifted);
|
||||
const lines: string[] = [
|
||||
`Drift check over active soft-band takes (weight 0.3–0.85) with timeline`,
|
||||
`evidence from the last ${cfg.lookbackDays} day(s). Judge model: ${modelId}.`,
|
||||
``,
|
||||
`**Report-only:** no takes were modified. \`dream.drift.auto_update\` is`,
|
||||
`${cfg.autoUpdate ? 'set but ignored in v1 (report-only posture)' : 'off'}; review and adjust weights manually.`,
|
||||
``,
|
||||
`${drifted.length} of ${judged.length} judged take(s) look drifted.`,
|
||||
``,
|
||||
];
|
||||
for (const { candidate: c, verdict: v } of judged) {
|
||||
lines.push(`## ${v.drifted ? 'DRIFTED' : 'stable'} — ${c.pageSlug} (take #${c.rowNum})`);
|
||||
lines.push(`- Claim: ${c.claim}`);
|
||||
lines.push(`- Weight: ${c.weight}${v.suggested_weight !== undefined ? ` → suggested ${v.suggested_weight}` : ''}`);
|
||||
lines.push(`- Confidence: ${v.confidence.toFixed(2)}`);
|
||||
if (v.reasoning) lines.push(`- Reasoning: ${v.reasoning}`);
|
||||
lines.push('');
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
function skipped(_reason: string, detail: string): DreamPhaseResult {
|
||||
return { name: 'drift', status: 'skipped', detail, duration_ms: 0 };
|
||||
}
|
||||
@@ -110,7 +275,7 @@ export async function runPhaseDrift(
|
||||
): Promise<DreamPhaseResult> {
|
||||
const start = Date.now();
|
||||
const config = await loadDriftConfig(engine);
|
||||
if (!config.enabled) {
|
||||
if (!config.enabled && !opts.forceEnabled) {
|
||||
return skipped('not_configured', 'dream.drift.enabled is false');
|
||||
}
|
||||
|
||||
@@ -125,9 +290,16 @@ export async function runPhaseDrift(
|
||||
};
|
||||
}
|
||||
|
||||
// Resolve model for the (future v0.29) LLM judge. For v0.28 we just
|
||||
// surface the candidates — the meter call is a no-op when we don't actually
|
||||
// submit, but resolveModel sets the right pricing key when v0.29 ships.
|
||||
if (opts.dryRun) {
|
||||
return {
|
||||
name: 'drift',
|
||||
status: 'skipped',
|
||||
detail: `dry-run: ${Math.min(candidates.length, config.maxPerCycle)} of ${candidates.length} candidates would be evaluated`,
|
||||
totals: { candidates: candidates.length },
|
||||
duration_ms: Date.now() - start,
|
||||
};
|
||||
}
|
||||
|
||||
const modelId = await resolveModel(engine, {
|
||||
configKey: 'models.drift',
|
||||
deprecatedConfigKey: 'dream.drift.model',
|
||||
@@ -139,27 +311,70 @@ export async function runPhaseDrift(
|
||||
phase: 'drift',
|
||||
auditPath: opts.auditPath,
|
||||
});
|
||||
const judge = opts.judge ?? defaultDriftJudge;
|
||||
const cutoffIso = lookbackCutoffIso(config.lookbackDays);
|
||||
|
||||
// v0.28 scaffold: write a candidate report. v0.29 wires LLM-driven weight
|
||||
// adjustment through autoUpdate. modelId + meter are wired now so the
|
||||
// ledger captures the gate state even when we don't submit.
|
||||
void modelId; void meter;
|
||||
|
||||
if (opts.dryRun) {
|
||||
return {
|
||||
name: 'drift',
|
||||
status: 'skipped',
|
||||
detail: `dry-run: ${candidates.length} candidates would be evaluated`,
|
||||
totals: { candidates: candidates.length },
|
||||
duration_ms: Date.now() - start,
|
||||
};
|
||||
const judged: JudgedCandidate[] = [];
|
||||
let budgetExhausted = false;
|
||||
let failed = 0;
|
||||
for (const candidate of candidates.slice(0, config.maxPerCycle)) {
|
||||
const check = meter.check({
|
||||
modelId,
|
||||
estimatedInputTokens: 1500,
|
||||
maxOutputTokens: 400,
|
||||
label: `drift:${candidate.pageSlug}#${candidate.rowNum}`,
|
||||
});
|
||||
if (!check.allowed) {
|
||||
budgetExhausted = true;
|
||||
break;
|
||||
}
|
||||
const evidence = await loadEvidence(engine, candidate.pageId, cutoffIso);
|
||||
try {
|
||||
const verdict = await judge({ candidate, evidence, modelHint: modelId });
|
||||
judged.push({ candidate, verdict });
|
||||
} catch (e) {
|
||||
failed += 1;
|
||||
process.stderr.write(`[drift] judge failed on take ${candidate.takeId}: ${(e as Error).message}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
const driftedCount = judged.filter(j => j.verdict.drifted).length;
|
||||
let reportSlug: string | undefined;
|
||||
if (judged.length > 0) {
|
||||
const date = new Date().toISOString().slice(0, 10);
|
||||
reportSlug = `reports/drift-${date}`;
|
||||
// Report-only v1: the report page is the ONLY write this phase makes.
|
||||
// Lands in the default source (brain-global artifact, same-day re-runs
|
||||
// upsert the same slug).
|
||||
await engine.putPage(reportSlug, {
|
||||
type: 'report',
|
||||
title: `Drift report ${date}`,
|
||||
compiled_truth: buildReportBody(judged, config, modelId),
|
||||
});
|
||||
}
|
||||
|
||||
const detail =
|
||||
`judged ${judged.length}/${candidates.length} candidates: ${driftedCount} drifted` +
|
||||
(reportSlug ? ` → ${reportSlug}` : '') +
|
||||
(budgetExhausted ? ' (budget exhausted)' : '') +
|
||||
(failed > 0 ? ` (${failed} judge failure(s))` : '') +
|
||||
`. Cumulative cost: $${meter.totalSpent.toFixed(4)} / $${config.budgetUsd.toFixed(2)}` +
|
||||
`. Report-only: auto_update=${config.autoUpdate} mutates nothing in v1.`;
|
||||
|
||||
return {
|
||||
name: 'drift',
|
||||
status: 'complete',
|
||||
detail: `surfaced ${candidates.length} drift candidates (LLM judge: v0.29 follow-up). autoUpdate=${config.autoUpdate}`,
|
||||
totals: { candidates: candidates.length },
|
||||
status: judged.length > 0
|
||||
? (budgetExhausted || failed > 0 ? 'partial' : 'complete')
|
||||
// Zero judged: budget capped before any judge ran → partial (capped,
|
||||
// not broken); otherwise every judge call failed → failed.
|
||||
: (budgetExhausted ? 'partial' : 'failed'),
|
||||
detail,
|
||||
totals: {
|
||||
candidates: candidates.length,
|
||||
judged: judged.length,
|
||||
drifted: driftedCount,
|
||||
failed,
|
||||
},
|
||||
duration_ms: Date.now() - start,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ import { join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { runPhaseAutoThink } from '../src/core/cycle/auto-think.ts';
|
||||
import { runPhaseDrift, __testing as driftTesting } from '../src/core/cycle/drift.ts';
|
||||
import { runPhaseDrift, parseDriftOutput, __testing as driftTesting } from '../src/core/cycle/drift.ts';
|
||||
import { _resetBudgetMeterWarningsForTest } from '../src/core/cycle/budget-meter.ts';
|
||||
import type { ThinkLLMClient } from '../src/core/think/index.ts';
|
||||
|
||||
@@ -153,6 +153,18 @@ describe('runPhaseAutoThink', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseDriftOutput', () => {
|
||||
test('parses fenced JSON with clamped fields', () => {
|
||||
const v = parseDriftOutput('```json\n{"drifted": true, "confidence": 1.7, "reasoning": "r", "suggested_weight": -0.2}\n```');
|
||||
expect(v).toEqual({ drifted: true, confidence: 1, reasoning: 'r', suggested_weight: 0 });
|
||||
});
|
||||
|
||||
test('returns null on garbage / missing drifted', () => {
|
||||
expect(parseDriftOutput('not json at all')).toBeNull();
|
||||
expect(parseDriftOutput('{"confidence": 0.5}')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('runPhaseDrift', () => {
|
||||
test('skipped when not enabled', async () => {
|
||||
const r = await runPhaseDrift(engine, { dryRun: false, auditPath: join(tmpDir, 'd0.jsonl') });
|
||||
@@ -166,16 +178,89 @@ describe('runPhaseDrift', () => {
|
||||
expect(cands.every(c => c.weight >= 0.3 && c.weight <= 0.85)).toBe(true);
|
||||
});
|
||||
|
||||
test('runs and surfaces candidates when enabled', async () => {
|
||||
test('judges candidates and writes a report-only drift report page (#2653)', async () => {
|
||||
await engine.setConfig('dream.drift.enabled', 'true');
|
||||
await engine.setConfig('dream.drift.lookback_days', '30');
|
||||
await engine.setConfig('dream.drift.budget', '1.0');
|
||||
const r = await runPhaseDrift(engine, { dryRun: false, auditPath: join(tmpDir, 'd1.jsonl') });
|
||||
const judgedRows: number[] = [];
|
||||
const r = await runPhaseDrift(engine, {
|
||||
dryRun: false,
|
||||
auditPath: join(tmpDir, 'd1.jsonl'),
|
||||
judge: async ({ candidate, evidence }) => {
|
||||
judgedRows.push(candidate.rowNum);
|
||||
expect(evidence).toContain('Funding round closed'); // real timeline evidence reached the judge
|
||||
return candidate.rowNum === 2
|
||||
? { drifted: true, confidence: 0.9, reasoning: 'evidence shifted', suggested_weight: 0.3 }
|
||||
: { drifted: false, confidence: 0.7, reasoning: 'consistent' };
|
||||
},
|
||||
});
|
||||
expect(r.status).toBe('complete');
|
||||
expect((r.totals as { candidates?: number }).candidates).toBeGreaterThanOrEqual(0);
|
||||
const totals = r.totals as { candidates: number; judged: number; drifted: number };
|
||||
expect(totals.judged).toBeGreaterThanOrEqual(2);
|
||||
expect(totals.drifted).toBe(1);
|
||||
expect(judgedRows).toContain(2);
|
||||
|
||||
// Report page landed.
|
||||
const date = new Date().toISOString().slice(0, 10);
|
||||
const page = await engine.getPage(`reports/drift-${date}`);
|
||||
expect(page).not.toBeNull();
|
||||
expect(page!.compiled_truth).toContain('DRIFTED');
|
||||
expect(page!.compiled_truth).toContain('Strong technical founder');
|
||||
|
||||
// Report-only v1: no take was mutated even though the judge suggested a weight.
|
||||
const rows = await engine.executeRaw<{ weight: number; resolved_at: string | null }>(
|
||||
'SELECT weight, resolved_at FROM takes WHERE page_id = $1 AND row_num = 2',
|
||||
[alicePageId],
|
||||
);
|
||||
expect(Number(rows[0]!.weight)).toBe(0.6);
|
||||
expect(rows[0]!.resolved_at).toBeNull();
|
||||
await engine.setConfig('dream.drift.enabled', 'false');
|
||||
});
|
||||
|
||||
test('auto_update mutates nothing in v1', async () => {
|
||||
await engine.setConfig('dream.drift.enabled', 'true');
|
||||
await engine.setConfig('dream.drift.auto_update', 'true');
|
||||
const r = await runPhaseDrift(engine, {
|
||||
dryRun: false,
|
||||
auditPath: join(tmpDir, 'd-auto.jsonl'),
|
||||
judge: async () => ({ drifted: true, confidence: 0.99, reasoning: 'x', suggested_weight: 0.1 }),
|
||||
});
|
||||
expect(r.status).toBe('complete');
|
||||
const rows = await engine.executeRaw<{ weight: number }>(
|
||||
'SELECT weight FROM takes WHERE page_id = $1 AND row_num = 3',
|
||||
[alicePageId],
|
||||
);
|
||||
expect(Number(rows[0]!.weight)).toBe(0.5); // untouched
|
||||
await engine.setConfig('dream.drift.auto_update', 'false');
|
||||
await engine.setConfig('dream.drift.enabled', 'false');
|
||||
});
|
||||
|
||||
test('budget exhaustion stops judging (partial, no judge calls)', async () => {
|
||||
await engine.setConfig('dream.drift.enabled', 'true');
|
||||
await engine.setConfig('dream.drift.budget', '0.0000001');
|
||||
let judgeCalls = 0;
|
||||
const r = await runPhaseDrift(engine, {
|
||||
dryRun: false,
|
||||
auditPath: join(tmpDir, 'd-budget.jsonl'),
|
||||
judge: async () => { judgeCalls += 1; return { drifted: false, confidence: 0.5, reasoning: '' }; },
|
||||
});
|
||||
expect(r.status).toBe('partial');
|
||||
expect(judgeCalls).toBe(0);
|
||||
await engine.setConfig('dream.drift.budget', '1.0');
|
||||
await engine.setConfig('dream.drift.enabled', 'false');
|
||||
});
|
||||
|
||||
test('forceEnabled (--once) bypasses the dream.drift.enabled gate', async () => {
|
||||
await engine.setConfig('dream.drift.enabled', 'false');
|
||||
const r = await runPhaseDrift(engine, {
|
||||
dryRun: false,
|
||||
auditPath: join(tmpDir, 'd-once.jsonl'),
|
||||
forceEnabled: true,
|
||||
judge: async () => ({ drifted: false, confidence: 0.5, reasoning: 'ok' }),
|
||||
});
|
||||
expect(r.status).toBe('complete');
|
||||
});
|
||||
|
||||
test('dry-run returns skipped with candidate count', async () => {
|
||||
await engine.setConfig('dream.drift.enabled', 'true');
|
||||
const r = await runPhaseDrift(engine, { dryRun: true, auditPath: join(tmpDir, 'd2.jsonl') });
|
||||
|
||||
@@ -394,7 +394,9 @@ describe('runCycle — yieldBetweenPhases hook', () => {
|
||||
// v0.41.11.0: 20 phases (added `conversation_facts_backfill` between consolidate and propose_takes).
|
||||
// v0.41.39 (#1700) + v0.42.0.0: 22 phases (added `enrich_thin` AND `skillopt`
|
||||
// between conversation_facts_backfill and embed — both landed in this merge).
|
||||
expect(hookCalls).toBe(22);
|
||||
// #2653: 23 phases (added `drift` between calibration_profile and
|
||||
// conversation_facts_backfill).
|
||||
expect(hookCalls).toBe(23);
|
||||
});
|
||||
|
||||
test('hook exceptions do not abort the cycle', async () => {
|
||||
@@ -409,7 +411,8 @@ describe('runCycle — yieldBetweenPhases hook', () => {
|
||||
// v0.39.0.0: 17 phases (T12 schema-suggest phase between orphans and purge).
|
||||
// v0.41.11.0: 20 phases (+extract_atoms, +synthesize_concepts, +conversation_facts_backfill).
|
||||
// v0.41.39 (#1700) + v0.42.0.0: 22 phases (+enrich_thin, +skillopt).
|
||||
expect(report.phases.length).toBe(22);
|
||||
// #2653: 23 phases (+drift).
|
||||
expect(report.phases.length).toBe(23);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -127,6 +127,7 @@ const EXPECTED_PHASES: CyclePhase[] = [
|
||||
'propose_takes', // v0.36.1.0 — hindsight calibration wave
|
||||
'grade_takes', // v0.36.1.0
|
||||
'calibration_profile', // v0.36.1.0
|
||||
'drift', // #2653 — drift detection (default OFF, report-only)
|
||||
'conversation_facts_backfill', // v0.41.11.0 — opt-in conversation backfill
|
||||
'enrich_thin', // v0.41.39 (#1700) — brain-internal stub enrichment (default OFF)
|
||||
'skillopt', // v0.42.0.0 — self-evolving skills (default OFF)
|
||||
|
||||
@@ -41,15 +41,16 @@ describe('PHASE_SCOPE coverage', () => {
|
||||
expect(invalid).toEqual([]);
|
||||
});
|
||||
|
||||
test('all 22 phases covered (regression on accidental omission)', () => {
|
||||
test('all 23 phases covered (regression on accidental omission)', () => {
|
||||
// Pin the count so a future PR that adds a phase to ALL_PHASES
|
||||
// without updating PHASE_SCOPE notices here too. The v0.39.1.0
|
||||
// master merge brought in the 17th phase (`schema-suggest`); v0.41
|
||||
// adds 'extract_atoms' + 'synthesize_concepts' (T9 lens packs) +
|
||||
// 'conversation_facts_backfill' (v0.41.11.0) for 20; v0.41.39 (#1700)
|
||||
// adds 'enrich_thin' and v0.42.0.0 adds 'skillopt' for a total of 22.
|
||||
expect(ALL_PHASES.length).toBe(22);
|
||||
expect(Object.keys(PHASE_SCOPE).length).toBe(22);
|
||||
// adds 'enrich_thin' and v0.42.0.0 adds 'skillopt' for a total of 22;
|
||||
// #2653 adds 'drift' for 23.
|
||||
expect(ALL_PHASES.length).toBe(23);
|
||||
expect(Object.keys(PHASE_SCOPE).length).toBe(23);
|
||||
});
|
||||
|
||||
test('embed remains global (the headline brain-wide phase)', () => {
|
||||
|
||||
Reference in New Issue
Block a user