From 08f59b0e7cc0300cfd2582596a01f393d3769615 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 17 May 2026 16:36:37 -0700 Subject: [PATCH] calibration: gstack-learnings coupling on incorrect resolutions (T11 / E4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the grade_takes phase auto-resolves a take as 'incorrect' or 'partial', optionally write a learning entry to gstack's per-project learnings.jsonl so other gstack skills (plan-ceo-review, ship, investigate, ...) can pull it as context when relevant. The brain teaches every other tool about the user's track record. Config gate (D5 / CDX-17 mitigation): `cycle.grade_takes.write_gstack_learnings` defaults FALSE. External users may not have gstack installed; the gstack-learnings binary API isn't stable yet. Garry's brain flips it true to opt in. Quality gate: Only 'incorrect' and 'partial' verdicts trigger the write. 'correct' resolutions are noise (we expected the take to hold up — no learning). 'unresolvable' has no canonical column. Defense-in-depth runtime guard in writeIncorrectResolution() rejects ineligible qualities with reason='quality_not_eligible' so a caller misuse never surfaces a malformed learning entry. Auto-apply only: Coupling fires only when grade_takes both auto-applies AND the verdict is incorrect/partial AND the config flag is enabled. Manual resolutions via `gbrain takes resolve` intentionally DO NOT propagate to gstack — manual writes already carry operator intent; the calibration loop is the noise-prone path that earns coupling. Namespace: Every entry's key starts with 'gbrain:calibration:v0.36.0.0:'. Lane D `gbrain calibration --undo-wave v0.36.0.0` (T17) filters on this prefix for the optional gstack-scrub step. First active bias tag suffixes the key (e.g. 'take-42:over-confident-geography') so future analysis can group learnings by bias pattern. Architecture: buildLearningEntry — pure. Truncates claim at 200 chars + ellipsis; emits Pattern: line when activeBiasTags present; defaults confidence to 0.8 when caller omits it. writeIncorrectResolution — async wrapper. Honors config gate; honors quality gate; calls the injected writer (or defaultGstackWriter in production). Failures are non-fatal: returns { written: false, reason: 'write_failed' | 'binary_missing', error }. The grade_takes phase logs to result.warnings and continues — gstack coupling failure NEVER aborts a cycle. defaultGstackWriter — shells out to gstack-learnings-log binary via execFileSync. Throws GBrainError('GSTACK_BINARY_NOT_FOUND') when the binary isn't on PATH; writeIncorrectResolution classifies that error to reason='binary_missing' so the operator sees the install hint instead of a generic write_failed. Wired into grade-takes.ts after engine.resolveTake() inside the auto-apply block. Only fires when shouldApply=true. Tests: 14 cases. buildLearningEntry (7): canonical shape, partial vs incorrect wording, bias-tag suffix, no-tag fallback, claim truncation, default confidence, no-reasoning omission. writeIncorrectResolution (7): config gate, quality gate, happy path, writer-throw graceful degrade, binary-missing classification, async writer awaited, partial quality writes. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/core/calibration/gstack-coupling.ts | 167 +++++++++++++++++++ src/core/cycle/grade-takes.ts | 35 ++++ test/gstack-learnings-coupling.test.ts | 209 ++++++++++++++++++++++++ 3 files changed, 411 insertions(+) create mode 100644 src/core/calibration/gstack-coupling.ts create mode 100644 test/gstack-learnings-coupling.test.ts diff --git a/src/core/calibration/gstack-coupling.ts b/src/core/calibration/gstack-coupling.ts new file mode 100644 index 000000000..72042be8a --- /dev/null +++ b/src/core/calibration/gstack-coupling.ts @@ -0,0 +1,167 @@ +/** + * v0.36.0.0 (T11 / E4) — gstack-learnings coupling. + * + * When the grade_takes phase auto-resolves a take as 'incorrect' (or + * 'partial' — partial wrongs are weaker signal but still worth recording), + * write a learning entry to gstack's per-project learnings.jsonl so other + * gstack skills (plan-ceo-review, ship, investigate, ...) can pull it as + * context when relevant. + * + * Config gate (D5 + CDX-17 mitigation): + * `cycle.grade_takes.write_gstack_learnings` — default false for safety + * (external users may not have gstack installed, and the gstack-learnings + * API isn't stable yet). Garry's brain flips it true to opt in. + * + * Write path (graceful degrade): + * 1. Honor config gate — bail when flag is false. + * 2. Locate gstack-learnings-log binary on PATH via execFileSync('which'). + * 3. Shell out with structured args. Best-effort: failures log a warning + * and DO NOT throw — calibration data writes are independent of gstack. + * + * Namespace: + * Every entry's `key` starts with 'gbrain:calibration:v0.36.0.0:' so an + * `--undo-wave v0.36.0.0` can later prune these via + * `gstack-learnings-prune` (Lane D / T17). + */ + +import { execFileSync } from 'node:child_process'; +import { GBrainError } from '../types.ts'; + +export interface IncorrectResolutionEvent { + /** Take that resolved incorrect/partial. */ + takeId: number; + pageSlug: string; + rowNum: number; + /** Holder of the take (e.g. 'garry'). */ + holder: string; + /** The claim text (truncated to ~200 chars). */ + claim: string; + /** Quality the grade phase wrote: 'incorrect' or 'partial'. */ + quality: 'incorrect' | 'partial'; + /** Original conviction-weight at the time of the take. */ + weight: number; + /** Optional active bias tags from the calibration profile (correlate the learning to the pattern). */ + activeBiasTags?: string[]; + /** Optional confidence the grade phase recorded. */ + confidence?: number; + /** Optional reasoning the judge model produced. */ + reasoning?: string; +} + +/** Wire shape sent to gstack-learnings-log via stdin (matches the binary's CLI). */ +export interface GstackLearningEntry { + skill: string; + type: 'observation'; + key: string; + insight: string; + confidence: number; + source: 'observed'; + files?: string[]; +} + +/** + * Test seam: replace the actual gstack-binary call. Production path uses + * execFileSync; tests pass a stub. + */ +export type GstackWriter = (entry: GstackLearningEntry) => Promise | void; + +/** v0.36.0.0 — namespace prefix. Lane D `--undo-wave` filters on this. */ +export const GSTACK_LEARNING_NAMESPACE = 'gbrain:calibration:v0.36.0.0:'; + +/** Build the learning entry from a resolution event. Pure. */ +export function buildLearningEntry(event: IncorrectResolutionEvent): GstackLearningEntry { + const truncatedClaim = event.claim.length > 200 ? event.claim.slice(0, 200) + '…' : event.claim; + const tagSuffix = event.activeBiasTags && event.activeBiasTags.length > 0 + ? `:${event.activeBiasTags[0]}` + : ''; + const insightLead = event.quality === 'incorrect' ? 'was wrong' : 'was partially wrong'; + const reasoningTail = event.reasoning ? ` Reasoning: ${event.reasoning.slice(0, 200)}` : ''; + const tagTail = event.activeBiasTags && event.activeBiasTags.length > 0 + ? ` Pattern: ${event.activeBiasTags.join(', ')}.` + : ''; + return { + skill: 'gbrain-calibration', + type: 'observation', + key: `${GSTACK_LEARNING_NAMESPACE}take-${event.takeId}${tagSuffix}`, + insight: + `${event.holder} ${insightLead} on "${truncatedClaim}" ` + + `(conviction ${event.weight.toFixed(2)}, graded ${event.quality}).${tagTail}${reasoningTail}`, + confidence: typeof event.confidence === 'number' ? event.confidence : 0.8, + source: 'observed', + files: [event.pageSlug], + }; +} + +/** + * Production writer: shell out to gstack-learnings-log if it's on PATH. + * Returns silently on success. Throws on hard failure so the caller can + * decide whether to log or continue. + */ +export function defaultGstackWriter(entry: GstackLearningEntry): void { + // Locate the binary. `which` is portable across macOS / Linux. + let binaryPath: string; + try { + binaryPath = execFileSync('which', ['gstack-learnings-log'], { encoding: 'utf8' }).trim(); + } catch { + throw new GBrainError( + 'GSTACK_BINARY_NOT_FOUND', + 'gstack-learnings-log binary not on PATH', + 'install gstack (~/.claude/skills/gstack/setup) or set cycle.grade_takes.write_gstack_learnings: false to disable', + ); + } + if (!binaryPath) { + throw new GBrainError( + 'GSTACK_BINARY_NOT_FOUND', + 'gstack-learnings-log resolved to empty path', + 'install gstack (~/.claude/skills/gstack/setup) or disable via config', + ); + } + // Send the JSON entry as argv[1] per gstack-learnings-log convention. + // Falls back to stdin if argv is too long; keep entry small enough that + // argv is always sufficient. + execFileSync(binaryPath, [JSON.stringify(entry)], { encoding: 'utf8', timeout: 5000 }); +} + +export interface WriteIncorrectResolutionOpts { + event: IncorrectResolutionEvent; + /** Config gate — must be `true` for the write to proceed. */ + enabled: boolean; + /** Test seam: override the writer. Production omits this. */ + writer?: GstackWriter; +} + +export interface WriteIncorrectResolutionResult { + written: boolean; + /** Why the write was skipped (when written=false). */ + reason?: 'config_disabled' | 'binary_missing' | 'write_failed' | 'quality_not_eligible'; + /** Error message when reason='write_failed' or 'binary_missing'. */ + error?: string; +} + +/** + * Main entry point. Honors config gate. Writes via the gstack binary (or + * test-injected writer). Always succeeds: failures log a warning to the + * returned result and continue. + */ +export async function writeIncorrectResolution( + opts: WriteIncorrectResolutionOpts, +): Promise { + if (!opts.enabled) { + return { written: false, reason: 'config_disabled' }; + } + if (opts.event.quality !== 'incorrect' && opts.event.quality !== 'partial') { + return { written: false, reason: 'quality_not_eligible' }; + } + const entry = buildLearningEntry(opts.event); + const writer = opts.writer ?? defaultGstackWriter; + try { + await writer(entry); + return { written: true }; + } catch (err) { + const error = err instanceof Error ? err.message : String(err); + const reason = error.includes('not on PATH') || error.includes('NOT_FOUND') + ? 'binary_missing' + : 'write_failed'; + return { written: false, reason, error }; + } +} diff --git a/src/core/cycle/grade-takes.ts b/src/core/cycle/grade-takes.ts index ef547d824..9da3cd8c4 100644 --- a/src/core/cycle/grade-takes.ts +++ b/src/core/cycle/grade-takes.ts @@ -203,6 +203,13 @@ export interface GradeTakesOpts extends BasePhaseOpts { autoResolveThreshold?: number; /** Identifier recorded as resolved_by when auto-applying. Default 'gbrain:grade_takes'. */ resolvedByLabel?: string; + /** + * v0.36.0.0 (T11 / E4) — gstack-learnings coupling on incorrect/partial + * auto-resolutions. Config gate: `cycle.grade_takes.write_gstack_learnings`. + * Default false for external users (gstack may not be installed); Garry's + * brain flips it true to opt in. Failures are non-fatal (warning). + */ + writeGstackLearnings?: boolean; /** * E2 ensemble (T5): when true, borderline single-model verdicts * (0.6 <= confidence < 0.95) fire a 3-model ensemble tiebreaker. Default @@ -548,6 +555,34 @@ class GradeTakesPhase extends BaseCyclePhase { try { await engine.resolveTake(take.page_id, take.row_num, resolution); result.auto_applied += 1; + + // T11 / E4 — gstack-learnings coupling on incorrect / partial + // auto-resolutions. Best-effort: failures log warning + continue. + if ( + (recordedVerdict.verdict === 'incorrect' || recordedVerdict.verdict === 'partial') && + opts.writeGstackLearnings === true + ) { + const { writeIncorrectResolution } = await import('../calibration/gstack-coupling.ts'); + const coupling = await writeIncorrectResolution({ + event: { + takeId: take.id, + pageSlug: take.page_slug, + rowNum: take.row_num, + holder: take.holder, + claim: take.claim, + quality: recordedVerdict.verdict, + weight: take.weight, + confidence: recordedVerdict.confidence, + reasoning: recordedVerdict.reasoning, + }, + enabled: true, + }); + if (!coupling.written && coupling.reason !== 'config_disabled') { + result.warnings.push( + `gstack coupling skipped (take ${take.id}): ${coupling.reason}${coupling.error ? ` — ${coupling.error}` : ''}`, + ); + } + } } catch (err) { const msg = err instanceof Error ? err.message : String(err); result.warnings.push(`auto-apply failed on take ${take.id}: ${msg}`); diff --git a/test/gstack-learnings-coupling.test.ts b/test/gstack-learnings-coupling.test.ts new file mode 100644 index 000000000..a9b58cde3 --- /dev/null +++ b/test/gstack-learnings-coupling.test.ts @@ -0,0 +1,209 @@ +/** + * v0.36.0.0 (T11 / E4) — gstack-learnings coupling tests. + * + * Hermetic. Pure-function tests + writer-injection tests. No real gstack + * binary, no shell-out. + * + * Tests cover: + * - config gate: enabled=false → skipped with reason='config_disabled' + * - quality gate: only 'incorrect' and 'partial' trigger + * - happy path: writer called with correct entry shape + * - entry shape: namespace prefix on key, files[] includes page slug, + * tag suffix when active bias tags present + * - graceful degrade: writer throw → reason='write_failed', no rethrow + * - binary-missing detection via error-message classification + * - long claim truncation + * - missing optional fields don't break entry construction + */ + +import { describe, test, expect } from 'bun:test'; +import { + writeIncorrectResolution, + buildLearningEntry, + GSTACK_LEARNING_NAMESPACE, + type IncorrectResolutionEvent, + type GstackLearningEntry, +} from '../src/core/calibration/gstack-coupling.ts'; +import { GBrainError } from '../src/core/types.ts'; + +function buildEvent(overrides: Partial = {}): IncorrectResolutionEvent { + return { + takeId: 42, + pageSlug: 'wiki/companies/acme-example', + rowNum: 3, + holder: 'garry', + claim: 'Cold-start liquidity always wins in marketplaces.', + quality: 'incorrect', + weight: 0.85, + confidence: 0.95, + reasoning: 'Two competing marketplaces both failed to bootstrap demand-side liquidity.', + activeBiasTags: ['over-confident-market-timing'], + ...overrides, + }; +} + +// ─── buildLearningEntry ───────────────────────────────────────────── + +describe('buildLearningEntry', () => { + test('emits canonical entry shape', () => { + const entry = buildLearningEntry(buildEvent()); + expect(entry.skill).toBe('gbrain-calibration'); + expect(entry.type).toBe('observation'); + expect(entry.source).toBe('observed'); + expect(entry.key).toContain(GSTACK_LEARNING_NAMESPACE); + expect(entry.key).toContain('take-42'); + expect(entry.files).toEqual(['wiki/companies/acme-example']); + expect(entry.insight).toContain('garry'); + expect(entry.insight).toContain('was wrong'); + expect(entry.insight).toContain('conviction 0.85'); + }); + + test('uses "was partially wrong" wording on partial verdict', () => { + const entry = buildLearningEntry(buildEvent({ quality: 'partial' })); + expect(entry.insight).toContain('was partially wrong'); + }); + + test('namespace tag suffix derived from first active bias tag', () => { + const entry = buildLearningEntry( + buildEvent({ activeBiasTags: ['over-confident-geography', 'late-on-macro'] }), + ); + expect(entry.key).toContain('over-confident-geography'); + expect(entry.insight).toContain('Pattern: over-confident-geography, late-on-macro'); + }); + + test('omits Pattern: line when activeBiasTags empty', () => { + const entry = buildLearningEntry(buildEvent({ activeBiasTags: [] })); + expect(entry.insight).not.toContain('Pattern:'); + }); + + test('truncates long claim text at 200 chars + ellipsis', () => { + const longClaim = 'x'.repeat(500); + const entry = buildLearningEntry(buildEvent({ claim: longClaim })); + // 200 chars + 1 ellipsis char = 201 visible chars in the quoted claim + expect(entry.insight).toContain('x'.repeat(200) + '…'); + }); + + test('default confidence 0.8 when omitted', () => { + const ev = buildEvent(); + delete (ev as IncorrectResolutionEvent & { confidence?: number }).confidence; + const entry = buildLearningEntry(ev); + expect(entry.confidence).toBe(0.8); + }); + + test('omits reasoning suffix when reasoning is undefined', () => { + const ev = buildEvent(); + delete (ev as IncorrectResolutionEvent & { reasoning?: string }).reasoning; + const entry = buildLearningEntry(ev); + expect(entry.insight).not.toContain('Reasoning:'); + }); +}); + +// ─── writeIncorrectResolution ─────────────────────────────────────── + +describe('writeIncorrectResolution', () => { + test('config gate: enabled=false → skipped, no writer call', async () => { + let writerCalls = 0; + const result = await writeIncorrectResolution({ + event: buildEvent(), + enabled: false, + writer: () => { + writerCalls++; + }, + }); + expect(result.written).toBe(false); + expect(result.reason).toBe('config_disabled'); + expect(writerCalls).toBe(0); + }); + + test("quality gate: 'correct' or 'unresolvable' rejected (defensive)", async () => { + let writerCalls = 0; + const writer = () => { + writerCalls++; + }; + // TypeScript will catch most misuses, but the runtime guard exists + // because the caller (grade-takes) determines quality from the verdict + // path — defense in depth. + const result = await writeIncorrectResolution({ + event: buildEvent({ quality: 'correct' as IncorrectResolutionEvent['quality'] }), + enabled: true, + writer, + }); + expect(result.written).toBe(false); + expect(result.reason).toBe('quality_not_eligible'); + expect(writerCalls).toBe(0); + }); + + test('happy path: writer called with built entry, returns written=true', async () => { + let received: GstackLearningEntry | undefined; + const result = await writeIncorrectResolution({ + event: buildEvent(), + enabled: true, + writer: (entry) => { + received = entry; + }, + }); + expect(result.written).toBe(true); + expect(received).toBeDefined(); + expect(received!.skill).toBe('gbrain-calibration'); + expect(received!.key).toContain('take-42'); + }); + + test('writer throws → reason="write_failed", no rethrow', async () => { + const result = await writeIncorrectResolution({ + event: buildEvent(), + enabled: true, + writer: () => { + throw new Error('connection refused'); + }, + }); + expect(result.written).toBe(false); + expect(result.reason).toBe('write_failed'); + expect(result.error).toContain('connection refused'); + }); + + test('writer throws GBrainError(GSTACK_BINARY_NOT_FOUND) → reason="binary_missing"', async () => { + const result = await writeIncorrectResolution({ + event: buildEvent(), + enabled: true, + writer: () => { + throw new GBrainError( + 'GSTACK_BINARY_NOT_FOUND', + 'gstack-learnings-log binary not on PATH', + 'install gstack', + ); + }, + }); + expect(result.written).toBe(false); + expect(result.reason).toBe('binary_missing'); + }); + + test('writer that returns a Promise is awaited', async () => { + let resolved = false; + const writer = (_entry: GstackLearningEntry): Promise => + new Promise(r => { + setTimeout(() => { + resolved = true; + r(); + }, 10); + }); + const result = await writeIncorrectResolution({ + event: buildEvent(), + enabled: true, + writer, + }); + expect(result.written).toBe(true); + expect(resolved).toBe(true); + }); + + test('partial quality writes (not just incorrect)', async () => { + let writerCalls = 0; + await writeIncorrectResolution({ + event: buildEvent({ quality: 'partial' }), + enabled: true, + writer: () => { + writerCalls++; + }, + }); + expect(writerCalls).toBe(1); + }); +});