From 8901dc0f452381dc3b8208591ec894b84c42ec49 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 13:23:36 -0700 Subject: [PATCH] fix(backlog): x-to-brain health check, propose_takes deadlines, capture title truncation, extract_atoms backlog + pooler direct-URL (part4-6) (#3165) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(recipes/x-to-brain): use /users/by/username for app-only bearer health check Takeover of #2343. /users/me requires user-context OAuth and always fails under the app-only bearer the recipe collects. Health check + setup curls now use /users/by/username/$X_HANDLE, with X_HANDLE declared in secrets so the installer prompts for it. Recipe version 0.8.1 -> 0.8.2. Co-authored-by: ethanbeard Co-Authored-By: Claude Fable 5 * fix(cycle): bound propose_takes with per-call timeout + phase deadline Takeover of #2262. The extractor's gateway.chat call had no abortSignal, so one stalled provider socket could pin the phase for the 300s gateway default per page; the nightly wrapper then SIGTERMed the whole phase mid-run. Each extractor call is now bounded at 90s (per-page failure already logs a warning and continues), and the page loop carries a 30-min wall-clock deadline that breaks cleanly into a partial result with deadline_hit:true + warn status. Unlike the original PR, the default pageLimit stays at 100 — shrinking it to 30 was an unrelated product-knob change that would permanently cut nightly take coverage. Co-authored-by: tschew72 Co-Authored-By: Claude Fable 5 * fix(capture): make fallback title truncation explicit and astral-safe Takeover of #2310. deriveTitle's silent .slice(0, 80) could split an astral surrogate pair mid-character and gave no signal the title was cut. Truncation is now codepoint-aware and appends an ellipsis (still capped at 80 codepoints). Unlike the original PR, this stays a three-line change: no whitespace normalization of every derived title, no word-boundary heuristics. Co-authored-by: xd-Neji Co-Authored-By: Claude Fable 5 * fix(doctor): clear extract_atoms raw source-holder backlog + normalize pooler direct-URL overrides Takeover of #2242, split to the two concerns that survive review: - extract_atoms: exclude source pages whose frontmatter declares a raw payload pointer from discovery AND the doctor backlog count (shared SQL fragment so they can't drift). Extraction on these yields zero atoms, so no atom row is ever written and they re-enter the backlog every cycle — a permanent no-progress doctor blocker. - connection-manager: a direct-URL override (opts/env) that still points at the Supavisor TRANSACTION pooler (port 6543, usually a copy-paste of the primary URL) is normalized to the real direct host via deriveDirectUrl. Session-mode pooler overrides (port 5432) pass through — they are a legitimate direct-ish target, which the original PR would have nulled out. Dropped from the original PR: orphan-reporting atom exclusions (master already excludes atoms/ and raw/ first segments plus /raw/ segments in src/commands/orphans.ts) and the drain dry-run status tweak. Co-authored-by: benjonp Co-Authored-By: Claude Fable 5 * fix(cycle): record propose_takes deadline break as a halt in the extract rollup A deadline-hit run breaks the page loop mid-list — same posture as budget exhaustion — but the rollup still counted it as a completed round with no halt, hiding chronic never-finishing nightly runs from extract-status/ doctor. Treat deadline_hit like budget_exhausted in the rollup deltas; deadline test now pins halt=1 / completed=0. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Garry Tan Co-authored-by: ethanbeard Co-authored-by: Claude Fable 5 Co-authored-by: tschew72 Co-authored-by: xd-Neji Co-authored-by: benjonp --- recipes/x-to-brain.md | 21 +++++--- skills/capture/SKILL.md | 2 +- src/commands/capture.ts | 8 ++- src/core/connection-manager.ts | 32 +++++++++++- src/core/cycle/extract-atoms.ts | 15 ++++++ src/core/cycle/propose-takes.ts | 42 +++++++++++++-- test/capture-build-content.test.ts | 18 ++++++- test/commands/capture.test.ts | 3 +- test/connection-manager.serial.test.ts | 63 +++++++++++++++++++++++ test/doctor-extract-atoms-backlog.test.ts | 11 ++++ test/extract-atoms-page-discovery.test.ts | 12 +++++ test/integrations.test.ts | 29 +++++++++++ test/propose-takes.test.ts | 41 +++++++++++++++ 13 files changed, 278 insertions(+), 19 deletions(-) diff --git a/recipes/x-to-brain.md b/recipes/x-to-brain.md index 67bda3c8e..9392862a7 100644 --- a/recipes/x-to-brain.md +++ b/recipes/x-to-brain.md @@ -1,7 +1,7 @@ --- id: x-to-brain name: X-to-Brain -version: 0.8.1 +version: 0.8.2 description: Twitter timeline, mentions, and keyword monitoring flow into brain pages. Tracks deletions, engagement velocity, OCR on images, and real-time alerts. category: sense requires: [] @@ -9,9 +9,12 @@ secrets: - name: X_BEARER_TOKEN description: X API v2 Bearer token (Basic tier minimum, $200/mo for full archive search) where: https://developer.x.com/en/portal/dashboard — create a project + app, copy the Bearer Token from "Keys and tokens" + - name: X_HANDLE + description: Your X username without the @ (used for the app-only health check — /users/me requires user-context OAuth, which app-only bearer tokens don't have) + where: Your X profile — the handle in your profile URL, e.g. x.com/yourhandle → yourhandle health_checks: - type: http - url: "https://api.x.com/2/users/me" + url: "https://api.x.com/2/users/by/username/$X_HANDLE" auth: bearer auth_token: "$X_BEARER_TOKEN" label: "X API" @@ -110,15 +113,17 @@ Tell the user: 4. Inside the project, create a new App 5. Go to the app's 'Keys and tokens' tab 6. Under 'Bearer Token', click 'Generate' (or 'Regenerate') -7. Copy the Bearer Token and paste it to me +7. Copy the Bearer Token and paste it to me, along with your X handle (without the @) Note: Free tier gives read-only access with low limits. Basic tier ($200/mo) gives search/recent endpoint and higher limits. Pro tier gets full archive search." -Validate immediately: +Set both `X_BEARER_TOKEN` and `X_HANDLE` in the environment. Validate immediately +(app-only bearer tokens cannot call `/users/me` — that endpoint requires +user-context OAuth — so validation uses the by-username lookup): ```bash curl -sf -H "Authorization: Bearer $X_BEARER_TOKEN" \ - "https://api.x.com/2/users/me" \ + "https://api.x.com/2/users/by/username/$X_HANDLE" \ && echo "PASS: X API connected" \ || echo "FAIL: X API token invalid" ``` @@ -134,10 +139,10 @@ starting with 'AAA...', (3) if you just created the app, the token is valid imme ```bash # Look up the user's X user ID from their handle curl -sf -H "Authorization: Bearer $X_BEARER_TOKEN" \ - "https://api.x.com/2/users/by/username/USERNAME" | grep -o '"id":"[^"]*"' + "https://api.x.com/2/users/by/username/$X_HANDLE" | grep -o '"id":"[^"]*"' ``` -Ask the user for their X handle (e.g., @yourhandle). Look up their user ID. +Look up the user ID from the handle collected in Step 1. Save it — the collector needs the numeric ID, not the handle. ### Step 3: Configure the Collector @@ -205,7 +210,7 @@ The agent should review collected data 2-3x daily and run enrichment. ```bash mkdir -p ~/.gbrain/integrations/x-to-brain -echo '{"ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","event":"setup_complete","source_version":"0.8.1","status":"ok","details":{"user_id":"X_USER_ID"}}' >> ~/.gbrain/integrations/x-to-brain/heartbeat.jsonl +echo '{"ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","event":"setup_complete","source_version":"0.8.2","status":"ok","details":{"user_id":"X_USER_ID"}}' >> ~/.gbrain/integrations/x-to-brain/heartbeat.jsonl ``` ## Production Patterns (v0.8.1) diff --git a/skills/capture/SKILL.md b/skills/capture/SKILL.md index 868534eb7..4335904d5 100644 --- a/skills/capture/SKILL.md +++ b/skills/capture/SKILL.md @@ -62,7 +62,7 @@ gbrain capture "..." --json # structured output for agents - **Slug:** `inbox/YYYY-MM-DD-` (stable for same content; the daemon's 24h dedup catches re-captures). - **Type:** `note` (override with `--type idea` etc.). - **Frontmatter stamps:** `captured_via: capture-cli`, `captured_at: `. -- **Title:** first non-empty line of the body, capped at 80 chars. +- **Title:** first non-empty line of the body, capped at 80 chars (truncation appends `…`). ## Output Format diff --git a/src/commands/capture.ts b/src/commands/capture.ts index 0b195e466..86919f8b3 100644 --- a/src/commands/capture.ts +++ b/src/commands/capture.ts @@ -233,14 +233,18 @@ export function maybeRewriteSourceFkError(err: unknown, sourceId: string | undef /** * Derive a title from the first non-empty, non-`---` line of the body, - * stripping leading markdown heading marks, capped at 80 chars. + * stripping leading markdown heading marks, capped at 80 chars. Truncation + * is codepoint-aware (never splits an astral surrogate pair) and appends an + * ellipsis so a cut title is visibly cut. * Falls back to 'Capture' when no usable line exists. */ function deriveTitle(rawBody: string): string { const firstLine = rawBody .split('\n') .find((l) => l.trim().length > 0 && l.trim() !== '---') ?? ''; - return firstLine.replace(/^#+\s*/, '').slice(0, 80) || 'Capture'; + const stripped = firstLine.replace(/^#+\s*/, ''); + const cps = [...stripped]; + return (cps.length > 80 ? cps.slice(0, 79).join('') + '…' : stripped) || 'Capture'; } /** diff --git a/src/core/connection-manager.ts b/src/core/connection-manager.ts index b299e2274..acabbf373 100644 --- a/src/core/connection-manager.ts +++ b/src/core/connection-manager.ts @@ -167,6 +167,33 @@ export function deriveDirectUrl(url: string): string | null { } } +/** True when the URL targets Supavisor transaction mode (port 6543). */ +function isTransactionPoolerUrl(url: string): boolean { + try { + return new URL(url.replace(/^postgres(ql)?:\/\//, 'http://')).port === '6543'; + } catch { + return false; + } +} + +/** + * Resolve the direct-pool URL from an explicit/env override + the primary URL. + * + * A direct override still pointing at the TRANSACTION-mode pooler (port 6543, + * usually a copy-paste of the primary URL) is a misconfiguration: the manager + * would believe DDL/bulk work runs on a long-timeout direct connection while + * still routing through Supavisor transaction mode (short timeouts, no + * prepared statements). Normalize it via deriveDirectUrl. Session-mode pooler + * URLs (pooler host, port 5432) pass through — they are a legitimate + * direct-ish target when the db. host is unreachable. + */ +export function normalizeDirectUrl(primaryUrl: string, override?: string | null): string | null { + const candidate = override ?? deriveDirectUrl(primaryUrl); + if (!candidate) return null; + if (!isTransactionPoolerUrl(candidate)) return candidate; + return deriveDirectUrl(candidate) ?? deriveDirectUrl(primaryUrl); +} + /** * Error codes that mean "the direct host is unreachable from this network" * (#1641). The auto-derived db..supabase.co host is IPv6-only without @@ -232,9 +259,10 @@ export class ConnectionManager { } else { this._killSwitch = readKillSwitchEnv(); this._isSupabase = isSupabasePoolerUrl(opts.url); - // Direct URL: explicit override > env > derive > null + // Direct URL: explicit override > env > derive > null. Pooler-shaped + // overrides are normalized to a real direct host (or dropped). const envOverride = process.env.GBRAIN_DIRECT_DATABASE_URL; - this._directUrl = opts.directUrl ?? envOverride ?? deriveDirectUrl(opts.url); + this._directUrl = normalizeDirectUrl(opts.url, opts.directUrl ?? envOverride); } } diff --git a/src/core/cycle/extract-atoms.ts b/src/core/cycle/extract-atoms.ts index 4c73ea400..7d9ea0ed4 100644 --- a/src/core/cycle/extract-atoms.ts +++ b/src/core/cycle/extract-atoms.ts @@ -83,6 +83,14 @@ const SYNTHESIS_OUTPUT_TYPES = new Set(['atom', 'concept']); const PAGE_DISCOVERY_BUDGET = 50; const MIN_PAGE_CHARS_FOR_EXTRACTION = 500; +// Source pages whose frontmatter declares a `raw` payload pointer hold raw +// import data, not extractable prose. Extraction on them yields zero atoms, +// so no atom row is ever written and they re-enter discovery + the doctor +// backlog count on every cycle — a permanent no-progress loop. Shared by +// discoverExtractablePages and countExtractAtomsBacklog so the phase and the +// doctor check can't drift. +const RAW_SOURCE_HOLDER_EXCLUSION_SQL = + `AND NOT (p.type = 'source' AND COALESCE(p.frontmatter ? 'raw', false))`; /** * Pure allowlist policy: the legacy floor UNION the pack's `extractable: true` @@ -207,6 +215,10 @@ interface DiscoveredPage { * participate in the NOT EXISTS check anyway. * #4 dream_generated exclusion — prevents the phase from chewing * its own output (e.g. dream-generated originals). + * #5 raw source-holder exclusion — source pages that only point at a raw + * import payload are not extractable prose; counting them creates a + * permanent backlog/no-progress loop (see + * RAW_SOURCE_HOLDER_EXCLUSION_SQL). */ export async function discoverExtractablePages( engine: BrainEngine, @@ -225,6 +237,7 @@ export async function discoverExtractablePages( AND p.content_hash IS NOT NULL AND COALESCE(p.frontmatter->>'imported_from', '') <> 'markdown-greenfield' AND COALESCE(p.frontmatter->>'dream_generated', '') <> 'true' + ${RAW_SOURCE_HOLDER_EXCLUSION_SQL} AND length(COALESCE(p.compiled_truth, '')) >= $3 ${hasFilter ? "AND p.slug = ANY($5::text[])" : ''} AND NOT EXISTS ( @@ -297,6 +310,7 @@ export async function countExtractAtomsBacklog( AND p.content_hash IS NOT NULL AND COALESCE(p.frontmatter->>'imported_from', '') <> 'markdown-greenfield' AND COALESCE(p.frontmatter->>'dream_generated', '') <> 'true' + ${RAW_SOURCE_HOLDER_EXCLUSION_SQL} AND length(COALESCE(p.compiled_truth, '')) >= $3 AND NOT EXISTS ( SELECT 1 FROM pages atom @@ -310,6 +324,7 @@ export async function countExtractAtomsBacklog( AND p.content_hash IS NOT NULL AND COALESCE(p.frontmatter->>'imported_from', '') <> 'markdown-greenfield' AND COALESCE(p.frontmatter->>'dream_generated', '') <> 'true' + ${RAW_SOURCE_HOLDER_EXCLUSION_SQL} AND length(COALESCE(p.compiled_truth, '')) >= $2 AND NOT EXISTS ( SELECT 1 FROM pages atom diff --git a/src/core/cycle/propose-takes.ts b/src/core/cycle/propose-takes.ts index 63ada141e..91cafead3 100644 --- a/src/core/cycle/propose-takes.ts +++ b/src/core/cycle/propose-takes.ts @@ -145,6 +145,8 @@ export interface ProposeTakesOpts extends BasePhaseOpts { model?: string; /** Skip pages that already have a complete takes fence. Default: true. */ skipPagesWithFence?: boolean; + /** Override the phase wall-clock deadline (tests). Default: 30 min. */ + deadlineMs?: number; } export interface ProposeTakesResult { @@ -153,6 +155,8 @@ export interface ProposeTakesResult { cache_misses: number; proposals_inserted: number; budget_exhausted: boolean; + /** True when the phase deadline fired before the page loop completed (partial result). */ + deadline_hit?: boolean; warnings: string[]; } @@ -210,6 +214,9 @@ export function extractExistingTakesForDedup(pageBody: string): Array<{ return rows; } +/** Per-call wall-clock timeout for the extractor LLM call. */ +const EXTRACTOR_CALL_TIMEOUT_MS = 90_000; + /** * Production extractor — calls gateway.chat with the EXTRACT_TAKES_PROMPT * and parses the JSON array output. Returns [] on parse failure (logged as @@ -227,10 +234,14 @@ export async function defaultExtractor( .replace('{EXISTING_TAKES_JSON}', JSON.stringify(input.existingTakes, null, 2)) .replace('{PAGE_BODY}', input.pageBody); + // Bound each call so one stalled provider socket can't pin the phase for the + // full gateway default (GBRAIN_AI_CHAT_TIMEOUT_MS, 300s) x pageLimit. The + // caller already catches per-page errors, logs a warning, and continues. const result = await gatewayChat({ messages: [{ role: 'user', content: prompt }], ...(input.modelHint ? { model: input.modelHint } : {}), maxTokens: 2048, + abortSignal: AbortSignal.timeout(EXTRACTOR_CALL_TIMEOUT_MS), }); // ChatResult.text is already the concatenated text content. @@ -287,6 +298,14 @@ class ProposeTakesPhase extends BaseCyclePhase { readonly name = 'propose_takes' as CyclePhase; protected readonly budgetUsdKey = 'cycle.propose_takes.budget_usd'; protected readonly budgetUsdDefault = 5.0; + /** + * Hard wall-clock deadline for the phase. Even with the per-call timeout in + * defaultExtractor, a long tail of slow-but-completing calls can accumulate. + * The phase breaks cleanly and returns a partial result with + * `deadline_hit: true` instead of being killed mid-write by an outer + * `timeout` wrapper (the recurring SIGTERM in nightly dream runs). + */ + private static readonly PHASE_DEADLINE_MS = 30 * 60 * 1000; protected override mapErrorCode(err: unknown): string { if (err instanceof GBrainError) return err.problem; @@ -307,6 +326,8 @@ class ProposeTakesPhase extends BaseCyclePhase { const promptVersion = opts.promptVersion ?? PROPOSE_TAKES_PROMPT_VERSION; const pageLimit = opts.pageLimit ?? 100; const skipPagesWithFence = opts.skipPagesWithFence ?? false; + const deadlineMs = opts.deadlineMs ?? ProposeTakesPhase.PHASE_DEADLINE_MS; + const phaseStartMs = Date.now(); const proposalRunId = `propose-${new Date().toISOString().slice(0, 19).replace(/[-:T]/g, '')}-${randomUUID().slice(0, 8)}`; const result: ProposeTakesResult = { @@ -333,6 +354,18 @@ class ProposeTakesPhase extends BaseCyclePhase { const modelId = opts.model ?? getChatModel(); for (const page of pages) { + // Phase deadline check. Break (not throw) so the phase returns a + // partial result with deadline_hit:true; work already banked stays. + const elapsedMs = Date.now() - phaseStartMs; + if (elapsedMs > deadlineMs) { + result.warnings.push( + `phase deadline hit at page ${result.pages_scanned}/${pages.length} ` + + `after ${(elapsedMs / 1000).toFixed(0)}s (cap ${(deadlineMs / 1000).toFixed(0)}s); partial completion`, + ); + result.deadline_hit = true; + break; + } + result.pages_scanned += 1; this.tick(opts); @@ -440,17 +473,20 @@ class ProposeTakesPhase extends BaseCyclePhase { console.error(`[propose_takes] receipt write failed: ${(err as Error).message}`); } } + // A deadline-hit run halted mid-list the same way a budget-exhausted one + // does — record it as a halt, not a completed round. + const halted = result.budget_exhausted || result.deadline_hit === true; await upsertExtractRollup(engine, { kind: 'takes.proposed', source_id: sourceIdForReceipt, - round_completed_delta: result.budget_exhausted ? 0 : 1, - halt_delta: result.budget_exhausted ? 1 : 0, + round_completed_delta: halted ? 0 : 1, + halt_delta: halted ? 1 : 0, }); return { summary: `propose_takes: scanned ${result.pages_scanned} pages, ${result.cache_hits} cached, ${result.proposals_inserted} new proposals (run ${proposalRunId})`, details: { ...result, proposal_run_id: proposalRunId, prompt_version: promptVersion }, - status: result.budget_exhausted ? 'warn' : 'ok', + status: result.budget_exhausted || result.deadline_hit ? 'warn' : 'ok', }; } } diff --git a/test/capture-build-content.test.ts b/test/capture-build-content.test.ts index 122222306..f71ae1e5c 100644 --- a/test/capture-build-content.test.ts +++ b/test/capture-build-content.test.ts @@ -190,9 +190,23 @@ describe('deriveTitle (no-frontmatter path)', () => { expect(deriveTitle('### Triple hash\nrest')).toBe('Triple hash'); }); - test('caps at 80 chars', () => { + test('caps at 80 chars with explicit ellipsis', () => { const long = 'a'.repeat(120); - expect(deriveTitle(long)).toBe('a'.repeat(80)); + expect(deriveTitle(long)).toBe('a'.repeat(79) + '…'); + expect([...deriveTitle(long)].length).toBe(80); + }); + + test('exactly 80 chars is not truncated', () => { + const exact = 'a'.repeat(80); + expect(deriveTitle(exact)).toBe(exact); + }); + + test('does not split astral Unicode while truncating', () => { + const title = deriveTitle('😀'.repeat(120)); + expect(title.endsWith('…')).toBe(true); + expect([...title].length).toBe(80); + // No lone surrogate halves left behind by the cut. + expect([...title].some((ch) => ch.length === 1 && /[\uD800-\uDFFF]/.test(ch))).toBe(false); }); test('falls back to Capture for empty input', () => { diff --git a/test/commands/capture.test.ts b/test/commands/capture.test.ts index 01553cf71..d7bcbf5bb 100644 --- a/test/commands/capture.test.ts +++ b/test/commands/capture.test.ts @@ -136,12 +136,13 @@ describe('capture — buildContent', () => { expect(parsed.data.title).toBe('Real first line'); }); - test('caps title at 80 chars', () => { + test('caps title at 80 chars with explicit ellipsis', () => { const longLine = 'x'.repeat(200); const result = __testing.buildContent(longLine, {}); const parsed = matter(result); expect(typeof parsed.data.title).toBe('string'); expect((parsed.data.title as string).length).toBeLessThanOrEqual(80); + expect((parsed.data.title as string).endsWith('…')).toBe(true); }); test('honors --source via captured_via', () => { diff --git a/test/connection-manager.serial.test.ts b/test/connection-manager.serial.test.ts index 598a0f332..b9848a44d 100644 --- a/test/connection-manager.serial.test.ts +++ b/test/connection-manager.serial.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test, beforeEach, afterEach } from 'bun:test'; import { isSupabasePoolerUrl, deriveDirectUrl, + normalizeDirectUrl, readKillSwitchEnv, isNetworkUnreachableError, resolveDirectPoolSize, @@ -79,6 +80,44 @@ describe('deriveDirectUrl', () => { }); }); +describe('normalizeDirectUrl', () => { + test('normalizes a transaction-pooler (6543) override to the real direct host', () => { + const direct = normalizeDirectUrl( + 'postgresql://postgres.abcxyz:p@aws-0-us-west-2.pooler.supabase.com:6543/postgres', + 'postgresql://postgres.abcxyz:p@aws-0-us-west-2.pooler.supabase.com:6543/postgres', + ); + expect(direct).toBe('postgresql://postgres:p@db.abcxyz.supabase.co:5432/postgres'); + }); + + test('keeps a non-pooler direct override', () => { + const direct = normalizeDirectUrl( + 'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db', + 'postgresql://u:p@custom-direct.example.com:5432/db', + ); + expect(direct).toBe('postgresql://u:p@custom-direct.example.com:5432/db'); + }); + + test('keeps a session-mode pooler override (pooler host, port 5432)', () => { + const sessionUrl = 'postgresql://postgres.abc:p@aws.pooler.supabase.com:5432/db'; + const direct = normalizeDirectUrl( + 'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db', + sessionUrl, + ); + expect(direct).toBe(sessionUrl); + }); + + test('no override: derives from the primary as before', () => { + const direct = normalizeDirectUrl( + 'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db', + ); + expect(direct).toContain('db.abc.supabase.co:5432'); + }); + + test('no override, non-Supabase primary: null', () => { + expect(normalizeDirectUrl('postgresql://u:p@localhost:5432/db')).toBeNull(); + }); +}); + describe('readKillSwitchEnv', () => { let original: string | undefined; beforeEach(() => { original = process.env.GBRAIN_DISABLE_DIRECT_POOL; }); @@ -146,13 +185,18 @@ describe('resolveDirectPoolSize', () => { describe('ConnectionManager — describeMode + dual-pool routing', () => { let originalKillSwitch: string | undefined; + let originalDirectUrl: string | undefined; beforeEach(() => { originalKillSwitch = process.env.GBRAIN_DISABLE_DIRECT_POOL; + originalDirectUrl = process.env.GBRAIN_DIRECT_DATABASE_URL; delete process.env.GBRAIN_DISABLE_DIRECT_POOL; + delete process.env.GBRAIN_DIRECT_DATABASE_URL; }); afterEach(() => { if (originalKillSwitch === undefined) delete process.env.GBRAIN_DISABLE_DIRECT_POOL; else process.env.GBRAIN_DISABLE_DIRECT_POOL = originalKillSwitch; + if (originalDirectUrl === undefined) delete process.env.GBRAIN_DIRECT_DATABASE_URL; + else process.env.GBRAIN_DIRECT_DATABASE_URL = originalDirectUrl; }); test('non-Supabase URL → single mode', () => { @@ -191,6 +235,25 @@ describe('ConnectionManager — describeMode + dual-pool routing', () => { expect(cm.resolveDirectUrl()).toContain('custom-direct.example.com'); }); + test('explicit transaction-pooler directUrl override is normalized to direct host', () => { + const cm = new ConnectionManager({ + url: 'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db', + directUrl: 'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db', + }); + expect(cm.resolveDirectUrl()).toContain('db.abc.supabase.co:5432'); + expect(cm.resolveDirectUrl()).not.toContain(':6543'); + }); + + test('env transaction-pooler directUrl override is normalized to direct host', () => { + process.env.GBRAIN_DIRECT_DATABASE_URL = + 'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db'; + const cm = new ConnectionManager({ + url: 'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db', + }); + expect(cm.resolveDirectUrl()).toContain('db.abc.supabase.co:5432'); + expect(cm.resolveDirectUrl()).not.toContain(':6543'); + }); + test('host string contains creds neither in describeMode nor resolveDirectUrl logging', () => { const cm = new ConnectionManager({ url: 'postgresql://postgres.abc:secret@aws.pooler.supabase.com:6543/db', diff --git a/test/doctor-extract-atoms-backlog.test.ts b/test/doctor-extract-atoms-backlog.test.ts index 83f502283..f37726d03 100644 --- a/test/doctor-extract-atoms-backlog.test.ts +++ b/test/doctor-extract-atoms-backlog.test.ts @@ -76,6 +76,17 @@ describe('countExtractAtomsBacklog (issue #1678)', () => { }); expect(await countExtractAtomsBacklog(engine)).toBe(0); }); + + it('ignores raw source-holder pages (permanent no-progress backlog otherwise)', async () => { + await engine.putPage('wiki/raw-email-source', { + type: 'source', + title: 'Raw email source', + compiled_truth: BODY, + frontmatter: { raw: 'raw/email/example.md' }, + }); + expect(await countExtractAtomsBacklog(engine)).toBe(0); + expect(await countExtractAtomsBacklog(engine, 'default')).toBe(0); + }); }); describe('computeExtractAtomsBacklogCheck (issue #1678)', () => { diff --git a/test/extract-atoms-page-discovery.test.ts b/test/extract-atoms-page-discovery.test.ts index 7289cc52e..d1b4fcefb 100644 --- a/test/extract-atoms-page-discovery.test.ts +++ b/test/extract-atoms-page-discovery.test.ts @@ -178,6 +178,18 @@ describe('v0.41.2.1: discoverExtractablePages SQL contract', () => { expect(discovered.map((d) => d.slug)).toEqual(['original/normal']); }); + test('raw source-holder pages excluded (#5 — no permanent no-progress backlog)', async () => { + await seedPage({ slug: 'source/normal', type: 'source' }); + await seedPage({ + slug: 'wiki/raw-email-source', + type: 'source', + frontmatter: { raw: 'raw/email/example.md' }, + }); + + const discovered = await discoverExtractablePages(engine, 'default'); + expect(discovered.map((d) => d.slug)).toEqual(['source/normal']); + }); + test('pages with NULL content_hash excluded (D9 #3 — no .slice crash)', async () => { await seedPage({ slug: 'meeting/with-hash', type: 'meeting' }); await seedPage({ slug: 'meeting/no-hash', type: 'meeting', content_hash: null }); diff --git a/test/integrations.test.ts b/test/integrations.test.ts index 2b4bc790c..0926d7c9d 100644 --- a/test/integrations.test.ts +++ b/test/integrations.test.ts @@ -263,6 +263,35 @@ describe('twilio-voice-brain recipe', () => { }); }); +describe('x-to-brain recipe', () => { + test('health check works with an app-only bearer token (#2343)', () => { + const { readFileSync } = require('fs'); + const content = readFileSync( + new URL('../recipes/x-to-brain.md', import.meta.url), + 'utf-8' + ); + const recipe = parseRecipe(content, 'x-to-brain.md'); + expect(recipe).not.toBeNull(); + const httpChecks = recipe!.frontmatter.health_checks + .filter((c: any) => typeof c === 'object' && c.type === 'http'); + expect(httpChecks.length).toBeGreaterThan(0); + const secretNames = new Set(recipe!.frontmatter.secrets.map((s: any) => s.name)); + for (const check of httpChecks as any[]) { + // /users/me requires user-context OAuth; the recipe only collects an + // app-only bearer token, so probing it always fails. + expect(check.url).not.toContain('/users/me'); + // Every $VAR the check expands must be declared in secrets, or the + // installer never prompts for it and the check fails for everyone. + const vars = [check.url, check.auth_token, check.auth_user, check.auth_pass] + .filter((v: unknown): v is string => typeof v === 'string') + .flatMap((v: string) => v.match(/\$[A-Z_][A-Z0-9_]*/g) ?? []) + .map((v: string) => v.slice(1)); + expect(vars.length).toBeGreaterThan(0); + for (const name of vars) expect(secretNames.has(name)).toBe(true); + } + }); +}); + // --- All recipes parse without error --- describe('all recipes', () => { diff --git a/test/propose-takes.test.ts b/test/propose-takes.test.ts index 3c0ccb68d..af5674a0d 100644 --- a/test/propose-takes.test.ts +++ b/test/propose-takes.test.ts @@ -368,6 +368,47 @@ New prose appended here.`; expect(extractorCalls).toBe(1); }); + test('phase deadline breaks the page loop with a partial result (deadline_hit)', async () => { + const pages = [ + buildPage({ slug: 'wiki/slow-a', body: 'page a' }), + buildPage({ slug: 'wiki/slow-b', body: 'page b' }), + ]; + const { engine, captured } = buildMockEngine({ pages }); + let extractorCalls = 0; + const extractor: ProposeTakesExtractor = async () => { + extractorCalls++; + await new Promise((r) => setTimeout(r, 10)); + return [{ claim_text: 'x', kind: 'take', holder: 'brain', weight: 0.5 }]; + }; + // 5ms deadline: page 1 processes (elapsed 0 at check), the 10ms extractor + // call pushes elapsed past the cap, page 2 is never scanned. + const result = await runPhaseProposeTakes(buildCtx(engine), { extractor, deadlineMs: 5 }); + + expect(result.status).toBe('warn'); + const details = result.details as Record; + expect(details.deadline_hit).toBe(true); + expect(details.pages_scanned).toBe(1); + expect(extractorCalls).toBe(1); + expect((details.warnings as string[]).some(w => w.includes('phase deadline hit'))).toBe(true); + + // Rollup records the deadline break as a halt, not a completed round + // (same posture as budget exhaustion). Params: $5 = halt, $8 = completed. + const rollup = captured.find((c) => c.sql.includes('extract_rollup_7d')); + expect(rollup).toBeDefined(); + expect(rollup!.params[4]).toBe(1); // halt_count delta + expect(rollup!.params[7]).toBe(0); // round_completed delta + }); + + test('default deadline does not fire on a fast run', async () => { + const pages = [buildPage({ slug: 'wiki/fast', body: 'quick page' })]; + const { engine } = buildMockEngine({ pages }); + const extractor: ProposeTakesExtractor = async () => []; + const result = await runPhaseProposeTakes(buildCtx(engine), { extractor }); + const details = result.details as Record; + expect(details.deadline_hit).toBeUndefined(); + expect(details.pages_scanned).toBe(1); + }); + test('proposal_run_id is stable across all proposals from one phase invocation', async () => { const pages = [ buildPage({ slug: 'wiki/a', body: 'page a' }),