diff --git a/recipes/x-to-brain.md b/recipes/x-to-brain.md index 67bda3c8e..9392862a7 100644 --- a/recipes/x-to-brain.md +++ b/recipes/x-to-brain.md @@ -1,7 +1,7 @@ --- id: x-to-brain name: X-to-Brain -version: 0.8.1 +version: 0.8.2 description: Twitter timeline, mentions, and keyword monitoring flow into brain pages. Tracks deletions, engagement velocity, OCR on images, and real-time alerts. category: sense requires: [] @@ -9,9 +9,12 @@ secrets: - name: X_BEARER_TOKEN description: X API v2 Bearer token (Basic tier minimum, $200/mo for full archive search) where: https://developer.x.com/en/portal/dashboard — create a project + app, copy the Bearer Token from "Keys and tokens" + - name: X_HANDLE + description: Your X username without the @ (used for the app-only health check — /users/me requires user-context OAuth, which app-only bearer tokens don't have) + where: Your X profile — the handle in your profile URL, e.g. x.com/yourhandle → yourhandle health_checks: - type: http - url: "https://api.x.com/2/users/me" + url: "https://api.x.com/2/users/by/username/$X_HANDLE" auth: bearer auth_token: "$X_BEARER_TOKEN" label: "X API" @@ -110,15 +113,17 @@ Tell the user: 4. Inside the project, create a new App 5. Go to the app's 'Keys and tokens' tab 6. Under 'Bearer Token', click 'Generate' (or 'Regenerate') -7. Copy the Bearer Token and paste it to me +7. Copy the Bearer Token and paste it to me, along with your X handle (without the @) Note: Free tier gives read-only access with low limits. Basic tier ($200/mo) gives search/recent endpoint and higher limits. Pro tier gets full archive search." -Validate immediately: +Set both `X_BEARER_TOKEN` and `X_HANDLE` in the environment. Validate immediately +(app-only bearer tokens cannot call `/users/me` — that endpoint requires +user-context OAuth — so validation uses the by-username lookup): ```bash curl -sf -H "Authorization: Bearer $X_BEARER_TOKEN" \ - "https://api.x.com/2/users/me" \ + "https://api.x.com/2/users/by/username/$X_HANDLE" \ && echo "PASS: X API connected" \ || echo "FAIL: X API token invalid" ``` @@ -134,10 +139,10 @@ starting with 'AAA...', (3) if you just created the app, the token is valid imme ```bash # Look up the user's X user ID from their handle curl -sf -H "Authorization: Bearer $X_BEARER_TOKEN" \ - "https://api.x.com/2/users/by/username/USERNAME" | grep -o '"id":"[^"]*"' + "https://api.x.com/2/users/by/username/$X_HANDLE" | grep -o '"id":"[^"]*"' ``` -Ask the user for their X handle (e.g., @yourhandle). Look up their user ID. +Look up the user ID from the handle collected in Step 1. Save it — the collector needs the numeric ID, not the handle. ### Step 3: Configure the Collector @@ -205,7 +210,7 @@ The agent should review collected data 2-3x daily and run enrichment. ```bash mkdir -p ~/.gbrain/integrations/x-to-brain -echo '{"ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","event":"setup_complete","source_version":"0.8.1","status":"ok","details":{"user_id":"X_USER_ID"}}' >> ~/.gbrain/integrations/x-to-brain/heartbeat.jsonl +echo '{"ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","event":"setup_complete","source_version":"0.8.2","status":"ok","details":{"user_id":"X_USER_ID"}}' >> ~/.gbrain/integrations/x-to-brain/heartbeat.jsonl ``` ## Production Patterns (v0.8.1) diff --git a/skills/capture/SKILL.md b/skills/capture/SKILL.md index 868534eb7..4335904d5 100644 --- a/skills/capture/SKILL.md +++ b/skills/capture/SKILL.md @@ -62,7 +62,7 @@ gbrain capture "..." --json # structured output for agents - **Slug:** `inbox/YYYY-MM-DD-` (stable for same content; the daemon's 24h dedup catches re-captures). - **Type:** `note` (override with `--type idea` etc.). - **Frontmatter stamps:** `captured_via: capture-cli`, `captured_at: `. -- **Title:** first non-empty line of the body, capped at 80 chars. +- **Title:** first non-empty line of the body, capped at 80 chars (truncation appends `…`). ## Output Format diff --git a/src/commands/capture.ts b/src/commands/capture.ts index 0b195e466..86919f8b3 100644 --- a/src/commands/capture.ts +++ b/src/commands/capture.ts @@ -233,14 +233,18 @@ export function maybeRewriteSourceFkError(err: unknown, sourceId: string | undef /** * Derive a title from the first non-empty, non-`---` line of the body, - * stripping leading markdown heading marks, capped at 80 chars. + * stripping leading markdown heading marks, capped at 80 chars. Truncation + * is codepoint-aware (never splits an astral surrogate pair) and appends an + * ellipsis so a cut title is visibly cut. * Falls back to 'Capture' when no usable line exists. */ function deriveTitle(rawBody: string): string { const firstLine = rawBody .split('\n') .find((l) => l.trim().length > 0 && l.trim() !== '---') ?? ''; - return firstLine.replace(/^#+\s*/, '').slice(0, 80) || 'Capture'; + const stripped = firstLine.replace(/^#+\s*/, ''); + const cps = [...stripped]; + return (cps.length > 80 ? cps.slice(0, 79).join('') + '…' : stripped) || 'Capture'; } /** diff --git a/src/core/connection-manager.ts b/src/core/connection-manager.ts index b299e2274..acabbf373 100644 --- a/src/core/connection-manager.ts +++ b/src/core/connection-manager.ts @@ -167,6 +167,33 @@ export function deriveDirectUrl(url: string): string | null { } } +/** True when the URL targets Supavisor transaction mode (port 6543). */ +function isTransactionPoolerUrl(url: string): boolean { + try { + return new URL(url.replace(/^postgres(ql)?:\/\//, 'http://')).port === '6543'; + } catch { + return false; + } +} + +/** + * Resolve the direct-pool URL from an explicit/env override + the primary URL. + * + * A direct override still pointing at the TRANSACTION-mode pooler (port 6543, + * usually a copy-paste of the primary URL) is a misconfiguration: the manager + * would believe DDL/bulk work runs on a long-timeout direct connection while + * still routing through Supavisor transaction mode (short timeouts, no + * prepared statements). Normalize it via deriveDirectUrl. Session-mode pooler + * URLs (pooler host, port 5432) pass through — they are a legitimate + * direct-ish target when the db. host is unreachable. + */ +export function normalizeDirectUrl(primaryUrl: string, override?: string | null): string | null { + const candidate = override ?? deriveDirectUrl(primaryUrl); + if (!candidate) return null; + if (!isTransactionPoolerUrl(candidate)) return candidate; + return deriveDirectUrl(candidate) ?? deriveDirectUrl(primaryUrl); +} + /** * Error codes that mean "the direct host is unreachable from this network" * (#1641). The auto-derived db..supabase.co host is IPv6-only without @@ -232,9 +259,10 @@ export class ConnectionManager { } else { this._killSwitch = readKillSwitchEnv(); this._isSupabase = isSupabasePoolerUrl(opts.url); - // Direct URL: explicit override > env > derive > null + // Direct URL: explicit override > env > derive > null. Pooler-shaped + // overrides are normalized to a real direct host (or dropped). const envOverride = process.env.GBRAIN_DIRECT_DATABASE_URL; - this._directUrl = opts.directUrl ?? envOverride ?? deriveDirectUrl(opts.url); + this._directUrl = normalizeDirectUrl(opts.url, opts.directUrl ?? envOverride); } } diff --git a/src/core/cycle/extract-atoms.ts b/src/core/cycle/extract-atoms.ts index 4c73ea400..7d9ea0ed4 100644 --- a/src/core/cycle/extract-atoms.ts +++ b/src/core/cycle/extract-atoms.ts @@ -83,6 +83,14 @@ const SYNTHESIS_OUTPUT_TYPES = new Set(['atom', 'concept']); const PAGE_DISCOVERY_BUDGET = 50; const MIN_PAGE_CHARS_FOR_EXTRACTION = 500; +// Source pages whose frontmatter declares a `raw` payload pointer hold raw +// import data, not extractable prose. Extraction on them yields zero atoms, +// so no atom row is ever written and they re-enter discovery + the doctor +// backlog count on every cycle — a permanent no-progress loop. Shared by +// discoverExtractablePages and countExtractAtomsBacklog so the phase and the +// doctor check can't drift. +const RAW_SOURCE_HOLDER_EXCLUSION_SQL = + `AND NOT (p.type = 'source' AND COALESCE(p.frontmatter ? 'raw', false))`; /** * Pure allowlist policy: the legacy floor UNION the pack's `extractable: true` @@ -207,6 +215,10 @@ interface DiscoveredPage { * participate in the NOT EXISTS check anyway. * #4 dream_generated exclusion — prevents the phase from chewing * its own output (e.g. dream-generated originals). + * #5 raw source-holder exclusion — source pages that only point at a raw + * import payload are not extractable prose; counting them creates a + * permanent backlog/no-progress loop (see + * RAW_SOURCE_HOLDER_EXCLUSION_SQL). */ export async function discoverExtractablePages( engine: BrainEngine, @@ -225,6 +237,7 @@ export async function discoverExtractablePages( AND p.content_hash IS NOT NULL AND COALESCE(p.frontmatter->>'imported_from', '') <> 'markdown-greenfield' AND COALESCE(p.frontmatter->>'dream_generated', '') <> 'true' + ${RAW_SOURCE_HOLDER_EXCLUSION_SQL} AND length(COALESCE(p.compiled_truth, '')) >= $3 ${hasFilter ? "AND p.slug = ANY($5::text[])" : ''} AND NOT EXISTS ( @@ -297,6 +310,7 @@ export async function countExtractAtomsBacklog( AND p.content_hash IS NOT NULL AND COALESCE(p.frontmatter->>'imported_from', '') <> 'markdown-greenfield' AND COALESCE(p.frontmatter->>'dream_generated', '') <> 'true' + ${RAW_SOURCE_HOLDER_EXCLUSION_SQL} AND length(COALESCE(p.compiled_truth, '')) >= $3 AND NOT EXISTS ( SELECT 1 FROM pages atom @@ -310,6 +324,7 @@ export async function countExtractAtomsBacklog( AND p.content_hash IS NOT NULL AND COALESCE(p.frontmatter->>'imported_from', '') <> 'markdown-greenfield' AND COALESCE(p.frontmatter->>'dream_generated', '') <> 'true' + ${RAW_SOURCE_HOLDER_EXCLUSION_SQL} AND length(COALESCE(p.compiled_truth, '')) >= $2 AND NOT EXISTS ( SELECT 1 FROM pages atom diff --git a/src/core/cycle/propose-takes.ts b/src/core/cycle/propose-takes.ts index 63ada141e..91cafead3 100644 --- a/src/core/cycle/propose-takes.ts +++ b/src/core/cycle/propose-takes.ts @@ -145,6 +145,8 @@ export interface ProposeTakesOpts extends BasePhaseOpts { model?: string; /** Skip pages that already have a complete takes fence. Default: true. */ skipPagesWithFence?: boolean; + /** Override the phase wall-clock deadline (tests). Default: 30 min. */ + deadlineMs?: number; } export interface ProposeTakesResult { @@ -153,6 +155,8 @@ export interface ProposeTakesResult { cache_misses: number; proposals_inserted: number; budget_exhausted: boolean; + /** True when the phase deadline fired before the page loop completed (partial result). */ + deadline_hit?: boolean; warnings: string[]; } @@ -210,6 +214,9 @@ export function extractExistingTakesForDedup(pageBody: string): Array<{ return rows; } +/** Per-call wall-clock timeout for the extractor LLM call. */ +const EXTRACTOR_CALL_TIMEOUT_MS = 90_000; + /** * Production extractor — calls gateway.chat with the EXTRACT_TAKES_PROMPT * and parses the JSON array output. Returns [] on parse failure (logged as @@ -227,10 +234,14 @@ export async function defaultExtractor( .replace('{EXISTING_TAKES_JSON}', JSON.stringify(input.existingTakes, null, 2)) .replace('{PAGE_BODY}', input.pageBody); + // Bound each call so one stalled provider socket can't pin the phase for the + // full gateway default (GBRAIN_AI_CHAT_TIMEOUT_MS, 300s) x pageLimit. The + // caller already catches per-page errors, logs a warning, and continues. const result = await gatewayChat({ messages: [{ role: 'user', content: prompt }], ...(input.modelHint ? { model: input.modelHint } : {}), maxTokens: 2048, + abortSignal: AbortSignal.timeout(EXTRACTOR_CALL_TIMEOUT_MS), }); // ChatResult.text is already the concatenated text content. @@ -287,6 +298,14 @@ class ProposeTakesPhase extends BaseCyclePhase { readonly name = 'propose_takes' as CyclePhase; protected readonly budgetUsdKey = 'cycle.propose_takes.budget_usd'; protected readonly budgetUsdDefault = 5.0; + /** + * Hard wall-clock deadline for the phase. Even with the per-call timeout in + * defaultExtractor, a long tail of slow-but-completing calls can accumulate. + * The phase breaks cleanly and returns a partial result with + * `deadline_hit: true` instead of being killed mid-write by an outer + * `timeout` wrapper (the recurring SIGTERM in nightly dream runs). + */ + private static readonly PHASE_DEADLINE_MS = 30 * 60 * 1000; protected override mapErrorCode(err: unknown): string { if (err instanceof GBrainError) return err.problem; @@ -307,6 +326,8 @@ class ProposeTakesPhase extends BaseCyclePhase { const promptVersion = opts.promptVersion ?? PROPOSE_TAKES_PROMPT_VERSION; const pageLimit = opts.pageLimit ?? 100; const skipPagesWithFence = opts.skipPagesWithFence ?? false; + const deadlineMs = opts.deadlineMs ?? ProposeTakesPhase.PHASE_DEADLINE_MS; + const phaseStartMs = Date.now(); const proposalRunId = `propose-${new Date().toISOString().slice(0, 19).replace(/[-:T]/g, '')}-${randomUUID().slice(0, 8)}`; const result: ProposeTakesResult = { @@ -333,6 +354,18 @@ class ProposeTakesPhase extends BaseCyclePhase { const modelId = opts.model ?? getChatModel(); for (const page of pages) { + // Phase deadline check. Break (not throw) so the phase returns a + // partial result with deadline_hit:true; work already banked stays. + const elapsedMs = Date.now() - phaseStartMs; + if (elapsedMs > deadlineMs) { + result.warnings.push( + `phase deadline hit at page ${result.pages_scanned}/${pages.length} ` + + `after ${(elapsedMs / 1000).toFixed(0)}s (cap ${(deadlineMs / 1000).toFixed(0)}s); partial completion`, + ); + result.deadline_hit = true; + break; + } + result.pages_scanned += 1; this.tick(opts); @@ -440,17 +473,20 @@ class ProposeTakesPhase extends BaseCyclePhase { console.error(`[propose_takes] receipt write failed: ${(err as Error).message}`); } } + // A deadline-hit run halted mid-list the same way a budget-exhausted one + // does — record it as a halt, not a completed round. + const halted = result.budget_exhausted || result.deadline_hit === true; await upsertExtractRollup(engine, { kind: 'takes.proposed', source_id: sourceIdForReceipt, - round_completed_delta: result.budget_exhausted ? 0 : 1, - halt_delta: result.budget_exhausted ? 1 : 0, + round_completed_delta: halted ? 0 : 1, + halt_delta: halted ? 1 : 0, }); return { summary: `propose_takes: scanned ${result.pages_scanned} pages, ${result.cache_hits} cached, ${result.proposals_inserted} new proposals (run ${proposalRunId})`, details: { ...result, proposal_run_id: proposalRunId, prompt_version: promptVersion }, - status: result.budget_exhausted ? 'warn' : 'ok', + status: result.budget_exhausted || result.deadline_hit ? 'warn' : 'ok', }; } } diff --git a/test/capture-build-content.test.ts b/test/capture-build-content.test.ts index 122222306..f71ae1e5c 100644 --- a/test/capture-build-content.test.ts +++ b/test/capture-build-content.test.ts @@ -190,9 +190,23 @@ describe('deriveTitle (no-frontmatter path)', () => { expect(deriveTitle('### Triple hash\nrest')).toBe('Triple hash'); }); - test('caps at 80 chars', () => { + test('caps at 80 chars with explicit ellipsis', () => { const long = 'a'.repeat(120); - expect(deriveTitle(long)).toBe('a'.repeat(80)); + expect(deriveTitle(long)).toBe('a'.repeat(79) + '…'); + expect([...deriveTitle(long)].length).toBe(80); + }); + + test('exactly 80 chars is not truncated', () => { + const exact = 'a'.repeat(80); + expect(deriveTitle(exact)).toBe(exact); + }); + + test('does not split astral Unicode while truncating', () => { + const title = deriveTitle('😀'.repeat(120)); + expect(title.endsWith('…')).toBe(true); + expect([...title].length).toBe(80); + // No lone surrogate halves left behind by the cut. + expect([...title].some((ch) => ch.length === 1 && /[\uD800-\uDFFF]/.test(ch))).toBe(false); }); test('falls back to Capture for empty input', () => { diff --git a/test/commands/capture.test.ts b/test/commands/capture.test.ts index 01553cf71..d7bcbf5bb 100644 --- a/test/commands/capture.test.ts +++ b/test/commands/capture.test.ts @@ -136,12 +136,13 @@ describe('capture — buildContent', () => { expect(parsed.data.title).toBe('Real first line'); }); - test('caps title at 80 chars', () => { + test('caps title at 80 chars with explicit ellipsis', () => { const longLine = 'x'.repeat(200); const result = __testing.buildContent(longLine, {}); const parsed = matter(result); expect(typeof parsed.data.title).toBe('string'); expect((parsed.data.title as string).length).toBeLessThanOrEqual(80); + expect((parsed.data.title as string).endsWith('…')).toBe(true); }); test('honors --source via captured_via', () => { diff --git a/test/connection-manager.serial.test.ts b/test/connection-manager.serial.test.ts index 598a0f332..b9848a44d 100644 --- a/test/connection-manager.serial.test.ts +++ b/test/connection-manager.serial.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test, beforeEach, afterEach } from 'bun:test'; import { isSupabasePoolerUrl, deriveDirectUrl, + normalizeDirectUrl, readKillSwitchEnv, isNetworkUnreachableError, resolveDirectPoolSize, @@ -79,6 +80,44 @@ describe('deriveDirectUrl', () => { }); }); +describe('normalizeDirectUrl', () => { + test('normalizes a transaction-pooler (6543) override to the real direct host', () => { + const direct = normalizeDirectUrl( + 'postgresql://postgres.abcxyz:p@aws-0-us-west-2.pooler.supabase.com:6543/postgres', + 'postgresql://postgres.abcxyz:p@aws-0-us-west-2.pooler.supabase.com:6543/postgres', + ); + expect(direct).toBe('postgresql://postgres:p@db.abcxyz.supabase.co:5432/postgres'); + }); + + test('keeps a non-pooler direct override', () => { + const direct = normalizeDirectUrl( + 'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db', + 'postgresql://u:p@custom-direct.example.com:5432/db', + ); + expect(direct).toBe('postgresql://u:p@custom-direct.example.com:5432/db'); + }); + + test('keeps a session-mode pooler override (pooler host, port 5432)', () => { + const sessionUrl = 'postgresql://postgres.abc:p@aws.pooler.supabase.com:5432/db'; + const direct = normalizeDirectUrl( + 'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db', + sessionUrl, + ); + expect(direct).toBe(sessionUrl); + }); + + test('no override: derives from the primary as before', () => { + const direct = normalizeDirectUrl( + 'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db', + ); + expect(direct).toContain('db.abc.supabase.co:5432'); + }); + + test('no override, non-Supabase primary: null', () => { + expect(normalizeDirectUrl('postgresql://u:p@localhost:5432/db')).toBeNull(); + }); +}); + describe('readKillSwitchEnv', () => { let original: string | undefined; beforeEach(() => { original = process.env.GBRAIN_DISABLE_DIRECT_POOL; }); @@ -146,13 +185,18 @@ describe('resolveDirectPoolSize', () => { describe('ConnectionManager — describeMode + dual-pool routing', () => { let originalKillSwitch: string | undefined; + let originalDirectUrl: string | undefined; beforeEach(() => { originalKillSwitch = process.env.GBRAIN_DISABLE_DIRECT_POOL; + originalDirectUrl = process.env.GBRAIN_DIRECT_DATABASE_URL; delete process.env.GBRAIN_DISABLE_DIRECT_POOL; + delete process.env.GBRAIN_DIRECT_DATABASE_URL; }); afterEach(() => { if (originalKillSwitch === undefined) delete process.env.GBRAIN_DISABLE_DIRECT_POOL; else process.env.GBRAIN_DISABLE_DIRECT_POOL = originalKillSwitch; + if (originalDirectUrl === undefined) delete process.env.GBRAIN_DIRECT_DATABASE_URL; + else process.env.GBRAIN_DIRECT_DATABASE_URL = originalDirectUrl; }); test('non-Supabase URL → single mode', () => { @@ -191,6 +235,25 @@ describe('ConnectionManager — describeMode + dual-pool routing', () => { expect(cm.resolveDirectUrl()).toContain('custom-direct.example.com'); }); + test('explicit transaction-pooler directUrl override is normalized to direct host', () => { + const cm = new ConnectionManager({ + url: 'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db', + directUrl: 'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db', + }); + expect(cm.resolveDirectUrl()).toContain('db.abc.supabase.co:5432'); + expect(cm.resolveDirectUrl()).not.toContain(':6543'); + }); + + test('env transaction-pooler directUrl override is normalized to direct host', () => { + process.env.GBRAIN_DIRECT_DATABASE_URL = + 'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db'; + const cm = new ConnectionManager({ + url: 'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db', + }); + expect(cm.resolveDirectUrl()).toContain('db.abc.supabase.co:5432'); + expect(cm.resolveDirectUrl()).not.toContain(':6543'); + }); + test('host string contains creds neither in describeMode nor resolveDirectUrl logging', () => { const cm = new ConnectionManager({ url: 'postgresql://postgres.abc:secret@aws.pooler.supabase.com:6543/db', diff --git a/test/doctor-extract-atoms-backlog.test.ts b/test/doctor-extract-atoms-backlog.test.ts index 83f502283..f37726d03 100644 --- a/test/doctor-extract-atoms-backlog.test.ts +++ b/test/doctor-extract-atoms-backlog.test.ts @@ -76,6 +76,17 @@ describe('countExtractAtomsBacklog (issue #1678)', () => { }); expect(await countExtractAtomsBacklog(engine)).toBe(0); }); + + it('ignores raw source-holder pages (permanent no-progress backlog otherwise)', async () => { + await engine.putPage('wiki/raw-email-source', { + type: 'source', + title: 'Raw email source', + compiled_truth: BODY, + frontmatter: { raw: 'raw/email/example.md' }, + }); + expect(await countExtractAtomsBacklog(engine)).toBe(0); + expect(await countExtractAtomsBacklog(engine, 'default')).toBe(0); + }); }); describe('computeExtractAtomsBacklogCheck (issue #1678)', () => { diff --git a/test/extract-atoms-page-discovery.test.ts b/test/extract-atoms-page-discovery.test.ts index 7289cc52e..d1b4fcefb 100644 --- a/test/extract-atoms-page-discovery.test.ts +++ b/test/extract-atoms-page-discovery.test.ts @@ -178,6 +178,18 @@ describe('v0.41.2.1: discoverExtractablePages SQL contract', () => { expect(discovered.map((d) => d.slug)).toEqual(['original/normal']); }); + test('raw source-holder pages excluded (#5 — no permanent no-progress backlog)', async () => { + await seedPage({ slug: 'source/normal', type: 'source' }); + await seedPage({ + slug: 'wiki/raw-email-source', + type: 'source', + frontmatter: { raw: 'raw/email/example.md' }, + }); + + const discovered = await discoverExtractablePages(engine, 'default'); + expect(discovered.map((d) => d.slug)).toEqual(['source/normal']); + }); + test('pages with NULL content_hash excluded (D9 #3 — no .slice crash)', async () => { await seedPage({ slug: 'meeting/with-hash', type: 'meeting' }); await seedPage({ slug: 'meeting/no-hash', type: 'meeting', content_hash: null }); diff --git a/test/integrations.test.ts b/test/integrations.test.ts index 2b4bc790c..0926d7c9d 100644 --- a/test/integrations.test.ts +++ b/test/integrations.test.ts @@ -263,6 +263,35 @@ describe('twilio-voice-brain recipe', () => { }); }); +describe('x-to-brain recipe', () => { + test('health check works with an app-only bearer token (#2343)', () => { + const { readFileSync } = require('fs'); + const content = readFileSync( + new URL('../recipes/x-to-brain.md', import.meta.url), + 'utf-8' + ); + const recipe = parseRecipe(content, 'x-to-brain.md'); + expect(recipe).not.toBeNull(); + const httpChecks = recipe!.frontmatter.health_checks + .filter((c: any) => typeof c === 'object' && c.type === 'http'); + expect(httpChecks.length).toBeGreaterThan(0); + const secretNames = new Set(recipe!.frontmatter.secrets.map((s: any) => s.name)); + for (const check of httpChecks as any[]) { + // /users/me requires user-context OAuth; the recipe only collects an + // app-only bearer token, so probing it always fails. + expect(check.url).not.toContain('/users/me'); + // Every $VAR the check expands must be declared in secrets, or the + // installer never prompts for it and the check fails for everyone. + const vars = [check.url, check.auth_token, check.auth_user, check.auth_pass] + .filter((v: unknown): v is string => typeof v === 'string') + .flatMap((v: string) => v.match(/\$[A-Z_][A-Z0-9_]*/g) ?? []) + .map((v: string) => v.slice(1)); + expect(vars.length).toBeGreaterThan(0); + for (const name of vars) expect(secretNames.has(name)).toBe(true); + } + }); +}); + // --- All recipes parse without error --- describe('all recipes', () => { diff --git a/test/propose-takes.test.ts b/test/propose-takes.test.ts index 3c0ccb68d..af5674a0d 100644 --- a/test/propose-takes.test.ts +++ b/test/propose-takes.test.ts @@ -368,6 +368,47 @@ New prose appended here.`; expect(extractorCalls).toBe(1); }); + test('phase deadline breaks the page loop with a partial result (deadline_hit)', async () => { + const pages = [ + buildPage({ slug: 'wiki/slow-a', body: 'page a' }), + buildPage({ slug: 'wiki/slow-b', body: 'page b' }), + ]; + const { engine, captured } = buildMockEngine({ pages }); + let extractorCalls = 0; + const extractor: ProposeTakesExtractor = async () => { + extractorCalls++; + await new Promise((r) => setTimeout(r, 10)); + return [{ claim_text: 'x', kind: 'take', holder: 'brain', weight: 0.5 }]; + }; + // 5ms deadline: page 1 processes (elapsed 0 at check), the 10ms extractor + // call pushes elapsed past the cap, page 2 is never scanned. + const result = await runPhaseProposeTakes(buildCtx(engine), { extractor, deadlineMs: 5 }); + + expect(result.status).toBe('warn'); + const details = result.details as Record; + expect(details.deadline_hit).toBe(true); + expect(details.pages_scanned).toBe(1); + expect(extractorCalls).toBe(1); + expect((details.warnings as string[]).some(w => w.includes('phase deadline hit'))).toBe(true); + + // Rollup records the deadline break as a halt, not a completed round + // (same posture as budget exhaustion). Params: $5 = halt, $8 = completed. + const rollup = captured.find((c) => c.sql.includes('extract_rollup_7d')); + expect(rollup).toBeDefined(); + expect(rollup!.params[4]).toBe(1); // halt_count delta + expect(rollup!.params[7]).toBe(0); // round_completed delta + }); + + test('default deadline does not fire on a fast run', async () => { + const pages = [buildPage({ slug: 'wiki/fast', body: 'quick page' })]; + const { engine } = buildMockEngine({ pages }); + const extractor: ProposeTakesExtractor = async () => []; + const result = await runPhaseProposeTakes(buildCtx(engine), { extractor }); + const details = result.details as Record; + expect(details.deadline_hit).toBeUndefined(); + expect(details.pages_scanned).toBe(1); + }); + test('proposal_run_id is stable across all proposals from one phase invocation', async () => { const pages = [ buildPage({ slug: 'wiki/a', body: 'page a' }),