Files
gbrain/test/cycle-extract-source.test.ts
T
7ffac65c62 fix(extract,ingest,cycle): source-provenance wave — thread source identity through ingest_capture, fs-walk links, and cycle extract (#1522 #1747 #1503) (#2920)
* fix(extract,ingest,cycle): source-provenance wave — thread source identity through ingest_capture, fs-walk links, and cycle extract (#1522 #1747 #1503)

Three fixes in the same invariant class (source identity silently dropped
on the write path, collapsing to the 'default' source):

- #1522: the ingest_capture Minion handler validated IngestionEvent
  provenance (source_id/source_kind/source_uri) then dropped it on the
  importFromContent call. Now threads source_kind/source_uri +
  ingested_via='ingest_capture' into the page write, and routes the write
  under event.source_id when it names a registered source AND the event
  is trusted (fail-closed: untrusted webhook payloads carry a
  caller-controlled x-gbrain-source-id header and must not choose their
  write source; unregistered emitter ids keep default routing so the
  webhook path can't FK-fail).

- #1747: the fs-walk extractors (extractLinksFromDir /
  extractTimelineFromDir / extractForSlugs) built batch rows with no
  source_id, so addLinksBatch/addTimelineEntriesBatch mapped missing →
  'default' and the pages JOIN dropped every row on a non-default source
  ("Links: created 0 from N pages", no error). ExtractOpts gains
  sourceId; the CLI fs path resolves it via resolveSourceId
  (--source-id > env > dotfile > registered path > sole-non-default)
  and rows are stamped from/to/origin_source_id + timeline source_id.

- #1503: the cycle's extract phase (runPhaseExtract) never passed a
  sourceId, so federated-brain dream/autopilot cycles persisted nothing
  every night. It now threads cycleSourceId (explicit --source or
  resolveSourceForDir(brainDir) — the same seam runPhaseSync uses) into
  runExtractCore for both the incremental and full-walk paths.

Regression tests: handler provenance write-through (registered/
unregistered/untrusted routing), fs-walk + CLI resolution + incremental
cycle route landing edges/timeline in the right source (all red on
unfixed master), and a negative control pinning the pre-fix JOIN-drop
shape.

Reuses the ExtractOpts.sourceId threading approach from PR #1719,
rebased onto the current signal-aware signatures and extended to the
cycle + timeline paths.

Fixes #1522
Fixes #1747
Fixes #1503

Co-authored-by: seungsu <kss530c@gmail.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* test: update cycle source pin for cycleSourceId threading

The #1972 source-pin test asserts the literal runPhaseExtract call site in
cycle.ts. The #1503 fix appended cycleSourceId after opts.signal; signal
threading is unchanged (still the 5th arg, forwarded to runExtractCore).
Pin updated to the new literal — invariant intact.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Sinabina <sinabina@Sinabinas-MacBook-Pro-4.local>
Co-authored-by: seungsu <kss530c@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-07-17 11:36:43 -07:00

115 lines
4.5 KiB
TypeScript

/**
* #1503 — the cycle's extract phase threads the resolved per-source id.
*
* Pre-fix, runPhaseExtract called runExtractCore with no sourceId, so on a
* federated brain (content in a non-'default' source) the fs-walk batch rows
* mapped to source_id='default', the pages JOIN dropped every row, and every
* dream/autopilot cycle logged "Links: created 0 from N pages" while
* persisting nothing. This pins that the cycle resolves the source from the
* brain dir (resolveSourceForDir — same seam runPhaseSync uses) and that the
* extracted link/timeline rows land in that source.
*
* GBRAIN_HOME is isolated per test because the PGLite cycle path takes a
* file lock at ~/.gbrain/cycle.lock (see cycle-last-full-cycle-at.test.ts).
*/
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test';
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { withEnv } from './helpers/with-env.ts';
import { runCycle } from '../src/core/cycle.ts';
let engine: PGLiteEngine;
let brainDir: string;
let gbrainHome: string;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
}, 30_000);
afterAll(async () => {
await engine.disconnect();
}, 30_000);
beforeEach(async () => {
await resetPgliteState(engine);
brainDir = mkdtempSync(join(tmpdir(), 'gbrain-cycle-extract-src-'));
gbrainHome = mkdtempSync(join(tmpdir(), 'gbrain-cycle-extract-src-home-'));
mkdirSync(join(brainDir, 'people'), { recursive: true });
writeFileSync(
join(brainDir, 'people', 'alice.md'),
'# Alice\n\nMet [[people/bob]] today.\n\n## Timeline\n\n- **2026-01-05** | meeting — Discussed the wiki\n',
);
writeFileSync(join(brainDir, 'people', 'bob.md'), '# Bob\n\nFriend of [[people/alice]].\n');
// Federated shape: the brain checkout is a registered non-default source
// and its pages live ONLY under that source.
await engine.executeRaw(
`INSERT INTO sources (id, name, local_path) VALUES ('wiki', 'wiki', $1)
ON CONFLICT (id) DO UPDATE SET local_path = EXCLUDED.local_path`,
[brainDir],
);
await engine.executeRaw(
`INSERT INTO pages (slug, source_id, type, title, compiled_truth, timeline)
VALUES
('people/alice', 'wiki', 'person', 'Alice', '', ''),
('people/bob', 'wiki', 'person', 'Bob', '', '')`,
);
});
afterEach(() => {
rmSync(brainDir, { recursive: true, force: true });
rmSync(gbrainHome, { recursive: true, force: true });
});
describe('cycle extract phase on a federated brain (#1503)', () => {
test('extract phase resolves the source from brainDir and writes links + timeline there', async () => {
await withEnv({ GBRAIN_HOME: gbrainHome }, async () => {
const report = await runCycle(engine, {
brainDir,
phases: ['extract'],
});
const extractPhase = report.phases.find(p => p.phase === 'extract');
expect(extractPhase?.status).toBe('ok');
// The #1503 symptom was exactly linksCreated: 0 on federated brains.
expect(Number(extractPhase?.details?.linksCreated ?? 0)).toBeGreaterThanOrEqual(2);
expect(Number(extractPhase?.details?.timelineCreated ?? 0)).toBeGreaterThanOrEqual(1);
});
const links = await engine.executeRaw<{ from_src: string; to_src: string }>(
`SELECT pf.source_id AS from_src, pt.source_id AS to_src
FROM links l
JOIN pages pf ON pf.id = l.from_page_id
JOIN pages pt ON pt.id = l.to_page_id`,
);
expect(links.length).toBeGreaterThanOrEqual(2);
for (const l of links) {
expect(l.from_src).toBe('wiki');
expect(l.to_src).toBe('wiki');
}
const tl = await engine.executeRaw<{ n: string }>(
`SELECT COUNT(*)::text AS n FROM timeline_entries t
JOIN pages p ON p.id = t.page_id AND p.source_id = 'wiki'`,
);
expect(Number(tl[0]?.n ?? 0)).toBeGreaterThanOrEqual(1);
});
test('explicit opts.sourceId wins (checkout-less --source shape)', async () => {
await withEnv({ GBRAIN_HOME: gbrainHome }, async () => {
const report = await runCycle(engine, {
brainDir,
sourceId: 'wiki',
phases: ['extract'],
});
const extractPhase = report.phases.find(p => p.phase === 'extract');
expect(extractPhase?.status).toBe('ok');
expect(Number(extractPhase?.details?.linksCreated ?? 0)).toBeGreaterThanOrEqual(2);
});
});
});