mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-28 14:59:47 +00:00
* fix(extract,ingest,cycle): source-provenance wave — thread source identity through ingest_capture, fs-walk links, and cycle extract (#1522 #1747 #1503) Three fixes in the same invariant class (source identity silently dropped on the write path, collapsing to the 'default' source): - #1522: the ingest_capture Minion handler validated IngestionEvent provenance (source_id/source_kind/source_uri) then dropped it on the importFromContent call. Now threads source_kind/source_uri + ingested_via='ingest_capture' into the page write, and routes the write under event.source_id when it names a registered source AND the event is trusted (fail-closed: untrusted webhook payloads carry a caller-controlled x-gbrain-source-id header and must not choose their write source; unregistered emitter ids keep default routing so the webhook path can't FK-fail). - #1747: the fs-walk extractors (extractLinksFromDir / extractTimelineFromDir / extractForSlugs) built batch rows with no source_id, so addLinksBatch/addTimelineEntriesBatch mapped missing → 'default' and the pages JOIN dropped every row on a non-default source ("Links: created 0 from N pages", no error). ExtractOpts gains sourceId; the CLI fs path resolves it via resolveSourceId (--source-id > env > dotfile > registered path > sole-non-default) and rows are stamped from/to/origin_source_id + timeline source_id. - #1503: the cycle's extract phase (runPhaseExtract) never passed a sourceId, so federated-brain dream/autopilot cycles persisted nothing every night. It now threads cycleSourceId (explicit --source or resolveSourceForDir(brainDir) — the same seam runPhaseSync uses) into runExtractCore for both the incremental and full-walk paths. Regression tests: handler provenance write-through (registered/ unregistered/untrusted routing), fs-walk + CLI resolution + incremental cycle route landing edges/timeline in the right source (all red on unfixed master), and a negative control pinning the pre-fix JOIN-drop shape. Reuses the ExtractOpts.sourceId threading approach from PR #1719, rebased onto the current signal-aware signatures and extended to the cycle + timeline paths. Fixes #1522 Fixes #1747 Fixes #1503 Co-authored-by: seungsu <kss530c@gmail.com> Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * test: update cycle source pin for cycleSourceId threading The #1972 source-pin test asserts the literal runPhaseExtract call site in cycle.ts. The #1503 fix appended cycleSourceId after opts.signal; signal threading is unchanged (still the 5th arg, forwarded to runExtractCore). Pin updated to the new literal — invariant intact. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Sinabina <sinabina@Sinabinas-MacBook-Pro-4.local> Co-authored-by: seungsu <kss530c@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
115 lines
4.5 KiB
TypeScript
115 lines
4.5 KiB
TypeScript
/**
|
|
* #1503 — the cycle's extract phase threads the resolved per-source id.
|
|
*
|
|
* Pre-fix, runPhaseExtract called runExtractCore with no sourceId, so on a
|
|
* federated brain (content in a non-'default' source) the fs-walk batch rows
|
|
* mapped to source_id='default', the pages JOIN dropped every row, and every
|
|
* dream/autopilot cycle logged "Links: created 0 from N pages" while
|
|
* persisting nothing. This pins that the cycle resolves the source from the
|
|
* brain dir (resolveSourceForDir — same seam runPhaseSync uses) and that the
|
|
* extracted link/timeline rows land in that source.
|
|
*
|
|
* GBRAIN_HOME is isolated per test because the PGLite cycle path takes a
|
|
* file lock at ~/.gbrain/cycle.lock (see cycle-last-full-cycle-at.test.ts).
|
|
*/
|
|
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test';
|
|
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'fs';
|
|
import { tmpdir } from 'os';
|
|
import { join } from 'path';
|
|
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
|
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
|
import { withEnv } from './helpers/with-env.ts';
|
|
import { runCycle } from '../src/core/cycle.ts';
|
|
|
|
let engine: PGLiteEngine;
|
|
let brainDir: string;
|
|
let gbrainHome: string;
|
|
|
|
beforeAll(async () => {
|
|
engine = new PGLiteEngine();
|
|
await engine.connect({});
|
|
await engine.initSchema();
|
|
}, 30_000);
|
|
|
|
afterAll(async () => {
|
|
await engine.disconnect();
|
|
}, 30_000);
|
|
|
|
beforeEach(async () => {
|
|
await resetPgliteState(engine);
|
|
brainDir = mkdtempSync(join(tmpdir(), 'gbrain-cycle-extract-src-'));
|
|
gbrainHome = mkdtempSync(join(tmpdir(), 'gbrain-cycle-extract-src-home-'));
|
|
mkdirSync(join(brainDir, 'people'), { recursive: true });
|
|
writeFileSync(
|
|
join(brainDir, 'people', 'alice.md'),
|
|
'# Alice\n\nMet [[people/bob]] today.\n\n## Timeline\n\n- **2026-01-05** | meeting — Discussed the wiki\n',
|
|
);
|
|
writeFileSync(join(brainDir, 'people', 'bob.md'), '# Bob\n\nFriend of [[people/alice]].\n');
|
|
|
|
// Federated shape: the brain checkout is a registered non-default source
|
|
// and its pages live ONLY under that source.
|
|
await engine.executeRaw(
|
|
`INSERT INTO sources (id, name, local_path) VALUES ('wiki', 'wiki', $1)
|
|
ON CONFLICT (id) DO UPDATE SET local_path = EXCLUDED.local_path`,
|
|
[brainDir],
|
|
);
|
|
await engine.executeRaw(
|
|
`INSERT INTO pages (slug, source_id, type, title, compiled_truth, timeline)
|
|
VALUES
|
|
('people/alice', 'wiki', 'person', 'Alice', '', ''),
|
|
('people/bob', 'wiki', 'person', 'Bob', '', '')`,
|
|
);
|
|
});
|
|
|
|
afterEach(() => {
|
|
rmSync(brainDir, { recursive: true, force: true });
|
|
rmSync(gbrainHome, { recursive: true, force: true });
|
|
});
|
|
|
|
describe('cycle extract phase on a federated brain (#1503)', () => {
|
|
test('extract phase resolves the source from brainDir and writes links + timeline there', async () => {
|
|
await withEnv({ GBRAIN_HOME: gbrainHome }, async () => {
|
|
const report = await runCycle(engine, {
|
|
brainDir,
|
|
phases: ['extract'],
|
|
});
|
|
const extractPhase = report.phases.find(p => p.phase === 'extract');
|
|
expect(extractPhase?.status).toBe('ok');
|
|
// The #1503 symptom was exactly linksCreated: 0 on federated brains.
|
|
expect(Number(extractPhase?.details?.linksCreated ?? 0)).toBeGreaterThanOrEqual(2);
|
|
expect(Number(extractPhase?.details?.timelineCreated ?? 0)).toBeGreaterThanOrEqual(1);
|
|
});
|
|
|
|
const links = await engine.executeRaw<{ from_src: string; to_src: string }>(
|
|
`SELECT pf.source_id AS from_src, pt.source_id AS to_src
|
|
FROM links l
|
|
JOIN pages pf ON pf.id = l.from_page_id
|
|
JOIN pages pt ON pt.id = l.to_page_id`,
|
|
);
|
|
expect(links.length).toBeGreaterThanOrEqual(2);
|
|
for (const l of links) {
|
|
expect(l.from_src).toBe('wiki');
|
|
expect(l.to_src).toBe('wiki');
|
|
}
|
|
|
|
const tl = await engine.executeRaw<{ n: string }>(
|
|
`SELECT COUNT(*)::text AS n FROM timeline_entries t
|
|
JOIN pages p ON p.id = t.page_id AND p.source_id = 'wiki'`,
|
|
);
|
|
expect(Number(tl[0]?.n ?? 0)).toBeGreaterThanOrEqual(1);
|
|
});
|
|
|
|
test('explicit opts.sourceId wins (checkout-less --source shape)', async () => {
|
|
await withEnv({ GBRAIN_HOME: gbrainHome }, async () => {
|
|
const report = await runCycle(engine, {
|
|
brainDir,
|
|
sourceId: 'wiki',
|
|
phases: ['extract'],
|
|
});
|
|
const extractPhase = report.phases.find(p => p.phase === 'extract');
|
|
expect(extractPhase?.status).toBe('ok');
|
|
expect(Number(extractPhase?.details?.linksCreated ?? 0)).toBeGreaterThanOrEqual(2);
|
|
});
|
|
});
|
|
});
|