Files
gbrain/test/sources-load.test.ts
T
df86ea5f1d v0.40.5.0 Federated Sync v2 — parallel source sync + push triggers + per-source health (#1322)
* wip: federated sync v2 pre-merge snapshot

* v0.40.5.0 Federated Sync v2 — parallel source sync + push triggers + per-source health

Bump VERSION + package.json + CHANGELOG header + migration walkthrough filename
to v0.40.5.0 (claiming the next free slot in the v0.40.x patch series after
master's v0.40.1.0).

What ships (6 components, all behind sync.federated_v2 feature flag default-on):
1. Per-source sync lock — syncLockId(sourceId), phantom-redirect parity
2. Parallel sync --all — pMapAllSettled fan-out, --max-sources N cap
3. embed-backfill minion handler — D2 per-source lock + D6 $10/job budget + D15.1
   fire-and-forget submission + D19 source-level cooldown + 24h $25 rolling cap
4. sync trigger CLI + POST /webhooks/github — HMAC-verified (60 req/min/IP),
   X-GitHub-Event=push + ref filter against tracked_branch
5. sources status + federation_health doctor — batched GROUP BY pipeline
   (4 queries instead of 6×N per-source roundtrips)
6. sources federate/unfederate hook — auto-submit embed-backfill on flip

Correctness fixes (unconditional):
- D21: sync.ts:959 facts backstop now passes sourceId to engine.getPage
- D15.4: redactSourceConfig + CI guard prevent webhook_secret leak
- D15.5: safeHexEqual extracted to src/core/timing-safe.ts

Schema:
- Migration v89 (sources_github_repo_index): partial expression index on
  config->>'github_repo' for fast webhook source-lookup

Tests:
- 14 new test files, 112 cases. 4 IRON-RULE regressions pinned (SYNC_LOCK_ID
  back-compat, phantom per-source lock, embed-backfill kill+resume,
  webhook HMAC prefix-strip). All 9449 unit tests pass.

Caught at test-write time: the webhook handler had a Buffer.from('sha256=...',
'hex') truncation bug — without the prefix-strip, every signature would have
"matched" empty buffers. Pinned by a test/sources-webhook.test.ts IRON-RULE.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* fix(check-source-config-leak): tighten regex to source-row patterns only

The v0.40.5.0 wave added scripts/check-source-config-leak.sh with a
too-broad pattern (JSON\.stringify\(.*config) that flagged any variable
named 'config' — catching the GLOBAL gbrain config.json serializers in
src/commands/init.ts (status envelopes) and src/core/config.ts (the
config-file write site). On the CI runner without rg installed, the
grep -rE fallback fired correctly and produced 4 false positives that
broke the `verify` script.

Tightened the patterns to specifically match `(source|src|row|s).config`
property access — the actual risk shape (a sources-table row being
serialized whole). The global gbrain config has a different shape and
threat model (file-mode 0o600 at the write site), so it's safe to
exempt at the regex level rather than per-file whitelist.

Also fixed a latent bug: the rg branch used `--include='*.ts'` (grep's
flag, not rg's). rg silently rejected it and CANDIDATES came back empty,
so the local-dev runs (which have rg) would never have caught a real
leak. Now branches on tool availability: `-g '*.ts'` for rg, `--include`
for grep -rE. Both branches verified against a synthetic leak fixture.

Also added init.ts + config.ts to the whitelist as a belt-and-suspenders
since they handle gbrain-global config (not source rows) and could
otherwise reflect-back via regex iteration.

CI: `bun run verify` exit 0 locally with both the original false-positive
fixture (clean repo) and a synthetic leak fixture (correctly caught,
exit 1).

---------

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-23 10:21:59 -07:00

135 lines
4.7 KiB
TypeScript

/**
* Tests for src/core/sources-load.ts (v0.40 D7).
*
* Validates the shared loader: ordering, filtering, config parsing, and
* defensive fallback when legacy brains lack the `archived` column.
*/
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import {
loadAllSources,
fetchSource,
parseSourceConfig,
isSourceFederated,
} from '../src/core/sources-load.ts';
let engine: PGLiteEngine;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
}, 30000);
afterAll(async () => {
await engine.disconnect();
});
beforeEach(async () => {
await resetPgliteState(engine);
});
async function insertSource(id: string, opts: { federated?: boolean; archived?: boolean; config?: Record<string, unknown> } = {}): Promise<void> {
const config = { ...opts.config, federated: opts.federated ?? false };
await engine.executeRaw(
`INSERT INTO sources (id, name, config, archived)
VALUES ($1, $1, $2::jsonb, $3)
ON CONFLICT (id) DO UPDATE SET config = EXCLUDED.config, archived = EXCLUDED.archived`,
[id, JSON.stringify(config), opts.archived ?? false],
);
}
describe('loadAllSources', () => {
test('returns default source first, then alphabetical', async () => {
await insertSource('zebra');
await insertSource('alpha');
const rows = await loadAllSources(engine);
expect(rows.map((r) => r.id)).toEqual(['default', 'alpha', 'zebra']);
});
test('excludes archived rows by default', async () => {
await insertSource('keep');
await insertSource('archived-one', { archived: true });
const rows = await loadAllSources(engine);
expect(rows.map((r) => r.id).sort()).toEqual(['default', 'keep']);
});
test('includeArchived: true returns archived rows', async () => {
await insertSource('keep');
await insertSource('archived-one', { archived: true });
const rows = await loadAllSources(engine, { includeArchived: true });
expect(rows.map((r) => r.id).sort()).toEqual(['archived-one', 'default', 'keep']);
});
test('federatedOnly: filters to config.federated=true', async () => {
await insertSource('iso', { federated: false });
await insertSource('fed-a', { federated: true });
await insertSource('fed-b', { federated: true });
const rows = await loadAllSources(engine, { federatedOnly: true });
// default source seeded by resetPgliteState has federated: true
expect(rows.map((r) => r.id).sort()).toEqual(['default', 'fed-a', 'fed-b']);
});
test('rows have the full SourceRow projection (no surprise drift)', async () => {
await insertSource('shapecheck', { federated: true, config: { tracked_branch: 'main' } });
const rows = await loadAllSources(engine);
const target = rows.find((r) => r.id === 'shapecheck');
expect(target).toBeDefined();
expect(target).toMatchObject({
id: 'shapecheck',
name: 'shapecheck',
local_path: null,
last_commit: null,
last_sync_at: null,
});
expect(target!.config).toBeDefined();
expect(target!.created_at).toBeDefined();
});
});
describe('fetchSource', () => {
test('returns row for known id', async () => {
await insertSource('mybrain', { federated: true });
const row = await fetchSource(engine, 'mybrain');
expect(row?.id).toBe('mybrain');
expect(isSourceFederated(row!.config)).toBe(true);
});
test('returns null for unknown id', async () => {
const row = await fetchSource(engine, 'does-not-exist');
expect(row).toBeNull();
});
});
describe('parseSourceConfig', () => {
test('passes through plain object', () => {
expect(parseSourceConfig({ federated: true })).toEqual({ federated: true });
});
test('parses JSON string', () => {
expect(parseSourceConfig('{"federated":true}')).toEqual({ federated: true });
});
test('returns empty object on null / undefined / non-object', () => {
expect(parseSourceConfig(null)).toEqual({});
expect(parseSourceConfig(undefined)).toEqual({});
expect(parseSourceConfig(42)).toEqual({});
});
test('returns empty object on malformed JSON string', () => {
expect(parseSourceConfig('{')).toEqual({});
});
});
describe('isSourceFederated', () => {
test('strict-true requirement', () => {
expect(isSourceFederated({ federated: true })).toBe(true);
expect(isSourceFederated({ federated: false })).toBe(false);
expect(isSourceFederated({ federated: 'true' })).toBe(false); // strict boolean
expect(isSourceFederated({ federated: 1 })).toBe(false);
expect(isSourceFederated({})).toBe(false);
expect(isSourceFederated(null)).toBe(false);
});
});