mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
* v0.40.4.0 feat(sync): parallel sync --all + per-source lock invariant + sources status dashboard (productionized from PR #1314) Lands the community-authored PR #1314 with the structural fixes Codex's outside-voice review caught: the original PR's lock-id change only fired inside the --all parallel path, which would have introduced a worse race than the global-lock contention it fixed (sync --all on per-source lock racing against sync --source foo on the still-global lock). The landed version makes the per-source lock the invariant for every source-scoped sync, paired with withRefreshingLock for sources that exceed 30 minutes. What's new - gbrain sync --all parallel fan-out via continuous worker pool (D2); --parallel N flag, default min(sourceCount, --workers, 4); per-source [<source-id>] line prefix via AsyncLocalStorage (D6 + D12 + D13); stable --json envelope {schema_version:1, ...} on stdout with banners on stderr (D4 + D14); --skip-failed/--retry-failed reject under --parallel > 1 (D15 — sync-failures.jsonl is brain-global today; source-scoping filed as v0.40.4 TODO). - gbrain sources status [--json] read-only dashboard (D3 — sibling to sources list/add/remove/archive, not a sync flag, so reads + writes don't share a verb). Counts pages + chunks + embedding coverage per source. Active embedding column resolved via the registry (D16) so Voyage / multimodal brains see the right column. Archived sources excluded by caller filter. - Connection-budget stderr warning when parallel × workers × 2 > 16 with the formula in the message text (D1 + D10 — Codex P0 #3: each per-file worker opens its own PostgresEngine with poolSize=2, so the multiplication factor is 2, not 1). The load-bearing structural fix - performSync defaults to per-source lock id (gbrain-sync:<sourceId>) whenever opts.sourceId is set + wraps in withRefreshingLock. Legacy single-default-source brains keep the bare tryAcquireDbLock(SYNC_LOCK_ID) path for back-compat. - Dashboard SQL is the canonical content_chunks ch JOIN pages pg ON pg.id = ch.page_id WHERE pg.deleted_at IS NULL shape — the original PR shipped chunks ch JOIN ON page_slug, which would have crashed on PGLite parse and silently zeroed on Postgres via a swallow-catch. Errors from the dashboard SQL propagate (no silent zero-counts on real DB errors). Tests - New test/console-prefix.test.ts — 8 cases pinning ALS propagation, nested wraps, embedded-newline prefixing, back-compat fast path. - New test/sync-all-parallel.test.ts (replaces PR's stubbed tests) — 16 cases covering resolveParallelism, per-source lock format, buildSyncStatusReport SQL math + error propagation + envelope shape, connection-budget math, per-source prefix routing. - New test/e2e/sync-status-pglite.test.ts — IRON RULE regression: real PGLite seeds 2 sources × pages × chunks (mixed embedded/unembedded, 1 soft-deleted, 1 archived source). Validates SQL excludes both AND the active embedding column is the one used. This is the case that would have caught the PR's original broken SQL. Compatibility - No schema changes. No new dependencies. - Single-source / non-`--all` paths: bit-for-bit identical to v0.40.2. - PGLite users get serial behavior (single-connection engine). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> Co-Authored-By: garrytan-agents <garrytan-agents@users.noreply.github.com> * v0.40.6.0 — version bump for ship (skipping 0.40.4 + 0.40.5 for in-flight work) Reserves v0.40.4 + v0.40.5 slots for parallel waves (salem's graph-signals work and any other in-flight branches) and lands this PR's parallel-sync work at v0.40.6.0. No code change beyond the version triple and the TODOS / CLAUDE.md / CHANGELOG cross-references which were updated from "v0.40.4" to "v0.41+" to match the new follow-up version. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com> Co-authored-by: garrytan-agents <garrytan-agents@users.noreply.github.com>
210 lines
6.4 KiB
TypeScript
210 lines
6.4 KiB
TypeScript
/**
|
|
* llms-config — single source of truth for llms.txt + llms-full.txt.
|
|
*
|
|
* Consumed by scripts/build-llms.ts (emits llms.txt, llms-full.txt) and
|
|
* test/build-llms.test.ts (asserts paths resolve, content contract holds).
|
|
*
|
|
* Adding a doc? Add it here and run `bun run build:llms`. The drift-detection
|
|
* test fails CI if you forget.
|
|
*
|
|
* Fork-friendliness: `rawBaseUrl` reads from `LLMS_REPO_BASE` so forks can
|
|
* regenerate without manual URL rewrites:
|
|
* LLMS_REPO_BASE=https://raw.githubusercontent.com/fork-org/gbrain/main bun run build:llms
|
|
*/
|
|
|
|
export type DocEntry = {
|
|
title: string;
|
|
description: string;
|
|
path: string;
|
|
includeInFull?: boolean;
|
|
};
|
|
|
|
export type DocSection = {
|
|
heading: string;
|
|
optional?: boolean;
|
|
entries: DocEntry[];
|
|
};
|
|
|
|
export const PROJECT = {
|
|
name: "GBrain",
|
|
summary:
|
|
"GBrain is a personal knowledge brain and GStack mod for agent platforms. Pluggable engines (PGLite default, Postgres+pgvector for scale), contract-first operations, 26 fat-markdown skills. Teaches agents brain ops, ingestion, enrichment, scheduling, identity, and access control.",
|
|
repoUrl: "https://github.com/garrytan/gbrain",
|
|
rawBaseUrl:
|
|
process.env.LLMS_REPO_BASE ??
|
|
"https://raw.githubusercontent.com/garrytan/gbrain/master",
|
|
};
|
|
|
|
export const SECTIONS: DocSection[] = [
|
|
{
|
|
heading: "Core entry points",
|
|
entries: [
|
|
{
|
|
title: "AGENTS.md",
|
|
description:
|
|
"Start here if you are not Claude Code. Install order, trust boundary, skill resolver, config/debug/migration pointers.",
|
|
path: "AGENTS.md",
|
|
},
|
|
{
|
|
title: "CLAUDE.md",
|
|
description:
|
|
"Architecture reference. Key files, trust boundaries, engine factory, test layout.",
|
|
path: "CLAUDE.md",
|
|
},
|
|
{
|
|
title: "INSTALL_FOR_AGENTS.md",
|
|
description: "9-step agent installation.",
|
|
path: "INSTALL_FOR_AGENTS.md",
|
|
},
|
|
{
|
|
title: "skills/RESOLVER.md",
|
|
description: "Skill dispatcher. Read first for any task.",
|
|
path: "skills/RESOLVER.md",
|
|
},
|
|
{
|
|
title: "README.md",
|
|
description: "Project overview, benchmarks, 30-minute setup.",
|
|
path: "README.md",
|
|
},
|
|
],
|
|
},
|
|
{
|
|
heading: "Configuration",
|
|
entries: [
|
|
{
|
|
title: "docs/ENGINES.md",
|
|
description: "PGLite vs Postgres trade-off and when to migrate.",
|
|
path: "docs/ENGINES.md",
|
|
},
|
|
{
|
|
title: "docs/GBRAIN_RECOMMENDED_SCHEMA.md",
|
|
description:
|
|
"MECE directory structure (people/, companies/, concepts/).",
|
|
path: "docs/GBRAIN_RECOMMENDED_SCHEMA.md",
|
|
// v0.40.6.0: 64KB reference doc. Web index entry stays; the single-fetch
|
|
// bundle gets the README + setup guides instead. Keeps llms-full.txt
|
|
// under the 600KB budget as CLAUDE.md grows with each release.
|
|
includeInFull: false,
|
|
},
|
|
{
|
|
title: "docs/guides/live-sync.md",
|
|
description: "Incremental markdown sync setup.",
|
|
path: "docs/guides/live-sync.md",
|
|
},
|
|
{
|
|
title: "docs/guides/cron-schedule.md",
|
|
description: "Recurring job scheduling.",
|
|
path: "docs/guides/cron-schedule.md",
|
|
},
|
|
{
|
|
title: "docs/guides/minions-deployment.md",
|
|
description:
|
|
"Deploying the gbrain jobs worker: crontab + watchdog, inline --follow, systemd/Procfile/fly.toml, upgrade checklist.",
|
|
path: "docs/guides/minions-deployment.md",
|
|
},
|
|
{
|
|
title: "docs/guides/quiet-hours.md",
|
|
description: "Notification hold + timezone-aware delivery.",
|
|
path: "docs/guides/quiet-hours.md",
|
|
},
|
|
{
|
|
title: "docs/mcp/DEPLOY.md",
|
|
description: "MCP server deployment.",
|
|
path: "docs/mcp/DEPLOY.md",
|
|
},
|
|
],
|
|
},
|
|
{
|
|
heading: "Debugging",
|
|
entries: [
|
|
{
|
|
title: "docs/GBRAIN_VERIFY.md",
|
|
description:
|
|
"7-check post-setup verification. Start here when something feels off.",
|
|
path: "docs/GBRAIN_VERIFY.md",
|
|
},
|
|
{
|
|
title: "docs/guides/minions-fix.md",
|
|
description: "Troubleshooting the Minions job queue.",
|
|
path: "docs/guides/minions-fix.md",
|
|
},
|
|
{
|
|
title: "docs/integrations/reliability-repair.md",
|
|
description: "Data integrity recovery.",
|
|
path: "docs/integrations/reliability-repair.md",
|
|
},
|
|
],
|
|
},
|
|
{
|
|
heading: "Migrations",
|
|
entries: [
|
|
{
|
|
title: "docs/UPGRADING_DOWNSTREAM_AGENTS.md",
|
|
description:
|
|
"Patches for downstream agent skill forks. One section per release.",
|
|
path: "docs/UPGRADING_DOWNSTREAM_AGENTS.md",
|
|
},
|
|
{
|
|
title: "skills/migrations/",
|
|
description:
|
|
"Per-version (v0.5.0 - v0.14.1) agent-executable migration instructions.",
|
|
path: "skills/migrations/",
|
|
},
|
|
{
|
|
title: "CHANGELOG.md",
|
|
description:
|
|
"Release-summary voice + itemized changes + self-repair block per version.",
|
|
path: "CHANGELOG.md",
|
|
includeInFull: false,
|
|
},
|
|
],
|
|
},
|
|
{
|
|
heading: "Philosophy",
|
|
optional: true,
|
|
entries: [
|
|
{
|
|
title: "docs/ethos/THIN_HARNESS_FAT_SKILLS.md",
|
|
description: "Why skills live in markdown.",
|
|
path: "docs/ethos/THIN_HARNESS_FAT_SKILLS.md",
|
|
includeInFull: false,
|
|
},
|
|
{
|
|
title: "docs/ethos/MARKDOWN_SKILLS_AS_RECIPES.md",
|
|
description: "Homebrew for Personal AI.",
|
|
path: "docs/ethos/MARKDOWN_SKILLS_AS_RECIPES.md",
|
|
includeInFull: false,
|
|
},
|
|
],
|
|
},
|
|
{
|
|
heading: "Optional",
|
|
optional: true,
|
|
entries: [
|
|
{
|
|
title: "docs/designs/",
|
|
description: "Forward-looking designs.",
|
|
path: "docs/designs/",
|
|
includeInFull: false,
|
|
},
|
|
{
|
|
title: "docs/architecture/infra-layer.md",
|
|
description: "Shared infra patterns.",
|
|
path: "docs/architecture/infra-layer.md",
|
|
includeInFull: false,
|
|
},
|
|
],
|
|
},
|
|
];
|
|
|
|
export const INLINE_TIPS = [
|
|
"`gbrain doctor [--json] [--fast] [--fix]` - built-in health checks.",
|
|
"`gbrain orphans [--json]` - pages with zero inbound wikilinks.",
|
|
"`gbrain repair-jsonb [--dry-run]` - repair v0.12.0 double-encoded JSONB rows.",
|
|
"`gbrain upgrade` runs post-upgrade + apply-migrations.",
|
|
];
|
|
|
|
// Target ~600KB so llms-full.txt fits in ~150k-token contexts with room to spare.
|
|
// Generator prints a WARN if exceeded; ship with includeInFull=false exclusions.
|
|
export const FULL_SIZE_BUDGET = 600_000;
|