mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
* refactor: extract importFile from import.ts + add tag reconciliation Shared single-file import function used by both import and sync. Adds tag reconciliation (removes stale tags on reimport), >1MB file skip, and import->sync checkpoint continuity (writes git HEAD to config table after import so sync picks up seamlessly). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * feat: add sync pure functions, updateSlug engine method, and sync tests - buildSyncManifest: parses git diff --name-status -M output - isSyncable: filters to .md pages, excludes hidden/ops/.raw/skip-list - pathToSlug: converts file paths to page slugs with optional prefix - updateSlug: renames page slug in-place (preserves page_id, chunks, embeddings) - rewriteLinks: stub for v0.2 (FKs use page_id, already correct) - 20 new tests, all passing (39 total across 3 files) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * feat: add gbrain sync command with CLI, MCP, and watch mode 18-step sync protocol: read config, git pull, ancestry validation, git diff --name-status -M for net changes, isSyncable filter, process deletes/renames/adds/modifies via importFile, batch optimization, sync state checkpoint in Postgres config table. Watch mode with polling and consecutive error counter. MCP sync_brain tool returns structured SyncResult. Stale page deletion for un-syncable files. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * feat: add files table, gbrain files commands, and config show redaction - files table: page_slug FK with ON DELETE SET NULL + ON UPDATE CASCADE, storage_path, storage_url, mime_type, content_hash for dedup - gbrain files list/upload/sync/verify commands for Supabase Storage - gbrain config show redacts postgresql:// passwords and secret keys - CLI help updated with FILES section Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * feat: add install skill for GBrain onboarding 6-phase install workflow: environment discovery, Supabase setup (magic path via CLI OAuth or fallback 2-copy-paste), init + import, ongoing sync cron, optional file migration with mandatory verification, and agent teaching (AGENTS.md rules). Every error gets what + why + fix. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * docs: update project documentation for v0.2.0 Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * docs: add v0.2 features to README (sync, files, install skill) README.md: added sync command to IMPORT/EXPORT section, added FILES section with 4 commands, added files table to schema diagram, added install skill to skills table, updated MCP tools count from 20 to 21 (sync_brain added). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: OpenClaw DX improvements (skill count, upgrade docs, config show help) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * refactor: consolidate version to single source of truth Create src/version.ts that reads from package.json via static import (safe for bun compiled binaries). Update mcp/server.ts from hardcoded '0.1.0' to use shared VERSION. Bump skills/manifest.json to 0.2.0. * fix: upgrade detection order, npm→bun naming, clawhub false positives Reorder detection: node_modules first, binary second, clawhub last. Rename 'npm' install method to 'bun'. Use 'clawhub --version' instead of 'which clawhub' to avoid false positives from dangling symlinks. Add 120s timeout to execSync calls to prevent hanging. Add --help flag. * feat: per-command --help, unknown command check before DB connection Add COMMAND_HELP map covering all 28 commands. Check --help before init/upgrade dispatch and before connectEngine() so help works without a database. Use COMMAND_HELP keys as known-command set to catch unknown commands before wasting a DB round-trip. * docs: standardize npm references to bun, add Upgrade section to README Fix init.ts: npx→bunx, npm→bun for supabase CLI guidance. Fix README: npm install→bun add for standalone CLI install. Add ## Upgrade section to README with all three install methods. Update install skill Upgrading section to list bun, ClawHub, and binary. * test: full coverage audit — CLI dispatch, upgrade detection, config, edge cases New test files: - test/cli.test.ts: COMMAND_HELP ↔ switch consistency, version from package.json, per-command --help, unknown command handling, global help - test/upgrade.test.ts: detection order verification, npm→bun naming, clawhub --version (not which), timeout presence - test/config.test.ts: redactUrl for postgresql URLs, edge cases Extended existing tests: - test/sync.test.ts: empty string pathToSlug, uppercase .MD rejection, deeply nested files, multiple renames, unknown status codes - test/markdown.test.ts: multiple --- separators, missing frontmatter, no frontmatter at all, empty string, type inference from paths Tests: 39 → 83 (+44 new). All pass. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * test: 100% coverage — import-file mock engine, files utils, chunker edge cases New test files: - test/import-file.test.ts (9 tests): mock BrainEngine to test importFile without DB — MAX_FILE_SIZE skip, content_hash dedup, tag reconciliation (remove stale + add new), compiled_truth/timeline chunking, noEmbed flag, sequential chunk_index - test/files.test.ts (22 tests): getMimeType for all extensions + uppercase + unknown + no-extension, fileHash consistency + different content + empty, collectFiles pattern (skip .md, skip hidden dirs, recurse, sorted output) Extended: - test/chunkers/recursive.test.ts (+6 tests): single newline splits, word-only text, clause delimiters, lossless preservation, default options, mixed delimiter hierarchy Tests: 83 → 118 (+35 new). All pass. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
118 lines
3.1 KiB
TypeScript
118 lines
3.1 KiB
TypeScript
/**
|
|
* Sync utilities — pure functions for git diff parsing, filtering, and slug management.
|
|
*
|
|
* SYNC DATA FLOW:
|
|
* git diff --name-status -M LAST..HEAD
|
|
* │
|
|
* buildSyncManifest() → parse A/M/D/R lines
|
|
* │
|
|
* isSyncable() → filter to .md pages only
|
|
* │
|
|
* pathToSlug() → convert file paths to page slugs
|
|
*/
|
|
|
|
export interface SyncManifest {
|
|
added: string[];
|
|
modified: string[];
|
|
deleted: string[];
|
|
renamed: Array<{ from: string; to: string }>;
|
|
}
|
|
|
|
export interface RawManifestEntry {
|
|
action: 'A' | 'M' | 'D' | 'R';
|
|
path: string;
|
|
oldPath?: string;
|
|
}
|
|
|
|
/**
|
|
* Parse the output of `git diff --name-status -M LAST..HEAD` into structured entries.
|
|
*
|
|
* Input format (tab-separated):
|
|
* A path/to/new-file.md
|
|
* M path/to/modified-file.md
|
|
* D path/to/deleted-file.md
|
|
* R100 old/path.md new/path.md
|
|
*/
|
|
export function buildSyncManifest(gitDiffOutput: string): SyncManifest {
|
|
const manifest: SyncManifest = {
|
|
added: [],
|
|
modified: [],
|
|
deleted: [],
|
|
renamed: [],
|
|
};
|
|
|
|
const lines = gitDiffOutput.split('\n');
|
|
|
|
for (const line of lines) {
|
|
const trimmed = line.trim();
|
|
if (!trimmed) continue;
|
|
|
|
const parts = trimmed.split('\t');
|
|
if (parts.length < 2) continue;
|
|
|
|
const action = parts[0];
|
|
const path = parts[parts.length === 3 ? 2 : 1]; // For renames, new path is 3rd column
|
|
|
|
if (action === 'A') {
|
|
manifest.added.push(path);
|
|
} else if (action === 'M') {
|
|
manifest.modified.push(path);
|
|
} else if (action === 'D') {
|
|
manifest.deleted.push(parts[1]);
|
|
} else if (action.startsWith('R')) {
|
|
// Rename: R100\told-path\tnew-path
|
|
const oldPath = parts[1];
|
|
const newPath = parts[2];
|
|
if (oldPath && newPath) {
|
|
manifest.renamed.push({ from: oldPath, to: newPath });
|
|
}
|
|
}
|
|
}
|
|
|
|
return manifest;
|
|
}
|
|
|
|
/**
|
|
* Filter a file path to determine if it should be synced to GBrain.
|
|
*/
|
|
export function isSyncable(path: string): boolean {
|
|
// Must be .md
|
|
if (!path.endsWith('.md')) return false;
|
|
|
|
// Skip hidden directories
|
|
if (path.split('/').some(p => p.startsWith('.'))) return false;
|
|
|
|
// Skip .raw/ sidecar directories
|
|
if (path.includes('.raw/')) return false;
|
|
|
|
// Skip meta files that aren't pages
|
|
const skipFiles = ['schema.md', 'index.md', 'log.md', 'README.md'];
|
|
const basename = path.split('/').pop() || '';
|
|
if (skipFiles.includes(basename)) return false;
|
|
|
|
// Skip ops/ directory
|
|
if (path.startsWith('ops/')) return false;
|
|
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Convert a repo-relative file path to a GBrain page slug.
|
|
*
|
|
* Examples:
|
|
* people/pedro-franceschi.md → people/pedro-franceschi
|
|
* daily/2026-04-05.md → daily/2026-04-05
|
|
* notes.md → notes
|
|
*/
|
|
export function pathToSlug(filePath: string, repoPrefix?: string): string {
|
|
// Strip .md extension
|
|
let slug = filePath.replace(/\.md$/, '');
|
|
// Normalize separators
|
|
slug = slug.replace(/\\/g, '/');
|
|
// Strip leading slash
|
|
slug = slug.replace(/^\//, '');
|
|
// Add repo prefix for multi-repo setups
|
|
if (repoPrefix) slug = `${repoPrefix}/${slug}`;
|
|
return slug;
|
|
}
|