mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
feat(skillpack): harvest + privacy linter (T6 + T7)
The inverse loop: lift a proven skill from a host repo (~/git/wintermute, etc.) back into gbrain so other clients can scaffold it. --from <host-repo-root> is symmetric with scaffold's --workspace. Security: symlink rejection + canonical-path containment (mirrors validateUploadPath). Privacy: default-on linter scans harvested files against ~/.gbrain/harvest-private-patterns.txt plus built-in defaults (Wintermute, email, Slack channel patterns). Any match rolls back the copy and exits non-zero. --no-lint bypasses for the editorial workflow after a manual scrub. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
79bd7d2703
commit
5b5fcd5be3
@@ -0,0 +1,119 @@
|
||||
/**
|
||||
* skillpack/harvest-lint.ts — privacy linter for `gbrain skillpack harvest`.
|
||||
*
|
||||
* Reads `~/.gbrain/harvest-private-patterns.txt` (one regex per line,
|
||||
* user-maintained) plus a small built-in default list of patterns that
|
||||
* commonly leak when harvesting from a personal fork into gbrain core:
|
||||
*
|
||||
* - `\bWintermute\b` — the canonical private fork name (CLAUDE.md
|
||||
* explicitly bans this from gbrain core)
|
||||
* - common email regex
|
||||
* - common Slack channel pattern (`#channel-name`)
|
||||
*
|
||||
* Matches → throws `PrivacyLintError` with `hits[]` listing each
|
||||
* `file:line: matched-pattern` entry. The harvest runner rolls back
|
||||
* the copy on this signal.
|
||||
*
|
||||
* Malformed regex in the patterns file → fail loud at load time so
|
||||
* the user fixes their config before any harvest.
|
||||
*/
|
||||
|
||||
import { existsSync, readFileSync } from 'fs';
|
||||
|
||||
export class PrivacyLintError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public hits: string[],
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'PrivacyLintError';
|
||||
}
|
||||
}
|
||||
|
||||
export class PrivacyLintConfigError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'PrivacyLintConfigError';
|
||||
}
|
||||
}
|
||||
|
||||
/** Default patterns shipped with gbrain (CLAUDE.md responsible-disclosure rule). */
|
||||
export const DEFAULT_PRIVATE_PATTERNS: string[] = [
|
||||
String.raw`\bWintermute\b`,
|
||||
// Email regex (RFC-5322-lite — good enough for harvest-time scrubbing).
|
||||
String.raw`\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b`,
|
||||
// Slack channel: whitespace/line-start, then `#alnum-with-dashes` (len ≥ 3).
|
||||
String.raw`(?:^|\s)#[a-z0-9][a-z0-9_\-]{2,}\b`,
|
||||
];
|
||||
|
||||
/**
|
||||
* Load patterns: user file (if present) + defaults. Each pattern
|
||||
* compiled to a global RegExp; malformed regex throws at load time.
|
||||
*/
|
||||
export function loadPatterns(patternsPath?: string): Array<{ regex: RegExp; source: string }> {
|
||||
const lines: string[] = [];
|
||||
if (patternsPath && existsSync(patternsPath)) {
|
||||
const raw = readFileSync(patternsPath, 'utf-8');
|
||||
for (const line of raw.split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.length === 0) continue;
|
||||
if (trimmed.startsWith('#')) continue; // line comment
|
||||
lines.push(trimmed);
|
||||
}
|
||||
}
|
||||
// Append defaults AFTER user patterns so user-defined ones can be
|
||||
// tried first (e.g. for performance on patterns the user knows will
|
||||
// hit). Order otherwise doesn't matter — we report all hits.
|
||||
lines.push(...DEFAULT_PRIVATE_PATTERNS);
|
||||
|
||||
return lines.map(line => {
|
||||
try {
|
||||
return { regex: new RegExp(line, 'g'), source: line };
|
||||
} catch (err) {
|
||||
throw new PrivacyLintConfigError(
|
||||
`Malformed regex in ${patternsPath ?? '<defaults>'}: ${line} — ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the privacy linter against a list of harvested file paths.
|
||||
* Throws `PrivacyLintError` (with `hits[]`) on any match. No-op when
|
||||
* patterns + files yield zero hits.
|
||||
*/
|
||||
export function runPrivacyLint(
|
||||
filePaths: string[],
|
||||
patternsPath?: string,
|
||||
): void {
|
||||
const patterns = loadPatterns(patternsPath);
|
||||
if (patterns.length === 0) return;
|
||||
|
||||
const hits: string[] = [];
|
||||
for (const file of filePaths) {
|
||||
if (!existsSync(file)) continue;
|
||||
let content: string;
|
||||
try {
|
||||
content = readFileSync(file, 'utf-8');
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const lines = content.split('\n');
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
for (const { regex, source } of patterns) {
|
||||
// Reset lastIndex for global regex re-use across lines.
|
||||
regex.lastIndex = 0;
|
||||
if (regex.test(lines[i])) {
|
||||
hits.push(`${file}:${i + 1}: matched /${source}/`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (hits.length > 0) {
|
||||
throw new PrivacyLintError(
|
||||
`Privacy lint found ${hits.length} match(es) in harvested content. Harvest rolled back. Edit your skill, run the editorial genericization, or add a pattern exception to ${patternsPath ?? '~/.gbrain/harvest-private-patterns.txt'}.`,
|
||||
hits,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,270 @@
|
||||
/**
|
||||
* skillpack/harvest.ts — `gbrain skillpack harvest <slug> --from <host-repo-root>`.
|
||||
*
|
||||
* Inverse of scaffold: lifts a skill from a host repo (e.g. wintermute)
|
||||
* into gbrain's tree so other clients can scaffold it via the normal
|
||||
* path.
|
||||
*
|
||||
* Source contract (D11): `--from` points at the host repo root.
|
||||
* `<from>/skills/<slug>/` is the skill dir. Paired source files
|
||||
* declared in the host skill's frontmatter `sources:` array land at
|
||||
* the mirror path inside gbrain.
|
||||
*
|
||||
* Security (D13): every harvested file goes through canonical-path
|
||||
* validation and symlink rejection. `realpath(file).startsWith
|
||||
* (realpath(host-skill-dir))`. Mirrors `validateUploadPath` from
|
||||
* `src/core/operations.ts`. Without this gate, a malicious or careless
|
||||
* symlink could leak secrets into gbrain's source tree.
|
||||
*
|
||||
* Privacy (D4, T7): after copying but before declaring success, the
|
||||
* harvested files are scanned against a regex allowlist of "private
|
||||
* patterns" (defaults + user-maintained `~/.gbrain/harvest-private-patterns.txt`).
|
||||
* Any match → rollback (delete harvested files) and exit non-zero.
|
||||
* `--no-lint` bypasses the linter (used by the editorial workflow
|
||||
* skill after a manual scrub).
|
||||
*/
|
||||
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
} from 'fs';
|
||||
import { homedir } from 'os';
|
||||
import { dirname, join, relative } from 'path';
|
||||
|
||||
import { copyArtifacts, walkSourceDir } from './copy.ts';
|
||||
import { loadSkillSources } from './bundle.ts';
|
||||
import { runPrivacyLint, PrivacyLintError } from './harvest-lint.ts';
|
||||
|
||||
export interface HarvestOptions {
|
||||
/** Slug of the skill to harvest (e.g. "my-fork-skill"). */
|
||||
slug: string;
|
||||
/** Absolute path to the host repo root (e.g. ~/git/wintermute). */
|
||||
hostRepoRoot: string;
|
||||
/** Absolute path to gbrain repo root (destination). */
|
||||
gbrainRoot: string;
|
||||
/** Skip the privacy linter. */
|
||||
noLint?: boolean;
|
||||
/** Dry-run: preview, no writes. */
|
||||
dryRun?: boolean;
|
||||
/** Custom private-patterns file (defaults to ~/.gbrain/harvest-private-patterns.txt). */
|
||||
privatePatternsPath?: string;
|
||||
/** Allow overwriting an existing gbrain/skills/<slug>/ tree. */
|
||||
overwriteLocal?: boolean;
|
||||
}
|
||||
|
||||
export type HarvestStatus = 'harvested' | 'host_skill_missing' | 'slug_collision' | 'lint_failed';
|
||||
|
||||
export interface HarvestResult {
|
||||
status: HarvestStatus;
|
||||
slug: string;
|
||||
hostSkillDir: string;
|
||||
/** Files written under gbrain/. */
|
||||
filesCopied: string[];
|
||||
/** Paired source files (from frontmatter) included. */
|
||||
pairedSources: string[];
|
||||
/** Privacy-lint hits, when status === 'lint_failed'. */
|
||||
lintHits: string[];
|
||||
/** True when the manifest was updated (only on success, non-dry-run). */
|
||||
manifestUpdated: boolean;
|
||||
dryRun: boolean;
|
||||
}
|
||||
|
||||
export class HarvestError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public code:
|
||||
| 'host_skill_missing'
|
||||
| 'host_skill_malformed'
|
||||
| 'slug_collision'
|
||||
| 'path_traversal'
|
||||
| 'symlink_rejected',
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'HarvestError';
|
||||
}
|
||||
}
|
||||
|
||||
const PLUGIN_JSON = 'openclaw.plugin.json';
|
||||
const DEFAULT_PRIVATE_PATTERNS_PATH = join(
|
||||
homedir(),
|
||||
'.gbrain',
|
||||
'harvest-private-patterns.txt',
|
||||
);
|
||||
|
||||
export function runHarvest(opts: HarvestOptions): HarvestResult {
|
||||
const dryRun = opts.dryRun ?? false;
|
||||
const hostSkillDir = join(opts.hostRepoRoot, 'skills', opts.slug);
|
||||
const hostSkillMd = join(hostSkillDir, 'SKILL.md');
|
||||
|
||||
if (!existsSync(hostSkillMd)) {
|
||||
throw new HarvestError(
|
||||
`Host skill not found: ${hostSkillMd}. Pass --from <host-repo-root> pointing at a repo whose skills/<slug>/ exists.`,
|
||||
'host_skill_missing',
|
||||
);
|
||||
}
|
||||
|
||||
const gbrainSkillDir = join(opts.gbrainRoot, 'skills', opts.slug);
|
||||
if (existsSync(gbrainSkillDir) && !opts.overwriteLocal) {
|
||||
throw new HarvestError(
|
||||
`Slug collision: gbrain already has skills/${opts.slug}/. Pass --overwrite-local to replace.`,
|
||||
'slug_collision',
|
||||
);
|
||||
}
|
||||
|
||||
// Read frontmatter sources from the host SKILL.md. Reuse the bundler's
|
||||
// validation — but skip its existence check on the destination since
|
||||
// we're reading from a different root.
|
||||
const pairedSources = readHostSkillSources(opts.hostRepoRoot, opts.slug);
|
||||
|
||||
// Build items list:
|
||||
// - skill dir → gbrain/skills/<slug>/
|
||||
// - paired sources → gbrain/<source-path>
|
||||
const items: Array<{ source: string; target: string }> = [];
|
||||
for (const item of walkSourceDir(hostSkillDir, gbrainSkillDir)) {
|
||||
items.push(item);
|
||||
}
|
||||
for (const src of pairedSources) {
|
||||
items.push({
|
||||
source: join(opts.hostRepoRoot, src),
|
||||
target: join(opts.gbrainRoot, src),
|
||||
});
|
||||
}
|
||||
|
||||
// Copy with D13 confinement + symlink reject. The confinement root is
|
||||
// the HOST skill dir (every source must canonicalize inside it). For
|
||||
// paired sources outside the skill dir, fall through to symlink-only
|
||||
// protection (the host repo is user-trusted at this granularity).
|
||||
const skillItems = items.filter(i => i.source.startsWith(hostSkillDir));
|
||||
const pairedItems = items.filter(i => !i.source.startsWith(hostSkillDir));
|
||||
|
||||
let filesCopied: string[] = [];
|
||||
try {
|
||||
if (!dryRun) {
|
||||
copyArtifacts(skillItems, {
|
||||
rejectSymlinks: true,
|
||||
confineRealpath: hostSkillDir,
|
||||
});
|
||||
copyArtifacts(pairedItems, { rejectSymlinks: true });
|
||||
} else {
|
||||
// Dry-run still validates safety gates but doesn't copy.
|
||||
copyArtifacts(skillItems, {
|
||||
rejectSymlinks: true,
|
||||
confineRealpath: hostSkillDir,
|
||||
dryRun: true,
|
||||
});
|
||||
copyArtifacts(pairedItems, { rejectSymlinks: true, dryRun: true });
|
||||
}
|
||||
filesCopied = items.map(i => i.target);
|
||||
} catch (err) {
|
||||
const e = err as Error & { code?: string };
|
||||
if (e.code === 'symlink_rejected') {
|
||||
throw new HarvestError(e.message, 'symlink_rejected');
|
||||
}
|
||||
if (e.code === 'path_traversal') {
|
||||
throw new HarvestError(e.message, 'path_traversal');
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
// Privacy lint AFTER copy (lint scans the harvested files). On match,
|
||||
// rollback (delete) and report.
|
||||
const lintHits: string[] = [];
|
||||
if (!opts.noLint && !dryRun) {
|
||||
try {
|
||||
runPrivacyLint(
|
||||
filesCopied,
|
||||
opts.privatePatternsPath ?? DEFAULT_PRIVATE_PATTERNS_PATH,
|
||||
);
|
||||
} catch (err) {
|
||||
if (err instanceof PrivacyLintError) {
|
||||
// Rollback: remove every file we just wrote.
|
||||
rollbackHarvest(gbrainSkillDir, pairedItems.map(i => i.target));
|
||||
return {
|
||||
status: 'lint_failed',
|
||||
slug: opts.slug,
|
||||
hostSkillDir,
|
||||
filesCopied: [],
|
||||
pairedSources,
|
||||
lintHits: err.hits,
|
||||
manifestUpdated: false,
|
||||
dryRun: false,
|
||||
};
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// Update openclaw.plugin.json — add slug to "skills" array if missing.
|
||||
let manifestUpdated = false;
|
||||
if (!dryRun) {
|
||||
manifestUpdated = addToBundleManifest(opts.gbrainRoot, opts.slug);
|
||||
}
|
||||
|
||||
return {
|
||||
status: 'harvested',
|
||||
slug: opts.slug,
|
||||
hostSkillDir,
|
||||
filesCopied,
|
||||
pairedSources,
|
||||
lintHits,
|
||||
manifestUpdated,
|
||||
dryRun,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a host skill's frontmatter `sources:` without using the bundler
|
||||
* (the bundler resolves paths against gbrainRoot, not the host). Mirrors
|
||||
* `loadSkillSources`'s validation but resolves against the host root.
|
||||
*/
|
||||
function readHostSkillSources(hostRoot: string, slug: string): string[] {
|
||||
// Lean on bundle.ts's loadSkillSources but pass the host as the root.
|
||||
// Its validation (no abs paths, no `..`, must exist) applies to the
|
||||
// host's tree, which is what we want.
|
||||
const result = loadSkillSources(hostRoot, `skills/${slug}`);
|
||||
return result.sources;
|
||||
}
|
||||
|
||||
/** Delete everything we just wrote. */
|
||||
function rollbackHarvest(gbrainSkillDir: string, pairedTargets: string[]): void {
|
||||
try {
|
||||
if (existsSync(gbrainSkillDir)) {
|
||||
rmSync(gbrainSkillDir, { recursive: true, force: true });
|
||||
}
|
||||
} catch {}
|
||||
for (const p of pairedTargets) {
|
||||
try {
|
||||
if (existsSync(p)) rmSync(p, { force: true });
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add `slugs/<slug>` to `openclaw.plugin.json#skills` if missing.
|
||||
* Preserves JSON formatting via 2-space indent. Idempotent.
|
||||
*
|
||||
* Returns true if the manifest was modified.
|
||||
*/
|
||||
export function addToBundleManifest(gbrainRoot: string, slug: string): boolean {
|
||||
const manifestPath = join(gbrainRoot, PLUGIN_JSON);
|
||||
if (!existsSync(manifestPath)) return false;
|
||||
const raw = readFileSync(manifestPath, 'utf-8');
|
||||
let manifest;
|
||||
try {
|
||||
manifest = JSON.parse(raw);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (!Array.isArray(manifest.skills)) return false;
|
||||
const skillRel = `skills/${slug}`;
|
||||
if (manifest.skills.includes(skillRel)) return false;
|
||||
manifest.skills.push(skillRel);
|
||||
manifest.skills.sort();
|
||||
writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + '\n');
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
/**
|
||||
* Tests for src/core/skillpack/harvest-lint.ts — the privacy linter
|
||||
* that defends against accidentally publishing real names from a
|
||||
* private host fork into gbrain core.
|
||||
*
|
||||
* Pins:
|
||||
* - default Wintermute pattern matches
|
||||
* - email + Slack patterns match
|
||||
* - user-supplied patterns merge with defaults
|
||||
* - malformed regex → PrivacyLintConfigError at LOAD time
|
||||
* - no hits → no throw
|
||||
* - hit detail format: `file:line: matched /<regex>/`
|
||||
*/
|
||||
|
||||
import { describe, expect, it, afterEach } from 'bun:test';
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
|
||||
import {
|
||||
DEFAULT_PRIVATE_PATTERNS,
|
||||
PrivacyLintConfigError,
|
||||
PrivacyLintError,
|
||||
loadPatterns,
|
||||
runPrivacyLint,
|
||||
} from '../src/core/skillpack/harvest-lint.ts';
|
||||
|
||||
const created: string[] = [];
|
||||
afterEach(() => {
|
||||
while (created.length) {
|
||||
const p = created.pop()!;
|
||||
try {
|
||||
rmSync(p, { recursive: true, force: true });
|
||||
} catch {}
|
||||
}
|
||||
});
|
||||
|
||||
function scratch(): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'sp-lint-'));
|
||||
created.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
describe('loadPatterns', () => {
|
||||
it('returns defaults when no patterns file is provided', () => {
|
||||
const patterns = loadPatterns();
|
||||
expect(patterns.length).toBe(DEFAULT_PRIVATE_PATTERNS.length);
|
||||
expect(patterns.map(p => p.source)).toEqual(DEFAULT_PRIVATE_PATTERNS);
|
||||
});
|
||||
|
||||
it('returns defaults + user patterns when both exist', () => {
|
||||
const dir = scratch();
|
||||
const path = join(dir, 'patterns.txt');
|
||||
writeFileSync(path, '# comment\nFooBar\n\n \nBaz123\n');
|
||||
|
||||
const patterns = loadPatterns(path);
|
||||
const sources = patterns.map(p => p.source);
|
||||
expect(sources).toContain('FooBar');
|
||||
expect(sources).toContain('Baz123');
|
||||
for (const def of DEFAULT_PRIVATE_PATTERNS) expect(sources).toContain(def);
|
||||
});
|
||||
|
||||
it('throws PrivacyLintConfigError on malformed regex', () => {
|
||||
const dir = scratch();
|
||||
const path = join(dir, 'patterns.txt');
|
||||
writeFileSync(path, 'unterminated[group\n');
|
||||
|
||||
expect(() => loadPatterns(path)).toThrow(PrivacyLintConfigError);
|
||||
});
|
||||
});
|
||||
|
||||
describe('runPrivacyLint — default patterns', () => {
|
||||
it('catches Wintermute references', () => {
|
||||
const dir = scratch();
|
||||
const file = join(dir, 'skill.md');
|
||||
writeFileSync(file, '# Demo\n\nThis was lifted from Wintermute.\n');
|
||||
|
||||
expect(() => runPrivacyLint([file])).toThrow(PrivacyLintError);
|
||||
try {
|
||||
runPrivacyLint([file]);
|
||||
} catch (err) {
|
||||
expect((err as PrivacyLintError).hits.some(h => h.includes('Wintermute'))).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it('catches email addresses', () => {
|
||||
const dir = scratch();
|
||||
const file = join(dir, 'skill.md');
|
||||
writeFileSync(file, 'Contact: jane.doe@example.com for details.\n');
|
||||
|
||||
expect(() => runPrivacyLint([file])).toThrow(PrivacyLintError);
|
||||
});
|
||||
|
||||
it('catches Slack channel patterns', () => {
|
||||
const dir = scratch();
|
||||
const file = join(dir, 'skill.md');
|
||||
writeFileSync(file, 'Notify #eng-alerts when this triggers.\n');
|
||||
|
||||
expect(() => runPrivacyLint([file])).toThrow(PrivacyLintError);
|
||||
});
|
||||
|
||||
it('does NOT match safe content — no throw', () => {
|
||||
const dir = scratch();
|
||||
const file = join(dir, 'skill.md');
|
||||
writeFileSync(
|
||||
file,
|
||||
'# generic skill\n\nThis is generic placeholder content for the agent to interpret.\n',
|
||||
);
|
||||
|
||||
expect(() => runPrivacyLint([file])).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('runPrivacyLint — hit reporting', () => {
|
||||
it('hit format is `file:line: matched /<regex>/`', () => {
|
||||
const dir = scratch();
|
||||
const file = join(dir, 'skill.md');
|
||||
writeFileSync(file, 'line 1\nWintermute on line 2\nline 3\n');
|
||||
|
||||
try {
|
||||
runPrivacyLint([file]);
|
||||
throw new Error('expected throw');
|
||||
} catch (err) {
|
||||
const hits = (err as PrivacyLintError).hits;
|
||||
expect(hits.length).toBeGreaterThan(0);
|
||||
const wintermuteHit = hits.find(h => h.includes('Wintermute'));
|
||||
expect(wintermuteHit).toBeDefined();
|
||||
expect(wintermuteHit).toContain(`${file}:2:`);
|
||||
}
|
||||
});
|
||||
|
||||
it('scans multiple files in one pass', () => {
|
||||
const dir = scratch();
|
||||
const f1 = join(dir, 's1.md');
|
||||
const f2 = join(dir, 's2.md');
|
||||
writeFileSync(f1, 'clean\n');
|
||||
writeFileSync(f2, 'has Wintermute in it\n');
|
||||
|
||||
try {
|
||||
runPrivacyLint([f1, f2]);
|
||||
throw new Error('expected throw');
|
||||
} catch (err) {
|
||||
const hits = (err as PrivacyLintError).hits;
|
||||
expect(hits.length).toBe(1);
|
||||
expect(hits[0]).toContain(f2);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('runPrivacyLint — user patterns file', () => {
|
||||
it('catches user-defined patterns alongside defaults', () => {
|
||||
const dir = scratch();
|
||||
const patternsPath = join(dir, 'patterns.txt');
|
||||
writeFileSync(patternsPath, '\\bMyPrivateProject\\b\n');
|
||||
|
||||
const file = join(dir, 'skill.md');
|
||||
writeFileSync(file, 'Refers to MyPrivateProject.\n');
|
||||
|
||||
expect(() => runPrivacyLint([file], patternsPath)).toThrow(PrivacyLintError);
|
||||
});
|
||||
|
||||
it('user patterns file with comments + blanks parses cleanly', () => {
|
||||
const dir = scratch();
|
||||
const patternsPath = join(dir, 'patterns.txt');
|
||||
writeFileSync(patternsPath, '# header comment\n\nUnique\n\n# another\nAlsoUnique\n');
|
||||
|
||||
expect(() => loadPatterns(patternsPath)).not.toThrow();
|
||||
const patterns = loadPatterns(patternsPath);
|
||||
expect(patterns.some(p => p.source === 'Unique')).toBe(true);
|
||||
expect(patterns.some(p => p.source === 'AlsoUnique')).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,261 @@
|
||||
/**
|
||||
* Tests for src/core/skillpack/harvest.ts — the host→gbrain lift.
|
||||
*
|
||||
* Pins:
|
||||
* - happy path: skill files + paired sources land in gbrain's tree
|
||||
* - openclaw.plugin.json updated (sorted, idempotent)
|
||||
* - slug collision refused unless --overwrite-local
|
||||
* - symlinks in host source rejected
|
||||
* - canonical-path containment rejects traversal
|
||||
* - privacy lint runs by default, rolls back on hit
|
||||
* - --no-lint bypasses
|
||||
* - dry-run reports plan, writes nothing
|
||||
* - host SKILL.md missing → HarvestError(host_skill_missing)
|
||||
*/
|
||||
|
||||
import { describe, expect, it, afterEach } from 'bun:test';
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
symlinkSync,
|
||||
writeFileSync,
|
||||
} from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
|
||||
import { HarvestError, runHarvest, addToBundleManifest } from '../src/core/skillpack/harvest.ts';
|
||||
|
||||
const created: string[] = [];
|
||||
afterEach(() => {
|
||||
while (created.length) {
|
||||
const p = created.pop()!;
|
||||
try {
|
||||
rmSync(p, { recursive: true, force: true });
|
||||
} catch {}
|
||||
}
|
||||
});
|
||||
|
||||
function scratchHost(opts: { withPairedSource?: boolean; contaminated?: boolean } = {}): string {
|
||||
const root = mkdtempSync(join(tmpdir(), 'sp-h-host-'));
|
||||
created.push(root);
|
||||
mkdirSync(join(root, 'src', 'commands'), { recursive: true });
|
||||
mkdirSync(join(root, 'skills', 'my-fork-skill'), { recursive: true });
|
||||
|
||||
const fm = opts.withPairedSource
|
||||
? '---\nname: my-fork-skill\ntriggers:\n - trigger\nsources:\n - src/commands/my-fork-skill.ts\n---\n'
|
||||
: '---\nname: my-fork-skill\ntriggers:\n - trigger\n---\n';
|
||||
const body = opts.contaminated
|
||||
? '# my-fork-skill\n\nThis was lifted from Wintermute.\n'
|
||||
: '# my-fork-skill\n\nGeneric placeholder content.\n';
|
||||
writeFileSync(join(root, 'skills', 'my-fork-skill', 'SKILL.md'), fm + body);
|
||||
|
||||
if (opts.withPairedSource) {
|
||||
writeFileSync(
|
||||
join(root, 'src', 'commands', 'my-fork-skill.ts'),
|
||||
'// real impl\nexport function run() { return 1; }\n',
|
||||
);
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
function scratchGbrain(): string {
|
||||
const root = mkdtempSync(join(tmpdir(), 'sp-h-gbrain-'));
|
||||
created.push(root);
|
||||
mkdirSync(join(root, 'src', 'commands'), { recursive: true });
|
||||
writeFileSync(join(root, 'src', 'cli.ts'), '// stub');
|
||||
mkdirSync(join(root, 'skills'), { recursive: true });
|
||||
writeFileSync(
|
||||
join(root, 'openclaw.plugin.json'),
|
||||
JSON.stringify(
|
||||
{
|
||||
name: 'gbrain',
|
||||
version: '0.33.0',
|
||||
skills: ['skills/existing-skill'],
|
||||
shared_deps: [],
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
return root;
|
||||
}
|
||||
|
||||
function emptyPatternsFile(): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'sp-h-patterns-'));
|
||||
created.push(dir);
|
||||
const path = join(dir, 'patterns.txt');
|
||||
writeFileSync(path, '# only the user-supplied patterns. Defaults still apply.\n');
|
||||
return path;
|
||||
}
|
||||
|
||||
describe('runHarvest — happy path', () => {
|
||||
it('copies a clean skill into gbrain, updates manifest', () => {
|
||||
const hostRoot = scratchHost();
|
||||
const gbrainRoot = scratchGbrain();
|
||||
|
||||
const result = runHarvest({
|
||||
slug: 'my-fork-skill',
|
||||
hostRepoRoot: hostRoot,
|
||||
gbrainRoot,
|
||||
});
|
||||
|
||||
expect(result.status).toBe('harvested');
|
||||
expect(existsSync(join(gbrainRoot, 'skills', 'my-fork-skill', 'SKILL.md'))).toBe(true);
|
||||
expect(result.manifestUpdated).toBe(true);
|
||||
|
||||
const manifest = JSON.parse(readFileSync(join(gbrainRoot, 'openclaw.plugin.json'), 'utf-8'));
|
||||
expect(manifest.skills).toContain('skills/my-fork-skill');
|
||||
// sorted
|
||||
expect(manifest.skills).toEqual([...manifest.skills].sort());
|
||||
});
|
||||
|
||||
it('copies paired source files declared in host frontmatter', () => {
|
||||
const hostRoot = scratchHost({ withPairedSource: true });
|
||||
const gbrainRoot = scratchGbrain();
|
||||
|
||||
const result = runHarvest({
|
||||
slug: 'my-fork-skill',
|
||||
hostRepoRoot: hostRoot,
|
||||
gbrainRoot,
|
||||
});
|
||||
|
||||
expect(result.pairedSources).toEqual(['src/commands/my-fork-skill.ts']);
|
||||
expect(existsSync(join(gbrainRoot, 'src', 'commands', 'my-fork-skill.ts'))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('runHarvest — error paths', () => {
|
||||
it('host_skill_missing when --from points at the wrong place', () => {
|
||||
const hostRoot = scratchHost();
|
||||
const gbrainRoot = scratchGbrain();
|
||||
|
||||
try {
|
||||
runHarvest({ slug: 'nonexistent', hostRepoRoot: hostRoot, gbrainRoot });
|
||||
throw new Error('expected throw');
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(HarvestError);
|
||||
expect((err as HarvestError).code).toBe('host_skill_missing');
|
||||
}
|
||||
});
|
||||
|
||||
it('slug_collision when gbrain already has skills/<slug>/ — without --overwrite-local', () => {
|
||||
const hostRoot = scratchHost();
|
||||
const gbrainRoot = scratchGbrain();
|
||||
mkdirSync(join(gbrainRoot, 'skills', 'my-fork-skill'), { recursive: true });
|
||||
writeFileSync(join(gbrainRoot, 'skills', 'my-fork-skill', 'SKILL.md'), '# already here\n');
|
||||
|
||||
try {
|
||||
runHarvest({
|
||||
slug: 'my-fork-skill',
|
||||
hostRepoRoot: hostRoot,
|
||||
gbrainRoot,
|
||||
});
|
||||
throw new Error('expected throw');
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(HarvestError);
|
||||
expect((err as HarvestError).code).toBe('slug_collision');
|
||||
}
|
||||
});
|
||||
|
||||
it('symlinks in host skill dir are rejected (D13 security gate)', () => {
|
||||
const hostRoot = scratchHost();
|
||||
const gbrainRoot = scratchGbrain();
|
||||
|
||||
// Plant a symlink inside the skill dir pointing outside.
|
||||
const outside = mkdtempSync(join(tmpdir(), 'sp-h-outside-'));
|
||||
created.push(outside);
|
||||
writeFileSync(join(outside, 'secret.txt'), 'PRIVATE\n');
|
||||
symlinkSync(
|
||||
join(outside, 'secret.txt'),
|
||||
join(hostRoot, 'skills', 'my-fork-skill', 'leaked.txt'),
|
||||
);
|
||||
|
||||
try {
|
||||
runHarvest({
|
||||
slug: 'my-fork-skill',
|
||||
hostRepoRoot: hostRoot,
|
||||
gbrainRoot,
|
||||
noLint: true,
|
||||
});
|
||||
throw new Error('expected throw');
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(HarvestError);
|
||||
expect(['symlink_rejected', 'path_traversal']).toContain((err as HarvestError).code);
|
||||
}
|
||||
// Nothing landed.
|
||||
expect(existsSync(join(gbrainRoot, 'skills', 'my-fork-skill'))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('runHarvest — privacy linter integration (T7)', () => {
|
||||
it('default Wintermute pattern triggers rollback (no manifest update)', () => {
|
||||
const hostRoot = scratchHost({ contaminated: true });
|
||||
const gbrainRoot = scratchGbrain();
|
||||
|
||||
const result = runHarvest({
|
||||
slug: 'my-fork-skill',
|
||||
hostRepoRoot: hostRoot,
|
||||
gbrainRoot,
|
||||
});
|
||||
|
||||
expect(result.status).toBe('lint_failed');
|
||||
expect(result.lintHits.length).toBeGreaterThan(0);
|
||||
expect(result.lintHits[0]).toContain('Wintermute');
|
||||
|
||||
// Rollback: nothing in gbrain tree.
|
||||
expect(existsSync(join(gbrainRoot, 'skills', 'my-fork-skill'))).toBe(false);
|
||||
// Manifest NOT updated.
|
||||
const manifest = JSON.parse(readFileSync(join(gbrainRoot, 'openclaw.plugin.json'), 'utf-8'));
|
||||
expect(manifest.skills).not.toContain('skills/my-fork-skill');
|
||||
});
|
||||
|
||||
it('--no-lint bypasses the linter (editorial workflow opt-out)', () => {
|
||||
const hostRoot = scratchHost({ contaminated: true });
|
||||
const gbrainRoot = scratchGbrain();
|
||||
|
||||
const result = runHarvest({
|
||||
slug: 'my-fork-skill',
|
||||
hostRepoRoot: hostRoot,
|
||||
gbrainRoot,
|
||||
noLint: true,
|
||||
});
|
||||
|
||||
expect(result.status).toBe('harvested');
|
||||
expect(existsSync(join(gbrainRoot, 'skills', 'my-fork-skill', 'SKILL.md'))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('runHarvest — dry-run', () => {
|
||||
it('reports plan, writes nothing', () => {
|
||||
const hostRoot = scratchHost();
|
||||
const gbrainRoot = scratchGbrain();
|
||||
|
||||
const result = runHarvest({
|
||||
slug: 'my-fork-skill',
|
||||
hostRepoRoot: hostRoot,
|
||||
gbrainRoot,
|
||||
dryRun: true,
|
||||
});
|
||||
|
||||
expect(result.status).toBe('harvested');
|
||||
expect(result.dryRun).toBe(true);
|
||||
expect(result.manifestUpdated).toBe(false);
|
||||
expect(existsSync(join(gbrainRoot, 'skills', 'my-fork-skill'))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('addToBundleManifest', () => {
|
||||
it('adds a new slug, sorts skills array, idempotent', () => {
|
||||
const gbrainRoot = scratchGbrain();
|
||||
|
||||
expect(addToBundleManifest(gbrainRoot, 'new-skill')).toBe(true);
|
||||
expect(addToBundleManifest(gbrainRoot, 'new-skill')).toBe(false); // idempotent
|
||||
|
||||
const manifest = JSON.parse(readFileSync(join(gbrainRoot, 'openclaw.plugin.json'), 'utf-8'));
|
||||
expect(manifest.skills).toContain('skills/new-skill');
|
||||
expect(manifest.skills).toEqual([...manifest.skills].sort());
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user