mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
v0.37.1.0 feat: brainstorm + lsd — bisociation idea generator grounded in your own brain (#1214)
* feat: brainstorm + lsd (v0.37 wave, pre-merge snapshot) Brainstorm + LSD bisociation idea generator. Will rebase + bump after master merge. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> * docs: CHANGELOG voice tightening — strip meta-content from v0.37.1.0 entry CLAUDE.md gains an IRON RULE for CHANGELOG entries: the changelog describes what the user gets, not how the work happened. No mentions of review processes, plan files, decision tags, migration version drama, or 'what we caught and fixed before merging.' If a fact only exists because of the development workflow, it does not belong in release notes. Rewrite v0.37.1.0 entry to comply: cut the 'what we caught' section (architectural review drama), the 'plan + reviews' bullet, and the migration-renumbering aside. Entry shrinks 67 → 56 lines, every sentence now answers 'what can I do / how do I use it / what should I watch for.' Regenerated llms-full.txt to absorb the CLAUDE.md voice update. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
bc9f7774bf
commit
39e14cd50e
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* v0.37.0 — domain-bank distance normalization (D6 + codex r2 #9).
|
||||
*
|
||||
* Pinned cases from the codex-r2 fix: same-vector → 0, orthogonal → 0.5,
|
||||
* opposite → 1, missing-vector → caller responsibility (skipped at retrieval).
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { normalizedCosineDistance } from '../../src/core/brainstorm/domain-bank.ts';
|
||||
|
||||
function v(...nums: number[]): Float32Array {
|
||||
return new Float32Array(nums);
|
||||
}
|
||||
|
||||
describe('normalizedCosineDistance — codex r2 #9 pinned cases', () => {
|
||||
test('same vector → distance 0 (identical)', () => {
|
||||
const a = v(0.6, 0.8, 0.0);
|
||||
expect(normalizedCosineDistance(a, a)).toBeCloseTo(0, 6);
|
||||
});
|
||||
|
||||
test('orthogonal unit vectors → distance 0.5 (neutral)', () => {
|
||||
const x = v(1, 0, 0);
|
||||
const y = v(0, 1, 0);
|
||||
expect(normalizedCosineDistance(x, y)).toBeCloseTo(0.5, 6);
|
||||
});
|
||||
|
||||
test('opposite unit vectors → distance 1 (maximally far)', () => {
|
||||
const x = v(1, 0, 0);
|
||||
const y = v(-1, 0, 0);
|
||||
expect(normalizedCosineDistance(x, y)).toBeCloseTo(1, 6);
|
||||
});
|
||||
|
||||
test('45-degree separation lands between 0 and 0.5', () => {
|
||||
// cos(45°) ≈ 0.707, so cosDist ≈ 0.293, halved → 0.146
|
||||
const x = v(1, 0);
|
||||
const y = v(Math.sqrt(0.5), Math.sqrt(0.5));
|
||||
const d = normalizedCosineDistance(x, y);
|
||||
expect(d).toBeGreaterThan(0.1);
|
||||
expect(d).toBeLessThan(0.2);
|
||||
});
|
||||
|
||||
test('zero-vector edge → 0.5 (neutral, no division-by-zero)', () => {
|
||||
const z = v(0, 0, 0);
|
||||
const a = v(1, 1, 1);
|
||||
expect(normalizedCosineDistance(z, a)).toBeCloseTo(0.5, 6);
|
||||
});
|
||||
|
||||
test('dimension mismatch throws', () => {
|
||||
expect(() => normalizedCosineDistance(v(1, 0), v(1, 0, 0))).toThrow(/dim mismatch/);
|
||||
});
|
||||
|
||||
test('result is symmetric: d(a,b) === d(b,a)', () => {
|
||||
const a = v(0.3, 0.4, 0.5);
|
||||
const b = v(0.7, 0.1, 0.2);
|
||||
expect(normalizedCosineDistance(a, b)).toBeCloseTo(normalizedCosineDistance(b, a), 6);
|
||||
});
|
||||
|
||||
test('result is bounded [0, 1] even on non-unit vectors', () => {
|
||||
const a = v(10, 0, 0);
|
||||
const b = v(0, 5, 0);
|
||||
const d = normalizedCosineDistance(a, b);
|
||||
expect(d).toBeGreaterThanOrEqual(0);
|
||||
expect(d).toBeLessThanOrEqual(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,201 @@
|
||||
/**
|
||||
* v0.37.0 — `gbrain eval brainstorm` pure-function tests (D3 + codex r2 #11).
|
||||
*
|
||||
* The orchestrator + judge themselves are exercised in E2E with a real
|
||||
* brain; here we pin the eval math (grounding rate + verdict computation +
|
||||
* threshold semantics) since those are what gate the eval suite.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
computeGroundingRate,
|
||||
computeVerdict,
|
||||
DEFAULT_BRAINSTORM_THRESHOLDS,
|
||||
readBrainstormEvalFixture,
|
||||
type PerFixtureResult,
|
||||
} from '../../src/commands/eval-brainstorm.ts';
|
||||
import { writeFileSync, mkdtempSync, rmSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
|
||||
describe('computeGroundingRate', () => {
|
||||
const real = new Set(['wiki/vc/alice', 'wiki/biology/bee', 'wiki/hardware/asic']);
|
||||
|
||||
test('100% grounding when every idea cites a real slug', () => {
|
||||
const ideas = [
|
||||
{ close_slug: 'wiki/vc/alice', far_slug: 'wiki/biology/bee' },
|
||||
{ close_slug: 'wiki/biology/bee', far_slug: 'wiki/hardware/asic' },
|
||||
];
|
||||
expect(computeGroundingRate(ideas, real)).toBe(1.0);
|
||||
});
|
||||
|
||||
test('50% grounding when half cite hallucinated slugs', () => {
|
||||
const ideas = [
|
||||
{ close_slug: 'wiki/vc/alice', far_slug: 'wiki/biology/bee' },
|
||||
{ close_slug: 'wiki/fake/no-such-page', far_slug: 'wiki/also-fake' },
|
||||
];
|
||||
expect(computeGroundingRate(ideas, real)).toBe(0.5);
|
||||
});
|
||||
|
||||
test('one-real-citation counts as grounded (close OR far real)', () => {
|
||||
const ideas = [
|
||||
{ close_slug: 'wiki/vc/alice', far_slug: 'wiki/hallucinated' },
|
||||
];
|
||||
expect(computeGroundingRate(ideas, real)).toBe(1.0);
|
||||
});
|
||||
|
||||
test('0% grounding when all slugs are hallucinated', () => {
|
||||
const ideas = [
|
||||
{ close_slug: 'wiki/fake/one', far_slug: 'wiki/fake/two' },
|
||||
];
|
||||
expect(computeGroundingRate(ideas, real)).toBe(0);
|
||||
});
|
||||
|
||||
test('empty ideas array → 0 (no division by zero)', () => {
|
||||
expect(computeGroundingRate([], real)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
function mkFixture(partial: Partial<PerFixtureResult>): PerFixtureResult {
|
||||
return {
|
||||
question: 'test',
|
||||
pass_count: 5,
|
||||
total_ideas: 5,
|
||||
mean_distance: 0.5,
|
||||
mean_usefulness: 4.0,
|
||||
grounding_rate: 1.0,
|
||||
short_of_target: false,
|
||||
cost_usd: 0.10,
|
||||
judge_failed: false,
|
||||
...partial,
|
||||
};
|
||||
}
|
||||
|
||||
describe('computeVerdict', () => {
|
||||
test('pass when all three axes clear', () => {
|
||||
const res = computeVerdict(
|
||||
[
|
||||
mkFixture({ mean_distance: 0.5, mean_usefulness: 4.0, grounding_rate: 1.0 }),
|
||||
mkFixture({ mean_distance: 0.6, mean_usefulness: 4.2, grounding_rate: 1.0 }),
|
||||
],
|
||||
DEFAULT_BRAINSTORM_THRESHOLDS,
|
||||
);
|
||||
expect(res.verdict).toBe('pass');
|
||||
});
|
||||
|
||||
test('fail when distance below threshold', () => {
|
||||
const res = computeVerdict(
|
||||
[
|
||||
mkFixture({ mean_distance: 0.2, mean_usefulness: 4.0, grounding_rate: 1.0 }),
|
||||
mkFixture({ mean_distance: 0.25, mean_usefulness: 4.0, grounding_rate: 1.0 }),
|
||||
],
|
||||
DEFAULT_BRAINSTORM_THRESHOLDS,
|
||||
);
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.reasons.some((r) => r.includes('distance'))).toBe(true);
|
||||
});
|
||||
|
||||
test('fail when usefulness below threshold (codex r2 #11 — distance alone is gameable)', () => {
|
||||
const res = computeVerdict(
|
||||
[
|
||||
mkFixture({ mean_distance: 0.6, mean_usefulness: 2.5, grounding_rate: 1.0 }),
|
||||
mkFixture({ mean_distance: 0.6, mean_usefulness: 2.8, grounding_rate: 1.0 }),
|
||||
],
|
||||
DEFAULT_BRAINSTORM_THRESHOLDS,
|
||||
);
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.reasons.some((r) => r.includes('usefulness'))).toBe(true);
|
||||
});
|
||||
|
||||
test('fail when grounding below 1.0 (every idea must cite a real slug)', () => {
|
||||
const res = computeVerdict(
|
||||
[
|
||||
mkFixture({ mean_distance: 0.6, mean_usefulness: 4.0, grounding_rate: 0.7 }),
|
||||
mkFixture({ mean_distance: 0.6, mean_usefulness: 4.0, grounding_rate: 0.9 }),
|
||||
],
|
||||
DEFAULT_BRAINSTORM_THRESHOLDS,
|
||||
);
|
||||
expect(res.verdict).toBe('fail');
|
||||
expect(res.reasons.some((r) => r.includes('grounding'))).toBe(true);
|
||||
});
|
||||
|
||||
test('inconclusive when <2 fixtures usable', () => {
|
||||
const res = computeVerdict(
|
||||
[mkFixture({ pass_count: 5 })],
|
||||
DEFAULT_BRAINSTORM_THRESHOLDS,
|
||||
);
|
||||
expect(res.verdict).toBe('inconclusive');
|
||||
});
|
||||
|
||||
test('inconclusive when all fixtures have judge_failed', () => {
|
||||
const res = computeVerdict(
|
||||
[
|
||||
mkFixture({ judge_failed: true }),
|
||||
mkFixture({ judge_failed: true }),
|
||||
],
|
||||
DEFAULT_BRAINSTORM_THRESHOLDS,
|
||||
);
|
||||
expect(res.verdict).toBe('inconclusive');
|
||||
});
|
||||
|
||||
test('threshold overrides honored', () => {
|
||||
const res = computeVerdict(
|
||||
[
|
||||
mkFixture({ mean_distance: 0.3, mean_usefulness: 4.0, grounding_rate: 1.0 }),
|
||||
mkFixture({ mean_distance: 0.35, mean_usefulness: 4.0, grounding_rate: 1.0 }),
|
||||
],
|
||||
{ distance_min: 0.25, usefulness_min: 3.5, grounding_min: 1.0 },
|
||||
);
|
||||
expect(res.verdict).toBe('pass');
|
||||
});
|
||||
});
|
||||
|
||||
describe('readBrainstormEvalFixture', () => {
|
||||
const tmpDir = mkdtempSync(join(tmpdir(), 'gbrain-eval-brainstorm-'));
|
||||
|
||||
test('parses valid JSONL with one question per line', () => {
|
||||
const f = join(tmpDir, 'good.jsonl');
|
||||
writeFileSync(f, [
|
||||
'{"question": "why is X"}',
|
||||
'{"question": "what about Y"}',
|
||||
'',
|
||||
'{"question": "and Z"}',
|
||||
].join('\n'));
|
||||
const out = readBrainstormEvalFixture(f);
|
||||
expect(out.length).toBe(3);
|
||||
expect(out[0].question).toBe('why is X');
|
||||
expect(out[2].question).toBe('and Z');
|
||||
});
|
||||
|
||||
test('skips malformed JSON lines', () => {
|
||||
const f = join(tmpDir, 'mixed.jsonl');
|
||||
writeFileSync(f, [
|
||||
'{"question": "good one"}',
|
||||
'not json at all',
|
||||
'{"question": "another good"}',
|
||||
'{"no_question_field": "skipped"}',
|
||||
].join('\n'));
|
||||
const out = readBrainstormEvalFixture(f);
|
||||
expect(out.length).toBe(2);
|
||||
expect(out.map((f) => f.question)).toEqual(['good one', 'another good']);
|
||||
});
|
||||
|
||||
test('honors expected_far_prefixes when present', () => {
|
||||
const f = join(tmpDir, 'prefixes.jsonl');
|
||||
writeFileSync(f, JSON.stringify({
|
||||
question: 'cross-pollinate this',
|
||||
expected_far_prefixes: ['wiki/biology', 'wiki/hardware'],
|
||||
}));
|
||||
const out = readBrainstormEvalFixture(f);
|
||||
expect(out[0].expected_far_prefixes).toEqual(['wiki/biology', 'wiki/hardware']);
|
||||
});
|
||||
|
||||
test('throws on missing file', () => {
|
||||
expect(() => readBrainstormEvalFixture('/no/such/path.jsonl')).toThrow(/not found/);
|
||||
});
|
||||
|
||||
// Cleanup at end
|
||||
test('_cleanup_', () => {
|
||||
rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,104 @@
|
||||
/**
|
||||
* v0.37.0 (D9 / D4) — dream cycle hook: synthesize phase MUST skip pages
|
||||
* with `mode: lsd` frontmatter (noise-by-design). Pinned via the same
|
||||
* `isDreamOutput` helper that drives the self-consumption guard.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
isDreamOutput,
|
||||
isLsdOutput,
|
||||
isBrainstormOutput,
|
||||
} from '../../src/core/cycle/transcript-discovery.ts';
|
||||
|
||||
const LSD_FRONTMATTER = `---
|
||||
title: "LSD: why are AI coding tools converging on the same UX?"
|
||||
mode: lsd
|
||||
generated_at: 2026-05-19T12:00:00Z
|
||||
question: "why are AI coding tools converging on the same UX?"
|
||||
---
|
||||
|
||||
# Some inverted-judge idea here.
|
||||
`;
|
||||
|
||||
const BRAINSTORM_FRONTMATTER = `---
|
||||
title: "Brainstorm: lab automation"
|
||||
mode: brainstorm
|
||||
generated_at: 2026-05-19T12:00:00Z
|
||||
question: "what's the bottleneck on lab automation?"
|
||||
---
|
||||
|
||||
# Some defensible idea here.
|
||||
`;
|
||||
|
||||
const DREAM_OUTPUT_FRONTMATTER = `---
|
||||
title: "Dream: monthly synthesis"
|
||||
dream_generated: true
|
||||
dream_cycle_date: 2026-05-01
|
||||
---
|
||||
|
||||
Body.
|
||||
`;
|
||||
|
||||
const REGULAR_TRANSCRIPT = `# Some meeting transcript
|
||||
|
||||
Person A: We should talk about lab automation.
|
||||
Person B: Agreed.
|
||||
`;
|
||||
|
||||
describe('v0.37.0 — LSD frontmatter skip in dream-cycle', () => {
|
||||
test('LSD page is detected by isLsdOutput', () => {
|
||||
expect(isLsdOutput(LSD_FRONTMATTER)).toBe(true);
|
||||
});
|
||||
|
||||
test('brainstorm page is NOT detected by isLsdOutput', () => {
|
||||
expect(isLsdOutput(BRAINSTORM_FRONTMATTER)).toBe(false);
|
||||
});
|
||||
|
||||
test('regular transcript is NOT detected by isLsdOutput', () => {
|
||||
expect(isLsdOutput(REGULAR_TRANSCRIPT)).toBe(false);
|
||||
});
|
||||
|
||||
test('brainstorm page is detected by isBrainstormOutput', () => {
|
||||
expect(isBrainstormOutput(BRAINSTORM_FRONTMATTER)).toBe(true);
|
||||
});
|
||||
|
||||
test('LSD page is NOT detected by isBrainstormOutput', () => {
|
||||
expect(isBrainstormOutput(LSD_FRONTMATTER)).toBe(false);
|
||||
});
|
||||
|
||||
test('isDreamOutput SKIPS LSD pages (D4 noise-by-design)', () => {
|
||||
expect(isDreamOutput(LSD_FRONTMATTER)).toBe(true);
|
||||
});
|
||||
|
||||
test('isDreamOutput still skips legitimate dream output', () => {
|
||||
expect(isDreamOutput(DREAM_OUTPUT_FRONTMATTER)).toBe(true);
|
||||
});
|
||||
|
||||
test('isDreamOutput does NOT skip brainstorm pages (they are user-validated content)', () => {
|
||||
expect(isDreamOutput(BRAINSTORM_FRONTMATTER)).toBe(false);
|
||||
});
|
||||
|
||||
test('isDreamOutput does NOT skip regular transcripts', () => {
|
||||
expect(isDreamOutput(REGULAR_TRANSCRIPT)).toBe(false);
|
||||
});
|
||||
|
||||
test('--unsafe-bypass-dream-guard does NOT bypass LSD skip', () => {
|
||||
// Bypass is for self-consumption recovery only; LSD must always be skipped.
|
||||
expect(isDreamOutput(LSD_FRONTMATTER, true)).toBe(true);
|
||||
});
|
||||
|
||||
test('--unsafe-bypass-dream-guard DOES bypass dream output skip', () => {
|
||||
expect(isDreamOutput(DREAM_OUTPUT_FRONTMATTER, true)).toBe(false);
|
||||
});
|
||||
|
||||
test('LSD marker tolerates double-quoted value', () => {
|
||||
const dq = LSD_FRONTMATTER.replace('mode: lsd', 'mode: "lsd"');
|
||||
expect(isLsdOutput(dq)).toBe(true);
|
||||
});
|
||||
|
||||
test('LSD marker tolerates single-quoted value', () => {
|
||||
const sq = LSD_FRONTMATTER.replace('mode: lsd', "mode: 'lsd'");
|
||||
expect(isLsdOutput(sq)).toBe(true);
|
||||
});
|
||||
});
|
||||
Vendored
+5
@@ -0,0 +1,5 @@
|
||||
{"question": "why are AI coding tools converging on the same UX?"}
|
||||
{"question": "the unspoken assumption in early-stage venture pricing"}
|
||||
{"question": "what's the real bottleneck on lab automation"}
|
||||
{"question": "how should we think about ML model fine-tuning in 2026"}
|
||||
{"question": "what makes a developer experience feel inevitable"}
|
||||
@@ -129,6 +129,11 @@ const REQUIRED_BOOTSTRAP_COVERAGE: ForwardReference[] = [
|
||||
{ kind: 'column', table: 'sources', column: 'archived' },
|
||||
{ kind: 'column', table: 'sources', column: 'archived_at' },
|
||||
{ kind: 'column', table: 'sources', column: 'archive_expires_at' },
|
||||
// v0.37.0 (v79) — forward-referenced by `CREATE INDEX
|
||||
// pages_last_retrieved_at_idx ON pages (last_retrieved_at)`. Pre-v79 brains
|
||||
// have pages without this column; bootstrap adds it before SCHEMA_SQL
|
||||
// replay creates the index.
|
||||
{ kind: 'column', table: 'pages', column: 'last_retrieved_at' },
|
||||
];
|
||||
|
||||
test('applyForwardReferenceBootstrap covers every forward reference declared in REQUIRED_BOOTSTRAP_COVERAGE', async () => {
|
||||
|
||||
Reference in New Issue
Block a user