diff --git a/src/core/context/retrieval-reflex.ts b/src/core/context/retrieval-reflex.ts index d1cb8e3af..74184b358 100644 --- a/src/core/context/retrieval-reflex.ts +++ b/src/core/context/retrieval-reflex.ts @@ -57,7 +57,7 @@ export interface ResolvePointersOpts { priorContextText?: string; } -interface PageRow { +export interface PageRow { slug: string; title: string; type: string | null; @@ -198,8 +198,11 @@ function displayForRow(row: PageRow, displayByNorm: Map): string * otherwise strip takes/private-fact fences from the body (the same boundary * get_page applies to untrusted readers) and take the first sentence. Never * returns raw compiled_truth. + * + * Exported for the MEMORY_VERBS v1 entity card (verbs/entity-card.ts) — the + * card's `summary` field runs through THIS boundary, not a parallel one. */ -function safeSynopsis(row: PageRow): string { +export function safeSynopsis(row: PageRow): string { const fmSummary = row.frontmatter?.summary; if (typeof fmSummary === 'string' && fmSummary.trim()) { return clip(collapse(fmSummary), SYNOPSIS_MAX); diff --git a/src/core/facts/fence-write.ts b/src/core/facts/fence-write.ts index febd2cb12..188f9cbcb 100644 --- a/src/core/facts/fence-write.ts +++ b/src/core/facts/fence-write.ts @@ -64,6 +64,12 @@ export interface FenceInputFact { /** Defaults to 1.0 when undefined (matches engine.insertFact behavior). */ confidence?: number; validFrom?: Date; + /** + * MEMORY_VERBS v1 (c5): remember's ttl → valid_until. Date-only in the + * fence cell; the DB column derives from it on the stamp step. + * Undefined/null = never expires (pre-v1 behavior unchanged). + */ + validUntil?: Date | null; embedding: Float32Array | null; sessionId: string | null; } @@ -223,7 +229,10 @@ export async function writeFactsToFence( visibility: f.visibility, notability: f.notability ?? 'medium', validFrom: validFromStr, - validUntil: undefined, + // MEMORY_VERBS v1 (c5): remember's ttl threads through to the fence + // cell — was hard-coded undefined, which silently dropped expiry on + // this path. extractFactsFromFenceText derives the DB column from it. + validUntil: f.validUntil ? f.validUntil.toISOString().slice(0, 10) : undefined, source: f.source, context: f.context ?? undefined, }); diff --git a/src/core/facts/write-single.ts b/src/core/facts/write-single.ts new file mode 100644 index 000000000..afe5a329b --- /dev/null +++ b/src/core/facts/write-single.ts @@ -0,0 +1,227 @@ +/** + * MEMORY_VERBS v1 — `writeSingleFact`: the zero-LLM single-fact write seam + * behind the `remember` verb [E1]. + * + * `runFactsPipeline` is extraction-first (LLM-gated in extract.ts) and cannot + * back a verb whose fact arrives pre-formed. This module reuses the pipeline's + * post-extraction stages directly: resolve → dedup (embedding cosine, same + * 0.95 threshold) → fence-first write (markdown durability) with the same + * legacy DB-only fallbacks (thin-client, unparented, stub-guard). + * + * Supersession [X1, frozen as implementation-defined]: minimal deterministic + * rule, zero LLM — when the top dedup candidate scores >= threshold with the + * SAME kind but DIFFERENT text, the new fact supersedes it (a near-duplicate + * with changed content is an update: "X at Acme" → "X left Acme"). Same text + * → plain duplicate (existing id returned, nothing written). + * + * Degradation (documented in the protocol doc): with no embedding provider, + * dedup/supersession are skipped on the fence path and near-duplicates may + * insert — `degraded_dedup: true` tells the caller. + * + * Provenance (c6): callers pass free-text provenance which lands on + * `NewFact.source` verbatim — this seam deliberately does NOT take a + * FactsBackstopCtx (whose `source` union is pipeline-internal). + */ + +import type { BrainEngine, FactInsertStatus, NewFact } from '../engine.ts'; + +const DEDUP_THRESHOLD = 0.95; +const DEDUP_CANDIDATE_LIMIT = 5; + +export interface SingleFactInput { + fact: string; + /** Free-text attribution, stored verbatim as the fact's `source`. */ + provenance: string; + kind?: NewFact['kind']; + /** Free-form entity ref; canonicalized via resolveEntitySlug. */ + entity?: string | null; + /** Facts-layer default 'private'; the remember VERB passes 'world' [F2]. */ + visibility?: 'private' | 'world'; + validUntil?: Date | null; + sessionId?: string | null; + confidence?: number; +} + +export interface SingleFactResult { + id: number; + status: FactInsertStatus; + entity_slug: string | null; + valid_until: Date | null; + /** True when no embedding provider — dedup/supersession skipped. */ + degraded_dedup: boolean; +} + +export async function writeSingleFact( + engine: BrainEngine, + sourceId: string, + input: SingleFactInput, +): Promise { + const { resolveEntitySlug } = await import('../entities/resolve.ts'); + const { cosineSimilarity } = await import('./classify.ts'); + const { writeFactsToFence, lookupSourceLocalPath } = await import('./fence-write.ts'); + const { isAvailable, embedOne } = await import('../ai/gateway.ts'); + + const factText = input.fact.trim(); + const kind = input.kind ?? 'fact'; + const visibility = input.visibility ?? 'private'; + const validUntil = input.validUntil ?? null; + + const resolvedSlug = input.entity + ? ((await resolveEntitySlug(engine, sourceId, input.entity)) ?? input.entity) + : null; + + // Embedding (NOT an LLM call): powers dedup + downstream recall. Fail-soft — + // a missing/failing provider degrades dedup, never the write. + let embedding: Float32Array | null = null; + let degradedDedup = false; + if (isAvailable('embedding')) { + try { + embedding = await embedOne(factText); + } catch { + degradedDedup = true; + } + } else { + degradedDedup = true; + } + + // Dedup + supersession decision (same candidates + threshold as the pipeline). + let supersedeId: number | null = null; + if (resolvedSlug && embedding) { + const candidates = await engine.findCandidateDuplicates(sourceId, resolvedSlug, factText, { + embedding, + k: DEDUP_CANDIDATE_LIMIT, + }); + let top: (typeof candidates)[number] | null = null; + let topScore = -1; + for (const c of candidates) { + if (!c.embedding) continue; + const s = cosineSimilarity(embedding, c.embedding); + if (s > topScore) { + topScore = s; + top = c; + } + } + if (top && topScore >= DEDUP_THRESHOLD) { + const textDiffers = collapse(top.fact) !== collapse(factText); + if (top.kind === kind && textDiffers) { + supersedeId = top.id; // X1: near-duplicate with changed content = update + } else { + return { + id: top.id, + status: 'duplicate', + entity_slug: resolvedSlug, + valid_until: top.valid_until ?? null, + degraded_dedup: false, + }; + } + } + } + + const newFact: NewFact = { + fact: factText, + kind, + entity_slug: resolvedSlug, + visibility, + source: input.provenance, + source_session: input.sessionId ?? null, + confidence: input.confidence ?? 1.0, + valid_until: validUntil, + embedding, + }; + + // Fence-first write (markdown durability — same policy as the pipeline): + // requires a resolved, prefixed entity slug and a local_path. Everything + // else takes the legacy DB-only insertFact path, which also handles the + // supersedeId bookkeeping engine-side. + const localPath = resolvedSlug ? await lookupSourceLocalPath(engine, sourceId) : null; + const fenceable = resolvedSlug !== null && localPath !== null; + + if (fenceable) { + const result = await writeFactsToFence( + engine, + { sourceId, localPath, slug: resolvedSlug }, + [ + { + fact: factText, + kind, + notability: 'medium', + source: input.provenance, + context: null, + visibility, + confidence: input.confidence ?? 1.0, + validFrom: new Date(), + validUntil, + embedding, + sessionId: input.sessionId ?? null, + }, + ], + ); + + if (result.fenceWriteFailed) { + // Parse-validate rejected the .tmp (quarantined). Hard failure — do NOT + // fall through to a DB row whose fence is broken (pipeline policy). + throw new Error( + `facts fence write failed for ${resolvedSlug} — .tmp quarantined; see the facts write-failure JSONL log`, + ); + } + if (!result.stubGuardBlocked && !result.legacyFallback) { + const newId = result.ids[0]; + if (supersedeId !== null && newId !== undefined) { + await expireSuperseded(engine, supersedeId, newId); + return { + id: newId, + status: 'superseded', + entity_slug: resolvedSlug, + valid_until: validUntil, + degraded_dedup: degradedDedup, + }; + } + return { + id: newId, + status: 'inserted', + entity_slug: resolvedSlug, + valid_until: validUntil, + degraded_dedup: degradedDedup, + }; + } + // stubGuardBlocked / defensive legacyFallback → DB-only path below. + } + + const inserted = await engine.insertFact(newFact, { + source_id: sourceId, + ...(supersedeId !== null ? { supersedeId } : {}), + }); // gbrain-allow-direct-insert: writeSingleFact legacy path for unparented / thin-client / stub-guarded facts (mirrors the pipeline's fallback buckets) + + return { + id: inserted.id, + status: supersedeId !== null ? 'superseded' : inserted.status, + entity_slug: resolvedSlug, + valid_until: validUntil, + degraded_dedup: degradedDedup, + }; +} + +/** + * Fence-path supersession bookkeeping: expire the old row through the fence + * (strikethrough + valid_until, the same surface `forget` uses) and link + * `superseded_by` for the audit trail. Both steps best-effort — the new fact + * is already durably written; a partial supersede is an audit gap, not data + * loss. + */ +async function expireSuperseded(engine: BrainEngine, oldId: number, newId: number): Promise { + try { + const { forgetFactInFence } = await import('./forget.ts'); + await forgetFactInFence(engine, oldId, { reason: `superseded by fact #${newId}` }); + } catch { + /* best-effort */ + } + try { + await engine.executeRaw(`UPDATE facts SET superseded_by = $1 WHERE id = $2`, [newId, oldId]); + } catch { + /* best-effort */ + } +} + +function collapse(s: string): string { + return s.replace(/\s+/g, ' ').trim().toLowerCase(); +} diff --git a/src/core/operations-descriptions.ts b/src/core/operations-descriptions.ts index 7cf77ae07..b43c70d81 100644 --- a/src/core/operations-descriptions.ts +++ b/src/core/operations-descriptions.ts @@ -74,7 +74,9 @@ export const SEARCH_DESCRIPTION = "without needing a search term. " + "For code-symbol questions (callers, callees, definitions, blast radius), use " + "code_callers / code_callees / code_def / code_refs instead — those return " + - "structural graph data, not text chunks."; + "structural graph data, not text chunks. " + + "For agent memory reads (saved facts + budget-packed retrieval), prefer the " + + "`recall` verb."; // ────────────────────────────────────────────────────────────────────────────── // v0.32.6 — contradiction probe MCP surface (M3) diff --git a/src/core/operations.ts b/src/core/operations.ts index aa6f41e41..ae7ad6b15 100644 --- a/src/core/operations.ts +++ b/src/core/operations.ts @@ -24,6 +24,10 @@ import { getContentFlag } from './quarantine.ts'; import { bumpLastRetrievedAt } from './last-retrieved.ts'; import { isSearchMode } from './search/mode.ts'; import { stampEvidence } from './search/evidence.ts'; +import { packToBudget, estimateTokens, resultTokens } from './search/token-budget.ts'; +import { isAvailable } from './ai/gateway.ts'; +import { verbOperations, MEMORY_VERBS_VERSION } from './verbs.ts'; +export { MEMORY_VERBS_VERSION }; import type { SearchResult } from './types.ts'; import { CJK_SLUG_CHARS } from './cjk.ts'; import * as db from './db.ts'; @@ -69,10 +73,27 @@ export type ErrorCode = | 'rate_limited' // v0.31: gateway rate-limit upstream | 'extraction_failed' // v0.31: facts extractor failed (refusal, parse, abort) | 'fact_not_found' // v0.31: forget_fact / recall on unknown id + // MEMORY_VERBS v1 protocol codes (frozen — docs/protocol/MEMORY_VERBS_v1.md). + // Coarse on purpose: codes are for branching (configure/retry vs caller bug + // vs server bug); the freeform `detail` field carries specifics. + | 'not_found' // verb-level: unknown fact id (forget) + | 'scope_denied' // verb-level: OAuth scope / trust-boundary refusal + | 'provenance_required' // remember: provenance missing or empty + | 'unavailable' // a required dependency cannot serve (no API key, gateway down, model refusal) + | 'budget_unsatisfiable' // RESERVED in v1 — schema-listed, never returned // eslint-disable-next-line @typescript-eslint/ban-types | (string & {}); // OPEN union for forward-compat (eE7 / D13) export class OperationError extends Error { + /** + * MEMORY_VERBS v1: verb handlers set `protocolVersion` (=1) and may set + * `detail` (freeform specifics, e.g. which dependency failed). Both are + * additive — non-verb ops never set them and their envelopes are unchanged + * (undefined keys drop out of JSON.stringify). + */ + public detail?: string; + public protocolVersion?: number; + constructor( public code: ErrorCode, message: string, @@ -89,10 +110,29 @@ export class OperationError extends Error { message: this.message, suggestion: this.suggestion, docs: this.docs, + detail: this.detail, + protocol_version: this.protocolVersion, }; } } +/** + * MEMORY_VERBS v1 error constructor. Every verb error carries a populated + * `suggestion` (problem + cause + fix — agents read it and self-correct; + * conformance asserts non-empty) and `protocol_version: 1`. + */ +export function verbError( + code: ErrorCode, + message: string, + suggestion: string, + detail?: string, +): OperationError { + const e = new OperationError(code, message, suggestion); + e.protocolVersion = MEMORY_VERBS_VERSION; + if (detail !== undefined) e.detail = detail; + return e; +} + // --- Upload validators (Fix 1 / B5 / H5 / M4) --- /** @@ -581,6 +621,22 @@ export interface Operation { */ scope?: 'read' | 'write' | 'admin' | 'sources_admin' | 'users_admin'; localOnly?: boolean; + /** + * MEMORY_VERBS v1: marks the five frozen protocol verbs (recall, remember, + * entity, synthesize, forget). `gbrain serve --surface verbs` exposes + * EXACTLY the ops with `verb: true`; `full` (default) exposes everything. + */ + verb?: boolean; + /** + * MCP ToolAnnotations passthrough (SDK 1.29+). Emitted by buildToolDefs + * ONLY when set — existing tools keep byte-identical definitions. + */ + annotations?: { + title?: string; + readOnlyHint?: boolean; + destructiveHint?: boolean; + idempotentHint?: boolean; + }; cliHints?: { name?: string; /** @@ -3627,7 +3683,7 @@ const sources_status: Operation = { const extract_facts: Operation = { name: 'extract_facts', description: - 'v0.31: extract personal-knowledge facts (events, preferences, commitments, beliefs) from a conversation turn into the per-source hot memory. Sanitizes turn_text via INJECTION_PATTERNS, calls Haiku to extract structured claims, runs the cosine fast-path + classifier dedup pipeline, INSERTs into facts. Returns counts by status. Skips extraction when the turn is dream-generated content (anti-loop).', + 'v0.31: extract personal-knowledge facts (events, preferences, commitments, beliefs) from a conversation turn into the per-source hot memory. Sanitizes turn_text via INJECTION_PATTERNS, calls Haiku to extract structured claims, runs the cosine fast-path + classifier dedup pipeline, INSERTs into facts. Returns counts by status. Skips extraction when the turn is dream-generated content (anti-loop). For agent memory writes of a SINGLE already-formed fact, prefer the `remember` verb (zero LLM, mandatory provenance).', params: { turn_text: { type: 'string', required: true, description: 'The user message or page body to extract facts from. Sanitized via INJECTION_PATTERNS before the LLM call.' }, session_id: { type: 'string', description: 'Opaque session id (e.g. topic-id from MCP _meta.session_id, or CLI --session). Stored on each fact for the recall --session filter. Not an auth surface.' }, @@ -3683,18 +3739,22 @@ const extract_facts: Operation = { const recall: Operation = { name: 'recall', description: - 'v0.31: query per-source hot memory (facts table). Filters by entity / since / session. Remote callers see only visibility=world facts. Returns most-recent first. v0.32 adds optional include_pending to return pending_consolidation_count alongside facts in one round trip.', + 'MEMORY VERB (v1): retrieve saved facts/snippets — the protocol read verb. Filters hot-memory facts by entity / since / session_id; pass `query` to ALSO run hybrid search over pages (results[] arm); pass `budget_tokens` for server-side packing (response reports budget_used + dropped_count — never trims client-side). Remote callers see visibility=world facts only. Routing: for ONE known person/company/project card use `entity` (zero LLM); for broad questions needing reasoning use `synthesize` (expensive). Branch on structured fields (status/kind/evidence), never on prose. Every response carries protocol_version.', params: { entity: { type: 'string', description: 'Entity slug (canonical). Returns facts about this entity newest first.' }, - since: { type: 'string', description: 'ISO datetime or duration shorthand (e.g. "8 hours ago"). Returns facts created since.' }, + query: { type: 'string', description: 'MEMORY_VERBS v1: free-text retrieval over pages (hybrid search arm). Response adds results[] (slug, title, chunk, evidence, create_safety, provenance). Combinable with entity (both arms run). Degrades to keyword-only search when no embedding provider is configured (search_degraded notes it; never an error).' }, + budget_tokens: { type: 'number', description: 'MEMORY_VERBS v1: server-side token budget (char/4 estimate). Facts pack first, then results. Response adds budget_tokens, budget_used, dropped_count.' }, + since: { type: 'string', description: 'ISO 8601 datetime or duration shorthand (e.g. "8 hours ago"). Filters the FACTS arm only.' }, session_id: { type: 'string', description: 'Source session id (e.g. topic-A). Returns facts captured in that session.' }, include_expired: { type: 'boolean', description: 'When true, include expired_at IS NOT NULL rows. Default false.' }, supersessions: { type: 'boolean', description: 'When true, return only the supersession audit log (expired_at + superseded_by both set).' }, - limit: { type: 'number', description: 'Max rows to return. Default 50, cap 100.' }, + limit: { type: 'number', description: 'Per-arm cap: max fact rows AND max search results. Default 50, cap 100.' }, grep: { type: 'string', description: 'Substring filter on fact text (case-insensitive). Applied client-side after recall.' }, include_pending: { type: 'boolean', description: 'v0.32: when true, response includes pending_consolidation_count (facts not yet promoted to takes by the dream-cycle consolidate phase). One round trip; backward-compatible (field omitted when false).' }, }, scope: 'read', + verb: true, + annotations: { title: 'recall (memory read)', readOnlyHint: true }, handler: async (ctx, p) => { const sourceId = ctx.sourceId ?? 'default'; const limit = typeof p.limit === 'number' ? p.limit : 50; @@ -3765,8 +3825,58 @@ const recall: Operation = { } } + // ── MEMORY_VERBS v1 — query arm (G1B superset). Hybrid search over pages + // when `query` is present; degrades to keyword-only with a note (never an + // error) when no embedding provider is configured [F-B]. + const queryText = typeof p.query === 'string' && p.query.trim().length > 0 ? p.query.trim() : null; + const budgetTokens = + typeof p.budget_tokens === 'number' && Number.isFinite(p.budget_tokens) && p.budget_tokens > 0 + ? Math.floor(p.budget_tokens) + : null; + + let searchResults: SearchResult[] = []; + let searchDegraded: string | undefined; + if (queryText) { + const searchScope = sourceScopeOpts(ctx); + if (!isAvailable('embedding')) { + const raw = await ctx.engine.searchKeyword(queryText, { limit, ...searchScope }); + searchResults = dedupResults(raw); + stampEvidenceSafe(searchResults); + await stampContentFlags(ctx.engine, searchResults); + searchDegraded = 'keyword_only_no_embedding_provider'; + } else { + searchResults = await hybridSearchCached(ctx.engine, queryText, { + limit, + expansion: false, + ...searchScope, + }); + } + bumpLastRetrievedAt(ctx.engine, searchResults.map(r => r.page_id)); + } + + // ── MEMORY_VERBS v1 — server-side budget packing. Facts pack first (cheap, + // high-precision one-liners, per-arm limit-capped so starvation is bounded), + // then search results take the remainder. packToBudget treats budget<=0 as + // a no-op, so an exhausted remainder must drop explicitly. + let packedFacts = rows; + let packedResults = searchResults; + let budgetUsed: number | undefined; + let droppedCount: number | undefined; + if (budgetTokens !== null) { + const factsPack = packToBudget(rows, r => estimateTokens(r.fact), budgetTokens); + packedFacts = factsPack.items; + const remaining = budgetTokens - factsPack.meta.used; + const resultsPack = + remaining > 0 + ? packToBudget(searchResults, resultTokens, remaining) + : { items: [] as SearchResult[], meta: { budget: 0, used: 0, dropped: searchResults.length, kept: 0 } }; + packedResults = resultsPack.items; + budgetUsed = factsPack.meta.used + resultsPack.meta.used; + droppedCount = factsPack.meta.dropped + resultsPack.meta.dropped; + } + return { - facts: rows.map(r => ({ + facts: packedFacts.map(r => ({ id: r.id, fact: r.fact, kind: r.kind, @@ -3786,9 +3896,33 @@ const recall: Operation = { source_session: r.source_session, confidence: r.confidence, created_at: r.created_at.toISOString(), + // MEMORY_VERBS v1 additive fields (G1B). `fact_id` is the opaque + // STRING id the `forget` verb accepts (legacy numeric `id` stays for + // pre-v1 consumers — legacy fields are frozen byte-equal). `provenance` + // is the protocol name for the stored source attribution. + fact_id: String(r.id), + provenance: r.source, })), - total: rows.length, + total: packedFacts.length, ...(pending_consolidation_count !== undefined ? { pending_consolidation_count } : {}), + // MEMORY_VERBS v1 envelope (G1B superset — additive on every response). + protocol_version: MEMORY_VERBS_VERSION, + ...(queryText + ? { + results: packedResults.map(r => ({ + slug: r.slug, + title: r.title, + chunk: r.chunk_text, + evidence: r.evidence, + create_safety: r.create_safety, + provenance: r.slug, + })), + ...(searchDegraded ? { search_degraded: searchDegraded } : {}), + } + : {}), + ...(budgetTokens !== null + ? { budget_tokens: budgetTokens, budget_used: budgetUsed, dropped_count: droppedCount } + : {}), }; }, }; @@ -3849,6 +3983,65 @@ function parseSinceParam(raw: unknown): Date | null { return null; } +/** + * MEMORY_VERBS v1 — parse the `remember` verb's `ttl` param into a + * `valid_until` Date. Sibling of parseSinceParam, pointed FORWARD. + * + * Accepted forms (frozen in docs/protocol/MEMORY_VERBS_v1.md): + * - relative duration shorthand: '30d', '12h', '45m', '90s' (also + * spelled-out: '30 days', '12 hours') → now + duration + * - absolute ISO 8601 date or datetime: '2026-07-12', '2026-07-12T00:00:00Z' + * + * Explicitly REJECTED with a self-correcting suggestion: ISO-8601 duration + * syntax ('P30D', 'PT12H') — agents that read "ISO 8601" as durations get a + * fix, not a mystery. Returns null for null/undefined/empty (= never expires). + * Throws verbError('invalid_params') on anything unparseable. + */ +export function parseTtlParam(raw: unknown): Date | null { + if (raw == null) return null; + if (typeof raw !== 'string') { + throw verbError( + 'invalid_params', + `ttl must be a string, got ${typeof raw}.`, + 'Pass a duration like "30d" or "12h", or an absolute ISO 8601 timestamp like "2026-07-12T00:00:00Z".', + ); + } + const s = raw.trim(); + if (!s) return null; + + // ISO-8601 DURATION syntax is a documented trap — reject with the fix. + if (/^P(T|\d)/i.test(s) && /^P(?:\d+[YMWD])*(?:T(?:\d+[HMS])+)?$/i.test(s)) { + throw verbError( + 'invalid_params', + `ttl "${s}" looks like an ISO-8601 duration, which is not accepted.`, + `Use the shorthand form instead (e.g. "${s.replace(/^PT?/i, '').toLowerCase()}" style: "30d", "12h"), or an absolute ISO 8601 expiry timestamp.`, + ); + } + + // Relative duration shorthand → now + duration. + const dur = s.match(/^(\d+)\s*(s|sec|seconds?|m|min|minutes?|h|hr|hours?|d|days?)$/i); + if (dur) { + const n = parseInt(dur[1], 10); + const unit = dur[2].toLowerCase(); + const ms = + unit.startsWith('s') ? n * 1000 : + unit.startsWith('m') ? n * 60 * 1000 : + unit.startsWith('h') ? n * 60 * 60 * 1000 : + n * 24 * 60 * 60 * 1000; + return new Date(Date.now() + ms); + } + + // Absolute ISO 8601 date or datetime. + const iso = Date.parse(s); + if (Number.isFinite(iso)) return new Date(iso); + + throw verbError( + 'invalid_params', + `Cannot parse ttl "${s}".`, + 'Pass a duration like "30d" or "12h", or an absolute ISO 8601 timestamp like "2026-07-12T00:00:00Z". Omit ttl for a fact that never expires.', + ); +} + // ────────────────────────────────────────────────────────────────────────────── // v0.34 Cathedral III — code-intelligence ops (MCP-exposed). // @@ -4807,6 +5000,10 @@ const run_skillopt: Operation = { }; export const operations: Operation[] = [ + // MEMORY_VERBS v1 (Cathedral 1) — remember/entity/synthesize/forget live in + // verbs.ts; the fifth verb is the extended `recall` op below. Spread first + // so `--surface verbs` agents see them at the top of the tool list. + ...verbOperations, // Page CRUD get_page, put_page, delete_page, list_pages, // v0.26.5 destructive-guard ops (page-level soft-delete + recovery + admin purge) diff --git a/src/core/search/token-budget.ts b/src/core/search/token-budget.ts index d945a54aa..85af26f3b 100644 --- a/src/core/search/token-budget.ts +++ b/src/core/search/token-budget.ts @@ -63,50 +63,66 @@ export interface TokenBudgetMeta { } /** - * Greedy top-down budget enforcement. Walks the input in order, accumulates - * token costs, and stops as soon as adding the next result would exceed - * the budget. Results are NOT re-ranked — caller's order is preserved. + * Generic greedy top-down budget packer (v1 memory-verbs protocol). Walks + * the input in order, accumulates per-item costs via the caller-supplied + * cost function, and stops as soon as adding the next item would exceed + * the budget. Items are NOT re-ranked — caller's order is preserved. * * Edge cases (all preserve the pre-v0.32 contract): * - budget undefined / <= 0: returns input unchanged; dropped=0, kept=N. - * - First result alone exceeds budget: returns []; dropped=N, kept=0. + * - First item alone exceeds budget: returns []; dropped=N, kept=0. * (Intentionally strict: the caller asked for a hard cap.) * - Input empty: returns []; budget unused. */ +export function packToBudget( + items: T[], + cost: (item: T) => number, + budget: number | undefined, +): { items: T[]; meta: TokenBudgetMeta } { + const safeBudget = typeof budget === 'number' && budget > 0 ? budget : 0; + + if (safeBudget === 0 || items.length === 0) { + return { + items, + meta: { + budget: safeBudget, + used: items.reduce((acc, it) => acc + cost(it), 0), + dropped: 0, + kept: items.length, + }, + }; + } + + const kept: T[] = []; + let used = 0; + for (const it of items) { + const c = cost(it); + if (used + c > safeBudget) break; + kept.push(it); + used += c; + } + + return { + items: kept, + meta: { + budget: safeBudget, + used, + dropped: items.length - kept.length, + kept: kept.length, + }, + }; +} + +/** + * Search-pipeline budget enforcement — a thin wrapper over packToBudget + * with the SearchResult cost model (title + chunk_text). Behavior is + * byte-identical to the pre-refactor implementation; pinned by + * test/token-budget.test.ts. + */ export function enforceTokenBudget( results: SearchResult[], budget: number | undefined, ): { results: SearchResult[]; meta: TokenBudgetMeta } { - const safeBudget = typeof budget === 'number' && budget > 0 ? budget : 0; - - if (safeBudget === 0 || results.length === 0) { - return { - results, - meta: { - budget: safeBudget, - used: results.reduce((acc, r) => acc + resultTokens(r), 0), - dropped: 0, - kept: results.length, - }, - }; - } - - const kept: SearchResult[] = []; - let used = 0; - for (const r of results) { - const cost = resultTokens(r); - if (used + cost > safeBudget) break; - kept.push(r); - used += cost; - } - - return { - results: kept, - meta: { - budget: safeBudget, - used, - dropped: results.length - kept.length, - kept: kept.length, - }, - }; + const { items, meta } = packToBudget(results, resultTokens, budget); + return { results: items, meta }; } diff --git a/src/core/think/index.ts b/src/core/think/index.ts index 4508631b1..765e63664 100644 --- a/src/core/think/index.ts +++ b/src/core/think/index.ts @@ -139,6 +139,13 @@ export interface ThinkResult { * pre-existing/test `ThinkResult` literals → treated as persistable (back-compat). */ synthesisOk?: boolean; + /** + * MEMORY_VERBS v1 [E2] — gateway token usage for the synthesis call(s), + * summed across rounds. Best-effort: null when no LLM ran (graceful stub), + * when a test client returns no usage, or when a provider omits accounting. + * The synthesize verb maps this to its frozen `cost` block. + */ + usage?: { input_tokens: number; output_tokens: number } | null; /** Only set when --save was true and the caller persisted a synthesis page. */ savedSlug?: string; /** Diagnostics for `--explain` callers (CLI surface for v0.29). */ @@ -421,6 +428,9 @@ export async function runThink( // sentinel, which is non-JSON) and on the no-client early return below; the final // return ANDs it with a non-empty-answer check (catches valid-but-empty JSON). let synthesisOk = true; + // [E2] best-effort usage aggregation across synthesis calls (single-pass in + // v0.28+, but summed so the round loop inherits it when gap-fill lands). + let usage: { input_tokens: number; output_tokens: number } | null = null; let response: ThinkResponse; if (opts.stubResponse) { response = opts.stubResponse; @@ -453,6 +463,7 @@ export async function runThink( rounds: 0, warnings, synthesisOk: false, // #1698: no LLM ran — never persist this + usage: null, // [E2] no LLM ran — no accounting diagnostics: { pagesFromHybrid: gather.diagnostics.pagesFromHybrid, takesFromKeyword: gather.diagnostics.takesFromKeyword, @@ -467,6 +478,17 @@ export async function runThink( system: systemPrompt, messages: [{ role: 'user', content: userMessage }], }); + // [E2] capture usage when the message carries it (test-injected clients + // and providers without accounting leave it null). + const u = (result as { usage?: { input_tokens?: number; output_tokens?: number } }).usage; + if (u && typeof u.input_tokens === 'number' && typeof u.output_tokens === 'number') { + // Single synthesis call in v0.28+; when gap-driven rounds land, sum here. + const prev = usage as { input_tokens: number; output_tokens: number } | null; + usage = { + input_tokens: (prev?.input_tokens ?? 0) + u.input_tokens, + output_tokens: (prev?.output_tokens ?? 0) + u.output_tokens, + }; + } const block = result.content.find(b => b.type === 'text'); const text = block && 'text' in block ? block.text : ''; const parsed = tryParseJSON(text); @@ -511,6 +533,7 @@ export async function runThink( // #1698: persistable only when a real synthesis produced a non-empty answer. // ANDs the not-JSON/sentinel flag with a content check (catches valid-but-empty JSON). synthesisOk: synthesisOk && response.answer.trim().length > 0, + usage, diagnostics: { pagesFromHybrid: gather.diagnostics.pagesFromHybrid, takesFromKeyword: gather.diagnostics.takesFromKeyword, diff --git a/src/core/verbs.ts b/src/core/verbs.ts new file mode 100644 index 000000000..f3440f53c --- /dev/null +++ b/src/core/verbs.ts @@ -0,0 +1,551 @@ +/** + * MEMORY_VERBS v1 — the frozen memory protocol verbs (Cathedral 1). + * + * Four first-class Operations (`remember`, `entity`, `synthesize`, `forget`) + * that join the extended `recall` op (operations.ts) as the five-verb façade + * over the operation catalog. Frozen contract: docs/protocol/MEMORY_VERBS_v1.md + * — field names and semantics in v1 never change; additions are forever- + * additive; `protocol_version` rides every response; errors carry enumerated + * codes + populated `suggestion` (agents read it and self-correct). + * + * These are ordinary Operations: they inherit trust-boundary fail-closed + * semantics (ctx.remote), scope enforcement, and source isolation like every + * other op. `gbrain serve --surface verbs` exposes exactly the ops marked + * `verb: true`. + * + * Import-cycle note: operations.ts spreads these into its `operations` array + * at MODULE-EVAL time, so this file must be a RUNTIME LEAF — it may import + * operations.ts types (erased) but never its values statically. Handlers load + * verbError/parseTtlParam/sourceScopeOpts via dynamic import (the file's + * existing style), which resolves after both modules finish evaluating. + * MEMORY_VERBS_VERSION lives HERE (operations.ts imports it from us) for the + * same reason. Violating this reintroduces the TDZ crash on whichever module + * evaluates second. + */ + +import type { Operation } from './operations.ts'; + +/** Frozen protocol version for the MEMORY_VERBS v1 verb set. Single source of truth. */ +export const MEMORY_VERBS_VERSION = 1; + +export const VERB_NAMES = ['recall', 'remember', 'entity', 'synthesize', 'forget'] as const; +export type VerbName = (typeof VERB_NAMES)[number]; + +const FACT_KINDS = ['event', 'preference', 'commitment', 'belief', 'fact'] as const; +const PROVENANCE_MAX = 500; + +// ─── remember ──────────────────────────────────────────────────────────────── + +const remember: Operation = { + name: 'remember', + description: + 'MEMORY VERB (v1): save one fact to durable agent memory — the protocol write verb. ' + + 'provenance is REQUIRED (free text, e.g. "conversation 2026-06-12", "user said in chat", "import: notes.md"). ' + + 'Set `entity` whenever the fact is about a specific person/company/project — entity-scoped recall will not find it otherwise. ' + + 'ttl accepts duration shorthand ("30d", "12h") or an absolute ISO 8601 timestamp; ISO-8601 durations like "P30D" are rejected with a fix. ' + + 'visibility defaults to "world" (readable by every agent connected to this brain; pass "private" for local-CLI-only facts). ' + + 'Response: branch on `status` (inserted|duplicate|superseded), never on `status_text` (human rendering only). ' + + 'On duplicate, `id` is the EXISTING fact\'s id. For bulk extraction from a raw transcript use extract_facts instead.', + params: { + fact: { type: 'string', required: true, description: 'The fact to remember, one claim per call.' }, + provenance: { + type: 'string', + required: true, + description: + 'Where this fact came from (REQUIRED, free text, max 500 chars). Examples: "conversation 2026-06-12", "user said in chat", "import: meeting-notes.md".', + }, + ttl: { + type: 'string', + description: + 'Optional expiry: duration shorthand ("30d", "12h", "45m") or absolute ISO 8601 timestamp ("2026-07-12T00:00:00Z"). NOT ISO-8601 durations ("P30D" is rejected). Omit = never expires.', + }, + entity: { + type: 'string', + description: + 'Person/company/project this fact is about (name or slug; canonicalized server-side). Set it whenever the fact has a subject — entity-scoped recall misses unattributed facts.', + }, + kind: { + type: 'string', + enum: [...FACT_KINDS], + description: 'Fact kind: event | preference | commitment | belief | fact (default).', + }, + visibility: { + type: 'string', + enum: ['world', 'private'], + description: + 'world (default): readable by every agent connected to this brain — required for the remote remember→recall round-trip. private: local CLI reads only.', + }, + }, + mutating: true, + scope: 'write', + verb: true, + annotations: { title: 'remember (memory write)', idempotentHint: true }, + handler: async (ctx, p) => { + const { verbError, parseTtlParam } = await import('./operations.ts'); + const fact = typeof p.fact === 'string' ? p.fact.trim() : ''; + if (!fact) { + throw verbError( + 'invalid_params', + 'fact must be a non-empty string.', + 'Pass the claim to remember, e.g. fact: "picked Stripe over Adyen — onboarding speed".', + ); + } + const provenance = typeof p.provenance === 'string' ? p.provenance.trim() : ''; + if (!provenance) { + throw verbError( + 'provenance_required', + 'provenance is required and must be non-empty.', + 'Pass where the fact came from, e.g. provenance: "user told me, 2026-06-12" or "import: notes.md".', + ); + } + if (provenance.length > PROVENANCE_MAX) { + throw verbError( + 'invalid_params', + `provenance exceeds ${PROVENANCE_MAX} chars (got ${provenance.length}).`, + 'Shorten the attribution — provenance is a pointer, not a transcript.', + ); + } + const kind = typeof p.kind === 'string' ? p.kind : 'fact'; + if (!FACT_KINDS.includes(kind as (typeof FACT_KINDS)[number])) { + throw verbError( + 'invalid_params', + `kind "${kind}" is not a fact kind.`, + `Use one of: ${FACT_KINDS.join(' | ')}.`, + ); + } + const visibility = typeof p.visibility === 'string' ? p.visibility : 'world'; + if (visibility !== 'world' && visibility !== 'private') { + throw verbError( + 'invalid_params', + `visibility "${visibility}" is not valid.`, + 'Use "world" (default — agents can recall it) or "private" (local CLI reads only).', + ); + } + const validUntil = parseTtlParam(p.ttl); // throws verbError(invalid_params) on bad input + + if (ctx.dryRun) { + return { + dry_run: true, + action: 'remember', + fact, + protocol_version: MEMORY_VERBS_VERSION, + }; + } + + const { writeSingleFact } = await import('./facts/write-single.ts'); + const result = await writeSingleFact(ctx.engine, ctx.sourceId ?? 'default', { + fact, + provenance, + kind: kind as (typeof FACT_KINDS)[number], + entity: typeof p.entity === 'string' && p.entity.trim() ? p.entity.trim() : null, + visibility, + validUntil, + }); + + const statusText = + result.status === 'inserted' + ? `remembered as fact #${result.id}` + : result.status === 'duplicate' + ? `already knew this — kept fact #${result.id}` + : `updated — fact #${result.id} supersedes the previous version`; + + return { + // Opaque STRING at the protocol level [T4]; gbrain serializes its ints. + id: String(result.id), + status: result.status, + status_text: statusText, + entity_slug: result.entity_slug ?? null, + valid_until: result.valid_until ? result.valid_until.toISOString() : null, + ...(result.degraded_dedup ? { degraded_dedup: true } : {}), + protocol_version: MEMORY_VERBS_VERSION, + }; + }, + cliHints: { name: 'remember', positional: ['fact'] }, +}; + +// ─── entity ────────────────────────────────────────────────────────────────── + +const entity: Operation = { + name: 'entity', + description: + 'MEMORY VERB (v1): inspect ONE known person/company/project card — zero LLM calls, sub-100ms. ' + + 'Resolution: alias > exact title > slug-suffix; ties break on most-recently-touched. ' + + 'NEVER errors on a miss: returns found:false plus near-miss suggestions with create_safety hints ' + + '(exists | probable | unknown — whether writing a new page would duplicate). ' + + 'Routing: for facts/snippets retrieval use recall; for broad questions needing reasoning use synthesize (expensive).', + params: { + name: { type: 'string', required: true, description: 'Free-text name, alias, or slug (e.g. "Alice Example", "people/alice-example").' }, + }, + scope: 'read', + verb: true, + annotations: { title: 'entity (card lookup, zero LLM)', readOnlyHint: true }, + handler: async (ctx, p) => { + const { verbError } = await import('./operations.ts'); + const name = typeof p.name === 'string' ? p.name.trim() : ''; + if (!name) { + throw verbError( + 'invalid_params', + 'name must be a non-empty string.', + 'Pass the entity to look up, e.g. name: "Alice Example" or name: "people/alice-example".', + ); + } + const t0 = Date.now(); + const { buildEntityCard } = await import('./verbs/entity-card.ts'); + const result = await buildEntityCard(ctx.engine, ctx.sourceId ?? 'default', name, { + remote: ctx.remote !== false, + }); + return { + protocol_version: MEMORY_VERBS_VERSION, + found: result.found, + latency_ms: Date.now() - t0, + ...(result.card ? { card: result.card } : {}), + ...(result.suggestions !== undefined ? { suggestions: result.suggestions } : {}), + }; + }, + cliHints: { name: 'entity', positional: ['name'] }, +}; + +// ─── synthesize ────────────────────────────────────────────────────────────── + +const synthesize: Operation = { + name: 'synthesize', + description: + '[EXPENSIVE / SLOW — makes LLM calls, seconds-to-minutes latency, costs money] ' + + 'MEMORY VERB (v1): answer a broad question using cross-page LLM reasoning with citations and gap analysis. ' + + 'Prefer recall (facts/snippets) or entity (one known card, zero LLM) for lookups — use synthesize only when the answer ' + + 'requires combining evidence across pages. Response carries a best-effort cost block (model, tokens, usd_estimate).', + params: { + question: { type: 'string', required: true, description: 'The question to answer.' }, + since: { type: 'string', description: 'Optional temporal window start (ISO 8601 date or datetime).' }, + until: { type: 'string', description: 'Optional temporal window end (ISO 8601 date or datetime).' }, + }, + scope: 'read', + verb: true, + annotations: { title: 'synthesize (slow, costly — LLM-backed)', readOnlyHint: true }, + handler: async (ctx, p) => { + const { verbError, sourceScopeOpts } = await import('./operations.ts'); + const question = typeof p.question === 'string' ? p.question.trim() : ''; + if (!question) { + throw verbError( + 'invalid_params', + 'question must be a non-empty string.', + 'Pass the question to synthesize an answer for, e.g. question: "what is our payments strategy?".', + ); + } + const scope = sourceScopeOpts(ctx); + const { runThink } = await import('./think/index.ts'); + // Remote-safe delegation: save/take are NEVER offered through this verb, + // for any caller — the verb is a pure read. + const result = await runThink(ctx.engine, { + question, + since: p.since ? String(p.since) : undefined, + until: p.until ? String(p.until) : undefined, + takesHoldersAllowList: ctx.takesHoldersAllowList, + ...(scope.sourceId !== undefined ? { sourceId: scope.sourceId } : {}), + ...(scope.sourceIds !== undefined ? { allowedSources: scope.sourceIds } : {}), + remote: ctx.remote === true, + }); + + // [c10] runThink degrades gracefully to a no-LLM stub RESULT; the protocol + // contract converts that state into an explicit `unavailable` error so + // agents branch on configure/retry instead of relaying a fake answer. + if (result.warnings.includes('NO_ANTHROPIC_API_KEY')) { + throw verbError( + 'unavailable', + 'synthesize needs an LLM and none is configured.', + 'Set an API key (e.g. `gbrain config set anthropic_api_key sk-...` or ANTHROPIC_API_KEY) and retry. recall and entity work without one.', + 'chat gateway unconfigured (NO_ANTHROPIC_API_KEY)', + ); + } + + // Best-effort cost block [E5/m3]: actual tokens when the gateway reported + // usage, priced via the canonical table; nulls when accounting is absent. + const { canonicalLookup } = await import('./model-pricing.ts'); + const usage = result.usage ?? null; + const pricing = canonicalLookup(result.modelUsed); + const usdEstimate = + usage && pricing + ? (usage.input_tokens * pricing.input + usage.output_tokens * pricing.output) / 1_000_000 + : null; + + return { + answer: result.answer, + sources: result.citations.map(c => c.page_slug), + gaps: result.gaps, + cost: { + model: result.modelUsed, + input_tokens: usage?.input_tokens ?? null, + output_tokens: usage?.output_tokens ?? null, + usd_estimate: usdEstimate, + }, + protocol_version: MEMORY_VERBS_VERSION, + }; + }, + cliHints: { name: 'synthesize', positional: ['question'] }, +}; + +// ─── forget ────────────────────────────────────────────────────────────────── + +const forget: Operation = { + name: 'forget', + description: + 'MEMORY VERB (v1): expire a remembered fact by id — the protocol delete verb. ' + + '`id` is the opaque string id returned by remember and recall (facts[].fact_id) — never a page slug. ' + + 'Idempotent: forgetting an already-expired fact returns expired:false (success), unknown id returns a not_found error. ' + + 'The fact is expired (audit trail kept), not deleted.', + params: { + id: { type: 'string', required: true, description: 'Opaque fact id from remember/recall (facts[].fact_id). Never a page slug.' }, + reason: { type: 'string', description: 'Optional reason, written to the fact\'s audit trail. Default: "forgotten".' }, + }, + mutating: true, + scope: 'write', + verb: true, + annotations: { title: 'forget (expire a fact)', destructiveHint: true, idempotentHint: true }, + handler: async (ctx, p) => { + const { verbError } = await import('./operations.ts'); + const rawId = typeof p.id === 'string' ? p.id.trim() : typeof p.id === 'number' ? String(p.id) : ''; + const numericId = Number(rawId); + if (!rawId || !Number.isInteger(numericId) || numericId <= 0) { + throw verbError( + 'not_found', + `No fact with id "${String(p.id)}".`, + 'Pass the opaque string id returned by remember or recall (facts[].fact_id) — page slugs are not fact ids.', + ); + } + const reason = typeof p.reason === 'string' && p.reason.trim() ? p.reason.trim() : null; + + if (ctx.dryRun) { + return { dry_run: true, action: 'forget', id: rawId, protocol_version: MEMORY_VERBS_VERSION }; + } + + const { forgetFactInFence } = await import('./facts/forget.ts'); + const result = await forgetFactInFence(ctx.engine, numericId, { + ...(reason ? { reason } : {}), + }); + + if (!result.ok && result.path === 'not_found') { + throw verbError( + 'not_found', + `No fact with id "${rawId}".`, + 'Ids come from remember/recall (facts[].fact_id). recall the entity first to find the right fact.', + ); + } + if (!result.ok && result.path === 'already_expired') { + // Idempotent re-forget: success, nothing changed. + return { + id: rawId, + expired: false, + reason, + protocol_version: MEMORY_VERBS_VERSION, + }; + } + + return { + id: rawId, + expired: true, + reason, + protocol_version: MEMORY_VERBS_VERSION, + }; + }, + cliHints: { name: 'forget', positional: ['id'] }, +}; + +export const verbOperations: Operation[] = [remember, entity, synthesize, forget]; + +// ─── RESPONSE_SCHEMAS — the protocol's response-shape registry [c8] ───────── +// +// `Operation` carries input params only; response envelopes live HERE, hand- +// authored, and conformance validates LIVE responses against this registry so +// registry-vs-code drift is caught by the same fixtures that certify servers. +// Field names and semantics are FROZEN (additive-forever); enum values are +// part of the contract. + +const EVIDENCE_ENUM = ['alias_hit', 'exact_title_match', 'high_vector_match', 'keyword_exact', 'weak_semantic']; +const CREATE_SAFETY_ENUM = ['exists', 'probable', 'unknown']; +const STATUS_ENUM = ['inserted', 'duplicate', 'superseded']; + +export const RESPONSE_SCHEMAS: Record> = { + recall: { + type: 'object', + required: ['facts', 'total', 'protocol_version'], + properties: { + protocol_version: { type: 'integer', const: MEMORY_VERBS_VERSION }, + total: { type: 'integer' }, + facts: { + type: 'array', + items: { + type: 'object', + required: ['id', 'fact', 'kind', 'fact_id', 'provenance'], + properties: { + id: { type: 'integer', description: 'LEGACY numeric id (pre-v1 consumers). Use fact_id.' }, + fact_id: { type: 'string', description: 'Opaque protocol id — the value forget accepts.' }, + fact: { type: 'string' }, + kind: { type: 'string', enum: FACT_KINDS as unknown as string[] }, + entity_slug: { type: ['string', 'null'] }, + provenance: { type: 'string' }, + valid_until: { type: ['string', 'null'] }, + visibility: { type: 'string', enum: ['private', 'world'] }, + }, + }, + }, + results: { + type: 'array', + description: 'Search arm — present only when `query` was passed.', + items: { + type: 'object', + required: ['slug', 'title', 'evidence', 'create_safety', 'provenance'], + properties: { + slug: { type: 'string' }, + title: { type: ['string', 'null'] }, + chunk: { type: ['string', 'null'] }, + evidence: { type: 'string', enum: EVIDENCE_ENUM }, + create_safety: { type: 'string', enum: CREATE_SAFETY_ENUM }, + provenance: { type: 'string', description: 'Origin page slug.' }, + }, + }, + }, + search_degraded: { type: 'string', description: 'Present when the search arm fell back to keyword-only (no embedding provider).' }, + budget_tokens: { type: 'integer', description: 'Present when budget_tokens was passed.' }, + budget_used: { type: 'integer' }, + dropped_count: { type: 'integer' }, + }, + }, + remember: { + type: 'object', + required: ['id', 'status', 'status_text', 'entity_slug', 'valid_until', 'protocol_version'], + properties: { + protocol_version: { type: 'integer', const: MEMORY_VERBS_VERSION }, + id: { type: 'string', description: 'Opaque fact id. On status=duplicate this is the EXISTING fact\'s id.' }, + status: { type: 'string', enum: STATUS_ENUM, description: 'Branch on THIS, never on status_text.' }, + status_text: { type: 'string', description: 'Human rendering of status. Display only — never branch on it.' }, + entity_slug: { type: ['string', 'null'] }, + valid_until: { type: ['string', 'null'], description: 'ISO 8601 or null (never expires).' }, + degraded_dedup: { type: 'boolean', description: 'Present (true) when no embedding provider — near-duplicates may insert.' }, + }, + }, + entity: { + type: 'object', + required: ['protocol_version', 'found', 'latency_ms'], + properties: { + protocol_version: { type: 'integer', const: MEMORY_VERBS_VERSION }, + found: { type: 'boolean' }, + latency_ms: { type: 'integer' }, + card: { + type: 'object', + required: ['entity', 'aka', 'summary', 'last_touched', 'open_threads', 'edges', 'backlink_count', 'active_fact_count'], + properties: { + entity: { + type: 'object', + required: ['slug', 'title', 'type'], + properties: { slug: { type: 'string' }, title: { type: 'string' }, type: { type: ['string', 'null'] } }, + }, + aka: { type: 'array', items: { type: 'string' } }, + summary: { type: 'string' }, + last_touched: { + type: 'object', + required: ['updated_at', 'last_retrieved_at', 'last_timeline_date'], + properties: { + updated_at: { type: ['string', 'null'] }, + last_retrieved_at: { type: ['string', 'null'] }, + last_timeline_date: { type: ['string', 'null'] }, + }, + }, + open_threads: { + type: 'array', + items: { + type: 'object', + required: ['kind', 'text', 'date'], + properties: { + kind: { type: 'string', enum: ['commitment', 'recent_event'] }, + text: { type: 'string' }, + date: { type: ['string', 'null'] }, + }, + }, + }, + edges: { + type: 'array', + items: { + type: 'object', + required: ['type', 'direction', 'slug'], + properties: { + type: { type: 'string' }, + direction: { type: 'string', enum: ['out', 'in'] }, + slug: { type: 'string' }, + context: { type: ['string', 'null'] }, + }, + }, + }, + backlink_count: { type: 'integer' }, + active_fact_count: { type: 'integer' }, + }, + }, + suggestions: { + type: 'array', + items: { + type: 'object', + required: ['slug', 'title', 'create_safety'], + properties: { + slug: { type: 'string' }, + title: { type: 'string' }, + create_safety: { type: 'string', enum: CREATE_SAFETY_ENUM }, + }, + }, + }, + }, + }, + synthesize: { + type: 'object', + required: ['answer', 'sources', 'cost', 'protocol_version'], + properties: { + protocol_version: { type: 'integer', const: MEMORY_VERBS_VERSION }, + answer: { type: 'string' }, + sources: { type: 'array', items: { type: 'string' } }, + gaps: { type: 'array', items: { type: 'string' } }, + cost: { + type: 'object', + required: ['model', 'input_tokens', 'output_tokens', 'usd_estimate'], + description: 'Best-effort aggregate (retries/multi-call flows sum; cache hits may undercount). Honest signal, not an invoice.', + properties: { + model: { type: 'string' }, + input_tokens: { type: ['integer', 'null'] }, + output_tokens: { type: ['integer', 'null'] }, + usd_estimate: { type: ['number', 'null'] }, + }, + }, + }, + }, + forget: { + type: 'object', + required: ['id', 'expired', 'reason', 'protocol_version'], + properties: { + protocol_version: { type: 'integer', const: MEMORY_VERBS_VERSION }, + id: { type: 'string' }, + expired: { type: 'boolean', description: 'true = this call expired the fact; false = it was ALREADY expired (idempotent re-forget).' }, + reason: { type: ['string', 'null'] }, + }, + }, +}; + +/** Error envelope schema (uniform across all five verbs). */ +export const ERROR_SCHEMA: Record = { + type: 'object', + required: ['error', 'message'], + properties: { + error: { + type: 'string', + enum: [ + 'invalid_params', + 'provenance_required', + 'not_found', + 'scope_denied', + 'unavailable', + 'budget_unsatisfiable', // RESERVED — schema-listed, never returned in v1 + 'internal', + ], + }, + message: { type: 'string' }, + suggestion: { type: 'string', description: 'Populated on every verb error: problem + cause + fix.' }, + detail: { type: 'string', description: 'Freeform specifics (e.g. which dependency failed).' }, + protocol_version: { type: 'integer', const: MEMORY_VERBS_VERSION }, + }, +}; diff --git a/src/core/verbs/entity-card.ts b/src/core/verbs/entity-card.ts new file mode 100644 index 000000000..4dc79a5a1 --- /dev/null +++ b/src/core/verbs/entity-card.ts @@ -0,0 +1,311 @@ +/** + * MEMORY_VERBS v1 — `entity(name)` card builder (zero LLM, p99 < 100ms). + * + * Resolves a free-text name to ONE brain page via the Retrieval Reflex's + * precision-biased arms (alias-first, then exact-title / exact-slug / + * slug-suffix), then assembles a compact self-describing card from parallel + * depth-1 indexed reads. Deliberately NOT the recursive-CTE traversal + * (traversePaths) — the card is a latency contract, not a graph walk. + * + * Resolution precedence (frozen): alias > exact title > slug-suffix; ties + * break on GREATEST(updated_at, last_retrieved_at) — "last_touched" is the + * card's OUTPUT name, not a column. Multi-hit → best match wins, runners-up + * land in `suggestions`. Miss → `found: false` + keyword near-misses with + * create_safety hints. NEVER throws for data reasons; each arm is guarded so + * a pre-page_aliases brain still resolves via arm 2 (same posture as the + * shipped reflex). + * + * Privacy: `summary` runs through safeSynopsis (the get_page fence boundary); + * facts respect visibility for remote callers (world-only). + */ + +import type { BrainEngine, FactRow } from '../engine.ts'; +import { normalizeAlias } from '../search/alias-normalize.ts'; +import { slugify } from '../entities/resolve.ts'; +import { safeSynopsis } from '../context/retrieval-reflex.ts'; +import { stampEvidence } from '../search/evidence.ts'; +import type { SearchResult } from '../types.ts'; + +const EDGE_CAP = 10; +const OPEN_THREADS_CAP = 3; +const OPEN_THREAD_TIMELINE_WINDOW_DAYS = 90; +const SUGGESTION_CAP = 3; +const FACT_FETCH_CAP = 100; + +export interface EntityCardEdge { + type: string; + direction: 'out' | 'in'; + slug: string; + context: string | null; +} + +export interface EntityOpenThread { + kind: 'commitment' | 'recent_event'; + text: string; + date: string | null; +} + +export interface EntityCard { + entity: { slug: string; title: string; type: string | null }; + /** page_aliases reverse lookup (normalized forms). Empty on pre-migration brains. */ + aka: string[]; + /** Privacy-safe synopsis — same fence boundary as get_page. */ + summary: string; + last_touched: { + updated_at: string | null; + last_retrieved_at: string | null; + last_timeline_date: string | null; + }; + /** Best-effort in v1: active commitment-kind facts + recent timeline entries. */ + open_threads: EntityOpenThread[]; + /** Top typed edges, mentions excluded, out-edges first. */ + edges: EntityCardEdge[]; + backlink_count: number; + /** Active facts about this entity (capped count; visibility-filtered for remote). */ + active_fact_count: number; +} + +export interface EntitySuggestion { + slug: string; + title: string; + create_safety: string; +} + +export interface EntityCardResult { + found: boolean; + card?: EntityCard; + suggestions?: EntitySuggestion[]; +} + +interface CardPageRow { + slug: string; + title: string; + type: string | null; + frontmatter: Record | null; + compiled_truth: string | null; + updated_at: Date | string | null; + last_retrieved_at: Date | string | null; +} + +/** Resolution arm rank: lower = higher confidence (frozen precedence ladder). */ +const ARM_ALIAS = 0; +const ARM_EXACT = 1; +const ARM_SUFFIX = 2; + +export async function buildEntityCard( + engine: BrainEngine, + sourceId: string, + name: string, + opts: { remote: boolean }, +): Promise { + const trimmed = (name ?? '').trim(); + if (!trimmed) return { found: false, suggestions: [] }; + + const norm = normalizeAlias(trimmed); + const titleLc = trimmed.toLowerCase(); + const slug = slugify(trimmed); + + // Candidate slugs with their best arm rank. + const rankBySlug = new Map(); + const consider = (s: string, rank: number) => { + if (!s) return; + const prev = rankBySlug.get(s); + if (prev === undefined || rank < prev) rankBySlug.set(s, rank); + }; + + // Arm 1 — alias-first. Guarded: pre-migration brains lack page_aliases. + if (norm) { + try { + const aliasMap = await engine.resolveAliases([norm], { sourceId }); + for (const hit of aliasMap.get(norm) ?? []) consider(hit.slug, ARM_ALIAS); + } catch { + /* no page_aliases table — degrade to arm 2 [E3] */ + } + } + + // Arm 2 — exact title / exact slug / slug-suffix, with the columns the + // card's tie-break needs. Guarded like the reflex. + let rows: CardPageRow[] = []; + try { + rows = await engine.executeRaw( + `SELECT slug, title, type, frontmatter, compiled_truth, updated_at, last_retrieved_at + FROM pages + WHERE deleted_at IS NULL + AND source_id = $1 + AND ( lower(title) = $2 + OR slug = $3 + OR slug LIKE $4 )`, + [sourceId, titleLc, slug || trimmed, `%/${slug || trimmed}`], + ); + } catch { + rows = []; + } + const rowBySlug = new Map(); + for (const r of rows) { + rowBySlug.set(r.slug, r); + const isExact = (r.title ?? '').toLowerCase() === titleLc || r.slug === slug; + consider(r.slug, isExact ? ARM_EXACT : ARM_SUFFIX); + } + + // Hydrate alias-resolved slugs that arm 2 didn't fetch. + const missing = [...rankBySlug.keys()].filter(s => !rowBySlug.has(s)); + if (missing.length) { + try { + const extra = await engine.executeRaw( + `SELECT slug, title, type, frontmatter, compiled_truth, updated_at, last_retrieved_at + FROM pages + WHERE deleted_at IS NULL AND source_id = $1 AND slug = ANY($2::text[])`, + [sourceId, missing], + ); + for (const r of extra) rowBySlug.set(r.slug, r); + } catch { + /* stale alias rows — drop */ + } + } + + // Rank candidates: arm rank asc, then GREATEST(updated_at, last_retrieved_at) desc. + const candidates = [...rankBySlug.entries()] + .map(([s, rank]) => ({ slug: s, rank, row: rowBySlug.get(s) })) + .filter((c): c is { slug: string; rank: number; row: CardPageRow } => c.row !== undefined) + .sort((a, b) => a.rank - b.rank || lastTouchedMs(b.row) - lastTouchedMs(a.row)); + + if (candidates.length === 0) { + return { found: false, suggestions: await nearMissSuggestions(engine, sourceId, trimmed) }; + } + + const best = candidates[0]; + const runnersUp: EntitySuggestion[] = candidates.slice(1, 1 + SUGGESTION_CAP).map(c => ({ + slug: c.slug, + title: c.row.title ?? c.slug, + // A page that resolved through the precision arms exists by definition. + create_safety: 'exists', + })); + + const card = await assembleCard(engine, sourceId, best.row, opts.remote); + return { + found: true, + card, + ...(runnersUp.length ? { suggestions: runnersUp } : {}), + }; +} + +async function assembleCard( + engine: BrainEngine, + sourceId: string, + row: CardPageRow, + remote: boolean, +): Promise { + const pageSlug = row.slug; + const visibility = remote ? (['world'] as ('private' | 'world')[]) : undefined; + + // Parallel depth-1 reads — every arm individually fail-soft so a partial + // brain (no aliases, no timeline) still returns a card. + const [aka, outLinks, inLinks, backlinkCounts, timeline, facts] = await Promise.all([ + engine + .executeRaw<{ alias_norm: string }>( + `SELECT alias_norm FROM page_aliases WHERE source_id = $1 AND slug = $2 ORDER BY alias_norm`, + [sourceId, pageSlug], + ) + .then(rs => rs.map(r => r.alias_norm)) + .catch(() => [] as string[]), + engine.getLinks(pageSlug, { sourceId }).catch(() => []), + engine.getBacklinks(pageSlug, { sourceId }).catch(() => []), + engine.getBacklinkCounts([pageSlug]).catch(() => new Map()), + engine.getTimeline(pageSlug, { limit: 5, sourceId }).catch(() => []), + engine + .listFactsByEntity(sourceId, pageSlug, { + activeOnly: true, + limit: FACT_FETCH_CAP, + ...(visibility ? { visibility } : {}), + }) + .catch(() => [] as FactRow[]), + ]); + + const edges: EntityCardEdge[] = []; + for (const l of outLinks) { + if (l.link_source === 'mentions') continue; + edges.push({ type: l.link_type, direction: 'out', slug: l.to_slug, context: l.context || null }); + if (edges.length >= EDGE_CAP) break; + } + if (edges.length < EDGE_CAP) { + for (const l of inLinks) { + if (l.link_source === 'mentions') continue; + edges.push({ type: l.link_type, direction: 'in', slug: l.from_slug, context: l.context || null }); + if (edges.length >= EDGE_CAP) break; + } + } + + // Open threads (best-effort v1): active commitments first, then recent + // timeline entries inside the window, capped together. + const openThreads: EntityOpenThread[] = []; + for (const f of facts) { + if (f.kind !== 'commitment') continue; + openThreads.push({ kind: 'commitment', text: f.fact, date: f.valid_from?.toISOString() ?? null }); + if (openThreads.length >= OPEN_THREADS_CAP) break; + } + if (openThreads.length < OPEN_THREADS_CAP) { + const cutoff = Date.now() - OPEN_THREAD_TIMELINE_WINDOW_DAYS * 24 * 60 * 60 * 1000; + for (const t of timeline) { + const ts = Date.parse(t.date); + if (!Number.isFinite(ts) || ts < cutoff) continue; + openThreads.push({ kind: 'recent_event', text: t.summary, date: t.date }); + if (openThreads.length >= OPEN_THREADS_CAP) break; + } + } + + return { + entity: { slug: pageSlug, title: row.title ?? pageSlug, type: row.type ?? null }, + aka, + summary: safeSynopsis(row), + last_touched: { + updated_at: toIso(row.updated_at), + last_retrieved_at: toIso(row.last_retrieved_at), + last_timeline_date: timeline.length ? timeline[0].date : null, + }, + open_threads: openThreads, + edges, + backlink_count: backlinkCounts.get(pageSlug) ?? 0, + active_fact_count: facts.length, + }; +} + +/** + * Near-miss suggestions on a total miss (E5 delight): keyword search top-N + * with evidence-derived create_safety so a typo'd name becomes a next move + * instead of a dead end. Zero LLM; fail-soft to []. + */ +async function nearMissSuggestions( + engine: BrainEngine, + sourceId: string, + name: string, +): Promise { + try { + const raw = await engine.searchKeyword(name, { limit: SUGGESTION_CAP, sourceId }); + const results = raw as SearchResult[]; + stampEvidence(results); + return results.map(r => ({ + slug: r.slug, + title: r.title ?? r.slug, + create_safety: r.create_safety ?? 'unknown', + })); + } catch { + return []; + } +} + +function lastTouchedMs(row: CardPageRow): number { + const u = toMs(row.updated_at); + const l = toMs(row.last_retrieved_at); + return Math.max(u, l); +} + +function toMs(v: Date | string | null): number { + if (v == null) return 0; + const ms = v instanceof Date ? v.getTime() : Date.parse(v); + return Number.isFinite(ms) ? ms : 0; +} + +function toIso(v: Date | string | null): string | null { + const ms = toMs(v); + return ms > 0 ? new Date(ms).toISOString() : null; +}