diff --git a/scripts/debug/agent-prepare-context-audit.mjs b/scripts/debug/agent-prepare-context-audit.mjs index cbb32e69b..75d9b5056 100644 --- a/scripts/debug/agent-prepare-context-audit.mjs +++ b/scripts/debug/agent-prepare-context-audit.mjs @@ -3,12 +3,18 @@ // // Drives real agent turns through JSON-RPC against an authenticated core, forces // the orchestrator to call `agent_prepare_context`, then reads the resulting -// session transcripts to surface — per query — the returned [context_bundle], -// the scout's step-by-step turns ("thoughts"), and tokens in/out/cached + cost. +// session transcripts to surface — per query — the returned [context_bundle] +// (including the new `recommended_skills` block), the scout's step-by-step turns +// ("thoughts"), which curated gathering tools it exercised, and tokens/cost. +// +// To prove the enrichment end-to-end on a live session, it also seeds a PRIOR +// chat thread with a distinctive canary fact and adds a "transcript/recall" +// case: the scout must call `transcript_search`, find that earlier message, and +// echo the canary into its bundle. Pass --no-seed-transcript to skip seeding. // // Unlike harness-cache-audit.mjs (which deliberately hides bodies), this script // PRINTS bundle + thought content so you can iterate on the context_scout -// prompt. Run it against a core built from this branch (so the tool exists) and +// prompt. Run it against a core built from this branch (so the tools exist) and // signed into your account (so the LLM calls bill to you). import { spawn } from "node:child_process"; import { randomBytes } from "node:crypto"; @@ -26,11 +32,29 @@ const SCOUT_AGENT = "context_scout"; // Default audit queries, each aimed at a different context source. const DEFAULT_CASES = [ - { name: "memory/projects", query: "What do you know about my current projects and what should I work on next?" }, - { name: "goals/profile", query: "Based on my stated goals, what should I prioritise this week?" }, - { name: "web/fresh-fact", query: "What is the latest stable Rust release and one notable change in it?" }, - { name: "integrations/email", query: "Summarise my most important unread emails from the last day." }, - { name: "mixed/plan", query: "Plan my day: combine my goals, recent activity, and anything time-sensitive." }, + { + name: "memory/projects", + query: + "What do you know about my current projects and what should I work on next?", + }, + { + name: "goals/profile", + query: "Based on my stated goals, what should I prioritise this week?", + }, + { + name: "web/fresh-fact", + query: + "What is the latest stable Rust release and one notable change in it?", + }, + { + name: "integrations/email", + query: "Summarise my most important unread emails from the last day.", + }, + { + name: "mixed/plan", + query: + "Plan my day: combine my goals, recent activity, and anything time-sensitive.", + }, ]; function usage() { @@ -51,6 +75,8 @@ Options: (writes a temporary workspace agent override; restored after the run unless --keep-workspace). Test your prompt. --thread-prefix Thread id prefix to isolate transcripts (default: random) + --no-seed-transcript Skip seeding a prior-chat thread (default: seed one with + a canary fact so the scout has past chat to search) --max-print-chars Truncate printed bundle/thought blocks (default: 4000) --rpc-timeout-ms Per-RPC timeout (default: 600000) --spawn-core Start \`cargo run --bin openhuman-core\` for the audit @@ -76,6 +102,7 @@ function parseArgs(argv) { raw: false, scoutPromptFile: "", threadPrefix: `apc-audit-${randomBytes(3).toString("hex")}`, + seedTranscript: true, maxPrintChars: 4000, rpcTimeoutMs: 600_000, spawnCore: false, @@ -93,20 +120,56 @@ function parseArgs(argv) { return value; }; switch (arg) { - case "--core-url": opts.coreUrl = next(); opts.coreUrlExplicit = true; break; - case "--token": opts.token = next(); break; - case "--workspace": opts.workspace = next(); opts.workspaceExplicit = true; break; - case "--model": opts.model = next(); break; - case "--query": opts.queries.push(next()); break; - case "--raw": opts.raw = true; break; - case "--scout-prompt-file": opts.scoutPromptFile = next(); break; - case "--thread-prefix": opts.threadPrefix = next(); break; - case "--max-print-chars": opts.maxPrintChars = parsePositiveInt(next(), "--max-print-chars"); break; - case "--rpc-timeout-ms": opts.rpcTimeoutMs = parsePositiveInt(next(), "--rpc-timeout-ms"); break; - case "--spawn-core": opts.spawnCore = true; break; - case "--keep-workspace": opts.keepWorkspace = true; break; - case "--json": opts.json = true; break; - case "--verbose": opts.verbose = true; break; + case "--core-url": + opts.coreUrl = next(); + opts.coreUrlExplicit = true; + break; + case "--token": + opts.token = next(); + break; + case "--workspace": + opts.workspace = next(); + opts.workspaceExplicit = true; + break; + case "--model": + opts.model = next(); + break; + case "--query": + opts.queries.push(next()); + break; + case "--raw": + opts.raw = true; + break; + case "--scout-prompt-file": + opts.scoutPromptFile = next(); + break; + case "--thread-prefix": + opts.threadPrefix = next(); + break; + case "--seed-transcript": + opts.seedTranscript = true; + break; + case "--no-seed-transcript": + opts.seedTranscript = false; + break; + case "--max-print-chars": + opts.maxPrintChars = parsePositiveInt(next(), "--max-print-chars"); + break; + case "--rpc-timeout-ms": + opts.rpcTimeoutMs = parsePositiveInt(next(), "--rpc-timeout-ms"); + break; + case "--spawn-core": + opts.spawnCore = true; + break; + case "--keep-workspace": + opts.keepWorkspace = true; + break; + case "--json": + opts.json = true; + break; + case "--verbose": + opts.verbose = true; + break; case "-h": case "--help": console.log(usage()); @@ -120,7 +183,8 @@ function parseArgs(argv) { function parsePositiveInt(raw, label) { const value = Number(raw); - if (!Number.isInteger(value) || value < 1) throw new Error(`${label} must be a positive integer`); + if (!Number.isInteger(value) || value < 1) + throw new Error(`${label} must be a positive integer`); return value; } @@ -151,9 +215,13 @@ async function defaultWorkspace() { if (process.env.OPENHUMAN_WORKSPACE) return process.env.OPENHUMAN_WORKSPACE; const openhumanDir = defaultOpenhumanDir(); try { - const active = await readFile(path.join(openhumanDir, "active_user.toml"), "utf8"); + const active = await readFile( + path.join(openhumanDir, "active_user.toml"), + "utf8", + ); const match = active.match(/^\s*user_id\s*=\s*"([^"]+)"\s*$/m); - if (match?.[1]) return path.join(openhumanDir, "users", match[1], "workspace"); + if (match?.[1]) + return path.join(openhumanDir, "users", match[1], "workspace"); } catch { // fall through to legacy root } @@ -162,7 +230,10 @@ async function defaultWorkspace() { async function readToken(opts) { if (opts.token.trim()) return opts.token.trim(); - const tokenPath = path.join(opts.workspace || (await defaultWorkspace()), "core.token"); + const tokenPath = path.join( + opts.workspace || (await defaultWorkspace()), + "core.token", + ); try { return (await readFile(tokenPath, "utf8")).trim(); } catch { @@ -180,7 +251,10 @@ async function rpc(coreUrl, token, method, params, timeoutMs = 600_000) { res = await fetch(coreUrl, { method: "POST", signal: controller.signal, - headers: { "content-type": "application/json", authorization: `Bearer ${token}` }, + headers: { + "content-type": "application/json", + authorization: `Bearer ${token}`, + }, body: JSON.stringify({ jsonrpc: "2.0", id: `apc-${Date.now()}-${Math.random().toString(16).slice(2)}`, @@ -189,7 +263,8 @@ async function rpc(coreUrl, token, method, params, timeoutMs = 600_000) { }), }); } catch (err) { - if (err?.name === "AbortError") throw new Error(`RPC ${method} timed out after ${timeoutMs}ms`); + if (err?.name === "AbortError") + throw new Error(`RPC ${method} timed out after ${timeoutMs}ms`); throw err; } finally { clearTimeout(timeout); @@ -199,10 +274,15 @@ async function rpc(coreUrl, token, method, params, timeoutMs = 600_000) { try { body = JSON.parse(bodyText); } catch { - throw new Error(`RPC ${method} returned non-JSON HTTP ${res.status}: ${bodyText.slice(0, 200)}`); + throw new Error( + `RPC ${method} returned non-JSON HTTP ${res.status}: ${bodyText.slice(0, 200)}`, + ); } if (!res.ok) throw new Error(`RPC ${method} HTTP ${res.status}`); - if (body.error) throw new Error(`RPC ${method} error: ${JSON.stringify(body.error).slice(0, 300)}`); + if (body.error) + throw new Error( + `RPC ${method} error: ${JSON.stringify(body.error).slice(0, 300)}`, + ); return body.result; } @@ -237,7 +317,7 @@ async function readTranscript(file) { const data = await readFile(file, "utf8"); const lines = data.split(/\r?\n/).filter((l) => l.trim()); if (lines.length === 0) throw new Error("empty transcript"); - const meta = (JSON.parse(lines[0])._meta) || {}; + const meta = JSON.parse(lines[0])._meta || {}; const messages = []; for (const line of lines.slice(1)) { try { @@ -281,7 +361,11 @@ function changedForThread(before, after, threadId) { for (const [file, cur] of after.entries()) { if (threadId && cur.threadId && cur.threadId !== threadId) continue; const prior = before.get(file); - const grew = !prior || cur.input !== prior.input || cur.output !== prior.output || cur.messages.length !== prior.messages.length; + const grew = + !prior || + cur.input !== prior.input || + cur.output !== prior.output || + cur.messages.length !== prior.messages.length; if (grew) rows.push(cur); } return rows; @@ -294,18 +378,51 @@ function extractBundle(text) { const open = text.indexOf("[context_bundle]"); if (open === -1) return null; const close = text.indexOf("[/context_bundle]", open); - const inner = text.slice(open + "[context_bundle]".length, close === -1 ? undefined : close).trim(); - const hasEnough = /has_enough_context\s*:\s*(true|false)/i.exec(inner)?.[1] ?? "(unspecified)"; + const inner = text + .slice(open + "[context_bundle]".length, close === -1 ? undefined : close) + .trim(); + const hasEnough = + /has_enough_context\s*:\s*(true|false)/i.exec(inner)?.[1] ?? + "(unspecified)"; // summary: everything between `summary:` and `recommended_tool_calls:` - const sumMatch = /summary\s*:\s*([\s\S]*?)(?:\n\s*recommended_tool_calls\s*:|$)/i.exec(inner); + const sumMatch = + /summary\s*:\s*([\s\S]*?)(?:\n\s*recommended_tool_calls\s*:|$)/i.exec( + inner, + ); const summary = (sumMatch?.[1] || "").trim(); + // recommended_tool_calls: between its header and recommended_skills (or end). const recIdx = inner.search(/recommended_tool_calls\s*:/i); - const recBlock = recIdx === -1 ? "" : inner.slice(recIdx).replace(/^[^\n]*\n?/, ""); + const skillsIdx = inner.search(/recommended_skills\s*:/i); + const toolsEnd = + skillsIdx > recIdx && skillsIdx !== -1 ? skillsIdx : undefined; + const recBlock = + recIdx === -1 + ? "" + : inner.slice(recIdx, toolsEnd).replace(/^[^\n]*\n?/, ""); const tools = []; for (const m of recBlock.matchAll(/(?:^|\n)\s*-?\s*tool\s*:\s*([^\n]+)/gi)) { tools.push(m[1].trim()); } - return { raw: text.slice(open, close === -1 ? undefined : close + "[/context_bundle]".length), hasEnough, summary, tools, full: inner }; + // recommended_skills: the new block — collect `skill:` names. + const skillBlock = + skillsIdx === -1 ? "" : inner.slice(skillsIdx).replace(/^[^\n]*\n?/, ""); + const skills = []; + for (const m of skillBlock.matchAll( + /(?:^|\n)\s*-?\s*skill\s*:\s*([^\n]+)/gi, + )) { + skills.push(m[1].trim()); + } + return { + raw: text.slice( + open, + close === -1 ? undefined : close + "[/context_bundle]".length, + ), + hasEnough, + summary, + tools, + skills, + full: inner, + }; } function clip(text, max) { @@ -314,6 +431,83 @@ function clip(text, max) { return `${text.slice(0, max)}\n…[truncated ${text.length - max} chars]`; } +// The scout's curated read-only gathering surface. We detect which of these it +// actually exercised by scanning its turn contents (tool calls render the tool +// name into the assistant turn; results carry recognizable output). This is the +// signal that the enrichment "went through chat messages / skills", not just +// memory + web. +const GATHERING_TOOLS = [ + "memory_recall", + "transcript_search", + "thread_list", + "thread_read", + "list_workflows", + "skill_registry_browse", + "skill_registry_search", + "web_search_tool", + "web_fetch", +]; + +function gatheringToolsUsed(scout) { + if (!scout) return []; + const blob = scout.messages + .filter((m) => m.role === "assistant" || m.role === "tool") + .map((m) => (typeof m.content === "string" ? m.content : "")) + .join("\n"); + return GATHERING_TOOLS.filter((t) => blob.includes(t)); +} + +// ── Transcript seeding ─────────────────────────────────────────────────────── + +// Seed a *prior* conversation the scout can find via `transcript_search`. Plants +// a distinctive canary fact so a transcript-recall case can prove the scout read +// past chat (the summary should echo the canary). Returns { threadId, canary, +// query } or null on failure (seeding is best-effort — the audit still runs). +async function seedTranscript(opts) { + const canary = `deploy-canary-${randomBytes(4).toString("hex")}`; + const nowIso = new Date().toISOString(); + try { + // create_new auto-generates the thread id; pull it from the envelope. + const created = await rpc( + opts.coreUrl, + opts.token, + "openhuman.threads_create_new", + { labels: ["apc-audit-seed"] }, + opts.rpcTimeoutMs, + ); + const threadId = created?.data?.id || created?.id; + if (!threadId) + throw new Error( + `no thread id in create_new envelope: ${JSON.stringify(created).slice(0, 200)}`, + ); + const message = { + id: `seed-${randomBytes(4).toString("hex")}`, + content: `Earlier I told you the staging deploy passphrase is "${canary}". Please remember it for later.`, + type: "text", + extraMetadata: {}, + sender: "user", + createdAt: nowIso, + }; + await rpc( + opts.coreUrl, + opts.token, + "openhuman.threads_message_append", + { thread_id: threadId, message }, + opts.rpcTimeoutMs, + ); + return { + threadId, + canary, + query: `In an earlier chat I told you the staging deploy passphrase. What was it?`, + }; + } catch (err) { + console.log( + `[apc-audit] WARN: transcript seeding failed (${err.message}); transcript-recall case skipped.`, + ); + return null; + } +} + // ── Prompt shaping ────────────────────────────────────────────────────────── function forcedPrompt(query) { @@ -335,9 +529,9 @@ function scoutOverrideToml(inlinePrompt) { display_name = "Context Scout (override)" when_to_use = "Pre-flight context collector (prompt override)." temperature = 0.3 -max_iterations = 6 +max_iterations = 8 iteration_policy = "extended" -max_result_chars = 4000 +max_result_chars = 5000 sandbox_mode = "read_only" agent_tier = "worker" omit_identity = true @@ -356,11 +550,21 @@ ${esc} """ [tools] -# Retrieval-only surface, matching the builtin context_scout. memory_tree is -# intentionally excluded: it bundles a write mode (ingest_document) under a -# ReadOnly wrapper, so it must not be reachable by the read-only scout — even -# via a prompt-override used in a "read-only" audit. -named = ["memory_recall", "web_search_tool", "web_fetch"] +# Curated read-only surface, mirroring the builtin context_scout so a prompt +# override under audit does not silently drop the scout's gathering reach. +# memory_tree and the write-capable thread/skill tools are intentionally +# excluded — the scout auto-runs on prompt-injectable input. +named = [ + "memory_recall", + "transcript_search", + "thread_list", + "thread_read", + "list_workflows", + "skill_registry_browse", + "skill_registry_search", + "web_search_tool", + "web_fetch", +] `; } @@ -388,16 +592,23 @@ async function startCore(opts) { // session (→ SESSION_EXPIRED). Leaving it unset lets the core resolve the // active user from ~/.openhuman/active_user.toml and load its live session; // transcripts then land in that same workspace the script reads. - if (opts.workspaceExplicit && opts.workspace) env.OPENHUMAN_WORKSPACE = opts.workspace; + if (opts.workspaceExplicit && opts.workspace) + env.OPENHUMAN_WORKSPACE = opts.workspace; const port = new URL(opts.coreUrl).port || "7788"; env.OPENHUMAN_CORE_PORT = port; env.OPENHUMAN_CORE_RPC_URL = opts.coreUrl; const args = ["run", "--host", "127.0.0.1", "--port", port, "--jsonrpc-only"]; - const child = spawn("cargo", ["run", "--quiet", "--bin", "openhuman-core", "--", ...args], { - cwd: path.resolve(SCRIPT_DIR, "../.."), - env, - stdio: opts.verbose ? ["ignore", "inherit", "inherit"] : ["ignore", "ignore", "pipe"], - }); + const child = spawn( + "cargo", + ["run", "--quiet", "--bin", "openhuman-core", "--", ...args], + { + cwd: path.resolve(SCRIPT_DIR, "../.."), + env, + stdio: opts.verbose + ? ["ignore", "inherit", "inherit"] + : ["ignore", "ignore", "pipe"], + }, + ); let stderr = ""; if (child.stderr) { child.stderr.on("data", (chunk) => { @@ -412,7 +623,10 @@ async function startCore(opts) { async function waitForCore(coreUrl, token, child, stderrFn) { const deadline = Date.now() + 180_000; while (Date.now() < deadline) { - if (child.exitCode !== null) throw new Error(`spawned core exited with ${child.exitCode}\n${stderrFn()}`); + if (child.exitCode !== null) + throw new Error( + `spawned core exited with ${child.exitCode}\n${stderrFn()}`, + ); try { await rpc(coreUrl, token, "core.ping", {}, 10_000); return; @@ -420,7 +634,9 @@ async function waitForCore(coreUrl, token, child, stderrFn) { await new Promise((r) => setTimeout(r, 750)); } } - throw new Error(`timed out waiting for spawned core at ${coreUrl}\n${stderrFn()}`); + throw new Error( + `timed out waiting for spawned core at ${coreUrl}\n${stderrFn()}`, + ); } async function stopChild(child) { @@ -432,7 +648,10 @@ async function stopChild(child) { ]); if (exited || child.exitCode !== null || child.signalCode !== null) return; child.kill("SIGKILL"); - await Promise.race([once(child, "exit"), new Promise((r) => setTimeout(r, 2_000))]); + await Promise.race([ + once(child, "exit"), + new Promise((r) => setTimeout(r, 2_000)), + ]); } // ── Reporting ──────────────────────────────────────────────────────────────── @@ -443,33 +662,69 @@ function printCase(opts, caseInfo, scout, root, ms) { console.log(` query: ${caseInfo.query}`); if (!scout) { - console.log(" ⚠ no context_scout transcript found — tool was not invoked."); - console.log(" (Is the core built from this branch? Is agent_prepare_context allowlisted?)"); + console.log( + " ⚠ no context_scout transcript found — tool was not invoked.", + ); + console.log( + " (Is the core built from this branch? Is agent_prepare_context allowlisted?)", + ); return; } + // Which curated gathering tools the scout actually exercised this turn. + const used = gatheringToolsUsed(scout); + console.log( + ` gathering tools used (${used.length}): ${used.join(", ") || "(none detected)"}`, + ); + if (caseInfo.canary) { + const hitCanary = scout.messages.some( + (m) => + typeof m.content === "string" && m.content.includes(caseInfo.canary), + ); + console.log( + ` transcript canary recalled: ${hitCanary ? `YES (${caseInfo.canary})` : "NO"}`, + ); + } + // Scout turns ("thoughts" — the model's step-by-step text + tool results). const assistantTurns = scout.messages.filter((m) => m.role === "assistant"); const toolTurns = scout.messages.filter((m) => m.role === "tool"); - console.log(`\n ── scout thoughts (${assistantTurns.length} assistant turn(s), ${toolTurns.length} tool result(s)) ──`); + console.log( + `\n ── scout thoughts (${assistantTurns.length} assistant turn(s), ${toolTurns.length} tool result(s)) ──`, + ); scout.messages .filter((m) => m.role === "assistant" || m.role === "tool") .forEach((m, i) => { const label = m.role === "assistant" ? "think" : "tool ←"; - console.log(` [${i}] ${label}: ${clip(m.content, opts.maxPrintChars).replace(/\n/g, "\n ")}`); + console.log( + ` [${i}] ${label}: ${clip(m.content, opts.maxPrintChars).replace(/\n/g, "\n ")}`, + ); }); // Parsed bundle from the scout's final assistant message. const finalText = assistantTurns.at(-1)?.content || ""; - const bundle = extractBundle(finalText) || extractBundle(scout.messages.at(-1)?.content || ""); + const bundle = + extractBundle(finalText) || + extractBundle(scout.messages.at(-1)?.content || ""); console.log("\n ── parsed context bundle ──"); if (bundle) { console.log(` has_enough_context: ${bundle.hasEnough}`); - console.log(` summary: ${clip(bundle.summary, opts.maxPrintChars).replace(/\n/g, "\n ")}`); - console.log(` recommended_tool_calls (${bundle.tools.length}): ${bundle.tools.join(", ") || "(none)"}`); + console.log( + ` summary: ${clip(bundle.summary, opts.maxPrintChars).replace(/\n/g, "\n ")}`, + ); + console.log( + ` recommended_tool_calls (${bundle.tools.length}): ${bundle.tools.join(", ") || "(none)"}`, + ); + console.log( + ` recommended_skills (${bundle.skills?.length || 0}): ${(bundle.skills || []).join(", ") || "(none)"}`, + ); } else { - console.log(" ⚠ scout output did not contain a [context_bundle] envelope. Raw final text:"); - console.log(` ${clip(finalText, opts.maxPrintChars).replace(/\n/g, "\n ")}`); + console.log( + " ⚠ scout output did not contain a [context_bundle] envelope. Raw final text:", + ); + console.log( + ` ${clip(finalText, opts.maxPrintChars).replace(/\n/g, "\n ")}`, + ); } // Token / cost breakdown. @@ -478,10 +733,13 @@ function printCase(opts, caseInfo, scout, root, ms) { if (root) rows.push(usageRow("orchestrator", root)); rows.push({ session: "TOTAL", - in: (scout.input + (root?.input || 0)), - out: (scout.output + (root?.output || 0)), - cached: (scout.cached + (root?.cached || 0)), - "cache%": cachePct(scout.input + (root?.input || 0), scout.cached + (root?.cached || 0)), + in: scout.input + (root?.input || 0), + out: scout.output + (root?.output || 0), + cached: scout.cached + (root?.cached || 0), + "cache%": cachePct( + scout.input + (root?.input || 0), + scout.cached + (root?.cached || 0), + ), cost_usd: round6(scout.charged + (root?.charged || 0)), }); console.log("\n ── tokens & cost ──"); @@ -513,7 +771,10 @@ async function main() { const opts = parseArgs(process.argv.slice(2)); if (!opts.workspace) opts.workspace = await defaultWorkspace(); - const cases = (opts.queries.length > 0 ? opts.queries.map((q, i) => ({ name: `custom-${i + 1}`, query: q })) : DEFAULT_CASES); + const cases = + opts.queries.length > 0 + ? opts.queries.map((q, i) => ({ name: `custom-${i + 1}`, query: q })) + : [...DEFAULT_CASES]; // Write the optional context_scout prompt override BEFORE the core starts, // so a spawned core loads it (the override is read at boot). For attached @@ -546,12 +807,34 @@ async function main() { opts.token = await readToken(opts); } + // Seed a prior-chat thread (with a canary fact) and append a transcript-recall + // case so the audit exercises the scout's new transcript_search reach. Only + // when running the default cases — custom --query runs are left untouched. + let seed = null; + if (opts.seedTranscript && opts.queries.length === 0) { + seed = await seedTranscript(opts); + if (seed) { + console.log( + `[apc-audit] seeded prior-chat thread ${seed.threadId} with canary ${seed.canary}`, + ); + cases.push({ + name: "transcript/recall", + query: seed.query, + canary: seed.canary, + }); + } + } + console.log("[apc-audit] starting live agent_prepare_context audit"); console.log(` rpc: ${opts.coreUrl}`); console.log(` workspace: ${opts.workspace}`); - console.log(` mode: ${opts.spawnCore ? "spawned-core (this branch)" : "attached-core"}`); + console.log( + ` mode: ${opts.spawnCore ? "spawned-core (this branch)" : "attached-core"}`, + ); console.log(` model: ${opts.model || "(account default)"}`); - console.log(` cases: ${cases.length}${opts.raw ? " (raw — not forcing the tool)" : " (forcing the tool)"}`); + console.log( + ` cases: ${cases.length}${opts.raw ? " (raw — not forcing the tool)" : " (forcing the tool)"}`, + ); const caseResults = []; try { @@ -559,26 +842,48 @@ async function main() { const c = cases[i]; const threadId = `${opts.threadPrefix}-${i}`; const before = await snapshot(opts.workspace); - const params = { message: opts.raw ? c.query : forcedPrompt(c.query), thread_id: threadId }; + const params = { + message: opts.raw ? c.query : forcedPrompt(c.query), + thread_id: threadId, + }; if (opts.model) params.model_override = opts.model; const started = Date.now(); let rpcError = ""; try { - await rpc(opts.coreUrl, opts.token, "openhuman.inference_agent_chat", params, opts.rpcTimeoutMs); + await rpc( + opts.coreUrl, + opts.token, + "openhuman.inference_agent_chat", + params, + opts.rpcTimeoutMs, + ); } catch (err) { rpcError = err.message; } const ms = Date.now() - started; const after = await snapshot(opts.workspace); const changed = changedForThread(before, after, threadId); - const scout = changed.find((t) => t.isSubagent && t.agent === SCOUT_AGENT) || changed.find((t) => t.agent === SCOUT_AGENT); + const scout = + changed.find((t) => t.isSubagent && t.agent === SCOUT_AGENT) || + changed.find((t) => t.agent === SCOUT_AGENT); const root = changed.find((t) => !t.isSubagent); if (rpcError) console.log(`\n▶ case: ${c.name} — RPC error: ${rpcError}`); printCase(opts, c, scout, root, ms); - const finalText = (scout?.messages.filter((m) => m.role === "assistant").at(-1)?.content) || ""; + const finalText = + scout?.messages.filter((m) => m.role === "assistant").at(-1)?.content || + ""; const bundle = scout ? extractBundle(finalText) : null; + const used = gatheringToolsUsed(scout); + const canaryRecalled = c.canary + ? Boolean( + scout?.messages.some( + (m) => + typeof m.content === "string" && m.content.includes(c.canary), + ), + ) + : null; caseResults.push({ name: c.name, query: c.query, @@ -586,16 +891,41 @@ async function main() { invoked: Boolean(scout), hasEnough: bundle?.hasEnough ?? null, recommended: bundle?.tools ?? [], + skills: bundle?.skills ?? [], + gatheringUsed: used, + canaryRecalled, scout: scout ? pick(scout) : null, orchestrator: root ? pick(root) : null, rpcError: rpcError || null, }); } } finally { + // Delete the seeded prior-chat thread BEFORE tearing down the core (RPC must + // still be reachable), so the audit leaves no fake "deploy passphrase" data + // in the user's live conversation index/memory surface. Best-effort — a + // failure here is logged, not fatal. --keep-workspace preserves it. + if (seed?.threadId && !opts.keepWorkspace) { + try { + await rpc( + opts.coreUrl, + opts.token, + "openhuman.threads_delete", + { thread_id: seed.threadId, deleted_at: new Date().toISOString() }, + opts.rpcTimeoutMs, + ); + console.log(`\n[apc-audit] cleaned up seeded thread ${seed.threadId}`); + } catch (err) { + console.log( + `\n[apc-audit] WARN: failed to delete seeded thread ${seed.threadId} (${err.message}); remove it manually.`, + ); + } + } if (spawned?.child) await stopChild(spawned.child); if (overridePath && !opts.keepWorkspace) { await rm(overridePath, { force: true }); - console.log(`\n[apc-audit] removed scout prompt override ${overridePath}`); + console.log( + `\n[apc-audit] removed scout prompt override ${overridePath}`, + ); } } @@ -620,28 +950,53 @@ async function main() { invoked: r.invoked ? "yes" : "NO", enough: r.hasEnough ?? "-", rec_tools: r.recommended.length, + rec_skills: r.skills.length, + gather: r.gatheringUsed.length, + canary: r.canaryRecalled === null ? "-" : r.canaryRecalled ? "YES" : "NO", ms: r.ms, - cost_usd: round6((r.scout?.charged || 0) + (r.orchestrator?.charged || 0)), + cost_usd: round6( + (r.scout?.charged || 0) + (r.orchestrator?.charged || 0), + ), })), ); console.log(` tool invoked: ${agg.invoked}/${caseResults.length}`); - console.log(` total tokens: ${agg.in} in / ${agg.out} out / ${agg.cached} cached (${cachePct(agg.in, agg.cached)} hit)`); + // Surface whether the transcript-recall case actually proved the new reach. + const recallCase = caseResults.find((r) => r.canaryRecalled !== null); + if (recallCase) { + console.log( + ` transcript recall: ${recallCase.canaryRecalled ? "PASS — scout recalled the seeded canary" : "MISS — scout did not surface the seeded canary"}` + + ` (gathering tools: ${recallCase.gatheringUsed.join(", ") || "none"})`, + ); + } + console.log( + ` total tokens: ${agg.in} in / ${agg.out} out / ${agg.cached} cached (${cachePct(agg.in, agg.cached)} hit)`, + ); console.log(` total cost: $${round6(agg.cost)}`); if (opts.json) { - console.log(`\n[apc-audit] json\n${JSON.stringify({ cases: caseResults, totals: { ...agg, cost: round6(agg.cost) } }, null, 2)}`); + console.log( + `\n[apc-audit] json\n${JSON.stringify({ cases: caseResults, totals: { ...agg, cost: round6(agg.cost) } }, null, 2)}`, + ); } const everInvoked = agg.invoked > 0; if (!everInvoked) { - console.error("\n[apc-audit] FAIL: agent_prepare_context was never invoked (no context_scout transcript)."); + console.error( + "\n[apc-audit] FAIL: agent_prepare_context was never invoked (no context_scout transcript).", + ); process.exit(1); } console.log("\n[apc-audit] done"); } function pick(t) { - return { input: t.input, output: t.output, cached: t.cached, charged: t.charged, file: t.file }; + return { + input: t.input, + output: t.output, + cached: t.cached, + charged: t.charged, + file: t.file, + }; } main().catch((err) => { diff --git a/scripts/debug/cli.sh b/scripts/debug/cli.sh index 2acb99014..55abe8fbb 100755 --- a/scripts/debug/cli.sh +++ b/scripts/debug/cli.sh @@ -26,7 +26,10 @@ Commands: Run live harness turns over JSON-RPC and summarize transcript token/cache deltas. agent-prepare-context-audit [options] Live-audit the agent_prepare_context tool: force it per query, print the - returned context bundle, scout thoughts, and tokens/cache/cost. + returned context bundle (incl. recommended_skills), scout thoughts, + gathering tools used, and tokens/cache/cost. Seeds a prior-chat thread + with a canary fact and adds a transcript-recall case to prove the scout + searches past chats (--no-seed-transcript to skip). goals-live [options] Live-test the memory_goals flow (list/add/edit/delete + reflect enrichment), printing the goals_agent's thoughts, tool calls, token usage and cost. diff --git a/src/openhuman/agent_orchestration/tools/agent_prepare_context.rs b/src/openhuman/agent_orchestration/tools/agent_prepare_context.rs index 1fc179764..c88b9e0ba 100644 --- a/src/openhuman/agent_orchestration/tools/agent_prepare_context.rs +++ b/src/openhuman/agent_orchestration/tools/agent_prepare_context.rs @@ -398,11 +398,12 @@ impl Tool for AgentPrepareContextTool { fn description(&self) -> &str { "Before answering or delegating, scout existing context. Runs a fast \ - read-only context-collector that checks memory, your goals/profile, \ - connected integrations, and the web, then returns whether there's \ - enough context to answer, a compact context summary, and an ordered \ - list of recommended next tool calls (your own tools, by exact name, \ - with args). Use at the start of non-trivial turns." + read-only context-collector that checks memory, past conversations \ + (transcripts), your goals/profile, installed/registry skills, connected \ + integrations, and the web, then returns whether there's enough context \ + to answer, a compact context summary, an ordered list of recommended \ + next tool calls (your own tools, by exact name, with args), and any \ + skills worth running. Use at the start of non-trivial turns." } fn parameters_schema(&self) -> serde_json::Value { diff --git a/src/openhuman/agent_registry/agents/context_scout/agent.toml b/src/openhuman/agent_registry/agents/context_scout/agent.toml index c2bb5b2d6..9c7b757d2 100644 --- a/src/openhuman/agent_registry/agents/context_scout/agent.toml +++ b/src/openhuman/agent_registry/agents/context_scout/agent.toml @@ -2,12 +2,17 @@ id = "context_scout" display_name = "Context Scout" when_to_use = "Pre-flight context collector. Reads memory, the user's goals/profile, connected integrations, and the web, then returns a tight context bundle plus a recommended plan of next tool calls. Read-only; produces a structured bundle, not actions." temperature = 0.3 -max_iterations = 6 +# Bumped from 6: the scout now has a wider read-only gathering surface +# (transcripts, threads, skills) so it may need a couple more steps to recall → +# check skills → assess before emitting the bundle. +max_iterations = 8 iteration_policy = "extended" -# ~1000-token cap on the returned bundle. The runner truncates the final -# output to this many characters (char-safe) before handing it back to the -# parent, so the orchestrator's context only ever grows by a bounded amount. -max_result_chars = 4000 +# Cap on the returned bundle. The runner truncates the final output to this +# many characters (char-safe) before handing it back to the parent, so the +# orchestrator's context only ever grows by a bounded amount. Bumped from 4000 +# to leave room for the `recommended_skills` block alongside the summary and +# `recommended_tool_calls`. +max_result_chars = 5000 sandbox_mode = "read_only" # Spawn hierarchy: leaf worker. The scout gathers context and stops — it @@ -32,8 +37,10 @@ omit_memory_md = false hint = "agentic" [tools] -# Strictly read-only context-gathering surface. No writes, no shell, no -# delegation — the scout collects and reports. +# Curated read-only context-gathering surface. No writes, no shell, no +# delegation — the scout collects and reports. Every tool here must be +# genuinely read-only: the scout auto-runs on prompt-injectable input, so a +# write-capable tool would be an injection foothold. named = [ # Targeted recall over the memory namespaces (global, background, …). # NOTE: `memory_tree` is intentionally NOT here. That tool bundles a @@ -42,6 +49,25 @@ named = [ # mutate memory ("remember this document") despite running read-only. # Retrieval-only `memory_recall` covers the scout's needs safely. "memory_recall", + # Transcripts: recall what the user said/decided in earlier chats. Backed + # by the cross-thread trigram index; read-only and workspace-scoped. + "transcript_search", + # Thread metadata: enumerate / read titles+labels to locate the right past + # conversation before pulling its transcript. Read-only (the create/update/ + # delete thread tools are deliberately excluded). + "thread_list", + "thread_read", + # Read the messages of a located thread (read-only). Complements + # `transcript_search` (content search) for title/label lookups like + # "summarize my Database work thread", where the thread is found by metadata + # but its transcript still has to be pulled. + "thread_message_list", + # Skills: discover which skills are installed and search the registry so the + # scout can recommend a relevant one back to the orchestrator. All three are + # read-only — `skill_registry_install` / `_uninstall` are deliberately out. + "list_workflows", + "skill_registry_browse", + "skill_registry_search", # Fresh external facts when the question needs them. "web_search_tool", "web_fetch", diff --git a/src/openhuman/agent_registry/agents/context_scout/prompt.md b/src/openhuman/agent_registry/agents/context_scout/prompt.md index 36acbf768..8deb09ed3 100644 --- a/src/openhuman/agent_registry/agents/context_scout/prompt.md +++ b/src/openhuman/agent_registry/agents/context_scout/prompt.md @@ -10,15 +10,28 @@ call next. 2. Gather only what's actually needed to act on it, drawing on: - **Memory** — `memory_recall` for relevant facts (search by namespace + query). This is read-only; you cannot and must not write to memory. + - **Past conversations (transcripts)** — `transcript_search` finds messages + the user sent in *earlier* chats (keyword/substring, recency-ranked). Use + it when the request leans on something the user said, asked, or decided + before ("the doc I mentioned", "like last time", a name/number from a prior + chat). `thread_list` / `thread_read` locate a specific past thread by + title/labels when a search term is too broad, and `thread_message_list` + reads that thread's messages once you've found it (e.g. "summarize my + Database work thread"). - **Goals / profile** — the user's `PROFILE.md` (their stated goals and preferences) and `MEMORY.md` are already in your prompt below. Mine them. + - **Skills** — `list_workflows` shows the skills already installed; + `skill_registry_search` / `skill_registry_browse` find skills in the + registry. If a skill clearly fits the request, surface it under + `recommended_skills` (below) so the orchestrator can run or install it. - **Connected integrations** — the Connected Integrations section below tells you which platforms (gmail, notion, slack, …) are actually wired up. - **The web** — `web_search_tool` / `web_fetch` for fresh external facts the request genuinely depends on. Skip the web when memory/goals already cover it; you are meant to be cheap. 3. Stop as soon as you have enough. Do **not** try to answer the request or - perform the task — that is the orchestrator's job. + perform the task — that is the orchestrator's job. Every tool you have is + read-only; never attempt to write, send, install, or otherwise act. ## What you return @@ -29,11 +42,18 @@ outside it. No preamble, no closing prose. Use exactly this shape: [context_bundle] has_enough_context: true|false summary: <≤ ~700 tokens of distilled, source-attributed context. Lead with what -matters. Attribute facts: (memory), (profile), (web: ), (integrations).> +matters. Attribute facts: (memory), (transcript: ), (profile), +(web: ), (integrations).> recommended_tool_calls: - tool: args: why: +recommended_skills: + - skill: + installed: true|false + why: [/context_bundle] ``` @@ -46,8 +66,15 @@ Rules for the bundle: "Orchestrator tools" list injected below — these are the tools the *orchestrator* can call, not the tools you used. Order them in the sequence the orchestrator should run them. +- `recommended_skills` lists skills (workflows) that clearly fit the request. + For an installed skill use its **runnable id** — the `dir_name` slug from + `list_workflows` (set `installed: true`); the orchestrator runs it with + `run_workflow`, which resolves by that id, not the display name. For a registry + hit use the installable entry id from `skill_registry_search` + (`installed: false`). Only include a skill when it genuinely matches; omit the + section entirely (or leave it empty) when none do. Never invent skill ids. - If no further tool calls are needed (you already have enough and the answer is knowledge-based), return an empty `recommended_tool_calls:` list and set `has_enough_context: true`. -- Keep it tight. The whole bundle is capped — spend the budget on the summary - and the plan, not on hedging. +- Keep it tight. The whole bundle is capped — spend the budget on the summary, + the plan, and any genuinely-matching skills, not on hedging. diff --git a/src/openhuman/agent_registry/agents/context_scout/prompt.rs b/src/openhuman/agent_registry/agents/context_scout/prompt.rs index aa3287acf..1113f2736 100644 --- a/src/openhuman/agent_registry/agents/context_scout/prompt.rs +++ b/src/openhuman/agent_registry/agents/context_scout/prompt.rs @@ -153,6 +153,25 @@ mod tests { assert!(body.contains("recommended_tool_calls")); } + #[test] + fn body_instructs_transcript_and_skill_gathering() { + // The enrichment is only real if the role prompt actually tells the + // scout to search past chats and recommend skills — lock that wiring. + let body = build(&test_ctx()).unwrap(); + assert!( + body.contains("transcript_search"), + "scout prompt must instruct searching past conversations" + ); + assert!( + body.contains("recommended_skills"), + "scout prompt must define the recommended_skills output block" + ); + assert!( + body.contains("list_workflows"), + "scout prompt must point at skill discovery" + ); + } + fn integration(toolkit: &str, connected: bool) -> ConnectedIntegration { ConnectedIntegration { toolkit: toolkit.to_string(), diff --git a/src/openhuman/agent_registry/agents/loader.rs b/src/openhuman/agent_registry/agents/loader.rs index 0f30801a7..58b4390cb 100644 --- a/src/openhuman/agent_registry/agents/loader.rs +++ b/src/openhuman/agent_registry/agents/loader.rs @@ -959,8 +959,9 @@ mod tests { let def = find("context_scout"); assert_eq!(def.agent_tier, AgentTier::Worker); assert_eq!(def.sandbox_mode, SandboxMode::ReadOnly); - // ~1000-token bundle cap — load-bearing for the parent's context budget. - assert_eq!(def.max_result_chars, Some(4000)); + // Bundle cap — load-bearing for the parent's context budget. Leaves + // room for the `recommended_skills` block alongside summary + plan. + assert_eq!(def.max_result_chars, Some(5000)); // Keeps goals/profile + long-term memory so it can ground the // orchestrator in who the user is and what they want. assert!(!def.omit_profile, "context_scout needs PROFILE.md (goals)"); @@ -968,7 +969,21 @@ mod tests { // Strictly read-only gathering surface — no writes / shell / delegation. match &def.tools { ToolScope::Named(tools) => { - for required in ["memory_recall", "web_search_tool", "web_fetch"] { + for required in [ + "memory_recall", + // Transcripts + thread metadata + message reader (read-only). + "transcript_search", + "thread_list", + "thread_read", + "thread_message_list", + // Skill discovery (read-only). + "list_workflows", + "skill_registry_browse", + "skill_registry_search", + // Web. + "web_search_tool", + "web_fetch", + ] { assert!( tools.iter().any(|t| t == required), "context_scout needs read-only gathering tool `{required}`" @@ -983,6 +998,12 @@ mod tests { // memory_tree bundles a write mode (ingest_document) under a // ReadOnly wrapper — must not be reachable by the auto-run scout. "memory_tree", + // Write-capable thread + skill tools must stay out of the + // auto-run, prompt-injectable scout. + "thread_create", + "thread_delete", + "skill_registry_install", + "skill_registry_uninstall", ] { assert!( !tools.iter().any(|t| t == forbidden), diff --git a/src/openhuman/agent_registry/agents/orchestrator/prompt.md b/src/openhuman/agent_registry/agents/orchestrator/prompt.md index 73f6c14e0..f173cf376 100644 --- a/src/openhuman/agent_registry/agents/orchestrator/prompt.md +++ b/src/openhuman/agent_registry/agents/orchestrator/prompt.md @@ -16,8 +16,9 @@ Follow this sequence for every user message: 0. **First message of a new conversation? Prepare context first.** - If this is the user's **first message in the thread** (there are no prior assistant turns above), call `agent_prepare_context` **once** with `question` set to the user's request, **before** doing anything else below. - - It runs a fast read-only scout over memory, your goals/profile, connected integrations, and the web, and returns a `[context_bundle]` with `has_enough_context`, a compact `summary`, and `recommended_tool_calls`. + - It runs a fast read-only scout over memory, past conversations (transcripts), your goals/profile, installed/registry skills, connected integrations, and the web, and returns a `[context_bundle]` with `has_enough_context`, a compact `summary`, `recommended_tool_calls`, and `recommended_skills`. - Use the `summary` to ground your reply, and treat `recommended_tool_calls` as a **suggested** plan — you decide whether to follow each one (route them through the normal delegation paths below; never bypass the approval gate). + - Treat `recommended_skills` the same way: a skill the scout flagged as a good fit. Run an installed one via `run_workflow` when it matches; for an uninstalled one, offer it rather than installing silently. You decide — it is a suggestion, not an instruction. - This is a one-time pass: call it **at most once per conversation**, only on the first message. On every later message skip straight to step 1. 1. **Can I answer directly without tools?** - Yes: reply directly (small talk, simple Q&A, basic factual answers). diff --git a/src/openhuman/mcp_server/resources.rs b/src/openhuman/mcp_server/resources.rs index ba8df4e96..2df85bd93 100644 --- a/src/openhuman/mcp_server/resources.rs +++ b/src/openhuman/mcp_server/resources.rs @@ -118,7 +118,7 @@ const RESOURCE_CATALOG: &[PromptResource] = &[ PromptResource { uri: "openhuman://prompts/agents/context_scout", name: "context_scout", - description: "Read-only pre-flight worker that gathers context (memory, goals, integrations, web) and returns a bounded context bundle.", + description: "Read-only pre-flight worker that gathers context (memory, transcripts, goals, skills, integrations, web) and returns a bounded context bundle.", content: include_str!("../agent_registry/agents/context_scout/prompt.md"), }, PromptResource { diff --git a/src/openhuman/threads/ops.rs b/src/openhuman/threads/ops.rs index efe55a3bf..156de2762 100644 --- a/src/openhuman/threads/ops.rs +++ b/src/openhuman/threads/ops.rs @@ -14,7 +14,7 @@ use crate::openhuman::memory::{ }; use crate::openhuman::memory_conversations::{ self as conversations, ConversationMessage, ConversationMessagePatch, ConversationStore, - ConversationThread, CreateConversationThread, + ConversationThread, CreateConversationThread, CrossThreadHit, }; use crate::openhuman::threads::title::{ build_title_prompt, is_auto_generated_thread_title, sanitize_generated_title, @@ -231,6 +231,34 @@ pub async fn messages_list( )) } +/// Search messages across **every** thread in the workspace for a query, +/// returning up to `limit` of the most-recent matches (newest first). Backed +/// by the trigram/CJK-bigram inverted index in `memory_conversations` — the +/// same cross-chat reader the durable-context pipeline uses (issue #1505). +/// +/// Read-only and workspace-scoped. `exclude_thread_id` lets a caller drop the +/// active chat from the results when it already has that context in hand. +pub async fn transcript_search( + query: &str, + limit: usize, + exclude_thread_id: Option<&str>, +) -> Result, String> { + let dir = workspace_dir().await?; + log::debug!( + "[threads][transcript_search] query_chars={} limit={} exclude={:?}", + query.chars().count(), + limit, + exclude_thread_id + ); + let hits = ConversationStore::new(dir).search_cross_thread_messages( + query, + limit, + exclude_thread_id, + )?; + log::debug!("[threads][transcript_search] hits={}", hits.len()); + Ok(hits) +} + /// Appends a message to a conversation thread. pub async fn message_append( request: AppendConversationMessageRequest, diff --git a/src/openhuman/threads/tools.rs b/src/openhuman/threads/tools.rs index 9f2a1205a..6e2f45035 100644 --- a/src/openhuman/threads/tools.rs +++ b/src/openhuman/threads/tools.rs @@ -281,6 +281,137 @@ impl Tool for ThreadMessageListTool { } } +/// Search past conversations (all threads) for messages matching a query. +pub struct ThreadTranscriptSearchTool; + +impl ThreadTranscriptSearchTool { + /// Default and ceiling for the number of matches returned. Kept small so a + /// context-gathering caller's prompt only grows by a bounded amount. + const DEFAULT_LIMIT: usize = 8; + const MAX_LIMIT: usize = 25; + /// Per-hit snippet cap (chars) — enough to recognise the message without + /// dumping whole turns into the caller's context. + const SNIPPET_CHARS: usize = 220; + + fn snippet(content: &str) -> String { + let collapsed: String = content.split_whitespace().collect::>().join(" "); + if collapsed.chars().count() <= Self::SNIPPET_CHARS { + return collapsed; + } + let cut = collapsed + .char_indices() + .nth(Self::SNIPPET_CHARS) + .map(|(i, _)| i) + .unwrap_or(collapsed.len()); + format!("{}…", &collapsed[..cut]) + } +} + +#[async_trait] +impl Tool for ThreadTranscriptSearchTool { + fn name(&self) -> &str { + "transcript_search" + } + + fn description(&self) -> &str { + "Search your PAST conversations across all threads for messages matching \ + a query (keyword/substring, recency-ranked). Returns the most relevant \ + recent matches — each with its thread id, sender, timestamp, and a \ + snippet. Use to recall what the user said, asked, or decided in earlier \ + chats before answering." + } + + fn parameters_schema(&self) -> serde_json::Value { + json!({ + "type": "object", + "required": ["query"], + "properties": { + "query": { + "type": "string", + "description": "Keywords or phrase to match against past messages. Be specific — short common words are ignored." + }, + "limit": { + "type": "integer", + "description": "Max matches to return (default 8, max 25).", + "minimum": 1, + "maximum": 25 + }, + "exclude_thread_id": { + "type": "string", + "description": "Thread id to omit from results. Defaults to the active thread when omitted, so the message the user just sent does not crowd out the prior chats you are trying to recall. Pass a specific id to override, or an empty string to search every thread." + } + } + }) + } + + fn permission_level(&self) -> PermissionLevel { + PermissionLevel::ReadOnly + } + + async fn execute(&self, args: serde_json::Value) -> anyhow::Result { + let query = read_required_str(&args, "query")?; + let limit = args + .get("limit") + .and_then(serde_json::Value::as_u64) + .map(|n| (n as usize).clamp(1, Self::MAX_LIMIT)) + .unwrap_or(Self::DEFAULT_LIMIT); + // Resolve the effective thread to exclude: + // - arg omitted → default to the ACTIVE thread, so the message the + // user just sent (persisted before inference starts) + // doesn't become the top "past conversation" hit and + // crowd out the real prior chats we're recalling. + // - arg = "" (empty) → explicit opt-out: search every thread. + // - arg = "" → exclude that specific thread. + let exclude_owned: Option = match args + .get("exclude_thread_id") + .and_then(serde_json::Value::as_str) + { + None => crate::openhuman::inference::provider::thread_context::current_thread_id(), + Some(s) if s.trim().is_empty() => None, + Some(s) => Some(s.trim().to_string()), + }; + let exclude = exclude_owned.as_deref(); + log::debug!( + "[tool][threads] transcript_search invoked query_chars={} limit={} exclude={:?}", + query.chars().count(), + limit, + exclude + ); + + let hits = ops::transcript_search(&query, limit, exclude) + .await + .map_err(|e| anyhow::anyhow!("transcript_search: {e}"))?; + + if hits.is_empty() { + return Ok(ToolResult::success(format!( + "No past messages matched `{query}`." + ))); + } + + let mut out = format!( + "{} past message(s) matched `{query}` (newest first):\n", + hits.len() + ); + for hit in &hits { + let _ = std::fmt::Write::write_fmt( + &mut out, + format_args!( + "- [thread {} · {} · {}] {}\n", + hit.thread_id, + hit.role, + hit.created_at, + Self::snippet(&hit.content) + ), + ); + } + Ok(ToolResult::success(out)) + } + + fn is_concurrency_safe(&self, _args: &serde_json::Value) -> bool { + true + } +} + /// Append a message to a thread. pub struct ThreadMessageAppendTool; @@ -705,6 +836,31 @@ mod tests { PermissionLevel::Write ); assert_eq!(ThreadListTool.scope(), ToolScope::All); + assert_eq!(ThreadTranscriptSearchTool.name(), "transcript_search"); + assert_eq!( + ThreadTranscriptSearchTool.permission_level(), + PermissionLevel::ReadOnly + ); + assert!(ThreadTranscriptSearchTool.is_concurrency_safe(&json!({}))); + } + + #[test] + fn transcript_search_requires_query() { + let schema = ThreadTranscriptSearchTool.parameters_schema(); + let required = schema["required"].as_array().expect("required array"); + assert!(required.iter().any(|v| v == "query")); + } + + #[test] + fn transcript_search_snippet_collapses_and_truncates() { + // Short content is returned with whitespace collapsed, untruncated. + let short = ThreadTranscriptSearchTool::snippet("hello there\n\nworld"); + assert_eq!(short, "hello there world"); + // Long content is cut to the cap with an ellipsis. + let long_src = "x ".repeat(400); + let long = ThreadTranscriptSearchTool::snippet(&long_src); + assert!(long.ends_with('…')); + assert!(long.chars().count() <= ThreadTranscriptSearchTool::SNIPPET_CHARS + 1); } #[test] diff --git a/src/openhuman/tools/ops.rs b/src/openhuman/tools/ops.rs index 0c5cafca2..4347683a9 100644 --- a/src/openhuman/tools/ops.rs +++ b/src/openhuman/tools/ops.rs @@ -363,6 +363,9 @@ pub fn all_tools_with_runtime( Box::new(ThreadUpdateTitleTool), Box::new(ThreadUpdateLabelsTool), Box::new(ThreadMessageListTool), + // Read-only cross-thread transcript search (trigram index). Lets the + // context scout and other agents recall what was said in earlier chats. + Box::new(ThreadTranscriptSearchTool), Box::new(ThreadMessageAppendTool), Box::new(ThreadMessageUpdateTool), Box::new(ThreadTitleGenerateTool), diff --git a/tests/transcript_search_e2e.rs b/tests/transcript_search_e2e.rs new file mode 100644 index 000000000..c2cecc013 --- /dev/null +++ b/tests/transcript_search_e2e.rs @@ -0,0 +1,336 @@ +//! End-to-end integration test for cross-thread transcript search. +//! +//! Proves the path the context scout (and any agent) actually walks when it +//! "goes through chat messages": persist real conversation threads + messages +//! via `ConversationStore`, then exercise both the `threads::ops::transcript_search` +//! op and the agent-facing `transcript_search` tool (`ThreadTranscriptSearchTool`) +//! against that on-disk data under a per-test temp `OPENHUMAN_WORKSPACE`. +//! +//! This is the Rust contract counterpart to the live-session audit in +//! `scripts/debug/agent-prepare-context-audit.mjs` (which drives the same path +//! through a real orchestrator turn over JSON-RPC). +//! +//! Run with: `cargo test --test transcript_search_e2e` + +use std::path::Path; +use std::sync::{Mutex, OnceLock}; + +use serde_json::json; +use tempfile::tempdir; + +use openhuman_core::openhuman::memory_conversations::{ + ConversationMessage, ConversationStore, CreateConversationThread, +}; +use openhuman_core::openhuman::threads::ops::transcript_search; +use openhuman_core::openhuman::threads::tools::ThreadTranscriptSearchTool; +use openhuman_core::openhuman::tools::traits::Tool; + +// ── Env isolation (mirrors tests/memory_roundtrip_e2e.rs) ──────────────────── + +struct EnvVarGuard { + key: &'static str, + old: Option, +} + +impl EnvVarGuard { + fn set_to_path(key: &'static str, path: &Path) -> Self { + let old = std::env::var(key).ok(); + // SAFETY: only used in tests that first acquire env_lock(), which + // serializes process-global env mutations. + unsafe { std::env::set_var(key, path.as_os_str()) }; + Self { key, old } + } +} + +impl Drop for EnvVarGuard { + fn drop(&mut self) { + match &self.old { + // SAFETY: teardown runs under the same env_lock() critical section. + Some(v) => unsafe { std::env::set_var(self.key, v) }, + None => unsafe { std::env::remove_var(self.key) }, + } + } +} + +/// Serialises tests: `HOME` + `OPENHUMAN_WORKSPACE` are process-global. +static ENV_LOCK: OnceLock> = OnceLock::new(); + +fn env_lock() -> std::sync::MutexGuard<'static, ()> { + ENV_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("env lock poisoned") +} + +// ── Fixture helpers ────────────────────────────────────────────────────────── + +fn thread(id: &str, title: &str) -> CreateConversationThread { + CreateConversationThread { + id: id.to_string(), + title: title.to_string(), + created_at: "2026-06-24T00:00:00Z".to_string(), + parent_thread_id: None, + labels: None, + personality_id: None, + } +} + +fn message(id: &str, sender: &str, content: &str, created_at: &str) -> ConversationMessage { + ConversationMessage { + id: id.to_string(), + content: content.to_string(), + message_type: "text".to_string(), + extra_metadata: json!({}), + sender: sender.to_string(), + created_at: created_at.to_string(), + } +} + +/// Seed two threads of realistic prior chat into a fresh workspace and return +/// the store + a kept-alive tempdir guard pair. The caller holds `env_lock()`. +fn seed_workspace(workspace: &Path) -> ConversationStore { + let store = ConversationStore::new(workspace.to_path_buf()); + + // Thread A — a past conversation about a Postgres migration. + store + .ensure_thread(thread("thread-pg", "Database work")) + .expect("ensure pg thread"); + store + .append_message( + "thread-pg", + message( + "pg-1", + "user", + "Remember the Postgres migration script lives in db/migrate_2026.sql", + "2026-06-20T09:00:00Z", + ), + ) + .expect("append pg-1"); + store + .append_message( + "thread-pg", + message( + "pg-2", + "assistant", + "Got it — I'll reference db/migrate_2026.sql for the migration.", + "2026-06-20T09:00:05Z", + ), + ) + .expect("append pg-2"); + + // Thread B — an unrelated past conversation about a vacation. + store + .ensure_thread(thread("thread-trip", "Vacation planning")) + .expect("ensure trip thread"); + store + .append_message( + "thread-trip", + message( + "trip-1", + "user", + "Book flights to Lisbon for the August holiday", + "2026-06-21T12:00:00Z", + ), + ) + .expect("append trip-1"); + + store +} + +// ── Tests ──────────────────────────────────────────────────────────────────── + +/// Happy path: the op surfaces a message from a *prior* thread by keyword, and +/// scopes the hit to the thread that actually contains it. +#[tokio::test] +async fn transcript_search_op_finds_message_in_prior_thread() { + let _lock = env_lock(); + let tmp = tempdir().expect("tempdir"); + let _home = EnvVarGuard::set_to_path("HOME", tmp.path()); + let workspace = tmp.path().join("workspace"); + std::fs::create_dir_all(&workspace).expect("create workspace"); + let _ws = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", &workspace); + seed_workspace(&workspace); + + let hits = transcript_search("Postgres migration script", 10, None) + .await + .expect("transcript_search op"); + + assert!( + !hits.is_empty(), + "expected at least one hit for the migration message" + ); + assert!( + hits.iter() + .any(|h| h.thread_id == "thread-pg" && h.content.contains("db/migrate_2026.sql")), + "the migration message from thread-pg should surface — got {hits:?}" + ); + assert!( + hits.iter().all(|h| h.thread_id != "thread-trip"), + "the unrelated vacation thread must not match a Postgres query — got {hits:?}" + ); +} + +/// `exclude_thread_id` drops the named thread from results — the knob the +/// orchestrator can use to omit the active chat it already has in hand. +#[tokio::test] +async fn transcript_search_op_honours_exclude_thread() { + let _lock = env_lock(); + let tmp = tempdir().expect("tempdir"); + let _home = EnvVarGuard::set_to_path("HOME", tmp.path()); + let workspace = tmp.path().join("workspace"); + std::fs::create_dir_all(&workspace).expect("create workspace"); + let _ws = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", &workspace); + seed_workspace(&workspace); + + let hits = transcript_search("migration", 10, Some("thread-pg")) + .await + .expect("transcript_search op"); + + assert!( + hits.iter().all(|h| h.thread_id != "thread-pg"), + "excluded thread must not appear in results — got {hits:?}" + ); +} + +/// A query that matches nothing returns no hits (not an error). +#[tokio::test] +async fn transcript_search_op_returns_empty_on_no_match() { + let _lock = env_lock(); + let tmp = tempdir().expect("tempdir"); + let _home = EnvVarGuard::set_to_path("HOME", tmp.path()); + let workspace = tmp.path().join("workspace"); + std::fs::create_dir_all(&workspace).expect("create workspace"); + let _ws = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", &workspace); + seed_workspace(&workspace); + + let hits = transcript_search("quantum chromodynamics zzz", 10, None) + .await + .expect("transcript_search op"); + + assert!(hits.is_empty(), "no message should match — got {hits:?}"); +} + +/// The agent-facing tool (`transcript_search`) — the exact entry point the +/// context scout calls — formats hits into a readable block that names the +/// source thread and quotes a snippet of the matched message. +#[tokio::test] +async fn transcript_search_tool_formats_hits_for_the_agent() { + let _lock = env_lock(); + let tmp = tempdir().expect("tempdir"); + let _home = EnvVarGuard::set_to_path("HOME", tmp.path()); + let workspace = tmp.path().join("workspace"); + std::fs::create_dir_all(&workspace).expect("create workspace"); + let _ws = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", &workspace); + seed_workspace(&workspace); + + let result = ThreadTranscriptSearchTool + .execute(json!({ "query": "Postgres migration", "limit": 5 })) + .await + .expect("transcript_search tool"); + assert!(!result.is_error, "tool should succeed: {}", result.output()); + let out = result.output(); + assert!( + out.contains("matched"), + "output should announce matches — got: {out}" + ); + assert!( + out.contains("thread-pg"), + "output should name the source thread — got: {out}" + ); + assert!( + out.contains("db/migrate_2026.sql"), + "output should quote the matched message snippet — got: {out}" + ); +} + +/// The tool reports a clean "no match" line (rather than an error) so the scout +/// can record "nothing in past chats" and move on. +#[tokio::test] +async fn transcript_search_tool_reports_no_match_cleanly() { + let _lock = env_lock(); + let tmp = tempdir().expect("tempdir"); + let _home = EnvVarGuard::set_to_path("HOME", tmp.path()); + let workspace = tmp.path().join("workspace"); + std::fs::create_dir_all(&workspace).expect("create workspace"); + let _ws = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", &workspace); + seed_workspace(&workspace); + + let result = ThreadTranscriptSearchTool + .execute(json!({ "query": "nonexistent-term-xyzzy" })) + .await + .expect("transcript_search tool"); + assert!( + !result.is_error, + "no-match is not an error: {}", + result.output() + ); + assert!( + result.output().contains("No past messages matched"), + "expected the clean no-match line — got: {}", + result.output() + ); +} + +/// Passing an explicit `exclude_thread_id` drops that thread from the tool's +/// results. "migration" lives only in thread-pg, so excluding it yields the +/// clean no-match line. +#[tokio::test] +async fn transcript_search_tool_excludes_named_thread() { + let _lock = env_lock(); + let tmp = tempdir().expect("tempdir"); + let _home = EnvVarGuard::set_to_path("HOME", tmp.path()); + let workspace = tmp.path().join("workspace"); + std::fs::create_dir_all(&workspace).expect("create workspace"); + let _ws = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", &workspace); + seed_workspace(&workspace); + + let result = ThreadTranscriptSearchTool + .execute(json!({ "query": "migration", "exclude_thread_id": "thread-pg" })) + .await + .expect("transcript_search tool"); + assert!(!result.is_error, "tool should succeed: {}", result.output()); + assert!( + result.output().contains("No past messages matched"), + "excluding the only matching thread should yield no matches — got: {}", + result.output() + ); +} + +/// An explicit empty `exclude_thread_id` is the opt-out: search every thread. +/// (With no active-thread context set in this test, the default path also +/// searches all — this pins the empty-string contract regardless.) +#[tokio::test] +async fn transcript_search_tool_empty_exclude_searches_all_threads() { + let _lock = env_lock(); + let tmp = tempdir().expect("tempdir"); + let _home = EnvVarGuard::set_to_path("HOME", tmp.path()); + let workspace = tmp.path().join("workspace"); + std::fs::create_dir_all(&workspace).expect("create workspace"); + let _ws = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", &workspace); + seed_workspace(&workspace); + + let result = ThreadTranscriptSearchTool + .execute(json!({ "query": "migration", "exclude_thread_id": "" })) + .await + .expect("transcript_search tool"); + assert!(!result.is_error, "tool should succeed: {}", result.output()); + assert!( + result.output().contains("thread-pg"), + "empty exclude must still surface the matching thread — got: {}", + result.output() + ); +} + +/// A missing `query` is a tool error, not a panic — guards the agent against +/// malformed calls. +#[tokio::test] +async fn transcript_search_tool_requires_query() { + let err = ThreadTranscriptSearchTool + .execute(json!({})) + .await + .expect_err("missing query must error"); + assert!( + err.to_string().contains("query"), + "error should mention `query`: {err}" + ); +}