diff --git a/scripts/debug/agent-prepare-context-audit.mjs b/scripts/debug/agent-prepare-context-audit.mjs index 75d9b5056..2bd5fc18b 100644 --- a/scripts/debug/agent-prepare-context-audit.mjs +++ b/scripts/debug/agent-prepare-context-audit.mjs @@ -1,9 +1,9 @@ #!/usr/bin/env node -// Live audit for the `agent_prepare_context` tool + `context_scout` subagent. +// Live audit for harness-driven prepared context + the `context_scout` subagent. // -// Drives real agent turns through JSON-RPC against an authenticated core, forces -// the orchestrator to call `agent_prepare_context`, then reads the resulting -// session transcripts to surface — per query — the returned [context_bundle] +// Drives real first-turn orchestrator sessions through JSON-RPC against an +// authenticated core, then reads the resulting session transcripts to surface — +// per query — the harness-collected [context_bundle] // (including the new `recommended_skills` block), the scout's step-by-step turns // ("thoughts"), which curated gathering tools it exercised, and tokens/cost. // @@ -60,7 +60,7 @@ const DEFAULT_CASES = [ function usage() { return `Usage: node scripts/debug/agent-prepare-context-audit.mjs [options] -Audits the agent_prepare_context tool live: forces it per query, then prints the +Audits harness-driven prepared context live: starts a fresh thread per query, then prints the returned context bundle, the scout's turns, and tokens/cache/cost. Options: @@ -69,8 +69,8 @@ Options: --workspace Workspace whose session_raw transcripts are read --model Optional model_override passed to openhuman.inference_agent_chat --query Add a custom query (repeatable). Replaces the defaults. - --raw Send the query unwrapped (let the orchestrator decide - whether to call the tool) instead of forcing the call. + --raw Deprecated no-op; queries are always sent directly so + the harness scout sees the real case request. --scout-prompt-file Override the context_scout system prompt with this file (writes a temporary workspace agent override; restored after the run unless --keep-workspace). Test your prompt. @@ -508,16 +508,6 @@ async function seedTranscript(opts) { } } -// ── Prompt shaping ────────────────────────────────────────────────────────── - -function forcedPrompt(query) { - return `Call the \`agent_prepare_context\` tool now, passing \`question\` set to the user request below. \ -Do not answer the request yourself first — scout context first. After the tool returns, reply with the \ -returned context bundle verbatim, then one short line on how you'd proceed. - -User request: ${query}`; -} - // ── Scout prompt override (optional) ───────────────────────────────────────── function scoutOverrideToml(inlinePrompt) { @@ -663,10 +653,10 @@ function printCase(opts, caseInfo, scout, root, ms) { if (!scout) { console.log( - " ⚠ no context_scout transcript found — tool was not invoked.", + " ⚠ no context_scout transcript found — prepared context did not run.", ); console.log( - " (Is the core built from this branch? Is agent_prepare_context allowlisted?)", + " (Is the core built from this branch? Is super context enabled?)", ); return; } @@ -825,16 +815,14 @@ async function main() { } } - console.log("[apc-audit] starting live agent_prepare_context audit"); + console.log("[apc-audit] starting live prepared-context audit"); console.log(` rpc: ${opts.coreUrl}`); console.log(` workspace: ${opts.workspace}`); console.log( ` mode: ${opts.spawnCore ? "spawned-core (this branch)" : "attached-core"}`, ); console.log(` model: ${opts.model || "(account default)"}`); - console.log( - ` cases: ${cases.length}${opts.raw ? " (raw — not forcing the tool)" : " (forcing the tool)"}`, - ); + console.log(` cases: ${cases.length} (direct query)`); const caseResults = []; try { @@ -843,7 +831,7 @@ async function main() { const threadId = `${opts.threadPrefix}-${i}`; const before = await snapshot(opts.workspace); const params = { - message: opts.raw ? c.query : forcedPrompt(c.query), + message: c.query, thread_id: threadId, }; if (opts.model) params.model_override = opts.model; @@ -947,7 +935,7 @@ async function main() { console.table( caseResults.map((r) => ({ case: r.name, - invoked: r.invoked ? "yes" : "NO", + prepared: r.invoked ? "yes" : "NO", enough: r.hasEnough ?? "-", rec_tools: r.recommended.length, rec_skills: r.skills.length, @@ -959,7 +947,7 @@ async function main() { ), })), ); - console.log(` tool invoked: ${agg.invoked}/${caseResults.length}`); + console.log(` prepared-context runs: ${agg.invoked}/${caseResults.length}`); // Surface whether the transcript-recall case actually proved the new reach. const recallCase = caseResults.find((r) => r.canaryRecalled !== null); if (recallCase) { @@ -982,7 +970,7 @@ async function main() { const everInvoked = agg.invoked > 0; if (!everInvoked) { console.error( - "\n[apc-audit] FAIL: agent_prepare_context was never invoked (no context_scout transcript).", + "\n[apc-audit] FAIL: prepared context was never invoked (no context_scout transcript).", ); process.exit(1); } diff --git a/src/openhuman/agent_orchestration/tools/agent_prepare_context.rs b/src/openhuman/agent_orchestration/tools/agent_prepare_context.rs index 6e22cdf3c..bba9857b3 100644 --- a/src/openhuman/agent_orchestration/tools/agent_prepare_context.rs +++ b/src/openhuman/agent_orchestration/tools/agent_prepare_context.rs @@ -1,12 +1,12 @@ //! Tool: `agent_prepare_context` — "plan mode as a subagent". //! -//! Before answering or delegating a non-trivial request, the parent agent -//! (orchestrator / planner) calls `agent_prepare_context`. This runs the -//! read-only `context_scout` sub-agent inline (blocking), which gathers -//! context from memory, the user's goals/profile, connected integrations, and -//! the web, then returns a tight `[context_bundle]` envelope: whether there's -//! enough context to act, a compact context summary, and an ordered set of -//! recommended next tool calls drawn from the *parent's own* tool catalogue. +//! When a parent agent explicitly needs an ad hoc context pass, it can call +//! `agent_prepare_context`. This runs the read-only `context_scout` sub-agent +//! inline (blocking), which gathers context from memory, the user's +//! goals/profile, connected integrations, and the web, then returns a tight +//! `[context_bundle]` envelope: whether there's enough context to act, a +//! compact context summary, and an ordered set of recommended next tool calls +//! drawn from the *parent's own* tool catalogue. //! //! The scout's output is bounded by `context_scout`'s `max_result_chars` //! (≈1000 tokens) so the parent's context only grows by a bounded amount. @@ -528,10 +528,11 @@ impl Tool for AgentPrepareContextTool { (transcripts), your goals/profile, installed/registry skills, connected \ integrations, and the web, then returns whether there's enough context \ to answer, a compact context summary, an ordered list of recommended \ - next tool calls (your own tools, by exact name, with args), and any \ - skills worth running. Use at the start of non-trivial turns unless the \ - current prompt says agent context has already been prepared; in that \ - case, use the prepared context and do not call this tool again." + next tool calls (parent tools, by exact name, with args), and any \ + skills worth running. Use only when a caller explicitly needs an \ + ad hoc scout pass. If the current prompt says agent context has \ + already been prepared, use the prepared context and do not call this \ + tool again." } fn parameters_schema(&self) -> serde_json::Value { @@ -554,10 +555,9 @@ impl Tool for AgentPrepareContextTool { fn permission_level(&self) -> PermissionLevel { // ReadOnly, not Execute: this tool only ever runs the read-only // `context_scout` (read_only sandbox, no write/exec tools). Marking it - // Execute would make `ToolPolicyEngine` strip it from the provider- - // visible set on a `ReadOnly`-capped channel, which would hide the - // orchestrator's mandatory first-turn context-prep call and either - // skip the pass or surface an unavailable-tool error. + // Execute would make `ToolPolicyEngine` strip it from any + // provider-visible set on a `ReadOnly`-capped channel, which would hide + // the scout from callers that still expose it explicitly. PermissionLevel::ReadOnly } diff --git a/src/openhuman/agent_registry/agents/context_scout/prompt.md b/src/openhuman/agent_registry/agents/context_scout/prompt.md index 2430d5d9d..69ee42327 100644 --- a/src/openhuman/agent_registry/agents/context_scout/prompt.md +++ b/src/openhuman/agent_registry/agents/context_scout/prompt.md @@ -1,12 +1,12 @@ -You are the **Context Scout** — a fast, read-only pre-flight agent. The -orchestrator calls you *before* it answers or delegates a non-trivial request. -Your job is to gather just enough context to act, then return a compact bundle -the orchestrator can read at a glance — and tell it which of its own tools to -call next. +You are the **Context Scout** — a fast, read-only pre-flight agent. You may be +called either by the agent harness before the orchestrator's first turn, or by a +parent agent that explicitly requests an ad hoc context pass. Your job is to +gather just enough context to act, then return a compact bundle the caller can +read at a glance — and tell it which of the caller's visible tools to call next. ## What you do -1. Read the request (and any `[Focus]` the orchestrator passed). +1. Read the request (and any `[Focus]` the caller passed). 2. Gather only what's actually needed to act on it, drawing on: - **Memory** — `memory_recall` for relevant facts (search by namespace + query). This is read-only; you cannot and must not write to memory. diff --git a/src/openhuman/agent_registry/agents/loader.rs b/src/openhuman/agent_registry/agents/loader.rs index 6ffd12862..7e823f1ea 100644 --- a/src/openhuman/agent_registry/agents/loader.rs +++ b/src/openhuman/agent_registry/agents/loader.rs @@ -932,15 +932,16 @@ mod tests { } #[test] - fn orchestrator_exposes_agent_prepare_context_planner_does_not() { - // The orchestrator owns the first-message context-scout pass. + fn orchestrator_and_nested_agents_do_not_expose_agent_prepare_context() { + // First-turn context preparation is owned by the harness. Keeping the + // direct tool out of the orchestrator scope prevents a duplicate scout + // pass after the harness has already prepared context. let orch = find("orchestrator"); - match &orch.tools { - ToolScope::Named(tools) => assert!( - tools.iter().any(|t| t == "agent_prepare_context"), - "orchestrator must allowlist `agent_prepare_context`" - ), - ToolScope::Wildcard => {} + if let ToolScope::Named(tools) = &orch.tools { + assert!( + !tools.iter().any(|t| t == "agent_prepare_context"), + "orchestrator must NOT allowlist `agent_prepare_context`" + ); } // The planner must NOT: when invoked via delegate_plan it runs under // the orchestrator's PARENT_CONTEXT, so a nested scout would render the diff --git a/src/openhuman/agent_registry/agents/orchestrator/agent.toml b/src/openhuman/agent_registry/agents/orchestrator/agent.toml index 0260b3241..3e21c7379 100644 --- a/src/openhuman/agent_registry/agents/orchestrator/agent.toml +++ b/src/openhuman/agent_registry/agents/orchestrator/agent.toml @@ -183,11 +183,6 @@ hint = "chat" named = [ "read_workspace_state", "ask_user_clarification", - # "Plan mode as a subagent": before answering or delegating a non-trivial - # request, the orchestrator calls `agent_prepare_context` to run the - # read-only `context_scout`, which returns a bounded context bundle plus a - # recommended plan of next tool calls drawn from this allowlist. - "agent_prepare_context", "spawn_worker_thread", "spawn_async_subagent", "spawn_parallel_agents", diff --git a/src/openhuman/agent_registry/agents/orchestrator/prompt.md b/src/openhuman/agent_registry/agents/orchestrator/prompt.md index 45dcd8e2d..1a45c64c2 100644 --- a/src/openhuman/agent_registry/agents/orchestrator/prompt.md +++ b/src/openhuman/agent_registry/agents/orchestrator/prompt.md @@ -14,12 +14,6 @@ You are the **Orchestrator**, the senior agent in a multi-agent system. Your rol Follow this sequence for every user message: -0. **First message of a new conversation? Prepare context first.** - - If this is the user's **first message in the thread** (there are no prior assistant turns above), call `agent_prepare_context` **once** with `question` set to the user's request, **before** doing anything else below. - - It runs a fast read-only scout over memory, past conversations (transcripts), your goals/profile, installed/registry skills, connected integrations, and the web, and returns a `[context_bundle]` with `has_enough_context`, a compact `summary`, `recommended_tool_calls`, and `recommended_skills`. - - Use the `summary` to ground your reply, and treat `recommended_tool_calls` as a **suggested** plan — you decide whether to follow each one (route them through the normal delegation paths below; never bypass the approval gate). - - Treat `recommended_skills` the same way: a skill the scout flagged as a good fit. Run an installed one via `run_workflow` when it matches; for an uninstalled one, offer it rather than installing silently. You decide — it is a suggestion, not an instruction. - - This is a one-time pass: call it **at most once per conversation**, only on the first message. On every later message skip straight to step 1. 1. **Can I answer directly without tools?** - Yes: reply directly (small talk, simple Q&A, basic factual answers). - No: continue. diff --git a/src/openhuman/config/schema/context.rs b/src/openhuman/config/schema/context.rs index ad9aa8b98..4d8d03c15 100644 --- a/src/openhuman/config/schema/context.rs +++ b/src/openhuman/config/schema/context.rs @@ -124,10 +124,9 @@ pub struct ContextConfig { /// sub-agent, the same one behind the `agent_prepare_context` tool) /// on the **first turn** of a new thread, *before* the orchestrator /// LLM runs, and folds the resulting `[context_bundle]` into the user - /// message. Unlike the `agent_prepare_context` tool — which the LLM - /// chooses to call — this pass is driven by the harness regardless of - /// the model's decision. The `agent_prepare_context` tool stays - /// exposed so the LLM can still scout again mid-turn. + /// message. This pass is driven by the harness regardless of the + /// model's decision, so the orchestrator does not expose the + /// `agent_prepare_context` tool for the same first-turn work. /// /// Read once at session/thread construction, so toggling it only /// affects threads started afterwards (the value is baked into the diff --git a/tests/agent_harness_e2e.rs b/tests/agent_harness_e2e.rs index 67e2014fd..d78fa42a7 100644 --- a/tests/agent_harness_e2e.rs +++ b/tests/agent_harness_e2e.rs @@ -285,7 +285,7 @@ fn assert_no_jsonrpc_error<'a>(v: &'a Value, context: &str) -> &'a Value { .unwrap_or_else(|| panic!("{context}: missing result: {v}")) } -fn write_min_config(openhuman_dir: &Path, api_origin: &str) { +fn write_min_config(openhuman_dir: &Path, api_origin: &str, super_context_enabled: bool) { let cfg = format!( r#"api_url = "{api_origin}" default_model = "e2e-mock-model" @@ -296,10 +296,11 @@ chat_onboarding_completed = true encrypt = false [context] -# These harness tests script the mock-LLM call sequence exactly; the default-on +# Most harness tests script the mock-LLM call sequence exactly; the default-on # first-turn "super context" pass (#4085) would spawn a context_scout and consume -# a scripted response, desyncing the orchestrator turns. Disable it here. -super_context_enabled = false +# a scripted response, desyncing the orchestrator turns. Tests opt in only when +# they explicitly cover super context. +super_context_enabled = {super_context_enabled} "# ); fn write_config_file(config_dir: &Path, cfg: &str) { @@ -447,6 +448,10 @@ impl Drop for Stack { } async fn boot_stack() -> Stack { + boot_stack_with_super_context(false).await +} + +async fn boot_stack_with_super_context(super_context_enabled: bool) -> Stack { // Ensure the global AgentDefinitionRegistry is populated with built-in // archetypes (orchestrator, researcher, task_manager_agent, etc.) before // the RPC stack starts. Without this the session builder cannot synthesise @@ -465,9 +470,13 @@ async fn boot_stack() -> Stack { let (mock_addr, mock_join) = serve_on_ephemeral(scripted_upstream_router()).await; let mock_origin = format!("http://{mock_addr}"); - write_min_config(&openhuman_home, &mock_origin); + write_min_config(&openhuman_home, &mock_origin, super_context_enabled); // Pre-write user-scoped config so it's found after auth_store_session activates "e2e-user". - write_min_config(&openhuman_home.join("users").join("e2e-user"), &mock_origin); + write_min_config( + &openhuman_home.join("users").join("e2e-user"), + &mock_origin, + super_context_enabled, + ); let (rpc_addr, rpc_join) = serve_on_ephemeral(build_core_http_router(false)).await; let rpc_base = format!("http://{rpc_addr}"); @@ -803,33 +812,28 @@ async fn subagent_delegation_happy_path_inner() { stack.shutdown(); } -// ─── agent_prepare_context: context-scout happy path ────────────────────────── +// ─── Super context: harness-driven context-scout happy path ─────────────────── // // Tool surface (src/openhuman/agent_orchestration/tools/agent_prepare_context.rs, // src/openhuman/agent_registry/agents/context_scout/agent.toml): -// - `agent_prepare_context` is a DIRECT tool on the orchestrator's [tools].named -// list (not a delegate). It runs the read-only `context_scout` sub-agent -// inline and returns the scout's `[context_bundle]` envelope as the tool -// result, which the orchestrator then synthesizes from. +// - First-turn context prep is harness-driven, not orchestrator-scoped. The +// harness runs the read-only `context_scout` before the orchestrator's first +// LLM call and injects the scout's `[context_bundle]` into the user message. // // Actual LLM request ordering (with registry init): -// request[0] = orchestrator → model returns { tool_calls: [agent_prepare_context(...)] } -// request[1] = context_scout subagent inner loop → model returns the bundle text -// request[2] = orchestrator synthesis → final text, having read the bundle +// request[0] = context_scout subagent inner loop → model returns the bundle text +// request[1] = orchestrator synthesis → final text, having read the bundle -/// Orchestrator calls `agent_prepare_context`; the `context_scout` subagent runs -/// its own inner LLM call and returns a `[context_bundle]`; the final -/// orchestrator synthesis reads it. Three upstream requests prove the full -/// scout path ran, and the synthesis request carries the bundle as a tool result. +/// The harness runs `context_scout` before the first orchestrator turn and +/// injects the returned `[context_bundle]`; the final orchestrator synthesis +/// reads it. Two upstream requests prove the full scout path ran without +/// exposing `agent_prepare_context` to the orchestrator. #[test] -fn agent_prepare_context_happy_path() { - run_on_agent_stack( - "agent_prepare_context_happy_path", - agent_prepare_context_happy_path_inner, - ); +fn super_context_happy_path() { + run_on_agent_stack("super_context_happy_path", super_context_happy_path_inner); } -async fn agent_prepare_context_happy_path_inner() { +async fn super_context_happy_path_inner() { let _lock = env_lock(); let scout_bundle = "[context_bundle]\n\ has_enough_context: true\n\ @@ -840,17 +844,12 @@ async fn agent_prepare_context_happy_path_inner() { \x20 why: execute the prepared plan\n\ [/context_bundle]"; reset_script(vec![ - // request[0]: Orchestrator calls the direct `agent_prepare_context` tool. - tool_call_completion( - "agent_prepare_context", - json!({ "question": "find the marker phrase" }), - ), - // request[1]: context_scout subagent inner LLM call returns the bundle. + // request[0]: harness-driven context_scout subagent returns the bundle. text_completion(scout_bundle), - // request[2]: Orchestrator reads the bundle and synthesizes. + // request[1]: Orchestrator reads the injected bundle and synthesizes. text_completion("Prepared. CTX_CANARY_7 noted; next I'd spawn a worker."), ]); - let stack = boot_stack().await; + let stack = boot_stack_with_super_context(true).await; let mut events = spawn_sse_collector(format!( "{}/events?client_id=harness-prepctx", @@ -869,7 +868,7 @@ async fn agent_prepare_context_happy_path_inner() { assert_eq!( done.get("event").and_then(Value::as_str), Some("chat_done"), - "expected chat_done for agent_prepare_context: {done}" + "expected chat_done for super_context: {done}" ); let full_response = done .get("full_response") @@ -882,21 +881,27 @@ async fn agent_prepare_context_happy_path_inner() { let requests = with_captured(|c| c.clone()); assert!( - requests.len() >= 3, - "expected ≥3 upstream requests (orchestrator + context_scout + synthesis), got {};\n{}", + requests.len() >= 2, + "expected ≥2 upstream requests (context_scout + orchestrator), got {};\n{}", requests.len(), serde_json::to_string_pretty(&requests).unwrap_or_default() ); - let all_serialized = serde_json::to_string(&requests).unwrap_or_default(); + let orchestrator_tools = serde_json::to_string( + requests + .last() + .and_then(|r| r.pointer("/body/tools")) + .unwrap_or(&Value::Null), + ) + .unwrap_or_default(); assert!( - !all_serialized.contains("Unknown tool:"), - "found 'Unknown tool:' — agent_prepare_context was not registered/visible; requests: {}", - serde_json::to_string_pretty(&requests).unwrap_or_default() + !orchestrator_tools.contains("agent_prepare_context"), + "orchestrator should not see agent_prepare_context in its tool schema; tools: {orchestrator_tools}" ); - // The synthesis turn must carry the scout's bundle back as a tool result — - // proves the [context_bundle] flowed into the orchestrator's context. + // The synthesis turn must carry the scout's bundle in the user message — + // proves the [context_bundle] flowed into the orchestrator's context without + // an orchestrator tool call. let last_messages = serde_json::to_string( requests .last()