From 1c58d478c50c5dc3724c71d03fa14178a5ee47d8 Mon Sep 17 00:00:00 2001 From: Steven Enamakel <31011319+senamakel@users.noreply.github.com> Date: Thu, 14 May 2026 05:11:06 -0700 Subject: [PATCH] test(harness): smart mock LLM provider + fake Composio backend + 21 new tests (#1729) --- scripts/mock-api/routes/integrations.mjs | 11 +- scripts/mock-api/routes/llm.mjs | 148 +++ scripts/mock-api/server.mjs | 5 + src/openhuman/agent/harness/bughunt_tests.rs | 557 ++++++++++ src/openhuman/agent/harness/mod.rs | 7 + src/openhuman/agent/harness/test_support.rs | 633 ++++++++++++ .../agent/harness/test_support_tests.rs | 978 ++++++++++++++++++ 7 files changed, 2329 insertions(+), 10 deletions(-) create mode 100644 scripts/mock-api/routes/llm.mjs create mode 100644 src/openhuman/agent/harness/bughunt_tests.rs create mode 100644 src/openhuman/agent/harness/test_support.rs create mode 100644 src/openhuman/agent/harness/test_support_tests.rs diff --git a/scripts/mock-api/routes/integrations.mjs b/scripts/mock-api/routes/integrations.mjs index 33c9f3049..57c2c373b 100644 --- a/scripts/mock-api/routes/integrations.mjs +++ b/scripts/mock-api/routes/integrations.mjs @@ -141,16 +141,7 @@ export function handleIntegrations(ctx) { return true; } - if (method === "POST" && /^\/openai\/v1\/chat\/completions\/?$/.test(url)) { - json(res, 200, { - choices: [ - { - message: { role: "assistant", content: "Hello from e2e mock agent" }, - }, - ], - }); - return true; - } + // (chat/completions is handled by routes/llm.mjs ahead of this route) // ── Composio ─────────────────────────────────────────────── if ( diff --git a/scripts/mock-api/routes/llm.mjs b/scripts/mock-api/routes/llm.mjs new file mode 100644 index 000000000..3eb2595cc --- /dev/null +++ b/scripts/mock-api/routes/llm.mjs @@ -0,0 +1,148 @@ +import { json } from "../http.mjs"; +import { behavior, parseBehaviorJson, setMockBehavior } from "../state.mjs"; + +/** + * Smart mock LLM endpoint. + * + * Drives keyword-based routing so unit/E2E tests can exercise the agent + * harness end-to-end without spinning up a real model. The mock looks + * at the latest user/tool message in the request and either: + * + * 1. Replays a forced response queue (`llmForcedResponses` behavior), + * 2. Matches a configured keyword rule (`llmKeywordRules` behavior), + * 3. Falls through to a sensible default ("Hello from e2e mock agent"). + * + * Keyword rules look like: + * + * [ + * { + * "keyword": "search", // case-insensitive substring + * "toolCalls": [ + * { "name": "search_tool", "arguments": {"q": "rust"} } + * ], + * "content": "Looking it up..." + * }, + * { + * "keyword": "search_tool-ok", + * "content": "Here's the answer." + * } + * ] + * + * Configure with: + * POST /__admin/behavior body: {"llmKeywordRules": ""} + * + * This mirrors the Rust-side `KeywordScriptedProvider` in + * `src/openhuman/agent/harness/test_support.rs` so the same testing + * mental model applies on both sides of the FFI. + */ + +function pickProbeText(parsedBody) { + if (!parsedBody || !Array.isArray(parsedBody.messages)) return ""; + for (let i = parsedBody.messages.length - 1; i >= 0; i -= 1) { + const m = parsedBody.messages[i]; + if (!m || typeof m !== "object") continue; + if (m.role === "user" || m.role === "tool") { + if (typeof m.content === "string") return m.content; + if (Array.isArray(m.content)) { + return m.content + .filter((c) => c && c.type === "text" && typeof c.text === "string") + .map((c) => c.text) + .join(" "); + } + } + } + return ""; +} + +function makeChoice({ content, toolCalls, callIdSeed }) { + const message = { role: "assistant", content: content ?? "" }; + if (Array.isArray(toolCalls) && toolCalls.length > 0) { + message.tool_calls = toolCalls.map((tc, idx) => ({ + id: tc.id ?? `call_${callIdSeed}_${idx}`, + type: "function", + function: { + name: String(tc.name ?? ""), + arguments: + typeof tc.arguments === "string" + ? tc.arguments + : JSON.stringify(tc.arguments ?? {}), + }, + })); + if (!content) message.content = null; + } + return { index: 0, message, finish_reason: toolCalls?.length ? "tool_calls" : "stop" }; +} + +function buildResponse({ model, content, toolCalls }) { + const seed = Date.now(); + return { + id: `chatcmpl-mock-${seed}`, + object: "chat.completion", + created: Math.floor(seed / 1000), + model: model || "e2e-mock-model", + choices: [makeChoice({ content, toolCalls, callIdSeed: seed })], + usage: { + prompt_tokens: 10, + completion_tokens: 10, + total_tokens: 20, + }, + }; +} + +/** + * Drive a mock OpenAI-compatible /v1/chat/completions endpoint with + * keyword-based responses. Returns true if the request was handled. + */ +export function handleLlmCompletions(ctx) { + const { method, url, parsedBody, res } = ctx; + if ( + method !== "POST" || + !/^\/openai\/v1\/chat\/completions\/?$/.test(url) + ) { + return false; + } + + const mockBehavior = behavior(); + const model = + typeof parsedBody?.model === "string" ? parsedBody.model : "e2e-mock-model"; + + // 1. Forced queue — replay exact ChatResponse objects in order. + const forced = parseBehaviorJson("llmForcedResponses", []); + if (Array.isArray(forced) && forced.length > 0) { + const next = forced.shift(); + // Persist the shrunk queue back so subsequent requests advance. + setMockBehavior("llmForcedResponses", JSON.stringify(forced)); + json(res, 200, buildResponse({ model, ...next })); + return true; + } + + // 2. Keyword rules. + const rules = parseBehaviorJson("llmKeywordRules", []); + const probe = pickProbeText(parsedBody).toLowerCase(); + if (Array.isArray(rules)) { + for (const rule of rules) { + if (!rule || typeof rule.keyword !== "string") continue; + if (probe.includes(rule.keyword.toLowerCase())) { + json( + res, + 200, + buildResponse({ + model, + content: rule.content ?? "", + toolCalls: rule.toolCalls ?? [], + }), + ); + return true; + } + } + } + + // 3. Default fallback. + const fallback = + typeof mockBehavior.llmFallbackContent === "string" && + mockBehavior.llmFallbackContent.length > 0 + ? mockBehavior.llmFallbackContent + : "Hello from e2e mock agent"; + json(res, 200, buildResponse({ model, content: fallback })); + return true; +} diff --git a/scripts/mock-api/server.mjs b/scripts/mock-api/server.mjs index 958fa6f49..119199b24 100644 --- a/scripts/mock-api/server.mjs +++ b/scripts/mock-api/server.mjs @@ -14,6 +14,7 @@ import { handleConversations } from "./routes/conversations.mjs"; import { handleCron } from "./routes/cron.mjs"; import { handleIntegrations } from "./routes/integrations.mjs"; import { handleInvites } from "./routes/invites.mjs"; +import { handleLlmCompletions } from "./routes/llm.mjs"; import { handleOAuth } from "./routes/oauth.mjs"; import { handlePayments } from "./routes/payments.mjs"; import { handleUser } from "./routes/user.mjs"; @@ -37,6 +38,10 @@ const ROUTE_HANDLERS = [ handleUser, handleInvites, handlePayments, + // LLM completions must run before the catch-all stub in + // `handleIntegrations` so keyword-driven test scripts can override + // the default "Hello from e2e mock agent" reply. + handleLlmCompletions, handleIntegrations, handleWebhooks, handleCron, diff --git a/src/openhuman/agent/harness/bughunt_tests.rs b/src/openhuman/agent/harness/bughunt_tests.rs new file mode 100644 index 000000000..dae7656ce --- /dev/null +++ b/src/openhuman/agent/harness/bughunt_tests.rs @@ -0,0 +1,557 @@ +//! Targeted bug-hunt tests for the agent harness + tool dispatch. +//! +//! These pair the [`super::test_support::KeywordScriptedProvider`] with +//! tightly-scoped tools to probe corner cases that aren't covered by +//! the broader behavioural suite. Each test documents the behaviour +//! observed, and any tests prefixed `documents_` describe a quirk +//! worth flagging in code review (silent data loss, surprising +//! precedence rules, etc.) rather than asserting correctness. + +use super::test_support::{KeywordRule, KeywordScriptedProvider, ScriptedToolCall}; +use super::tool_loop::run_tool_call_loop; +use crate::openhuman::providers::{ChatMessage, ChatResponse, ToolCall}; +use crate::openhuman::tools::traits::{Tool, ToolResult}; +use async_trait::async_trait; +use parking_lot::Mutex; +use serde_json::json; +use std::sync::Arc; + +fn mm() -> crate::openhuman::config::MultimodalConfig { + crate::openhuman::config::MultimodalConfig::default() +} + +struct ArgsCapturingTool { + name_str: String, + captured: Arc>>, + output: String, +} + +impl ArgsCapturingTool { + fn new(name: &str, output: &str) -> (Self, Arc>>) { + let captured = Arc::new(Mutex::new(Vec::new())); + ( + Self { + name_str: name.to_string(), + captured: captured.clone(), + output: output.to_string(), + }, + captured, + ) + } +} + +#[async_trait] +impl Tool for ArgsCapturingTool { + fn name(&self) -> &str { + &self.name_str + } + fn description(&self) -> &str { + "captures args" + } + fn parameters_schema(&self) -> serde_json::Value { + json!({"type":"object","additionalProperties":true}) + } + async fn execute(&self, args: serde_json::Value) -> anyhow::Result { + self.captured.lock().push(args); + Ok(ToolResult::success(self.output.clone())) + } +} + +// ── 1. Native tool call with a JSON-encoded string of args ──────── +// +// Real OpenAI/Anthropic providers send `arguments` as a *string* +// containing JSON. The harness must transparently decode it before +// passing to the tool. + +#[tokio::test] +async fn native_tool_call_decodes_json_encoded_arguments_string() { + let provider = + KeywordScriptedProvider::new(vec![KeywordRule::final_reply("captured-ok", "done")]) + .with_native_tools(true); + + // Forced first turn: native tool_call with arguments as a STRING. + provider.push_forced_response(ChatResponse { + text: None, + tool_calls: vec![ToolCall { + id: "c1".into(), + name: "captured".into(), + arguments: "{\"city\":\"Berlin\",\"n\":3}".to_string(), + }], + usage: None, + }); + + let (tool, captured) = ArgsCapturingTool::new("captured", "captured-ok"); + let tools: Vec> = vec![Box::new(tool)]; + let mut history = vec![ChatMessage::user("anything")]; + + let out = run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock", + "m", + 0.0, + true, + None, + "channel", + &mm(), + 3, + None, + None, + &[], + None, + None, + ) + .await + .unwrap(); + + assert_eq!(out, "done"); + let args = captured.lock(); + assert_eq!(args.len(), 1); + assert_eq!(args[0]["city"], "Berlin"); + assert_eq!(args[0]["n"], 3); +} + +// ── 2. SILENT FAILURE: non-JSON args string is replaced with `{}` ── +// +// `parse_arguments_value` calls `serde_json::from_str` on the string +// payload and silently falls back to `{}` on parse failure. This +// means: if a model emits `arguments: "world"` (not valid JSON), the +// tool sees `{}` — the user's intent is silently dropped and there's +// no signal to the LLM that anything went wrong. +// +// This test documents the behaviour so future refactors don't +// "accidentally" fix it without considering downstream impact, and +// flags the behaviour for follow-up. + +#[tokio::test] +async fn documents_silent_drop_of_non_json_arguments_string() { + let provider = + KeywordScriptedProvider::new(vec![KeywordRule::final_reply("captured-ok", "done")]) + .with_native_tools(true); + + provider.push_forced_response(ChatResponse { + text: None, + tool_calls: vec![ToolCall { + id: "c1".into(), + name: "captured".into(), + // Not valid JSON — the model "meant" a plain string. + arguments: "world".to_string(), + }], + usage: None, + }); + + let (tool, captured) = ArgsCapturingTool::new("captured", "captured-ok"); + let tools: Vec> = vec![Box::new(tool)]; + let mut history = vec![ChatMessage::user("hi")]; + + run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock", + "m", + 0.0, + true, + None, + "channel", + &mm(), + 3, + None, + None, + &[], + None, + None, + ) + .await + .unwrap(); + + let args = captured.lock(); + assert_eq!(args.len(), 1); + // BUG-CANDIDATE: the LLM's intent ("world") is silently dropped. + // The tool receives an empty object with no indication the args + // were unparseable. A more defensive design would surface a + // structured error back to the model instead. + assert_eq!(args[0], json!({})); +} + +// ── 3. Parallel tool calls in a single iteration ────────────────── +// +// The model may emit multiple `` blocks at once. They +// should all execute in order, each result threaded into history. + +#[tokio::test] +async fn parallel_tool_calls_in_single_iteration_all_execute() { + let provider = + KeywordScriptedProvider::new(vec![KeywordRule::final_reply("tool_b-ok", "all done")]); + + // Both tool calls share one assistant turn (XML path). + provider.push_forced_response(ChatResponse { + text: Some( + "{\"name\":\"tool_a\",\"arguments\":{\"k\":1}}\n\ + {\"name\":\"tool_b\",\"arguments\":{\"k\":2}}" + .into(), + ), + tool_calls: vec![], + usage: None, + }); + + let (a, a_calls) = ArgsCapturingTool::new("tool_a", "tool_a-ok"); + let (b, b_calls) = ArgsCapturingTool::new("tool_b", "tool_b-ok"); + let tools: Vec> = vec![Box::new(a), Box::new(b)]; + let mut history = vec![ChatMessage::user("do both")]; + + let out = run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock", + "m", + 0.0, + true, + None, + "channel", + &mm(), + 5, + None, + None, + &[], + None, + None, + ) + .await + .unwrap(); + + assert_eq!(out, "all done"); + assert_eq!(a_calls.lock().len(), 1); + assert_eq!(b_calls.lock().len(), 1); + assert_eq!(a_calls.lock()[0]["k"], 1); + assert_eq!(b_calls.lock()[0]["k"], 2); +} + +// ── 4. Same-named tools: first match in registry wins ───────────── + +#[tokio::test] +async fn same_named_tool_in_registry_first_match_wins() { + let provider = KeywordScriptedProvider::new(vec![ + KeywordRule::tool_call("go", ScriptedToolCall::new("dupe", json!({}))), + KeywordRule::final_reply("first-output", "got first"), + ]); + + let (first, first_calls) = ArgsCapturingTool::new("dupe", "first-output"); + let (second, second_calls) = ArgsCapturingTool::new("dupe", "second-output"); + let tools: Vec> = vec![Box::new(first), Box::new(second)]; + let mut history = vec![ChatMessage::user("go ahead")]; + + let out = run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock", + "m", + 0.0, + true, + None, + "channel", + &mm(), + 5, + None, + None, + &[], + None, + None, + ) + .await + .unwrap(); + + assert_eq!(out, "got first"); + assert_eq!(first_calls.lock().len(), 1); + assert_eq!(second_calls.lock().len(), 0); +} + +// ── 5. Markdown-fenced tool call (```tool_call ... ```) ─────────── +// +// Some OpenRouter-mediated models emit fenced markdown blocks +// instead of XML tags. `parse_tool_calls` is supposed to handle this. + +#[tokio::test] +async fn markdown_fenced_tool_call_block_is_parsed() { + let provider = + KeywordScriptedProvider::new(vec![KeywordRule::final_reply("tool_a-ok", "ok done")]); + + provider.push_forced_response(ChatResponse { + text: Some( + "Here's the call:\n\ + ```tool_call\n\ + {\"name\":\"tool_a\",\"arguments\":{\"x\":42}}\n\ + ```" + .into(), + ), + tool_calls: vec![], + usage: None, + }); + + let (a, a_calls) = ArgsCapturingTool::new("tool_a", "tool_a-ok"); + let tools: Vec> = vec![Box::new(a)]; + let mut history = vec![ChatMessage::user("anything")]; + + let out = run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock", + "m", + 0.0, + true, + None, + "channel", + &mm(), + 5, + None, + None, + &[], + None, + None, + ) + .await + .unwrap(); + + assert_eq!(out, "ok done"); + assert_eq!(a_calls.lock().len(), 1); + assert_eq!(a_calls.lock()[0]["x"], 42); +} + +// ── 6. Native vs prompt-guided precedence ───────────────────────── +// +// When a response carries BOTH native `tool_calls` *and* an XML +// `` block in the text, the native calls are authoritative +// and the XML must NOT also fire (else the same logical call could +// execute twice). + +#[tokio::test] +async fn native_tool_calls_take_precedence_over_xml_in_text() { + let provider = + KeywordScriptedProvider::new(vec![KeywordRule::final_reply("tool_a-ok", "done")]) + .with_native_tools(true); + + provider.push_forced_response(ChatResponse { + text: Some("{\"name\":\"tool_a\",\"arguments\":{}}".into()), + tool_calls: vec![ToolCall { + id: "c1".into(), + name: "tool_a".into(), + arguments: "{\"src\":\"native\"}".into(), + }], + usage: None, + }); + + let (a, a_calls) = ArgsCapturingTool::new("tool_a", "tool_a-ok"); + let tools: Vec> = vec![Box::new(a)]; + let mut history = vec![ChatMessage::user("call it")]; + + run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock", + "m", + 0.0, + true, + None, + "channel", + &mm(), + 5, + None, + None, + &[], + None, + None, + ) + .await + .unwrap(); + + // Tool ran exactly once, using the native args (not the XML block). + assert_eq!(a_calls.lock().len(), 1); + assert_eq!(a_calls.lock()[0]["src"], "native"); +} + +// ── 7. Big tool output: per-tool cap truncation ─────────────────── + +struct CappedBigTool; + +#[async_trait] +impl Tool for CappedBigTool { + fn name(&self) -> &str { + "cap_big" + } + fn description(&self) -> &str { + "emits a big payload but caps it" + } + fn parameters_schema(&self) -> serde_json::Value { + json!({"type":"object"}) + } + fn max_result_size_chars(&self) -> Option { + Some(50) + } + async fn execute(&self, _args: serde_json::Value) -> anyhow::Result { + Ok(ToolResult::success("X".repeat(500))) + } +} + +#[tokio::test] +async fn per_tool_max_result_size_caps_history_payload() { + let provider = KeywordScriptedProvider::new(vec![ + KeywordRule::tool_call("go", ScriptedToolCall::new("cap_big", json!({}))), + KeywordRule::final_reply("truncated by tool cap", "ok"), + ]); + + let tools: Vec> = vec![Box::new(CappedBigTool)]; + let mut history = vec![ChatMessage::user("go big")]; + + run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock", + "m", + 0.0, + true, + None, + "channel", + &mm(), + 5, + None, + None, + &[], + None, + None, + ) + .await + .unwrap(); + + let tool_results = history + .iter() + .find(|msg| msg.role == "user" && msg.content.contains("[Tool results]")) + .expect("tool results should land in history"); + assert!( + tool_results.content.contains("truncated by tool cap"), + "cap marker missing: {}", + tool_results.content + ); + assert!( + tool_results.content.len() < 500, + "raw 500-char body must not flow through (got {} chars)", + tool_results.content.len() + ); +} + +// ── 8. Empty assistant response with no tool calls terminates loop + +#[tokio::test] +async fn empty_response_with_no_tool_calls_terminates_with_empty_text() { + let provider = KeywordScriptedProvider::new(vec![]); + provider.push_forced_response(ChatResponse { + text: Some(String::new()), + tool_calls: vec![], + usage: None, + }); + + let tools: Vec> = vec![]; + let mut history = vec![ChatMessage::user("hi")]; + + let out = run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock", + "m", + 0.0, + true, + None, + "channel", + &mm(), + 5, + None, + None, + &[], + None, + None, + ) + .await + .unwrap(); + + assert!(out.is_empty()); + // Loop must still record the (empty) assistant turn. + assert!(history.iter().any(|m| m.role == "assistant")); +} + +// ── 9. Progress sink receives ordered turn lifecycle events ─────── + +#[tokio::test] +async fn progress_sink_emits_lifecycle_events_in_order() { + use crate::openhuman::agent::progress::AgentProgress; + + let provider = KeywordScriptedProvider::new(vec![ + KeywordRule::tool_call("go", ScriptedToolCall::new("p_tool", json!({}))), + KeywordRule::final_reply("p_tool-ok", "all done"), + ]); + + let (tool, _) = ArgsCapturingTool::new("p_tool", "p_tool-ok"); + let tools: Vec> = vec![Box::new(tool)]; + + let (tx, mut rx) = tokio::sync::mpsc::channel::(32); + + let mut history = vec![ChatMessage::user("go go")]; + run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock", + "m", + 0.0, + true, + None, + "channel", + &mm(), + 5, + None, + None, + &[], + Some(tx), + None, + ) + .await + .unwrap(); + + let mut events = Vec::new(); + while let Ok(e) = rx.try_recv() { + events.push(e); + } + + // Must see TurnStarted, at least 2 IterationStarted, ToolCallStarted, + // ToolCallCompleted, and TurnCompleted. + let kinds: Vec<&'static str> = events + .iter() + .map(|e| match e { + AgentProgress::TurnStarted => "TurnStarted", + AgentProgress::IterationStarted { .. } => "IterationStarted", + AgentProgress::ToolCallStarted { .. } => "ToolCallStarted", + AgentProgress::ToolCallCompleted { .. } => "ToolCallCompleted", + AgentProgress::TurnCompleted { .. } => "TurnCompleted", + _ => "Other", + }) + .collect(); + + assert_eq!(kinds.first().copied(), Some("TurnStarted")); + assert_eq!(kinds.last().copied(), Some("TurnCompleted")); + assert!(kinds.contains(&"IterationStarted")); + assert!(kinds.contains(&"ToolCallStarted")); + assert!(kinds.contains(&"ToolCallCompleted")); + + // ToolCallStarted must precede its matching ToolCallCompleted. + let started = kinds.iter().position(|k| *k == "ToolCallStarted").unwrap(); + let completed = kinds + .iter() + .position(|k| *k == "ToolCallCompleted") + .unwrap(); + assert!(started < completed); +} diff --git a/src/openhuman/agent/harness/mod.rs b/src/openhuman/agent/harness/mod.rs index c2b591723..16b61b95f 100644 --- a/src/openhuman/agent/harness/mod.rs +++ b/src/openhuman/agent/harness/mod.rs @@ -51,5 +51,12 @@ pub(crate) use instructions::build_tool_instructions_filtered; pub(crate) use parse::parse_tool_calls; pub(crate) use tool_loop::run_tool_call_loop; +#[cfg(test)] +mod bughunt_tests; +#[cfg(test)] +pub(crate) mod test_support; +#[cfg(test)] +mod test_support_tests; + #[cfg(test)] mod tests; diff --git a/src/openhuman/agent/harness/test_support.rs b/src/openhuman/agent/harness/test_support.rs new file mode 100644 index 000000000..77e9778e3 --- /dev/null +++ b/src/openhuman/agent/harness/test_support.rs @@ -0,0 +1,633 @@ +//! Smart-mock test support for the agent harness. +//! +//! This module provides a reusable "fake LLM" that drives the real +//! [`run_tool_call_loop`] without needing any network access. Two +//! building blocks are exposed: +//! +//! 1. [`KeywordScriptedProvider`] — a [`Provider`] implementation that +//! inspects the latest `user` message of the conversation and emits +//! canned tool calls (or a final reply) when a configured keyword +//! matches. The first turn that has *no* matching rule returns a +//! plain "done" reply, which terminates the loop deterministically. +//! +//! Compared to the hand-rolled `ScriptedProvider` in +//! [`super::tool_loop_tests`], this provider: +//! * Reacts to the rolling conversation state instead of replaying +//! a fixed queue — so tests can exercise iterative loops where +//! what the LLM does next depends on what tools returned. +//! * Supports both **native** OpenAI-style `tool_calls` and +//! **prompt-guided** `…` text — flipping +//! a single flag toggles which surface the harness exercises. +//! * Records the messages it saw and the responses it returned for +//! post-hoc assertion. +//! +//! 2. [`spawn_fake_composio_backend`] — boots a minimal axum app that +//! responds to the `/agent-integrations/composio/*` routes with +//! realistic-looking fixture data (real-world toolkit/action shapes, +//! not synthetic gibberish). Tests can pair this with a +//! [`crate::openhuman::composio::ComposioClient`] to exercise the +//! full agent → tool → backend → response flow against a hermetic +//! in-process server. +//! +//! Both helpers are `#[cfg(test)]`-only so they never leak into release +//! binaries. + +use std::collections::VecDeque; +use std::sync::Arc; + +use async_trait::async_trait; +use parking_lot::Mutex; +use serde_json::json; + +use crate::openhuman::providers::traits::ProviderCapabilities; +use crate::openhuman::providers::{ChatMessage, ChatRequest, ChatResponse, Provider, ToolCall}; + +/// One scripted reaction the [`KeywordScriptedProvider`] can emit when +/// it sees its keyword in the latest user/tool turn. +#[derive(Debug, Clone)] +pub struct KeywordRule { + /// Substring matched (case-insensitive) against the latest user + /// or tool message in the conversation. + pub keyword: String, + /// Tool calls to emit. Empty ⇒ no tool calls, only `final_text`. + pub tool_calls: Vec, + /// Optional plain-text body to include alongside any tool calls. + /// When `tool_calls` is empty, this becomes the loop-terminating + /// final response. + pub final_text: Option, + /// How many times this rule may fire. `None` ⇒ unlimited. + pub max_fires: Option, +} + +#[derive(Debug, Clone)] +pub struct ScriptedToolCall { + pub name: String, + pub arguments: serde_json::Value, +} + +impl ScriptedToolCall { + pub fn new(name: impl Into, arguments: serde_json::Value) -> Self { + Self { + name: name.into(), + arguments, + } + } +} + +impl KeywordRule { + pub fn final_reply(keyword: impl Into, text: impl Into) -> Self { + Self { + keyword: keyword.into(), + tool_calls: Vec::new(), + final_text: Some(text.into()), + max_fires: None, + } + } + + pub fn tool_call(keyword: impl Into, call: ScriptedToolCall) -> Self { + Self { + keyword: keyword.into(), + tool_calls: vec![call], + final_text: None, + max_fires: Some(1), + } + } + + pub fn with_text(mut self, text: impl Into) -> Self { + self.final_text = Some(text.into()); + self + } + + pub fn unlimited(mut self) -> Self { + self.max_fires = None; + self + } +} + +/// Snapshot of one turn the provider served — handy for tests that +/// want to assert what the LLM "saw" without coupling to the harness +/// internals. +#[derive(Debug, Clone)] +pub struct ProviderTurn { + pub messages: Vec, + pub rule_keyword: Option, + pub emitted_tool_calls: Vec, + pub emitted_text: Option, +} + +struct ProviderState { + rules: Vec, + fired: Vec, + turns: Vec, + fallback_text: String, + next_call_id: usize, + /// Optional queue of scripted responses to consume *before* the + /// keyword rules run — useful when a test wants the first turn to + /// behave deterministically regardless of the user message. + forced: VecDeque, +} + +/// Smart provider that reacts to conversation state via keyword rules. +pub struct KeywordScriptedProvider { + state: Arc>, + native_tools: bool, + vision: bool, +} + +impl KeywordScriptedProvider { + pub fn new(rules: Vec) -> Self { + Self { + state: Arc::new(Mutex::new(ProviderState { + rules, + fired: Vec::new(), + turns: Vec::new(), + fallback_text: "done".to_string(), + next_call_id: 0, + forced: VecDeque::new(), + })), + native_tools: false, + vision: false, + } + } + + pub fn with_native_tools(mut self, enabled: bool) -> Self { + self.native_tools = enabled; + self + } + + pub fn with_vision(mut self, enabled: bool) -> Self { + self.vision = enabled; + self + } + + pub fn with_fallback(self, text: impl Into) -> Self { + { + let mut guard = self.state.lock(); + guard.fallback_text = text.into(); + } + self + } + + pub fn push_forced_response(&self, resp: ChatResponse) { + self.state.lock().forced.push_back(resp); + } + + pub fn turns(&self) -> Vec { + self.state.lock().turns.clone() + } + + pub fn turn_count(&self) -> usize { + self.state.lock().turns.len() + } +} + +fn latest_user_or_tool_msg(messages: &[ChatMessage]) -> Option<&ChatMessage> { + messages + .iter() + .rev() + .find(|m| m.role == "user" || m.role == "tool") +} + +#[async_trait] +impl Provider for KeywordScriptedProvider { + fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + native_tool_calling: self.native_tools, + vision: self.vision, + } + } + + async fn chat_with_system( + &self, + _system_prompt: Option<&str>, + _message: &str, + _model: &str, + _temperature: f64, + ) -> anyhow::Result { + Ok("fallback".into()) + } + + async fn chat( + &self, + request: ChatRequest<'_>, + _model: &str, + _temperature: f64, + ) -> anyhow::Result { + let messages = request.messages.to_vec(); + let mut state = self.state.lock(); + + // Forced queue wins, regardless of keyword matching. + if let Some(resp) = state.forced.pop_front() { + state.turns.push(ProviderTurn { + messages, + rule_keyword: None, + emitted_tool_calls: resp.tool_calls.clone(), + emitted_text: resp.text.clone(), + }); + return Ok(resp); + } + + let probe = latest_user_or_tool_msg(&messages) + .map(|m| m.content.to_lowercase()) + .unwrap_or_default(); + + let mut chosen: Option = None; + for (idx, rule) in state.rules.iter().enumerate() { + let fired = *state.fired.get(idx).unwrap_or(&0); + if let Some(cap) = rule.max_fires { + if fired >= cap { + continue; + } + } + if probe.contains(&rule.keyword.to_lowercase()) { + chosen = Some(idx); + break; + } + } + + let (rule_keyword, tool_calls, text) = if let Some(idx) = chosen { + while state.fired.len() <= idx { + state.fired.push(0); + } + state.fired[idx] += 1; + let rule = state.rules[idx].clone(); + let tool_calls: Vec = if self.native_tools { + rule.tool_calls + .iter() + .map(|c| { + let id = state.next_call_id; + state.next_call_id += 1; + ToolCall { + id: format!("call_{id}"), + name: c.name.clone(), + arguments: c.arguments.to_string(), + } + }) + .collect() + } else { + Vec::new() + }; + + let text = if self.native_tools { + rule.final_text.clone() + } else if !rule.tool_calls.is_empty() { + // Prompt-guided: emit XML-wrapped tool calls in text. + let mut body = String::new(); + if let Some(prefix) = &rule.final_text { + body.push_str(prefix); + if !prefix.ends_with('\n') { + body.push('\n'); + } + } + for c in &rule.tool_calls { + body.push_str(""); + body.push_str(&json!({"name": c.name, "arguments": c.arguments}).to_string()); + body.push_str("\n"); + } + Some(body) + } else { + rule.final_text.clone() + }; + + (Some(rule.keyword.clone()), tool_calls, text) + } else { + // No rule matched — emit the fallback as the final reply + // so the loop terminates rather than hanging. + (None, Vec::new(), Some(state.fallback_text.clone())) + }; + + let resp = ChatResponse { + text: text.clone(), + tool_calls: tool_calls.clone(), + usage: None, + }; + + state.turns.push(ProviderTurn { + messages, + rule_keyword, + emitted_tool_calls: tool_calls, + emitted_text: text, + }); + + Ok(resp) + } +} + +// ── Fake Composio backend ────────────────────────────────────────── + +use crate::openhuman::composio::ComposioClient; +use crate::openhuman::integrations::IntegrationClient; +use axum::{ + extract::{Path, State}, + routing::{delete, get, post}, + Json, Router, +}; + +/// Realistic-looking Composio fixture data (toolkit slugs, action +/// names, and parameter shapes lifted from the production catalog — +/// just enough to exercise the schemas without coupling tests to the +/// upstream API). +#[derive(Default, Clone)] +pub struct ComposioFixture { + pub toolkits: Vec, + pub connections: Vec, + pub tools: Vec, + /// Per-action canned execute responses, keyed by action slug. + pub execute_responses: std::collections::HashMap, +} + +impl ComposioFixture { + /// A reasonable default fixture: GMail/Notion/GitHub connections + /// with two actions each. Use this when a test just needs *some* + /// Composio data and doesn't care about the exact shape. + pub fn realistic() -> Self { + Self { + toolkits: vec![ + "gmail".to_string(), + "notion".to_string(), + "github".to_string(), + "slack".to_string(), + ], + connections: vec![ + json!({ + "id": "conn_gmail_1", + "toolkit": "gmail", + "status": "ACTIVE", + "createdAt": "2026-04-01T12:00:00Z", + }), + json!({ + "id": "conn_notion_1", + "toolkit": "notion", + "status": "ACTIVE", + "createdAt": "2026-04-02T08:00:00Z", + }), + json!({ + "id": "conn_github_1", + "toolkit": "github", + "status": "ACTIVE", + "createdAt": "2026-04-03T15:30:00Z", + }), + ], + tools: vec![ + json!({ + "name": "GMAIL_SEND_EMAIL", + "description": "Send an email via Gmail", + "parameters": { + "type": "object", + "properties": { + "recipient_email": {"type": "string"}, + "subject": {"type": "string"}, + "body": {"type": "string"}, + }, + "required": ["recipient_email", "subject", "body"], + }, + }), + json!({ + "name": "GMAIL_FETCH_EMAILS", + "description": "Fetch emails from Gmail", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "max_results": {"type": "integer"}, + }, + }, + }), + json!({ + "name": "NOTION_CREATE_PAGE", + "description": "Create a new Notion page", + "parameters": { + "type": "object", + "properties": { + "parent_id": {"type": "string"}, + "title": {"type": "string"}, + }, + "required": ["parent_id", "title"], + }, + }), + json!({ + "name": "GITHUB_CREATE_ISSUE", + "description": "Open a GitHub issue", + "parameters": { + "type": "object", + "properties": { + "owner": {"type": "string"}, + "repo": {"type": "string"}, + "title": {"type": "string"}, + "body": {"type": "string"}, + }, + "required": ["owner", "repo", "title"], + }, + }), + ], + execute_responses: [ + ( + "GMAIL_SEND_EMAIL".to_string(), + json!({"message_id": "gmail-msg-1234", "thread_id": "gmail-thread-9999"}), + ), + ( + "GMAIL_FETCH_EMAILS".to_string(), + json!({ + "messages": [ + {"id": "m1", "subject": "Welcome", "from": "team@openhuman.com"}, + {"id": "m2", "subject": "Invoice", "from": "billing@stripe.com"}, + ] + }), + ), + ( + "NOTION_CREATE_PAGE".to_string(), + json!({"page_id": "notion-page-abc123", "url": "https://notion.so/abc123"}), + ), + ( + "GITHUB_CREATE_ISSUE".to_string(), + json!({ + "number": 42, + "html_url": "https://github.com/example/repo/issues/42", + }), + ), + ] + .into_iter() + .collect(), + } + } +} + +#[derive(Clone)] +struct FakeComposioState { + fixture: Arc>, + requests: Arc>>, +} + +/// Handle to a spawned in-process Composio backend. +pub struct FakeComposioBackend { + pub base_url: String, + state: FakeComposioState, +} + +impl FakeComposioBackend { + /// All `(method, path, body)` requests received in order. + pub fn requests(&self) -> Vec<(String, String, serde_json::Value)> { + self.state.requests.lock().clone() + } + + /// Build a `ComposioClient` pointed at this backend. + pub fn client(&self) -> ComposioClient { + let inner = Arc::new(IntegrationClient::new( + self.base_url.clone(), + "test-token".into(), + )); + ComposioClient::new(inner) + } +} + +async fn record( + requests: &Arc>>, + method: &str, + path: &str, + body: Option<&B>, +) { + let body_v = match body { + Some(b) => serde_json::to_value(b).unwrap_or(serde_json::Value::Null), + None => serde_json::Value::Null, + }; + requests + .lock() + .push((method.to_string(), path.to_string(), body_v)); +} + +/// Spawn an in-process Composio backend on `127.0.0.1:0`. +pub async fn spawn_fake_composio_backend(fixture: ComposioFixture) -> FakeComposioBackend { + let state = FakeComposioState { + fixture: Arc::new(Mutex::new(fixture)), + requests: Arc::new(Mutex::new(Vec::new())), + }; + + let app = Router::new() + .route( + "/agent-integrations/composio/toolkits", + get({ + let st = state.clone(); + move || async move { + record::<()>(&st.requests, "GET", "/toolkits", None).await; + let toolkits = st.fixture.lock().toolkits.clone(); + Json(json!({ + "success": true, + "data": { "toolkits": toolkits } + })) + } + }), + ) + .route( + "/agent-integrations/composio/connections", + get({ + let st = state.clone(); + move || async move { + record::<()>(&st.requests, "GET", "/connections", None).await; + let connections = st.fixture.lock().connections.clone(); + Json(json!({ + "success": true, + "data": { "connections": connections } + })) + } + }), + ) + .route( + "/agent-integrations/composio/tools", + get({ + let st = state.clone(); + move || async move { + record::<()>(&st.requests, "GET", "/tools", None).await; + let tools = st.fixture.lock().tools.clone(); + Json(json!({ + "success": true, + "data": { "tools": tools } + })) + } + }), + ) + .route( + "/agent-integrations/composio/authorize", + post({ + let st = state.clone(); + move |Json(body): Json| async move { + record(&st.requests, "POST", "/authorize", Some(&body)).await; + let toolkit = body + .get("toolkit") + .and_then(|v| v.as_str()) + .unwrap_or("unknown"); + Json(json!({ + "success": true, + "data": { + "connectUrl": format!("https://composio.dev/auth/{toolkit}"), + "connectionId": format!("conn_{toolkit}_pending"), + } + })) + } + }), + ) + .route( + "/agent-integrations/composio/execute", + post({ + let st = state.clone(); + move |Json(body): Json| async move { + record(&st.requests, "POST", "/execute", Some(&body)).await; + // ComposioClient::execute_tool sends `{tool, arguments}`. + let action = body + .get("tool") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let fx = st.fixture.lock(); + let response = fx + .execute_responses + .get(&action) + .cloned() + .unwrap_or_else(|| json!({"ok": true, "action": action.clone()})); + // Wrap in the BackendResponse envelope expected by + // IntegrationClient, with the inner shape matching + // ComposioExecuteResponse. + Json(json!({ + "success": true, + "data": { + "data": response, + "successful": true, + "costUsd": 0.0, + } + })) + } + }), + ) + .route( + "/agent-integrations/composio/connections/{id}", + delete({ + let st = state.clone(); + move |Path(id): Path| async move { + record::<()>(&st.requests, "DELETE", &format!("/connections/{id}"), None).await; + let mut fx = st.fixture.lock(); + fx.connections + .retain(|c| c.get("id").and_then(|v| v.as_str()).unwrap_or("") != id); + Json(json!({ + "success": true, + "data": {"deleted": true} + })) + } + }), + ) + .with_state(state.clone()); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + + FakeComposioBackend { + base_url: format!("http://127.0.0.1:{}", addr.port()), + state, + } +} + +// A handler signature placeholder to silence unused warnings when the +// State extractor isn't reached (e.g. when only sub-routes fire). +#[allow(dead_code)] +async fn _unused(_state: State) {} diff --git a/src/openhuman/agent/harness/test_support_tests.rs b/src/openhuman/agent/harness/test_support_tests.rs new file mode 100644 index 000000000..76ddf9f72 --- /dev/null +++ b/src/openhuman/agent/harness/test_support_tests.rs @@ -0,0 +1,978 @@ +//! Behavioural tests of the agent harness driven by the smart mock +//! provider in [`super::test_support`]. These exercise the real +//! [`run_tool_call_loop`] path end-to-end — no provider stubbing inside +//! the test bodies — and surface regressions in tool dispatch, parsing, +//! and history threading. + +use super::test_support::{ + spawn_fake_composio_backend, ComposioFixture, KeywordRule, KeywordScriptedProvider, + ScriptedToolCall, +}; +use super::tool_loop::run_tool_call_loop; +use crate::openhuman::providers::{ChatMessage, ChatResponse}; +use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolCategory, ToolResult, ToolScope}; +use async_trait::async_trait; +use serde_json::json; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +fn mm() -> crate::openhuman::config::MultimodalConfig { + crate::openhuman::config::MultimodalConfig::default() +} + +/// Generic test tool: records the args it was called with and returns +/// whatever was wired at construction. +struct RecordingTool { + name_str: String, + description_str: String, + result: ToolResult, + calls: Arc>>, + permission: PermissionLevel, + scope_v: ToolScope, + category_v: ToolCategory, +} + +impl RecordingTool { + fn echo(name: &str) -> (Self, Arc>>) { + let calls = Arc::new(parking_lot::Mutex::new(Vec::new())); + let tool = Self { + name_str: name.to_string(), + description_str: format!("recording tool {name}"), + result: ToolResult::success(format!("{name}-ok")), + calls: calls.clone(), + permission: PermissionLevel::ReadOnly, + scope_v: ToolScope::All, + category_v: ToolCategory::System, + }; + (tool, calls) + } +} + +#[async_trait] +impl Tool for RecordingTool { + fn name(&self) -> &str { + &self.name_str + } + fn description(&self) -> &str { + &self.description_str + } + fn parameters_schema(&self) -> serde_json::Value { + json!({"type": "object", "additionalProperties": true}) + } + fn permission_level(&self) -> PermissionLevel { + self.permission + } + fn scope(&self) -> ToolScope { + self.scope_v + } + fn category(&self) -> ToolCategory { + self.category_v + } + async fn execute(&self, args: serde_json::Value) -> anyhow::Result { + self.calls.lock().push(args); + Ok(self.result.clone()) + } +} + +// ── 1. Keyword-driven loop: prompt-guided XML path ──────────────── + +#[tokio::test] +async fn keyword_provider_drives_prompt_guided_tool_loop_to_completion() { + let provider = KeywordScriptedProvider::new(vec![ + KeywordRule::tool_call( + "search", + ScriptedToolCall::new("search_tool", json!({"q": "rust"})), + ) + .with_text("Looking it up."), + KeywordRule::final_reply("search_tool-ok", "Here is the answer."), + ]); + + let (search_tool, search_calls) = RecordingTool::echo("search_tool"); + let tools: Vec> = vec![Box::new(search_tool)]; + + let mut history = vec![ChatMessage::user("please search the web for rust news")]; + + let result = run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock", + "test-model", + 0.0, + true, + None, + "channel", + &mm(), + 5, + None, + None, + &[], + None, + None, + ) + .await + .expect("loop should complete"); + + assert_eq!(result, "Here is the answer."); + assert_eq!( + search_calls.lock().len(), + 1, + "tool should fire exactly once" + ); + assert_eq!(search_calls.lock()[0]["q"], "rust"); + assert!(provider.turn_count() >= 2); +} + +// ── 2. Keyword-driven loop: native tool_calls path ──────────────── + +#[tokio::test] +async fn keyword_provider_drives_native_tool_calls_path() { + let provider = KeywordScriptedProvider::new(vec![ + KeywordRule::tool_call( + "weather", + ScriptedToolCall::new("weather_tool", json!({"city": "Berlin"})), + ), + KeywordRule::final_reply("weather_tool-ok", "It's sunny."), + ]) + .with_native_tools(true); + + let (weather_tool, weather_calls) = RecordingTool::echo("weather_tool"); + let tools: Vec> = vec![Box::new(weather_tool)]; + + let mut history = vec![ChatMessage::user("what's the weather in Berlin?")]; + + let out = run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock-native", + "test-model", + 0.0, + true, + None, + "channel", + &mm(), + 5, + None, + None, + &[], + None, + None, + ) + .await + .expect("loop should complete"); + + assert_eq!(out, "It's sunny."); + assert_eq!(weather_calls.lock().len(), 1); + // History should contain a tool role message (native path) referencing the call id. + assert!(history.iter().any(|m| m.role == "tool")); + let tool_msg = history.iter().find(|m| m.role == "tool").unwrap(); + assert!( + tool_msg.content.contains("weather_tool-ok"), + "tool result should be threaded through history: {}", + tool_msg.content + ); +} + +// ── 3. Multi-tool chain via successive keyword matches ──────────── + +#[tokio::test] +async fn keyword_provider_chains_multiple_tools_across_iterations() { + let provider = KeywordScriptedProvider::new(vec![ + KeywordRule::tool_call( + "draft an email", + ScriptedToolCall::new("draft_tool", json!({"to": "alice@example.com"})), + ), + KeywordRule::tool_call( + "draft_tool-ok", + ScriptedToolCall::new("send_tool", json!({"draft_id": "d-1"})), + ), + KeywordRule::final_reply("send_tool-ok", "Email sent to alice."), + ]); + + let (draft_tool, draft_calls) = RecordingTool::echo("draft_tool"); + let (send_tool, send_calls) = RecordingTool::echo("send_tool"); + let tools: Vec> = vec![Box::new(draft_tool), Box::new(send_tool)]; + + let mut history = vec![ChatMessage::user("draft an email to alice")]; + + let out = run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock", + "test-model", + 0.0, + true, + None, + "channel", + &mm(), + 10, + None, + None, + &[], + None, + None, + ) + .await + .unwrap(); + + assert_eq!(out, "Email sent to alice."); + assert_eq!(draft_calls.lock().len(), 1); + assert_eq!(send_calls.lock().len(), 1); +} + +// ── 4. Unknown tool name handled gracefully ─────────────────────── + +#[tokio::test] +async fn keyword_provider_unknown_tool_surfaces_error_and_loop_continues() { + let provider = KeywordScriptedProvider::new(vec![ + KeywordRule::tool_call("go", ScriptedToolCall::new("nonexistent_tool", json!({}))), + // After we see "Unknown tool" in the role=tool injection, give up. + KeywordRule::final_reply("unknown tool", "Sorry, I can't do that."), + ]); + + let tools: Vec> = vec![]; + + let mut history = vec![ChatMessage::user("go go go")]; + + let out = run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock", + "test-model", + 0.0, + true, + None, + "channel", + &mm(), + 5, + None, + None, + &[], + None, + None, + ) + .await + .unwrap(); + + assert_eq!(out, "Sorry, I can't do that."); + // The Tool Results message should record the "Unknown tool: nonexistent_tool". + assert!(history.iter().any(|m| m.content.contains("Unknown tool"))); +} + +// ── 5. Max iterations guard ─────────────────────────────────────── + +#[tokio::test] +async fn run_tool_call_loop_returns_max_iterations_error() { + // Configure a rule that keeps firing forever — but cap iterations. + let provider = KeywordScriptedProvider::new(vec![KeywordRule { + keyword: "echo-ok".to_string(), // matches tool result, so it loops + tool_calls: vec![ScriptedToolCall::new("echo", json!({}))], + final_text: None, + max_fires: None, + }]) + // First turn: kick it off + .with_fallback("end"); + provider.push_forced_response(ChatResponse { + text: Some("{\"name\":\"echo\",\"arguments\":{}}".into()), + tool_calls: vec![], + usage: None, + }); + + let (echo_tool, _) = RecordingTool::echo("echo"); + let tools: Vec> = vec![Box::new(echo_tool)]; + let mut history = vec![ChatMessage::user("loop forever")]; + + let err = run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock", + "test-model", + 0.0, + true, + None, + "channel", + &mm(), + 3, + None, + None, + &[], + None, + None, + ) + .await + .expect_err("should hit max iterations"); + + let s = err.to_string(); + assert!( + s.contains("3") && s.to_lowercase().contains("iteration"), + "expected MaxIterationsExceeded with 3, got: {s}" + ); +} + +// ── 6. CliRpcOnly tools are blocked in the agent loop ───────────── + +struct CliOnlyTool { + calls: Arc, +} + +#[async_trait] +impl Tool for CliOnlyTool { + fn name(&self) -> &str { + "cli_only_tool" + } + fn description(&self) -> &str { + "cli-only" + } + fn parameters_schema(&self) -> serde_json::Value { + json!({"type": "object"}) + } + fn scope(&self) -> ToolScope { + ToolScope::CliRpcOnly + } + async fn execute(&self, _args: serde_json::Value) -> anyhow::Result { + self.calls.fetch_add(1, Ordering::SeqCst); + Ok(ToolResult::success("ran")) + } +} + +#[tokio::test] +async fn agent_loop_refuses_clirpconly_tools() { + let calls = Arc::new(AtomicUsize::new(0)); + let tool = CliOnlyTool { + calls: calls.clone(), + }; + + let provider = KeywordScriptedProvider::new(vec![ + KeywordRule::tool_call("use", ScriptedToolCall::new("cli_only_tool", json!({}))), + KeywordRule::final_reply("only available via", "Denied as expected."), + ]); + + let tools: Vec> = vec![Box::new(tool)]; + let mut history = vec![ChatMessage::user("use the tool")]; + + let out = run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock", + "test-model", + 0.0, + true, + None, + "channel", + &mm(), + 5, + None, + None, + &[], + None, + None, + ) + .await + .unwrap(); + + assert_eq!(out, "Denied as expected."); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "CliRpcOnly tool must never execute in the agent loop" + ); +} + +// ── 7. Tool error result is threaded back as `Error: …` ─────────── + +struct FailingTool; + +#[async_trait] +impl Tool for FailingTool { + fn name(&self) -> &str { + "fail_tool" + } + fn description(&self) -> &str { + "always fails" + } + fn parameters_schema(&self) -> serde_json::Value { + json!({"type": "object"}) + } + async fn execute(&self, _args: serde_json::Value) -> anyhow::Result { + Ok(ToolResult::error("boom")) + } +} + +#[tokio::test] +async fn tool_error_result_is_surfaced_to_next_iteration() { + let provider = KeywordScriptedProvider::new(vec![ + KeywordRule::tool_call("try", ScriptedToolCall::new("fail_tool", json!({}))), + KeywordRule::final_reply("boom", "got the error"), + ]); + + let tools: Vec> = vec![Box::new(FailingTool)]; + let mut history = vec![ChatMessage::user("try the broken tool")]; + + let out = run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock", + "test-model", + 0.0, + true, + None, + "channel", + &mm(), + 5, + None, + None, + &[], + None, + None, + ) + .await + .unwrap(); + + assert_eq!(out, "got the error"); + assert!(history.iter().any(|m| m.content.contains("Error: boom"))); +} + +// ── 8. Tool that bails with anyhow::Error ───────────────────────── + +struct PanickyTool; + +#[async_trait] +impl Tool for PanickyTool { + fn name(&self) -> &str { + "panicky" + } + fn description(&self) -> &str { + "raises anyhow" + } + fn parameters_schema(&self) -> serde_json::Value { + json!({"type": "object"}) + } + async fn execute(&self, _args: serde_json::Value) -> anyhow::Result { + anyhow::bail!("kaboom") + } +} + +#[tokio::test] +async fn tool_anyhow_error_surfaces_in_history() { + let provider = KeywordScriptedProvider::new(vec![ + KeywordRule::tool_call("run", ScriptedToolCall::new("panicky", json!({}))), + KeywordRule::final_reply("kaboom", "tool blew up"), + ]); + + let tools: Vec> = vec![Box::new(PanickyTool)]; + let mut history = vec![ChatMessage::user("run it")]; + + let out = run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock", + "test-model", + 0.0, + true, + None, + "channel", + &mm(), + 5, + None, + None, + &[], + None, + None, + ) + .await + .unwrap(); + + assert_eq!(out, "tool blew up"); + assert!(history.iter().any(|m| m.content.contains("kaboom"))); +} + +// ── 9. visible_tool_names whitelist hides tools from the model ──── + +#[tokio::test] +async fn visible_tool_names_whitelist_rejects_filtered_out_tools() { + let provider = KeywordScriptedProvider::new(vec![ + // Model asks for a tool that *exists* but is filtered out. + KeywordRule::tool_call("go", ScriptedToolCall::new("hidden", json!({}))), + KeywordRule::final_reply("unknown tool", "Cannot reach hidden tool."), + ]); + + let (visible_tool, visible_calls) = RecordingTool::echo("visible"); + let (hidden_tool, hidden_calls) = RecordingTool::echo("hidden"); + let tools: Vec> = vec![Box::new(visible_tool), Box::new(hidden_tool)]; + + let mut visible = std::collections::HashSet::new(); + visible.insert("visible".to_string()); + + let mut history = vec![ChatMessage::user("go please")]; + + let out = run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock", + "test-model", + 0.0, + true, + None, + "channel", + &mm(), + 5, + None, + Some(&visible), + &[], + None, + None, + ) + .await + .unwrap(); + + assert_eq!(out, "Cannot reach hidden tool."); + assert_eq!(visible_calls.lock().len(), 0); + assert_eq!( + hidden_calls.lock().len(), + 0, + "hidden tool must not execute when filtered out" + ); +} + +// ── 10. extra_tools are reachable alongside the registry ────────── + +#[tokio::test] +async fn extra_tools_are_invokable_alongside_registry() { + let provider = KeywordScriptedProvider::new(vec![ + KeywordRule::tool_call("delegate", ScriptedToolCall::new("extra", json!({"x": 1}))), + KeywordRule::final_reply("extra-ok", "delegated"), + ]); + + let (extra_tool, extra_calls) = RecordingTool::echo("extra"); + let extras: Vec> = vec![Box::new(extra_tool)]; + + let tools: Vec> = vec![]; + let mut history = vec![ChatMessage::user("delegate the work")]; + + let out = run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock", + "test-model", + 0.0, + true, + None, + "channel", + &mm(), + 5, + None, + None, + &extras, + None, + None, + ) + .await + .unwrap(); + + assert_eq!(out, "delegated"); + assert_eq!(extra_calls.lock().len(), 1); +} + +// ── 11. Composio fixture backend: end-to-end client wiring ──────── + +#[tokio::test] +async fn fake_composio_backend_serves_realistic_toolkits() { + let backend = spawn_fake_composio_backend(ComposioFixture::realistic()).await; + let client = backend.client(); + + let toolkits = client.list_toolkits().await.unwrap(); + assert!(toolkits.toolkits.contains(&"gmail".to_string())); + assert!(toolkits.toolkits.contains(&"github".to_string())); + + let conns = client.list_connections().await.unwrap(); + assert!( + conns.connections.iter().any(|c| c.toolkit == "gmail"), + "expected a gmail connection: {:?}", + conns.connections + ); + + let exec = client + .execute_tool( + "GMAIL_SEND_EMAIL", + Some(json!({"recipient_email": "a@b.com", "subject": "hi", "body": "hello"})), + ) + .await + .unwrap(); + assert!(exec.successful, "execute should report success"); + let resp_json = serde_json::to_value(&exec.data).unwrap(); + assert!( + resp_json.to_string().contains("gmail-msg-1234"), + "expected fixture response, got: {resp_json}" + ); + + let reqs = backend.requests(); + // Authorize wasn't called; toolkits + connections + execute were. + assert!(reqs.iter().any(|(_, p, _)| p == "/toolkits")); + assert!(reqs.iter().any(|(_, p, _)| p == "/connections")); + assert!(reqs.iter().any(|(_, p, _)| p == "/execute")); +} + +// ── 12. End-to-end: harness drives a Composio tool against fake backend + +#[tokio::test] +async fn harness_invokes_composio_action_tool_against_fake_backend() { + use crate::openhuman::composio::ComposioActionTool; + + let backend = spawn_fake_composio_backend(ComposioFixture::realistic()).await; + let client = backend.client(); + + let tool = ComposioActionTool::new( + client, + "GMAIL_SEND_EMAIL".to_string(), + "Send a Gmail email".to_string(), + Some(json!({"type": "object"})), + ); + + let provider = KeywordScriptedProvider::new(vec![ + KeywordRule::tool_call( + "send email", + ScriptedToolCall::new( + "GMAIL_SEND_EMAIL", + json!({"recipient_email": "alice@example.com", "subject": "hi", "body": "hello"}), + ), + ), + KeywordRule::final_reply("gmail-msg-1234", "Email sent."), + ]); + + let tools: Vec> = vec![Box::new(tool)]; + let mut history = vec![ChatMessage::user("send email to alice")]; + + let out = run_tool_call_loop( + &provider, + &mut history, + &tools, + "mock", + "test-model", + 0.0, + true, + None, + "channel", + &mm(), + 5, + None, + None, + &[], + None, + None, + ) + .await + .unwrap(); + + assert_eq!(out, "Email sent."); + // Backend should have received the execute POST. + let reqs = backend.requests(); + let exec = reqs + .iter() + .find(|(m, p, _)| m == "POST" && p == "/execute") + .expect("execute call must have hit the backend"); + assert_eq!(exec.2["tool"], "GMAIL_SEND_EMAIL"); + assert_eq!(exec.2["arguments"]["recipient_email"], "alice@example.com"); +} + +// ── 13. Orchestrator-prompt → delegation → Composio round-trip ──── +// +// End-to-end test that ties together the three things the user needs +// confidence in: +// +// 1. The orchestrator's system prompt (built by +// `orchestrator::prompt::build`) advertises a connected toolkit +// via the collapsed `delegate_to_integrations_agent` tool. +// 2. Given that prompt and a user task that mentions the toolkit, +// the LLM (mocked) emits a tool call that satisfies the real +// `SkillDelegationTool` schema. +// 3. Delegation reaches the integrations side, where a +// `ComposioActionTool` is dispatched against a real +// `ComposioClient` pointed at a hermetic fake backend — and the +// backend records the action with the orchestrator-provided args. +// +// To avoid pulling in the full sub-agent runner, the test substitutes +// a `TestDelegationTool` that mirrors `SkillDelegationTool`'s contract +// (same tool name, same schema validation against the connected +// toolkit list) but runs a *nested* `run_tool_call_loop` for the +// integrations side instead of calling `dispatch_subagent`. The +// nested loop is the same code path the real integrations_agent uses, +// so the wiring under test is the orchestrator → delegation arg → +// integrations LLM → ComposioActionTool → backend chain. + +struct TestDelegationTool { + connected_toolkits: Vec<(String, String)>, + nested_tools: Arc>>>>, + inner_provider: Arc, +} + +impl TestDelegationTool { + fn new( + connected_toolkits: Vec<(String, String)>, + nested_tools: Vec>, + inner_provider: Arc, + ) -> Self { + Self { + connected_toolkits, + nested_tools: Arc::new(parking_lot::Mutex::new(Some(nested_tools))), + inner_provider, + } + } +} + +#[async_trait] +impl Tool for TestDelegationTool { + fn name(&self) -> &str { + // Mirror SkillDelegationTool's canonical name so the orchestrator + // system prompt's references resolve. + "delegate_to_integrations_agent" + } + fn description(&self) -> &str { + "Delegate to integrations_agent (test stand-in for SkillDelegationTool)." + } + fn parameters_schema(&self) -> serde_json::Value { + // Same shape as the real SkillDelegationTool. + let slugs: Vec<&str> = self + .connected_toolkits + .iter() + .map(|(s, _)| s.as_str()) + .collect(); + json!({ + "type": "object", + "required": ["toolkit", "prompt"], + "properties": { + "toolkit": {"type": "string", "enum": slugs}, + "prompt": {"type": "string"} + } + }) + } + async fn execute(&self, args: serde_json::Value) -> anyhow::Result { + let toolkit = args + .get("toolkit") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim() + .to_string(); + let prompt = args + .get("prompt") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim() + .to_string(); + if toolkit.is_empty() { + return Ok(ToolResult::error("`toolkit` is required")); + } + let known = self + .connected_toolkits + .iter() + .any(|(slug, _)| slug == &toolkit); + if !known { + return Ok(ToolResult::error(format!( + "toolkit `{toolkit}` is not connected" + ))); + } + if prompt.is_empty() { + return Ok(ToolResult::error("`prompt` is required")); + } + + // Take ownership of the nested tool list (one-shot). + let nested_tools = self + .nested_tools + .lock() + .take() + .expect("nested tools already consumed"); + + // Run a NESTED tool loop — same code path the integrations_agent + // uses inside the real sub-agent runner. + let mut nested_history = vec![ChatMessage::user(format!("[toolkit={toolkit}] {prompt}"))]; + let out = run_tool_call_loop( + self.inner_provider.as_ref(), + &mut nested_history, + &nested_tools, + "test-integrations", + "test-model", + 0.0, + true, + None, + "channel", + &mm(), + 5, + None, + None, + &[], + None, + None, + ) + .await?; + + Ok(ToolResult::success(out)) + } +} + +#[tokio::test] +async fn orchestrator_prompt_drives_composio_call_via_delegation_chain() { + use crate::openhuman::agent::agents::orchestrator::prompt as orch_prompt; + use crate::openhuman::agent::prompts::types::ConnectedIntegration; + use crate::openhuman::composio::ComposioActionTool; + + // ── 1. Build the orchestrator's system prompt with gmail wired in. + let integrations = vec![ConnectedIntegration { + toolkit: "gmail".into(), + description: "Email send/fetch via Gmail.".into(), + tools: Vec::new(), + connected: true, + }]; + let ctx = { + use crate::openhuman::context::prompt::{LearnedContextData, ToolCallFormat}; + use std::collections::HashSet; + use std::sync::OnceLock; + static EMPTY: OnceLock> = OnceLock::new(); + crate::openhuman::context::prompt::PromptContext { + workspace_dir: std::path::Path::new("."), + model_name: "test", + agent_id: "orchestrator", + tools: &[], + skills: &[], + dispatcher_instructions: "", + learned: LearnedContextData::default(), + visible_tool_names: EMPTY.get_or_init(HashSet::new), + tool_call_format: ToolCallFormat::PFormat, + connected_integrations: &integrations, + connected_identities_md: String::new(), + include_profile: false, + include_memory_md: false, + curated_snapshot: None, + user_identity: None, + } + }; + let system_prompt = orch_prompt::build(&ctx).expect("build orchestrator prompt"); + // The prompt must explicitly route gmail tasks via the collapsed + // delegation tool — otherwise the orchestrator can't know to call it. + assert!(system_prompt.contains("## Connected Integrations")); + assert!(system_prompt.contains("delegate_to_integrations_agent")); + assert!(system_prompt.contains("toolkit: \"gmail\"")); + + // ── 2. Spawn the fake Composio backend + wire a ComposioActionTool. + let backend = spawn_fake_composio_backend(ComposioFixture::realistic()).await; + let composio_client = backend.client(); + let gmail_action_tool: Box = Box::new(ComposioActionTool::new( + composio_client, + "GMAIL_SEND_EMAIL".to_string(), + "Send a Gmail email".to_string(), + Some(json!({"type": "object"})), + )); + + // ── 3. Inner (integrations-side) provider: emit a ComposioActionTool call, + // then a final reply once it sees the action's success marker. + let inner_provider = Arc::new(KeywordScriptedProvider::new(vec![ + KeywordRule::tool_call( + "toolkit=gmail", + ScriptedToolCall::new( + "GMAIL_SEND_EMAIL", + json!({ + "recipient_email": "alice@example.com", + "subject": "hi", + "body": "hello from orchestrator", + }), + ), + ), + KeywordRule::final_reply("gmail-msg-1234", "delivered"), + ])); + + let nested_tools: Vec> = vec![gmail_action_tool]; + + // ── 4. Outer (orchestrator) provider: when the user asks to email Alice + // via gmail, emit the delegation tool call. After the delegation + // returns "delivered", produce the final user-facing reply. + let outer_provider = KeywordScriptedProvider::new(vec![ + KeywordRule::tool_call( + "send an email to alice", + ScriptedToolCall::new( + "delegate_to_integrations_agent", + json!({ + "toolkit": "gmail", + "prompt": "Send an email to alice@example.com saying hi" + }), + ), + ), + KeywordRule::final_reply("delivered", "I've sent the email to Alice."), + ]); + + // ── 5. Wire the test delegation tool that bridges outer -> inner loop. + let delegation_tool: Box = Box::new(TestDelegationTool::new( + vec![( + "gmail".to_string(), + "Email send/fetch via Gmail.".to_string(), + )], + nested_tools, + inner_provider.clone(), + )); + let outer_tools: Vec> = vec![delegation_tool]; + + let mut history = vec![ + ChatMessage::system(system_prompt), + ChatMessage::user("Please send an email to alice@example.com saying hi via gmail."), + ]; + + // ── 6. Drive the outer (orchestrator) loop. + let final_reply = run_tool_call_loop( + &outer_provider, + &mut history, + &outer_tools, + "orchestrator-mock", + "test-model", + 0.0, + true, + None, + "channel", + &mm(), + 5, + None, + None, + &[], + None, + None, + ) + .await + .expect("orchestrator loop should complete"); + + assert_eq!(final_reply, "I've sent the email to Alice."); + + // ── 7. Assert the Composio backend actually received the action with + // the orchestrator-routed arguments. + let backend_reqs = backend.requests(); + let exec = backend_reqs + .iter() + .find(|(m, p, _)| m == "POST" && p == "/execute") + .expect("Composio /execute must have been called via the orchestrator chain"); + assert_eq!(exec.2["tool"], "GMAIL_SEND_EMAIL"); + assert_eq!(exec.2["arguments"]["recipient_email"], "alice@example.com"); + assert_eq!(exec.2["arguments"]["subject"], "hi"); + + // ── 8. Both sides of the chain should have seen exactly one turn that + // emitted the expected call. + let outer_turns = outer_provider.turns(); + assert!( + outer_turns + .iter() + .any(|t| t.rule_keyword.as_deref() == Some("send an email to alice")), + "orchestrator must have matched the delegation rule" + ); + let inner_turns = inner_provider.turns(); + assert!( + inner_turns + .iter() + .any(|t| t.rule_keyword.as_deref() == Some("toolkit=gmail")), + "integrations agent must have matched its tool-call rule" + ); +}