From 351d561318bc756e90b2dde3a83bf0c29ff4bdb7 Mon Sep 17 00:00:00 2001 From: Steven Enamakel <31011319+senamakel@users.noreply.github.com> Date: Sat, 16 May 2026 13:22:49 -0700 Subject: [PATCH] test(rust): expand smart harness coverage (#1961) --- src/openhuman/agent/harness/mod.rs | 2 +- src/openhuman/agent/harness/test_support.rs | 69 ++++- ..._support_tests.rs => test_support_test.rs} | 293 +++++++++++++++++- src/openhuman/channels/tests/memory.rs | 95 +++++- tests/memory_roundtrip_e2e.rs | 43 ++- 5 files changed, 490 insertions(+), 12 deletions(-) rename src/openhuman/agent/harness/{test_support_tests.rs => test_support_test.rs} (77%) diff --git a/src/openhuman/agent/harness/mod.rs b/src/openhuman/agent/harness/mod.rs index 0c75b3349..882416fba 100644 --- a/src/openhuman/agent/harness/mod.rs +++ b/src/openhuman/agent/harness/mod.rs @@ -56,7 +56,7 @@ mod bughunt_tests; #[cfg(test)] pub(crate) mod test_support; #[cfg(test)] -mod test_support_tests; +mod test_support_test; #[cfg(test)] mod harness_gap_tests; diff --git a/src/openhuman/agent/harness/test_support.rs b/src/openhuman/agent/harness/test_support.rs index 7a92db882..ec44869dc 100644 --- a/src/openhuman/agent/harness/test_support.rs +++ b/src/openhuman/agent/harness/test_support.rs @@ -334,6 +334,37 @@ pub struct ComposioFixture { pub tools: Vec, /// Per-action canned execute responses, keyed by action slug. pub execute_responses: std::collections::HashMap, + /// Ordered request-aware execute overrides. The first matching rule wins. + pub execute_rules: Vec, +} + +#[derive(Debug, Clone)] +pub struct ComposioExecuteRule { + pub action: String, + pub argument_path: Option, + pub argument_contains: Option, + pub response: serde_json::Value, +} + +impl ComposioExecuteRule { + pub fn new(action: impl Into, response: serde_json::Value) -> Self { + Self { + action: action.into(), + argument_path: None, + argument_contains: None, + response, + } + } + + pub fn when_argument_contains( + mut self, + path: impl Into, + needle: impl Into, + ) -> Self { + self.argument_path = Some(path.into()); + self.argument_contains = Some(needle.into()); + self + } } impl ComposioFixture { @@ -448,10 +479,42 @@ impl ComposioFixture { ] .into_iter() .collect(), + execute_rules: Vec::new(), } } } +fn json_path<'a>(value: &'a serde_json::Value, path: &str) -> Option<&'a serde_json::Value> { + path.split('.') + .filter(|segment| !segment.is_empty()) + .try_fold(value, |current, segment| current.get(segment)) +} + +fn match_execute_rule( + rules: &[ComposioExecuteRule], + action: &str, + body: &serde_json::Value, +) -> Option { + rules.iter().find_map(|rule| { + if rule.action != action { + return None; + } + if let Some(path) = rule.argument_path.as_deref() { + let actual = json_path(body, path)?; + if let Some(needle) = rule.argument_contains.as_deref() { + if !actual + .to_string() + .to_ascii_lowercase() + .contains(&needle.to_ascii_lowercase()) + { + return None; + } + } + } + Some(rule.response.clone()) + }) +} + #[derive(Clone)] struct FakeComposioState { fixture: Arc>, @@ -635,10 +698,8 @@ pub async fn spawn_fake_composio_backend(fixture: ComposioFixture) -> FakeCompos .unwrap_or("") .to_string(); let fx = st.fixture.lock(); - let response = fx - .execute_responses - .get(&action) - .cloned() + let response = match_execute_rule(&fx.execute_rules, &action, &body) + .or_else(|| fx.execute_responses.get(&action).cloned()) .unwrap_or_else(|| json!({"ok": true, "action": action.clone()})); // Wrap in the BackendResponse envelope expected by // IntegrationClient, with the inner shape matching diff --git a/src/openhuman/agent/harness/test_support_tests.rs b/src/openhuman/agent/harness/test_support_test.rs similarity index 77% rename from src/openhuman/agent/harness/test_support_tests.rs rename to src/openhuman/agent/harness/test_support_test.rs index fd2e7c83b..810864a7c 100644 --- a/src/openhuman/agent/harness/test_support_tests.rs +++ b/src/openhuman/agent/harness/test_support_test.rs @@ -5,11 +5,11 @@ //! and history threading. use super::test_support::{ - spawn_fake_composio_backend, ComposioFixture, KeywordRule, KeywordScriptedProvider, - ScriptedToolCall, + spawn_fake_composio_backend, ComposioExecuteRule, ComposioFixture, KeywordRule, + KeywordScriptedProvider, ScriptedToolCall, }; use super::tool_loop::run_tool_call_loop; -use crate::openhuman::providers::{ChatMessage, ChatResponse}; +use crate::openhuman::providers::{ChatMessage, ChatRequest, ChatResponse, Provider}; use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolCategory, ToolResult, ToolScope}; use async_trait::async_trait; use serde_json::json; @@ -22,6 +22,244 @@ fn mm() -> crate::openhuman::config::MultimodalConfig { crate::openhuman::config::MultimodalConfig::default() } +#[tokio::test] +async fn keyword_provider_records_forced_then_fallback_turns() { + let provider = + KeywordScriptedProvider::new(vec![KeywordRule::final_reply("matched", "final answer")]) + .with_native_tools(true) + .with_vision(true) + .with_fallback("fallback reply"); + + let caps = provider.capabilities(); + assert!(caps.native_tool_calling); + assert!(caps.vision); + + provider.push_forced_response(ChatResponse { + text: Some("forced reply".into()), + tool_calls: vec![], + usage: None, + }); + + let messages = vec![ChatMessage::user("nothing should match here")]; + let forced = provider + .chat( + ChatRequest { + messages: &messages, + tools: None, + stream: None, + }, + "test-model", + 0.0, + ) + .await + .expect("forced response"); + assert_eq!(forced.text.as_deref(), Some("forced reply")); + + let fallback = provider + .chat( + ChatRequest { + messages: &messages, + tools: None, + stream: None, + }, + "test-model", + 0.0, + ) + .await + .expect("fallback response"); + assert_eq!(fallback.text.as_deref(), Some("fallback reply")); + + let turns = provider.turns(); + assert_eq!(turns.len(), 2); + assert_eq!(turns[0].rule_keyword, None); + assert_eq!(turns[0].emitted_text.as_deref(), Some("forced reply")); + assert_eq!(turns[1].rule_keyword, None); + assert_eq!(turns[1].emitted_text.as_deref(), Some("fallback reply")); +} + +#[tokio::test] +async fn keyword_provider_prompt_guided_text_wraps_tool_calls_and_honors_fire_limit() { + let provider = KeywordScriptedProvider::new(vec![KeywordRule::tool_call( + "search please", + ScriptedToolCall::new("search_tool", json!({"q": "rust"})), + ) + .with_text("Looking it up.")]); + + let messages = vec![ + ChatMessage::assistant("earlier assistant turn"), + ChatMessage::tool("search please from a tool result"), + ]; + + let first = provider + .chat( + ChatRequest { + messages: &messages, + tools: None, + stream: None, + }, + "test-model", + 0.0, + ) + .await + .expect("prompt-guided response"); + + let text = first.text.expect("prompt-guided text body"); + assert!(first.tool_calls.is_empty()); + assert!(text.starts_with("Looking it up.\n")); + assert!(text.contains("")); + assert!(text.contains("\"name\":\"search_tool\"")); + assert!(text.contains("\"q\":\"rust\"")); + + let second = provider + .chat( + ChatRequest { + messages: &messages, + tools: None, + stream: None, + }, + "test-model", + 0.0, + ) + .await + .expect("fallback after max_fires"); + assert_eq!(second.text.as_deref(), Some("done")); + assert_eq!(provider.turn_count(), 2); +} + +#[tokio::test] +async fn fake_composio_backend_serves_routes_and_uses_response_fallbacks() { + let mut fixture = ComposioFixture::realistic(); + fixture.execute_rules = vec![ComposioExecuteRule::new( + "GMAIL_FETCH_EMAILS", + json!({"messages": [{"id": "gmail-priority-2"}]}), + ) + .when_argument_contains("arguments.query", "release blocker")]; + + let backend = spawn_fake_composio_backend(fixture).await; + let http = reqwest::Client::new(); + + let toolkits: serde_json::Value = http + .get(format!( + "{}/agent-integrations/composio/toolkits", + backend.base_url + )) + .send() + .await + .expect("toolkits request") + .json() + .await + .expect("toolkits json"); + assert_eq!(toolkits["data"]["toolkits"][0], "gmail"); + + let authorize: serde_json::Value = http + .post(format!( + "{}/agent-integrations/composio/authorize", + backend.base_url + )) + .json(&json!({"toolkit": "gmail"})) + .send() + .await + .expect("authorize request") + .json() + .await + .expect("authorize json"); + assert_eq!(authorize["data"]["connectionId"], "conn_gmail_pending",); + + let rule_match: serde_json::Value = http + .post(format!( + "{}/agent-integrations/composio/execute", + backend.base_url + )) + .json(&json!({ + "tool": "GMAIL_FETCH_EMAILS", + "arguments": {"query": "Need RELEASE BLOCKER updates"}, + })) + .send() + .await + .expect("execute request") + .json() + .await + .expect("execute json"); + assert_eq!( + rule_match["data"]["data"]["messages"][0]["id"], + "gmail-priority-2", + ); + + let execute_fallback: serde_json::Value = http + .post(format!( + "{}/agent-integrations/composio/execute", + backend.base_url + )) + .json(&json!({ + "tool": "GMAIL_FETCH_EMAILS", + "arguments": {"page": 1}, + })) + .send() + .await + .expect("execute fallback request") + .json() + .await + .expect("execute fallback json"); + assert_eq!(execute_fallback["data"]["data"]["messages"][0]["id"], "m1",); + + let default_execute: serde_json::Value = http + .post(format!( + "{}/agent-integrations/composio/execute", + backend.base_url + )) + .json(&json!({ + "tool": "UNKNOWN_ACTION", + "arguments": {"topic": "ops"}, + })) + .send() + .await + .expect("default execute request") + .json() + .await + .expect("default execute json"); + assert_eq!(default_execute["data"]["data"]["ok"], true); + assert_eq!(default_execute["data"]["data"]["action"], "UNKNOWN_ACTION"); + + let delete: serde_json::Value = http + .delete(format!( + "{}/agent-integrations/composio/connections/conn_gmail_1", + backend.base_url + )) + .send() + .await + .expect("delete request") + .json() + .await + .expect("delete json"); + assert_eq!(delete["data"]["deleted"], true); + + let requests = backend.requests(); + assert!( + requests + .iter() + .any(|(m, p, _)| m == "GET" && p == "/toolkits"), + "expected toolkits route to be recorded" + ); + assert!( + requests + .iter() + .any(|(m, p, _)| m == "POST" && p == "/authorize"), + "expected authorize route to be recorded" + ); + assert!( + requests + .iter() + .any(|(m, p, body)| m == "POST" && p == "/execute" && body["tool"] == "UNKNOWN_ACTION"), + "expected execute route to record unknown action body" + ); + assert!( + requests + .iter() + .any(|(m, p, _)| m == "DELETE" && p == "/connections/conn_gmail_1"), + "expected delete route to be recorded" + ); +} + /// Generic test tool: records the args it was called with and returns /// whatever was wired at construction. struct RecordingTool { @@ -623,6 +861,55 @@ async fn fake_composio_backend_serves_realistic_toolkits() { assert!(reqs.iter().any(|(_, p, _)| p == "/execute")); } +#[tokio::test] +async fn fake_composio_backend_can_match_execute_rules_by_argument_content() { + let mut fixture = ComposioFixture::realistic(); + fixture.execute_rules.push( + ComposioExecuteRule::new( + "GMAIL_FETCH_EMAILS", + json!({ + "messages": [ + { + "id": "gmail-priority-1", + "subject": "Release blocker", + "snippet": "The release blocker is the broken memory recall spec." + } + ] + }), + ) + .when_argument_contains("arguments.query", "release blocker"), + ); + let backend = spawn_fake_composio_backend(fixture).await; + let client = backend.client(); + + let exec = client + .execute_tool( + "GMAIL_FETCH_EMAILS", + Some(json!({ + "query": "label:inbox release blocker", + "max_results": 5 + })), + ) + .await + .unwrap(); + + assert!(exec.successful, "execute should report success"); + let resp_json = serde_json::to_value(&exec.data).unwrap(); + assert!( + resp_json.to_string().contains("gmail-priority-1"), + "expected rule-driven response, got: {resp_json}" + ); + let reqs = backend.requests(); + let exec_req = reqs + .iter() + .find(|(method, path, _)| method == "POST" && path == "/execute") + .expect("execute request should be recorded"); + assert_eq!( + exec_req.2["arguments"]["query"], + "label:inbox release blocker" + ); +} + // ── 12. End-to-end: harness drives a Composio tool against fake backend #[tokio::test] diff --git a/src/openhuman/channels/tests/memory.rs b/src/openhuman/channels/tests/memory.rs index 12ef5dff2..64b55c3c0 100644 --- a/src/openhuman/channels/tests/memory.rs +++ b/src/openhuman/channels/tests/memory.rs @@ -1,6 +1,6 @@ use super::super::context::{ - build_memory_context, conversation_memory_key, ChannelRuntimeContext, - CHANNEL_MESSAGE_TIMEOUT_SECS, + build_memory_context, clear_sender_history, conversation_history_key, conversation_memory_key, + ChannelRuntimeContext, CHANNEL_MESSAGE_TIMEOUT_SECS, }; use super::super::runtime::process_channel_message; use super::super::{traits, Channel}; @@ -205,4 +205,95 @@ async fn process_channel_message_restores_per_sender_history_on_follow_ups() { assert!(calls[1][3].1.contains("follow up")); } +#[tokio::test] +async fn process_channel_message_uses_autosaved_memory_after_history_is_cleared() { + let _bus_guard = super::common::use_real_agent_handler().await; + let channel_impl = Arc::new(RecordingChannel::default()); + let channel: Arc = channel_impl.clone(); + + let mut channels_by_name = HashMap::new(); + channels_by_name.insert(channel.name().to_string(), channel); + + let provider_impl = Arc::new(HistoryCaptureProvider::default()); + let tmp = TempDir::new().unwrap(); + let memory = Arc::new(UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap()); + + let runtime_ctx = Arc::new(ChannelRuntimeContext { + channels_by_name: Arc::new(channels_by_name), + provider: provider_impl.clone(), + default_provider: Arc::new("test-provider".to_string()), + memory, + tools_registry: Arc::new(vec![]), + system_prompt: Arc::new("test-system-prompt".to_string()), + model: Arc::new("test-model".to_string()), + temperature: 0.0, + auto_save_memory: true, + max_tool_iterations: 5, + min_relevance_score: 0.0, + conversation_histories: Arc::new(Mutex::new(HashMap::new())), + provider_cache: Arc::new(Mutex::new(HashMap::new())), + route_overrides: Arc::new(Mutex::new(HashMap::new())), + api_url: None, + inference_url: None, + reliability: Arc::new(crate::openhuman::config::ReliabilityConfig::default()), + provider_runtime_options: providers::ProviderRuntimeOptions::default(), + workspace_dir: Arc::new(std::env::temp_dir()), + message_timeout_secs: CHANNEL_MESSAGE_TIMEOUT_SECS, + multimodal: crate::openhuman::config::MultimodalConfig::default(), + }); + + let first = traits::ChannelMessage { + id: "msg-memory-a".to_string(), + sender: "alice".to_string(), + reply_target: "chat-1".to_string(), + content: "My launch code is phoenix-773.".to_string(), + channel: "test-channel".to_string(), + timestamp: 1, + thread_ts: None, + }; + let history_key = conversation_history_key(&first); + + process_channel_message(runtime_ctx.clone(), first).await; + + clear_sender_history(&runtime_ctx, &history_key); + + process_channel_message( + runtime_ctx, + traits::ChannelMessage { + id: "msg-memory-b".to_string(), + sender: "alice".to_string(), + reply_target: "chat-1".to_string(), + content: "What is my launch code?".to_string(), + channel: "test-channel".to_string(), + timestamp: 2, + thread_ts: None, + }, + ) + .await; + + let calls = provider_impl + .calls + .lock() + .unwrap_or_else(|e| e.into_inner()); + assert_eq!(calls.len(), 2); + assert_eq!(calls[1].len(), 2); + assert_eq!(calls[1][0].0, "system"); + assert_eq!(calls[1][1].0, "user"); + assert!( + calls[1][1].1.contains("[Memory context]"), + "second turn should include recalled memory context: {:?}", + calls[1][1] + ); + assert!( + calls[1][1].1.contains("phoenix-773"), + "second turn should surface the autosaved fact: {:?}", + calls[1][1] + ); + assert!( + calls[1][1].1.contains("What is my launch code?"), + "current user question should remain in the final prompt: {:?}", + calls[1][1] + ); +} + // ── AIEOS Identity Tests (Issue #168) ───────────────────────── diff --git a/tests/memory_roundtrip_e2e.rs b/tests/memory_roundtrip_e2e.rs index 53b698774..8ff79b8a3 100644 --- a/tests/memory_roundtrip_e2e.rs +++ b/tests/memory_roundtrip_e2e.rs @@ -16,9 +16,10 @@ use std::sync::{Mutex, OnceLock}; use tempfile::tempdir; use openhuman_core::openhuman::memory::ops::{ - clear_namespace, doc_put, memory_recall_memories, ClearNamespaceParams, PutDocParams, + clear_namespace, doc_put, memory_recall_context, memory_recall_memories, ClearNamespaceParams, + PutDocParams, }; -use openhuman_core::openhuman::memory::rpc_models::RecallMemoriesRequest; +use openhuman_core::openhuman::memory::rpc_models::{RecallContextRequest, RecallMemoriesRequest}; // ── Env isolation ──────────────────────────────────────────────────── @@ -91,6 +92,15 @@ fn recall_request() -> RecallMemoriesRequest { } } +fn recall_context_request() -> RecallContextRequest { + RecallContextRequest { + namespace: NS.to_string(), + include_references: Some(true), + limit: Some(10), + max_chunks: None, + } +} + // ── Tests ──────────────────────────────────────────────────────────── /// 8.1.1 store + 8.1.2 recall — the happy-path round-trip. @@ -122,6 +132,35 @@ async fn doc_put_then_recall_memories_returns_canary() { ); } +/// `recall_context` should surface the same document as an LLM-ready prompt +/// block, not only in the raw memory list view. +#[tokio::test] +async fn doc_put_then_recall_context_renders_llm_context_message() { + let _lock = env_lock(); + let tmp = tempdir().expect("tempdir"); + let _home = EnvVarGuard::set_to_path("HOME", tmp.path()); + let workspace_path = tmp.path().join("workspace"); + std::fs::create_dir_all(&workspace_path).expect("create workspace dir"); + let _ws = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", &workspace_path); + + doc_put(put_params()).await.expect("doc_put rpc"); + + let recall_outcome = memory_recall_context(recall_context_request()) + .await + .expect("memory_recall_context rpc"); + let llm_context = recall_outcome + .value + .data + .as_ref() + .and_then(|data| data.llm_context_message.as_ref()) + .cloned() + .unwrap_or_default(); + assert!( + llm_context.contains(CONTENT) || llm_context.contains(KEY), + "llm context should reference the canary content/key — got {llm_context}" + ); +} + /// 8.1.3 forget — clear_namespace must scrub the namespace so subsequent /// recalls do not see the canary content. Failure-path / edge-case assertion /// required by gitbooks/developing/testing-strategy.md.