From 090c987fff34283729edd2ed3910cb2da54a59eb Mon Sep 17 00:00:00 2001 From: Steven Enamakel <31011319+senamakel@users.noreply.github.com> Date: Thu, 4 Jun 2026 18:36:58 -0400 Subject: [PATCH] Test subagent cache reuse and chat tier routing (#3389) --- .../settings/panels/AgentEditorPage.tsx | 2 +- src/openhuman/agent/cost.rs | 13 +- .../agents/orchestrator/agent.toml | 2 +- src/openhuman/config/schema/identity_cost.rs | 10 +- src/openhuman/config/schema/types.rs | 27 +- src/openhuman/inference/model_context.rs | 5 +- src/openhuman/inference/provider/factory.rs | 10 +- .../inference/provider/factory_tests.rs | 5 +- src/openhuman/inference/provider/router.rs | 7 +- .../inference/provider/router_tests.rs | 3 +- src/openhuman/migrations/README.md | 4 +- src/openhuman/migrations/mod.rs | 7 +- .../migrations/retire_chat_v1_model.rs | 54 ++-- src/openhuman/routing/README.md | 4 +- src/openhuman/routing/provider.rs | 8 +- src/openhuman/routing/provider_tests.rs | 10 +- tests/agent_harness_raw_coverage_e2e.rs | 239 +++++++++++++++++- tests/inference_agent_raw_coverage_e2e.rs | 2 +- 18 files changed, 314 insertions(+), 98 deletions(-) diff --git a/app/src/components/settings/panels/AgentEditorPage.tsx b/app/src/components/settings/panels/AgentEditorPage.tsx index fedee49f2..271384924 100644 --- a/app/src/components/settings/panels/AgentEditorPage.tsx +++ b/app/src/components/settings/panels/AgentEditorPage.tsx @@ -39,8 +39,8 @@ const MODEL_HINTS = [ ]; const MODEL_TIERS = [ 'reasoning-v1', - 'reasoning-quick-v1', 'chat-v1', + 'reasoning-quick-v1', 'agentic-v1', 'coding-v1', 'summarization-v1', diff --git a/src/openhuman/agent/cost.rs b/src/openhuman/agent/cost.rs index af9bfd37c..c66ba4ecb 100644 --- a/src/openhuman/agent/cost.rs +++ b/src/openhuman/agent/cost.rs @@ -69,9 +69,16 @@ pub const PRICING_TABLE: &[ModelPricing] = &[ cached_input_per_mtok_usd: 1.50, output_per_mtok_usd: 75.00, }, - // Quick reasoning tier — Kimi K2.6 Turbo on Fireworks (backend PR - // #760). Low TTFT, 128k context, `supportsThinking: false`. Rates - // track Fireworks' published Kimi turbo pricing at time of writing. + // Chat tier — Kimi K2.6 Turbo on Fireworks (backend PR #760). + // Low TTFT, 128k context, `supportsThinking: false`. Rates track + // Fireworks' published Kimi turbo pricing at time of writing. + ModelPricing { + model: "chat-v1", + input_per_mtok_usd: 0.60, + cached_input_per_mtok_usd: 0.06, + output_per_mtok_usd: 2.50, + }, + // Legacy chat tier slug retained for older transcripts/configs. ModelPricing { model: "reasoning-quick-v1", input_per_mtok_usd: 0.60, diff --git a/src/openhuman/agent_registry/agents/orchestrator/agent.toml b/src/openhuman/agent_registry/agents/orchestrator/agent.toml index ac0eea36d..b70858f5b 100644 --- a/src/openhuman/agent_registry/agents/orchestrator/agent.toml +++ b/src/openhuman/agent_registry/agents/orchestrator/agent.toml @@ -120,7 +120,7 @@ subagents = [ [model] # Front-line conversational agent: TTFT dominates UX. `hint:chat` resolves -# to the fast chat tier (`reasoning-quick-v1` / Kimi K2.6 Turbo on +# to the fast chat tier (`chat-v1` / Kimi K2.6 Turbo on # Fireworks via backend PR #760, 128k context, `supportsThinking: false`). # The orchestrator is a planner/router that picks a delegate and # synthesises sub-agent output — workload that doesn't benefit from the diff --git a/src/openhuman/config/schema/identity_cost.rs b/src/openhuman/config/schema/identity_cost.rs index 18d014388..2f99a239d 100644 --- a/src/openhuman/config/schema/identity_cost.rs +++ b/src/openhuman/config/schema/identity_cost.rs @@ -150,7 +150,8 @@ impl Default for CostConfig { /// Default pricing for popular models (USD per 1M tokens) fn get_default_pricing() -> HashMap { use super::types::{ - MODEL_AGENTIC_V1, MODEL_CODING_V1, MODEL_REASONING_QUICK_V1, MODEL_REASONING_V1, + MODEL_AGENTIC_V1, MODEL_CHAT_V1, MODEL_CODING_V1, MODEL_REASONING_QUICK_V1, + MODEL_REASONING_V1, }; let mut prices = HashMap::new(); @@ -163,6 +164,13 @@ fn get_default_pricing() -> HashMap { }, ); // Kimi K2.6 Turbo on Fireworks — see backend PR #760. + prices.insert( + MODEL_CHAT_V1.into(), + ModelPricing { + input: 0.60, + output: 2.50, + }, + ); prices.insert( MODEL_REASONING_QUICK_V1.into(), ModelPricing { diff --git a/src/openhuman/config/schema/types.rs b/src/openhuman/config/schema/types.rs index a90ba80e8..db655ab90 100644 --- a/src/openhuman/config/schema/types.rs +++ b/src/openhuman/config/schema/types.rs @@ -9,32 +9,19 @@ use std::path::PathBuf; /// Standard model identifiers matching the backend model registry. pub const MODEL_AGENTIC_V1: &str = "agentic-v1"; pub const MODEL_REASONING_V1: &str = "reasoning-v1"; -/// Conversational tier (deprecated — retired from the backend strict model -/// registry in migration 2→3). Do not use for new sessions; the backend now -/// returns 400 for threads that send `chat-v1`. Retained here only for -/// migration code that needs to identify and replace the old model identifier. -/// Use [`MODEL_REASONING_QUICK_V1`] or [`DEFAULT_MODEL`] instead. +/// Low-latency conversational tier. pub const MODEL_CHAT_V1: &str = "chat-v1"; -/// Low-latency chat tier. Backend maps this to Kimi K2.6 Turbo on -/// Fireworks (128k context, `supportsThinking: false`) — tuned for -/// time-to-first-token on conversational turns. See backend PR #760. -/// The orchestrator (user-facing front-line agent) rides on this tier -/// by default (via `hint:chat`) so chat responses feel snappy; reach -/// for the slower `reasoning-v1` (DeepSeek V4 Pro) only when deep -/// reasoning is needed. +/// Legacy low-latency chat tier slug retained for older persisted configs. pub const MODEL_REASONING_QUICK_V1: &str = "reasoning-quick-v1"; pub const MODEL_CODING_V1: &str = "coding-v1"; pub const MODEL_SUMMARIZATION_V1: &str = "summarization-v1"; /// Default model used when no explicit model is configured. /// -/// Set to `reasoning-quick-v1` (Kimi K2.6 Turbo on Fireworks — low-latency, -/// 128k context, tuned for time-to-first-token). `chat-v1` was the previous -/// value here but was retired from the backend strict model registry; new -/// session threads that sent `chat-v1` received a 400 error. Existing threads -/// had it silently remapped to `reasoning-v1` by the backend, but sub-agent -/// spawns (new threads) failed. Migration 2 → 3 (`retire_chat_v1_model`) -/// upgrades any persisted `config.toml` that still holds `chat-v1`. -pub const DEFAULT_MODEL: &str = MODEL_REASONING_QUICK_V1; +/// Set to `chat-v1`, the backend's low-latency conversational tier. The +/// orchestrator (user-facing front-line agent) rides on this tier by default +/// via `hint:chat`; reach for the slower `reasoning-v1` only when deep +/// reasoning is needed. +pub const DEFAULT_MODEL: &str = MODEL_CHAT_V1; #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct ModelRegistryEntry { diff --git a/src/openhuman/inference/model_context.rs b/src/openhuman/inference/model_context.rs index 6642ea9b1..3b467295f 100644 --- a/src/openhuman/inference/model_context.rs +++ b/src/openhuman/inference/model_context.rs @@ -6,7 +6,7 @@ //! metadata is not yet available. use crate::openhuman::config::{ - MODEL_AGENTIC_V1, MODEL_CODING_V1, MODEL_REASONING_QUICK_V1, MODEL_REASONING_V1, + MODEL_AGENTIC_V1, MODEL_CHAT_V1, MODEL_CODING_V1, MODEL_REASONING_QUICK_V1, MODEL_REASONING_V1, }; /// Conservative default for OpenHuman abstract tier models (tokens). @@ -99,7 +99,7 @@ fn tier_context_window(model: &str) -> Option { match model { MODEL_REASONING_V1 | MODEL_AGENTIC_V1 | MODEL_CODING_V1 => Some(TIER_LARGE_CONTEXT), "summarization-v1" => Some(TIER_SUMMARIZATION_CONTEXT), - MODEL_REASONING_QUICK_V1 | "chat" => Some(TIER_STANDARD_CONTEXT), + MODEL_CHAT_V1 | MODEL_REASONING_QUICK_V1 | "chat" => Some(TIER_STANDARD_CONTEXT), m if m.starts_with("gemma") || m.contains(":1b") || m.contains("270m") => { Some(TIER_LOCAL_CONTEXT) } @@ -179,6 +179,7 @@ mod tests { fn tier_aliases_resolve() { assert_eq!(context_window_for_model("reasoning-v1"), Some(200_000)); assert_eq!(context_window_for_model("agentic-v1"), Some(200_000)); + assert_eq!(context_window_for_model("chat-v1"), Some(128_000)); assert_eq!( context_window_for_model("reasoning-quick-v1"), Some(128_000) diff --git a/src/openhuman/inference/provider/factory.rs b/src/openhuman/inference/provider/factory.rs index 3ab4a3273..490446d27 100644 --- a/src/openhuman/inference/provider/factory.rs +++ b/src/openhuman/inference/provider/factory.rs @@ -62,7 +62,8 @@ pub const BYOK_INCOMPLETE_SENTINEL: &str = "__byok_incomplete__"; fn is_abstract_tier_model(model: &str) -> bool { use crate::openhuman::config::{ - MODEL_AGENTIC_V1, MODEL_CODING_V1, MODEL_REASONING_QUICK_V1, MODEL_REASONING_V1, + MODEL_AGENTIC_V1, MODEL_CHAT_V1, MODEL_CODING_V1, MODEL_REASONING_QUICK_V1, + MODEL_REASONING_V1, }; // No dedicated constant for the summarization tier yet; keep the literal // in sync with the tier name used by the summarizer sub-agent. @@ -70,6 +71,7 @@ fn is_abstract_tier_model(model: &str) -> bool { let trimmed = model.trim(); trimmed == MODEL_REASONING_V1 || trimmed == MODEL_REASONING_QUICK_V1 + || trimmed == MODEL_CHAT_V1 || trimmed == MODEL_AGENTIC_V1 || trimmed == MODEL_CODING_V1 || trimmed == MODEL_SUMMARIZATION_V1 @@ -91,17 +93,17 @@ pub fn auth_key_for_slug(slug: &str) -> String { pub fn resolve_model_for_hint(hint_or_tier: &str, config: &Config) -> String { let hint_to_tier: &[(&str, &str)] = &[ ("reasoning", crate::openhuman::config::MODEL_REASONING_V1), - ("chat", crate::openhuman::config::MODEL_REASONING_QUICK_V1), + ("chat", crate::openhuman::config::MODEL_CHAT_V1), ("agentic", crate::openhuman::config::MODEL_AGENTIC_V1), ("coding", crate::openhuman::config::MODEL_CODING_V1), ("summarization", "summarization-v1"), ]; let tier_to_role: &[(&str, &str)] = &[ (crate::openhuman::config::MODEL_REASONING_V1, "reasoning"), + (crate::openhuman::config::MODEL_CHAT_V1, "chat"), (crate::openhuman::config::MODEL_REASONING_QUICK_V1, "chat"), (crate::openhuman::config::MODEL_AGENTIC_V1, "agentic"), (crate::openhuman::config::MODEL_CODING_V1, "coding"), - ("chat-v1", "chat"), ("summarization-v1", "summarization"), ]; @@ -616,7 +618,7 @@ fn make_openhuman_backend(config: &Config) -> anyhow::Result<(Box, // "deepseek-v4-pro", "claude-opus-4-7") fall back to the platform default. let model = match model.strip_prefix("hint:") { Some("reasoning") => crate::openhuman::config::MODEL_REASONING_V1.to_string(), - Some("chat") => crate::openhuman::config::MODEL_REASONING_QUICK_V1.to_string(), + Some("chat") => crate::openhuman::config::MODEL_CHAT_V1.to_string(), Some("agentic") => crate::openhuman::config::MODEL_AGENTIC_V1.to_string(), Some("coding") => crate::openhuman::config::MODEL_CODING_V1.to_string(), Some("summarization") => crate::openhuman::config::MODEL_SUMMARIZATION_V1.to_string(), diff --git a/src/openhuman/inference/provider/factory_tests.rs b/src/openhuman/inference/provider/factory_tests.rs index 8f64d2363..b597f8617 100644 --- a/src/openhuman/inference/provider/factory_tests.rs +++ b/src/openhuman/inference/provider/factory_tests.rs @@ -1772,10 +1772,7 @@ fn resolve_model_for_hint_maps_known_hints_to_tiers() { resolve_model_for_hint("hint:reasoning", &config), "reasoning-v1" ); - assert_eq!( - resolve_model_for_hint("hint:chat", &config), - "reasoning-quick-v1" - ); + assert_eq!(resolve_model_for_hint("hint:chat", &config), "chat-v1"); assert_eq!( resolve_model_for_hint("hint:agentic", &config), "agentic-v1" diff --git a/src/openhuman/inference/provider/router.rs b/src/openhuman/inference/provider/router.rs index ee3e25818..ecb8e0024 100644 --- a/src/openhuman/inference/provider/router.rs +++ b/src/openhuman/inference/provider/router.rs @@ -3,13 +3,14 @@ use super::Provider; use async_trait::async_trait; use std::collections::HashMap; -/// Maps OpenHuman's abstract tier model names (`reasoning-v1`, +/// Maps OpenHuman's abstract tier model names (`reasoning-v1`, `chat-v1`, /// `reasoning-quick-v1`, `agentic-v1`, `coding-v1`, `summarization-v1`) /// to the hint slot in `model_routes`. Returns `None` for any model the /// router shouldn't rewrite. fn openhuman_tier_to_hint(model: &str) -> Option<&'static str> { match model { "reasoning-v1" => Some("reasoning"), + "chat-v1" => Some("chat"), "reasoning-quick-v1" => Some("chat"), "agentic-v1" => Some("agentic"), "coding-v1" => Some("coding"), @@ -90,8 +91,8 @@ impl RouterProvider { /// /// Resolution order: /// 1. `hint:` — direct hint lookup (e.g. `hint:reasoning`). - /// 2. OpenHuman abstract tier names — `reasoning-v1`, `agentic-v1`, - /// `coding-v1`, `summarization-v1` map onto the corresponding hints + /// 2. OpenHuman abstract tier names — `reasoning-v1`, `chat-v1`, + /// `agentic-v1`, `coding-v1`, `summarization-v1` map onto the corresponding hints /// so a custom provider gets the user-configured model id instead of /// the literal tier name (which is only meaningful to the OpenHuman /// backend and would 404 on OpenAI/Anthropic/etc.). diff --git a/src/openhuman/inference/provider/router_tests.rs b/src/openhuman/inference/provider/router_tests.rs index f217eba4b..64bcf6399 100644 --- a/src/openhuman/inference/provider/router_tests.rs +++ b/src/openhuman/inference/provider/router_tests.rs @@ -199,7 +199,7 @@ fn resolve_translates_openhuman_tier_aliases_via_route_table() { assert_eq!(reasoning_idx, 1); assert_eq!(reasoning_model, "gpt-5.5"); - let (chat_idx, chat_model) = router.resolve("reasoning-quick-v1"); + let (chat_idx, chat_model) = router.resolve("chat-v1"); assert_eq!(chat_idx, 1); assert_eq!(chat_model, "gpt-5.5-mini"); @@ -249,6 +249,7 @@ fn every_tier_alias_falls_back_to_default_model_when_unrouted() { for alias in [ "reasoning-v1", + "chat-v1", "reasoning-quick-v1", "agentic-v1", "coding-v1", diff --git a/src/openhuman/migrations/README.md b/src/openhuman/migrations/README.md index bd14da6ee..051bdff3c 100644 --- a/src/openhuman/migrations/README.md +++ b/src/openhuman/migrations/README.md @@ -19,7 +19,7 @@ Startup data-migration runner gated by `Config::schema_version`. Each migration | `src/openhuman/migrations/mod.rs` | Module docstring + the `run_pending` runner and `CURRENT_SCHEMA_VERSION` constant; declares each migration as a private `mod`. | | `src/openhuman/migrations/phase_out_profile_md.rs` | **0→1.** Deletes `{workspace}/PROFILE.md` and strips `### PROFILE.md` blocks from persisted JSONL session transcripts (`session_raw/**`) and `.md` companions (`sessions/**`). Blocking fs I/O. | | `src/openhuman/migrations/unify_ai_provider_settings.rs` | **1→2.** Consolidates scattered AI-provider settings into per-workload provider strings; seeds `cloud_providers` (always an `Openhuman` entry) and migrates a non-backend `inference_url` into a `Custom` cloud entry. Pure in-memory. | -| `src/openhuman/migrations/retire_chat_v1_model.rs` | **2→3.** Remaps a persisted `default_model = "chat-v1"` to `"reasoning-quick-v1"` (backend dropped `chat-v1` from its strict registry; sub-agent spawns 400'd). Pure in-memory. | +| `src/openhuman/migrations/retire_chat_v1_model.rs` | **2→3.** Legacy chat-v1 migration hook retained for schema-version progression. No longer remaps `chat-v1`, which is the canonical low-latency chat slug again. Pure in-memory. | | `src/openhuman/migrations/expand_autonomy_defaults.rs` | **3→4.** Additively merges expanded `autonomy.allowed_commands` / `auto_approve` defaults (PR #2500) and bumps `max_actions_per_hour` from the old `20` to `u32::MAX` only when still exactly `20`. Pure in-memory. | | `src/openhuman/migrations/remove_write_auto_approve.rs` | **4→5.** Removes `file_write` / `edit_file` from `autonomy.auto_approve` so Supervised mode resumes its ask-before-edit prompt. Pure in-memory. | | `src/openhuman/migrations/repair_http_request_limits.rs` | **5→6 (a).** Coerces stale-zero `[http_request]` `timeout_secs` / `max_response_size` to schema defaults (30s / 1 MB); a persisted `0` is an instant timeout / empty-body cap that serde defaults don't repair. Pure in-memory. | @@ -55,7 +55,7 @@ Does not own a `store.rs`. Its effects are written through other layers: ## Dependencies -- `crate::openhuman::config` (`Config`, `HttpRequestConfig`, `schema::cloud_providers::{CloudProviderCreds, AuthStyle, CloudProviderType, generate_provider_id}`, `schema::{MODEL_CHAT_V1, MODEL_REASONING_QUICK_V1}`) — the migrations read and mutate the config struct; `schema_version` is the gating field. +- `crate::openhuman::config` (`Config`, `HttpRequestConfig`, `schema::cloud_providers::{CloudProviderCreds, AuthStyle, CloudProviderType, generate_provider_id}`, `schema::MODEL_CHAT_V1`) — the migrations read and mutate the config struct; `schema_version` is the gating field. - `crate::openhuman::agent::harness::session::transcript` (`transcript`, `SessionTranscript`, `write_transcript`) — `phase_out_profile_md` parses and rewrites persisted session transcripts byte-compatibly. - `crate::openhuman::inference::provider::factory` — `reconcile_orphaned_providers` mirrors the factory's exact, case-sensitive provider-string grammar so "resolvable here" matches "resolvable at inference time"; `unify_ai_provider_settings` references the factory's provider-string format. (`inference::provider::ChatMessage` is imported only by tests.) diff --git a/src/openhuman/migrations/mod.rs b/src/openhuman/migrations/mod.rs index 1ffcacb5a..283cf6258 100644 --- a/src/openhuman/migrations/mod.rs +++ b/src/openhuman/migrations/mod.rs @@ -147,10 +147,9 @@ pub async fn run_pending(config: &mut Config) { } } - // 2 -> 3: retire `chat-v1` as the default model. The backend removed - // `chat-v1` from its strict model registry; sub-agent spawns (new - // threads) that sent this literal model ID received a 400. Remap any - // persisted `default_model = "chat-v1"` to `"reasoning-quick-v1"`. + // 2 -> 3: legacy chat-v1 migration hook retained for schema-version + // progression. `chat-v1` is now the canonical low-latency chat slug, so + // the migration no longer rewrites default_model. // Guard on `== 2` so a failed 1→2 migration doesn't skip this step. if config.schema_version == 2 { match retire_chat_v1_model::run(config) { diff --git a/src/openhuman/migrations/retire_chat_v1_model.rs b/src/openhuman/migrations/retire_chat_v1_model.rs index b85bc1c7f..ac60657f9 100644 --- a/src/openhuman/migrations/retire_chat_v1_model.rs +++ b/src/openhuman/migrations/retire_chat_v1_model.rs @@ -1,56 +1,36 @@ -//! Migration 2 → 3: retire `chat-v1` as the default model. +//! Migration 2 → 3: legacy `chat-v1` default-model migration. //! -//! The backend removed `chat-v1` from its strict model registry. New inference -//! threads (sub-agent spawns) that sent the literal `"chat-v1"` model ID -//! received a 400 error ("Model 'chat-v1' is not available"), while existing -//! session threads continued to work because the backend silently remapped -//! `chat-v1` → `reasoning-v1` for backward compatibility on old threads only. -//! -//! This migration upgrades any workspace whose `config.default_model` is still -//! `"chat-v1"` to `"reasoning-quick-v1"` — the same Kimi K2.6 Turbo backend -//! model that `chat-v1` was previously aliased to. The code constant -//! `DEFAULT_MODEL` was updated in the same change. +//! `chat-v1` is once again the canonical low-latency conversational tier. This +//! migration is retained as a no-op so already-upgraded workspaces keep their +//! schema-version progression without rewriting the model slug away from +//! `chat-v1`. //! //! ## Behaviour //! //! - Pure in-memory mutation of `Config`. The caller (`migrations::run_pending`) //! persists the result via `Config::save()` and bumps `schema_version`. -//! - Idempotent: only remaps when `default_model == Some("chat-v1")`. +//! - Idempotent: does not remap `default_model`. //! - Does not touch any other config fields, API keys, or session files. -use crate::openhuman::config::schema::MODEL_CHAT_V1; -use crate::openhuman::config::schema::MODEL_REASONING_QUICK_V1; use crate::openhuman::config::Config; /// Counters returned by [`run`] for diagnostics. #[derive(Debug, Default, Clone)] pub struct MigrationStats { - /// `true` when `default_model` was remapped from `chat-v1`. + /// Always false; retained for backward-compatible migration telemetry. pub default_model_remapped: bool, } -/// Run the `chat-v1` retirement migration on the given `Config`. +/// Run the legacy `chat-v1` migration hook on the given `Config`. /// /// Synchronous — pure config mutation, no I/O. Caller persists via /// `Config::save()` once `schema_version` is also bumped. pub fn run(config: &mut Config) -> anyhow::Result { - let mut stats = MigrationStats::default(); - - if config.default_model.as_deref() == Some(MODEL_CHAT_V1) { - log::info!( - "[migrations][retire-chat-v1] remapping default_model chat-v1 -> {}", - MODEL_REASONING_QUICK_V1 - ); - config.default_model = Some(MODEL_REASONING_QUICK_V1.to_string()); - stats.default_model_remapped = true; - } else { - log::debug!( - "[migrations][retire-chat-v1] default_model={:?} — no remap needed", - config.default_model - ); - } - - Ok(stats) + log::debug!( + "[migrations][retire-chat-v1] default_model={:?} — no remap needed", + config.default_model + ); + Ok(MigrationStats::default()) } #[cfg(test)] @@ -59,17 +39,17 @@ mod tests { use crate::openhuman::config::Config; #[test] - fn remaps_chat_v1_to_reasoning_quick_v1() { + fn leaves_chat_v1_default_model_unchanged() { let mut config = Config::default(); config.default_model = Some("chat-v1".to_string()); let stats = run(&mut config).expect("migration should succeed"); - assert!(stats.default_model_remapped); + assert!(!stats.default_model_remapped); assert_eq!( config.default_model.as_deref(), - Some("reasoning-quick-v1"), - "default_model must be remapped" + Some("chat-v1"), + "chat-v1 is the canonical chat tier and must not be remapped" ); } diff --git a/src/openhuman/routing/README.md b/src/openhuman/routing/README.md index bce6143dd..1479d56ec 100644 --- a/src/openhuman/routing/README.md +++ b/src/openhuman/routing/README.md @@ -10,7 +10,7 @@ Intelligent model routing — a policy-driven layer that sits between callers (a - Probe and cache local model-server health (`GET {base}/api/tags` for Ollama, `GET {base}/models` for OpenAI-compat backends) with a 30 s TTL and 3 s probe timeout. - On a local primary: dispatch locally, and on error **or** low-quality output retry on remote — unless `privacy_required` forbids leaving the device. - Heuristically score local responses for low quality (length floor, empty-noise utterances, refusal phrases) to drive fallback. -- Normalize heavy `hint:*` model strings to backend-valid model IDs (`reasoning` → `MODEL_REASONING_V1`, `chat` → `MODEL_REASONING_QUICK_V1`, `agentic` → `MODEL_AGENTIC_V1`, `coding` → `MODEL_CODING_V1`). +- Normalize heavy `hint:*` model strings to backend-valid model IDs (`reasoning` -> `MODEL_REASONING_V1`, `chat` -> `MODEL_CHAT_V1`, `agentic` -> `MODEL_AGENTIC_V1`, `coding` -> `MODEL_CODING_V1`). - Force remote when native tool-calling is required (tools present) and refuse to silently bypass local routing for streaming. - Emit a structured `RoutingRecord` (category, target, resolved model, health, fallback flag, latency, tokens, cost) per completed call. @@ -60,7 +60,7 @@ None (no `store.rs`). The only state is the in-memory, TTL'd health cache inside - `LocalAiConfig` (`openhuman::config`): `runtime_enabled`, `provider`, `base_url`, `api_key`, `chat_model_id` drive local-provider selection and whether local routing is active at all. - `OPENHUMAN_LOCAL_INFERENCE_URL` (env): full `/v1` base URL of a local OpenAI-compatible server; when set, takes precedence over `config.base_url` and switches the health probe to `GET {base}/models`. -- Model-ID constants from `openhuman::config`: `MODEL_REASONING_V1`, `MODEL_REASONING_QUICK_V1`, `MODEL_AGENTIC_V1`, `MODEL_CODING_V1`. +- Model-ID constants from `openhuman::config`: `MODEL_REASONING_V1`, `MODEL_CHAT_V1`, `MODEL_REASONING_QUICK_V1` (legacy), `MODEL_AGENTIC_V1`, `MODEL_CODING_V1`. ## Dependencies diff --git a/src/openhuman/routing/provider.rs b/src/openhuman/routing/provider.rs index f595d5518..60f6ea122 100644 --- a/src/openhuman/routing/provider.rs +++ b/src/openhuman/routing/provider.rs @@ -18,7 +18,7 @@ use anyhow::Result; use async_trait::async_trait; use crate::openhuman::config::{ - MODEL_AGENTIC_V1, MODEL_CODING_V1, MODEL_REASONING_QUICK_V1, MODEL_REASONING_V1, + MODEL_AGENTIC_V1, MODEL_CHAT_V1, MODEL_CODING_V1, MODEL_REASONING_V1, }; use crate::openhuman::inference::provider::traits::{ ChatMessage, ChatRequest, ChatResponse, Provider, ProviderCapabilities, StreamChunk, @@ -96,10 +96,8 @@ impl IntelligentRoutingProvider { // Keep remote model naming aligned with backend modelRegistry. match requested_model.strip_prefix("hint:") { Some("reasoning") => MODEL_REASONING_V1.to_string(), - // Orchestrator's low-TTFT chat tier — Kimi K2.6 Turbo on the - // backend's `reasoning-quick-v1`. Backend support added in - // tinyhumansai/backend#760. - Some("chat") => MODEL_REASONING_QUICK_V1.to_string(), + // Orchestrator's low-TTFT chat tier. + Some("chat") => MODEL_CHAT_V1.to_string(), Some("agentic") => MODEL_AGENTIC_V1.to_string(), Some("coding") => MODEL_CODING_V1.to_string(), _ => requested_model.to_string(), diff --git a/src/openhuman/routing/provider_tests.rs b/src/openhuman/routing/provider_tests.rs index 90094f30c..571b6ee35 100644 --- a/src/openhuman/routing/provider_tests.rs +++ b/src/openhuman/routing/provider_tests.rs @@ -388,7 +388,7 @@ async fn regression_reasoning_hint_routes_remote_with_backend_model_name() { } #[tokio::test] -async fn regression_chat_hint_routes_remote_as_reasoning_quick_v1() { +async fn regression_chat_hint_routes_remote_as_chat_v1() { let local = MockProvider::new("local", "l"); let remote = MockProvider::new("remote", "r"); let health = LocalHealthChecker::seeded(true); @@ -403,10 +403,10 @@ async fn regression_chat_hint_routes_remote_as_reasoning_quick_v1() { .await .unwrap(); - // hint:chat must be translated to the backend's reasoning-quick-v1 tier - // (Kimi K2.6 Turbo). Sending the literal "hint:chat" would 400 on the - // backend since modelRegistry has no `hint:*` aliases. - assert_eq!(remote.last_model(), "reasoning-quick-v1"); + // hint:chat must be translated to the backend's chat-v1 tier. Sending the + // literal "hint:chat" would 400 on the backend since modelRegistry has no + // `hint:*` aliases. + assert_eq!(remote.last_model(), "chat-v1"); assert_eq!(local.calls(), 0); } diff --git a/tests/agent_harness_raw_coverage_e2e.rs b/tests/agent_harness_raw_coverage_e2e.rs index 4827a1cf4..87a7275e7 100644 --- a/tests/agent_harness_raw_coverage_e2e.rs +++ b/tests/agent_harness_raw_coverage_e2e.rs @@ -1,11 +1,13 @@ use anyhow::Result; use async_trait::async_trait; use openhuman_core::openhuman::agent::dispatcher::NativeToolDispatcher; +use openhuman_core::openhuman::agent::harness::definition::AgentDefinitionRegistry; use openhuman_core::openhuman::agent::harness::session::Agent; use openhuman_core::openhuman::agent::harness::{ run_subagent, with_parent_context, AgentDefinition, ParentExecutionContext, PromptSource, SandboxMode, SubagentRunOptions, ToolScope, }; +use openhuman_core::openhuman::agent::progress::AgentProgress; use openhuman_core::openhuman::config::AgentConfig; use openhuman_core::openhuman::context::prompt::ToolCallFormat; use openhuman_core::openhuman::inference::provider::traits::ProviderCapabilities; @@ -13,6 +15,7 @@ use openhuman_core::openhuman::inference::provider::{ ChatMessage, ChatRequest, ChatResponse, Provider, ToolCall, UsageInfo, }; use openhuman_core::openhuman::memory::{Memory, MemoryCategory, MemoryEntry, NamespaceSummary}; +use openhuman_core::openhuman::tools::SpawnSubagentTool; use openhuman_core::openhuman::tools::{Tool, ToolResult}; use parking_lot::Mutex; use serde_json::json; @@ -166,11 +169,15 @@ impl Tool for EchoTool { } fn usage(input_tokens: u64, output_tokens: u64) -> UsageInfo { + usage_with_cached(input_tokens, output_tokens, input_tokens / 2) +} + +fn usage_with_cached(input_tokens: u64, output_tokens: u64, cached_input_tokens: u64) -> UsageInfo { UsageInfo { input_tokens, output_tokens, context_window: 8_192, - cached_input_tokens: input_tokens / 2, + cached_input_tokens, charged_amount_usd: 0.001, } } @@ -197,6 +204,21 @@ fn response( } } +fn response_with_cached( + text: Option<&str>, + tool_calls: Vec, + input: u64, + output: u64, + cached: u64, +) -> ChatResponse { + ChatResponse { + text: text.map(str::to_string), + tool_calls, + usage: Some(usage_with_cached(input, output, cached)), + reasoning_content: None, + } +} + fn agent_config() -> AgentConfig { AgentConfig { max_tool_iterations: 4, @@ -209,10 +231,19 @@ fn build_agent( workspace: &Path, provider: Arc, agent_name: &str, +) -> Result { + build_agent_with_tools(workspace, provider, agent_name, vec![Box::new(EchoTool)]) +} + +fn build_agent_with_tools( + workspace: &Path, + provider: Arc, + agent_name: &str, + tools: Vec>, ) -> Result { let mut agent = Agent::builder() .provider_arc(provider) - .tools(vec![Box::new(EchoTool)]) + .tools(tools) .memory(Arc::new(StubMemory)) .tool_dispatcher(Box::new(NativeToolDispatcher)) .config(agent_config()) @@ -411,3 +442,207 @@ async fn run_subagent_filters_tools_runs_inner_loop_and_writes_child_transcript( Ok(()) } + +#[tokio::test] +async fn repeated_subagent_spawns_keep_cacheable_prefix_and_record_provider_cache_hit() -> Result<()> +{ + let workspace = tempfile::tempdir()?; + let provider = Arc::new(ScriptedProvider::new(vec![ + response_with_cached(Some("first"), Vec::new(), 100, 5, 0), + response_with_cached(Some("second"), Vec::new(), 100, 5, 88), + ])); + let parent = parent_context(workspace.path().to_path_buf(), provider.clone()); + let definition = coverage_definition(); + + let (first, second) = with_parent_context(parent, async { + let first = run_subagent( + &definition, + "Answer the stable cache probe.", + SubagentRunOptions { + task_id: Some("cache-probe-a".to_string()), + ..SubagentRunOptions::default() + }, + ) + .await?; + let second = run_subagent( + &definition, + "Answer the stable cache probe.", + SubagentRunOptions { + task_id: Some("cache-probe-b".to_string()), + ..SubagentRunOptions::default() + }, + ) + .await?; + Ok::<_, openhuman_core::openhuman::agent::harness::SubagentRunError>((first, second)) + }) + .await?; + + assert_eq!(first.output, "first"); + assert_eq!(second.output, "second"); + + let requests = provider.requests(); + assert_eq!(requests.len(), 2); + let first_system = requests[0] + .iter() + .find(|message| message.role == "system") + .expect("first subagent request should include a system prompt"); + let second_system = requests[1] + .iter() + .find(|message| message.role == "system") + .expect("second subagent request should include a system prompt"); + assert_eq!( + first_system.content, second_system.content, + "repeated subagent spawns of the same definition must preserve the byte-identical \ + system prefix the backend can cache" + ); + + let transcripts = transcript_jsonl_files(workspace.path()); + assert_eq!( + transcripts.len(), + 2, + "each subagent run should persist its own raw transcript: {transcripts:?}" + ); + let joined = transcripts + .iter() + .map(std::fs::read_to_string) + .collect::>>()? + .join("\n"); + assert!( + joined.contains("\"cached_input_tokens\":88"), + "provider-reported cached input tokens from the second child run should be preserved \ + in subagent transcript accounting:\n{joined}" + ); + + Ok(()) +} + +#[tokio::test] +async fn orchestrator_spawn_subagent_round_trip_streams_child_events_and_returns_result( +) -> Result<()> { + let workspace = tempfile::tempdir()?; + let agents_dir = workspace.path().join("agents"); + std::fs::create_dir_all(&agents_dir)?; + std::fs::write( + agents_dir.join("cache_probe_child.toml"), + r#" +id = "cache_probe_child" +display_name = "Cache Probe Child" +when_to_use = "Deterministic child agent used by harness cache tests." +temperature = 0.0 +max_iterations = 3 +omit_identity = true +omit_memory_context = true +omit_safety_preamble = true +omit_skills_catalog = true +omit_profile = true +omit_memory_md = true + +[system_prompt] +inline = "Answer the delegated cache probe directly." +"#, + )?; + let _ = AgentDefinitionRegistry::init_global(workspace.path()); + + let child_answer = "child-cache-observation: prefix was reusable"; + let parent_final = "orchestrator final: child-cache-observation accepted"; + let provider = Arc::new(ScriptedProvider::new(vec![ + response( + Some("delegating to child"), + vec![tool_call( + "spawn-child-1", + "spawn_subagent", + json!({ + "agent_id": "cache_probe_child", + "prompt": "Inspect whether the child turn can answer a cache probe.", + "context": "Parent observed request id cache-42.", + }), + )], + 140, + 9, + ), + response_with_cached(Some(child_answer), Vec::new(), 96, 7, 64), + response(Some(parent_final), Vec::new(), 120, 10), + ])); + let mut agent = build_agent_with_tools( + workspace.path(), + provider.clone(), + "coverage_orchestrator", + vec![Box::new(SpawnSubagentTool::new())], + )?; + let (progress_tx, mut progress_rx) = tokio::sync::mpsc::channel(1024); + agent.set_on_progress(Some(progress_tx)); + + let answer = agent + .turn("Ask a child agent for the cache observation, then respond.") + .await?; + assert_eq!( + answer, parent_final, + "orchestrator should synthesize after receiving the child result" + ); + + let mut progress = Vec::new(); + while let Ok(event) = progress_rx.try_recv() { + progress.push(event); + } + assert!( + progress.iter().any(|event| matches!( + event, + AgentProgress::SubagentSpawned { agent_id, task_id, .. } + if agent_id == "cache_probe_child" && task_id.starts_with("sub-") + )), + "parent progress should announce the child spawn: {progress:#?}" + ); + assert!( + progress.iter().any(|event| matches!( + event, + AgentProgress::SubagentCompleted { agent_id, output_chars, .. } + if agent_id == "cache_probe_child" && *output_chars == child_answer.chars().count() + )), + "parent progress should announce child completion: {progress:#?}" + ); + + let requests = provider.requests(); + assert_eq!( + requests.len(), + 3, + "expected parent request, child request, then parent synthesis request" + ); + assert!( + requests[0] + .iter() + .any(|message| message.role == "user" && message.content.contains("Ask a child agent")), + "first request should be the orchestrator turn: {:#?}", + requests[0] + ); + assert!( + requests[1].iter().any(|message| message.role == "system" + && message.content.contains("Sub-agent Role Contract")) + && requests[1].iter().any(|message| message.role == "user" + && message + .content + .contains("Parent observed request id cache-42")), + "second request should be the child subagent turn with parent-supplied context: {:#?}", + requests[1] + ); + assert!( + requests[2] + .iter() + .any(|message| message.role == "tool" && message.content.contains(child_answer)), + "third request should return the child result to the orchestrator as a tool result: {:#?}", + requests[2] + ); + let transcripts = transcript_jsonl_files(workspace.path()); + let joined = transcripts + .iter() + .map(std::fs::read_to_string) + .collect::>>()? + .join("\n"); + assert!( + joined.contains("\"agent\":\"cache_probe_child\"") + && joined.contains(child_answer) + && joined.contains("\"cached_input_tokens\":64"), + "child transcript should be persisted alongside the parent turn:\n{joined}" + ); + + Ok(()) +} diff --git a/tests/inference_agent_raw_coverage_e2e.rs b/tests/inference_agent_raw_coverage_e2e.rs index d59a44170..b7d2e6256 100644 --- a/tests/inference_agent_raw_coverage_e2e.rs +++ b/tests/inference_agent_raw_coverage_e2e.rs @@ -4386,7 +4386,7 @@ async fn inference_router_provider_covers_hint_tier_and_passthrough_routing() { assert!(routed_hint.contains("model=fast-chat")); let routed_tier = router - .chat_with_history(&[ChatMessage::user("tier")], "reasoning-quick-v1", 0.3) + .chat_with_history(&[ChatMessage::user("tier")], "chat-v1", 0.3) .await .expect("tier route"); assert!(routed_tier.contains("model=fast-chat"));