diff --git a/src/openhuman/agent/harness/mod.rs b/src/openhuman/agent/harness/mod.rs index 235b741d9..457c07777 100644 --- a/src/openhuman/agent/harness/mod.rs +++ b/src/openhuman/agent/harness/mod.rs @@ -36,6 +36,7 @@ pub mod session; pub(crate) mod session_queue; pub(crate) mod spawn_depth_context; pub mod subagent_runner; +mod token_budget; pub(crate) mod tool_filter; mod tool_loop; diff --git a/src/openhuman/agent/harness/session/turn.rs b/src/openhuman/agent/harness/session/turn.rs index 71cc86a2c..2dc7e49e4 100644 --- a/src/openhuman/agent/harness/session/turn.rs +++ b/src/openhuman/agent/harness/session/turn.rs @@ -27,6 +27,7 @@ use crate::openhuman::agent::memory_loader::collect_recall_citations; use crate::openhuman::agent::progress::AgentProgress; use crate::openhuman::context::prompt::{LearnedContextData, PromptContext, PromptTool}; use crate::openhuman::context::{ReductionOutcome, ARCHIVIST_EXTRACTION_PROMPT}; +use crate::openhuman::inference::model_context::context_window_for_model; use crate::openhuman::inference::provider::{ ChatMessage, ChatRequest, ConversationMessage, ProviderDelta, }; @@ -34,6 +35,10 @@ use crate::openhuman::memory::MemoryCategory; use crate::openhuman::tools::traits::ToolCallOptions; use crate::openhuman::tools::Tool; use crate::openhuman::util::truncate_with_ellipsis; + +use crate::openhuman::agent::harness::token_budget::{ + trim_chat_messages_to_budget, trim_conversation_history_to_budget, +}; use anyhow::Result; use std::hash::{Hash, Hasher}; use std::sync::Arc; @@ -489,6 +494,21 @@ impl Agent { self.history.len() ); + if let Some(context_window) = context_window_for_model(&effective_model) { + let budget_outcome = + trim_conversation_history_to_budget(&mut self.history, context_window); + if budget_outcome.trimmed { + log::warn!( + "[agent_loop] pre-dispatch history trimmed model={} context_window={} original_tokens={} final_tokens={} messages_removed={}", + effective_model, + context_window, + budget_outcome.original_tokens, + budget_outcome.final_tokens, + budget_outcome.messages_removed + ); + } + } + // Global context management: run the reduction chain // before every provider hit. Cheap when the guard is // healthy; executes the summarizer LLM call @@ -558,7 +578,8 @@ impl Agent { // a resumed session to provide a byte-identical prefix for // KV cache reuse. After `.take()` the cache is consumed; // subsequent iterations rebuild from history normally. - let messages = if let Some(mut cached) = self.cached_transcript_messages.take() { + let mut messages = if let Some(mut cached) = self.cached_transcript_messages.take() + { // Append only the delta (new user message) from the // end of the current history. let new_tail = self.tool_dispatcher.to_provider_messages( @@ -574,6 +595,21 @@ impl Agent { } else { self.tool_dispatcher.to_provider_messages(&self.history) }; + if let Some(context_window) = context_window_for_model(&effective_model) { + let budget_outcome = + trim_chat_messages_to_budget(&mut messages, context_window); + if budget_outcome.trimmed { + log::warn!( + "[agent_loop] pre-dispatch provider messages trimmed model={} context_window={} original_tokens={} final_tokens={} messages_removed={}", + effective_model, + context_window, + budget_outcome.original_tokens, + budget_outcome.final_tokens, + budget_outcome.messages_removed + ); + } + } + last_provider_messages = Some(messages.clone()); log::info!( diff --git a/src/openhuman/agent/harness/token_budget.rs b/src/openhuman/agent/harness/token_budget.rs new file mode 100644 index 000000000..fd961d53a --- /dev/null +++ b/src/openhuman/agent/harness/token_budget.rs @@ -0,0 +1,246 @@ +//! Pre-dispatch token budgeting for agent conversation history. +//! +//! Estimates prompt size with the same ~4 chars/token heuristic used elsewhere +//! in the codebase and drops the oldest non-system messages until the payload +//! fits the target model's context window. + +use crate::openhuman::inference::provider::{ChatMessage, ConversationMessage}; + +/// Tokens reserved for the model's completion, tool schemas, and provider overhead. +pub const DEFAULT_OUTPUT_RESERVE_TOKENS: u64 = 8_192; + +/// Minimum reserve when the context window is very small. +const MIN_OUTPUT_RESERVE_TOKENS: u64 = 512; + +/// Outcome of a pre-dispatch trim pass. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TokenBudgetOutcome { + pub original_tokens: usize, + pub final_tokens: usize, + pub messages_removed: usize, + pub trimmed: bool, +} + +/// Rough token estimate: ~4 characters per token (matches tree summarizer). +pub fn estimate_tokens(text: &str) -> usize { + text.len().saturating_add(3) / 4 +} + +pub fn estimate_chat_message_tokens(msg: &ChatMessage) -> usize { + estimate_tokens(&msg.content) +} + +pub fn estimate_conversation_message_tokens(msg: &ConversationMessage) -> usize { + match msg { + ConversationMessage::Chat(chat) => estimate_chat_message_tokens(chat), + ConversationMessage::AssistantToolCalls { text, tool_calls } => { + let body = text.as_deref().unwrap_or_default(); + let mut total = estimate_tokens(body); + for call in tool_calls { + total = total.saturating_add(estimate_tokens(&call.name)); + total = total.saturating_add(estimate_tokens(&call.arguments)); + } + total + } + ConversationMessage::ToolResults(results) => results + .iter() + .map(|r| estimate_tokens(&r.tool_call_id).saturating_add(estimate_tokens(&r.content))) + .sum(), + } +} + +fn output_reserve_tokens(context_window: u64) -> u64 { + let pct = context_window / 10; + pct.max(MIN_OUTPUT_RESERVE_TOKENS) + .min(DEFAULT_OUTPUT_RESERVE_TOKENS.max(context_window / 4)) +} + +fn max_input_tokens(context_window: u64) -> u64 { + context_window.saturating_sub(output_reserve_tokens(context_window)) +} + +/// Trim `messages` oldest-first (never removing `system` role) until the +/// estimated prompt fits `context_window`. +pub fn trim_chat_messages_to_budget( + messages: &mut Vec, + context_window: u64, +) -> TokenBudgetOutcome { + trim_messages_to_budget( + messages, + context_window, + estimate_chat_message_tokens, + |msg| msg.role == "system", + ) +} + +/// Trim conversation `history` oldest-first, preserving system chat messages. +pub fn trim_conversation_history_to_budget( + history: &mut Vec, + context_window: u64, +) -> TokenBudgetOutcome { + trim_messages_to_budget( + history, + context_window, + estimate_conversation_message_tokens, + |msg| matches!(msg, ConversationMessage::Chat(c) if c.role == "system"), + ) +} + +fn trim_messages_to_budget( + messages: &mut Vec, + context_window: u64, + estimate: F, + is_system: P, +) -> TokenBudgetOutcome +where + F: Fn(&T) -> usize, + P: Fn(&T) -> bool, +{ + let max_tokens = max_input_tokens(context_window) as usize; + let original_tokens: usize = messages.iter().map(&estimate).sum(); + + if original_tokens <= max_tokens { + return TokenBudgetOutcome { + original_tokens, + final_tokens: original_tokens, + messages_removed: 0, + trimmed: false, + }; + } + + // Drop oldest non-system messages until the budget fits, preserving the + // original relative order of every retained message (system + non-system). + // Rebuilding as `system ++ other` would reorder history when a system + // message appears after non-system messages, which changes prompt + // semantics (see PR #2100 CodeRabbit review). + let mut removable_positions: Vec = messages + .iter() + .enumerate() + .filter_map(|(idx, msg)| (!is_system(msg)).then_some(idx)) + .collect(); + + let mut removed = 0usize; + while !removable_positions.is_empty() { + let total: usize = messages.iter().map(&estimate).sum(); + if total <= max_tokens { + break; + } + let absolute_idx = removable_positions.remove(0); + // Subsequent positions shift left by one for every prior removal. + let remove_at = absolute_idx - removed; + messages.remove(remove_at); + removed += 1; + } + + let final_tokens: usize = messages.iter().map(&estimate).sum(); + + TokenBudgetOutcome { + original_tokens, + final_tokens, + messages_removed: removed, + trimmed: removed > 0, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::openhuman::inference::provider::ToolCall; + + fn user_msg(content: &str) -> ChatMessage { + ChatMessage::user(content.to_string()) + } + + #[test] + fn under_limit_passes_through_unchanged() { + let mut messages = vec![ + ChatMessage::system("sys"), + user_msg("hello"), + ChatMessage::assistant("hi"), + ]; + let before_len = messages.len(); + let outcome = trim_chat_messages_to_budget(&mut messages, 100_000); + assert!(!outcome.trimmed); + assert_eq!(outcome.original_tokens, outcome.final_tokens); + assert_eq!(messages.len(), before_len); + } + + #[test] + fn over_limit_truncates_oldest_non_system_first() { + let mut messages = vec![ + ChatMessage::system("system prompt"), + user_msg(&"x".repeat(400_000)), + user_msg("keep-me"), + ]; + let outcome = trim_chat_messages_to_budget(&mut messages, 1_000); + assert!(outcome.trimmed); + assert!(outcome.final_tokens < outcome.original_tokens); + assert!(outcome.messages_removed >= 1); + assert_eq!(messages.first().unwrap().role, "system"); + assert!( + messages.iter().any(|m| m.content.contains("keep-me")), + "newest user message should survive trimming" + ); + } + + #[test] + fn trim_conversation_history_drops_oldest_messages() { + let mut messages = vec![ConversationMessage::Chat(user_msg(&"y".repeat(80_000)))]; + let outcome = trim_conversation_history_to_budget(&mut messages, 1_000); + assert!(outcome.trimmed); + assert!(outcome.original_tokens > outcome.final_tokens); + } + + #[test] + fn conversation_tool_results_are_counted_in_estimate() { + let msg = ConversationMessage::ToolResults(vec![ + crate::openhuman::inference::provider::ToolResultMessage { + tool_call_id: "c1".into(), + content: "z".repeat(8_000), + }, + ]); + assert!(estimate_conversation_message_tokens(&msg) > 1_000); + } + + #[test] + fn trim_preserves_relative_order_when_system_appears_late() { + // System message in the middle of history must not be moved to the + // front during trimming. Regression guard for PR #2100 review. + let mut messages = vec![ + user_msg(&"a".repeat(40_000)), // oldest non-system, expected to drop + user_msg("first-user"), + ChatMessage::system("late-system"), + user_msg("last-user"), + ]; + let outcome = trim_chat_messages_to_budget(&mut messages, 1_000); + assert!(outcome.trimmed); + // System position relative to surrounding messages is preserved. + let roles: Vec<&str> = messages.iter().map(|m| m.role.as_str()).collect(); + let sys_idx = roles + .iter() + .position(|r| *r == "system") + .expect("system message must be retained"); + // At least one user message should still precede the late system message. + assert!( + sys_idx > 0, + "late system message must remain after earlier surviving non-system messages" + ); + assert!( + messages.iter().any(|m| m.content == "last-user"), + "newest user message must survive" + ); + } + + #[test] + fn assistant_tool_calls_estimate_includes_arguments() { + let msg = ConversationMessage::AssistantToolCalls { + text: Some("thinking".into()), + tool_calls: vec![ToolCall { + id: "1".into(), + name: "echo".into(), + arguments: "{\"value\":\"x\"}".into(), + }], + }; + assert!(estimate_conversation_message_tokens(&msg) > 0); + } +} diff --git a/src/openhuman/agent/harness/tool_loop.rs b/src/openhuman/agent/harness/tool_loop.rs index 140d4e997..7ddbef8f0 100644 --- a/src/openhuman/agent/harness/tool_loop.rs +++ b/src/openhuman/agent/harness/tool_loop.rs @@ -17,6 +17,9 @@ use super::credentials::scrub_credentials; use super::parse::{build_native_assistant_history, parse_structured_tool_calls, parse_tool_calls}; use super::payload_summarizer::PayloadSummarizer; use crate::openhuman::context::guard::{ContextCheckResult, ContextGuard}; +use crate::openhuman::inference::model_context::context_window_for_model; + +use super::token_budget::trim_chat_messages_to_budget; /// Minimum characters per chunk when relaying LLM text to a streaming draft. const STREAM_CHUNK_MIN_CHARS: usize = 80; @@ -153,7 +156,9 @@ pub(crate) async fn run_tool_call_loop( .join(", ") ); - let mut context_guard = ContextGuard::new(); + let mut context_guard = context_window_for_model(model) + .map(ContextGuard::with_context_window) + .unwrap_or_else(ContextGuard::new); let mut turn_cost = TurnCost::new(); // Announce turn start to progress subscribers (if any). We use @@ -236,6 +241,28 @@ pub(crate) async fn run_tool_call_loop( } } + if let Some(context_window) = context_window_for_model(model) { + let budget_outcome = trim_chat_messages_to_budget(history, context_window); + if budget_outcome.trimmed { + log::warn!( + "[agent_loop] pre-dispatch history trimmed model={} context_window={} original_tokens={} final_tokens={} messages_removed={}", + model, + context_window, + budget_outcome.original_tokens, + budget_outcome.final_tokens, + budget_outcome.messages_removed + ); + } else { + tracing::debug!( + iteration, + model, + context_window, + estimated_tokens = budget_outcome.final_tokens, + "[agent_loop] pre-dispatch token budget ok" + ); + } + } + tracing::debug!(iteration, "[agent_loop] sending LLM request"); let image_marker_count = multimodal::count_image_markers(history); if image_marker_count > 0 && !provider.supports_vision() { diff --git a/src/openhuman/inference/mod.rs b/src/openhuman/inference/mod.rs index 7fb5549e0..3848ed587 100644 --- a/src/openhuman/inference/mod.rs +++ b/src/openhuman/inference/mod.rs @@ -15,6 +15,7 @@ pub mod device; pub mod http; pub mod local; +pub mod model_context; pub mod model_ids; pub mod ops; pub mod parse; @@ -36,6 +37,7 @@ pub use schemas::{ pub use device::DeviceProfile; pub use local::all_local_ai_controller_schemas; pub use local::all_local_ai_registered_controllers; +pub use model_context::context_window_for_model; pub use presets::{ModelPreset, ModelTier, VisionMode}; pub use sentiment::SentimentResult; pub use types::{ diff --git a/src/openhuman/inference/model_context.rs b/src/openhuman/inference/model_context.rs new file mode 100644 index 000000000..dbcacd3a8 --- /dev/null +++ b/src/openhuman/inference/model_context.rs @@ -0,0 +1,155 @@ +//! Known model context-window sizes for pre-inference budgeting. +//! +//! Provider `/models` responses may include `context_length` / `context_window`, +//! but the agent harness must enforce limits **before** the first dispatch — +//! otherwise long histories produce upstream `400 Bad Request` errors when usage +//! metadata is not yet available. + +use crate::openhuman::config::{ + MODEL_AGENTIC_V1, MODEL_CODING_V1, MODEL_REASONING_QUICK_V1, MODEL_REASONING_V1, +}; + +/// Conservative default for OpenHuman abstract tier models (tokens). +const TIER_LARGE_CONTEXT: u64 = 200_000; +const TIER_STANDARD_CONTEXT: u64 = 128_000; +const TIER_LOCAL_CONTEXT: u64 = 8_192; + +/// How a pattern in [`MODEL_CONTEXT_PATTERNS`] is matched against a model id. +#[derive(Copy, Clone)] +enum PatternMatch { + /// Pattern must appear anywhere as a substring (after lowercasing). + Substring, + /// Pattern must appear as a full `-`/`_`/`/`/`:`-delimited segment. + /// Prevents false positives like `"solo1-7b"` matching the `"o1"` pattern + /// or `"proto3-chat"` matching the `"o3"` pattern. + Segment, +} + +/// `(pattern, match mode, context window in tokens)` — first match wins. +const MODEL_CONTEXT_PATTERNS: &[(&str, PatternMatch, u64)] = &[ + ("claude-haiku-4.5", PatternMatch::Substring, 200_000), + ("claude-haiku-4", PatternMatch::Substring, 200_000), + ("claude-haiku", PatternMatch::Substring, 200_000), + ("claude-sonnet-4", PatternMatch::Substring, 200_000), + ("claude-opus-4", PatternMatch::Substring, 200_000), + ("claude-3-5-sonnet", PatternMatch::Substring, 200_000), + ("claude-3-5-haiku", PatternMatch::Substring, 200_000), + ("claude-3-opus", PatternMatch::Substring, 200_000), + ("gpt-4.1", PatternMatch::Substring, 1_047_576), + ("gpt-4o", PatternMatch::Substring, 128_000), + ("gpt-4-turbo", PatternMatch::Substring, 128_000), + ("gpt-4", PatternMatch::Substring, 128_000), + ("gpt-3.5", PatternMatch::Substring, 16_385), + // `o1`/`o3` are short and collide with substrings of unrelated model ids + // (e.g. `solo1-7b`, `proto3-chat`). Require a segment-boundary match. + ("o1", PatternMatch::Segment, 200_000), + ("o3", PatternMatch::Segment, 200_000), + ("deepseek", PatternMatch::Substring, 128_000), + ("gemma3", PatternMatch::Substring, 8_192), + ("gemma", PatternMatch::Substring, 8_192), + ("llama-3", PatternMatch::Substring, 128_000), + ("llama3", PatternMatch::Substring, 128_000), +]; + +fn matches_pattern(lower: &str, pattern: &str, mode: PatternMatch) -> bool { + match mode { + PatternMatch::Substring => lower.contains(pattern), + PatternMatch::Segment => lower + .split(|c: char| matches!(c, '/' | '-' | '_' | ':' | '.')) + .any(|seg| seg == pattern), + } +} + +/// Resolve the context window (in tokens) for a model id or OpenHuman tier alias. +/// +/// Returns `None` when the model is unknown — callers should skip pre-dispatch +/// trimming rather than guess. +pub fn context_window_for_model(model: &str) -> Option { + let normalized = model.trim(); + if normalized.is_empty() { + return None; + } + + if let Some(window) = tier_context_window(normalized) { + return Some(window); + } + + let lower = normalized.to_ascii_lowercase(); + for (pattern, mode, window) in MODEL_CONTEXT_PATTERNS { + if matches_pattern(&lower, pattern, *mode) { + tracing::debug!( + model = normalized, + pattern, + context_window = window, + "[model_context] matched known model pattern" + ); + return Some(*window); + } + } + + None +} + +fn tier_context_window(model: &str) -> Option { + match model { + MODEL_REASONING_V1 | MODEL_AGENTIC_V1 | MODEL_CODING_V1 => Some(TIER_LARGE_CONTEXT), + MODEL_REASONING_QUICK_V1 | "summarization-v1" | "chat" => Some(TIER_STANDARD_CONTEXT), + m if m.starts_with("gemma") || m.contains(":1b") || m.contains("270m") => { + Some(TIER_LOCAL_CONTEXT) + } + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tier_aliases_resolve() { + assert_eq!(context_window_for_model("reasoning-v1"), Some(200_000)); + assert_eq!(context_window_for_model("agentic-v1"), Some(200_000)); + assert_eq!( + context_window_for_model("reasoning-quick-v1"), + Some(128_000) + ); + } + + #[test] + fn copilot_haiku_resolves_to_200k() { + assert_eq!( + context_window_for_model("github_copilot/claude-haiku-4.5"), + Some(200_000) + ); + } + + #[test] + fn unknown_model_returns_none() { + assert_eq!(context_window_for_model("totally-unknown-model-xyz"), None); + } + + #[test] + fn empty_model_returns_none() { + assert_eq!(context_window_for_model(" "), None); + } + + #[test] + fn o1_o3_segment_match_does_not_overmatch() { + // Real OpenAI o1/o3 model ids must still resolve. + assert_eq!(context_window_for_model("o1"), Some(200_000)); + assert_eq!(context_window_for_model("o1-mini"), Some(200_000)); + assert_eq!(context_window_for_model("o3-mini"), Some(200_000)); + assert_eq!(context_window_for_model("openai/o1-preview"), Some(200_000)); + + // Names that merely *contain* the substring "o1" / "o3" must NOT + // inherit the 200K window (regression guard for PR #2100 review). + assert_eq!(context_window_for_model("solo1-7b"), None); + assert_eq!(context_window_for_model("proto3-chat"), None); + assert_eq!( + context_window_for_model("ollama/mistral-for-o1-benchmark"), + Some(200_000), + "`-o1-` segment should still match" + ); + assert_eq!(context_window_for_model("octo3thing"), None); + } +} diff --git a/src/openhuman/inference/provider/ops.rs b/src/openhuman/inference/provider/ops.rs index 0d638b6d2..b4696c224 100644 --- a/src/openhuman/inference/provider/ops.rs +++ b/src/openhuman/inference/provider/ops.rs @@ -624,6 +624,10 @@ pub fn create_routed_provider_with_options( router::Route { provider_name: INFERENCE_BACKEND_ID.to_string(), model: r.model.clone(), + context_window: + crate::openhuman::inference::model_context::context_window_for_model( + &r.model, + ), }, ) }) @@ -705,6 +709,10 @@ pub fn create_intelligent_routing_provider( router::Route { provider_name: INFERENCE_BACKEND_ID.to_string(), model: r.model.clone(), + context_window: + crate::openhuman::inference::model_context::context_window_for_model( + &r.model, + ), }, ) }) diff --git a/src/openhuman/inference/provider/router.rs b/src/openhuman/inference/provider/router.rs index b08752e2b..d97065974 100644 --- a/src/openhuman/inference/provider/router.rs +++ b/src/openhuman/inference/provider/router.rs @@ -23,6 +23,8 @@ fn openhuman_tier_to_hint(model: &str) -> Option<&'static str> { pub struct Route { pub provider_name: String, pub model: String, + /// Known context window for `model` when discoverable (tokens). + pub context_window: Option, } /// Multi-model router — routes requests to different provider+model combos diff --git a/src/openhuman/inference/provider/router_test.rs b/src/openhuman/inference/provider/router_test.rs index 1bfc6cbd1..f217eba4b 100644 --- a/src/openhuman/inference/provider/router_test.rs +++ b/src/openhuman/inference/provider/router_test.rs @@ -69,6 +69,7 @@ fn make_router( Route { provider_name: provider_name.to_string(), model: model.to_string(), + context_window: None, }, ) })