fix(agent): guard agent prompts against model max-context limit (#2074) (#2100)

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Steven Enamakel <enamakel@tinyhumans.ai>

src/openhuman/agent/harness/mod.rs

@@ -36,6 +36,7 @@ pub mod session;
 pub(crate) mod session_queue;
 pub(crate) mod spawn_depth_context;
 pub mod subagent_runner;
+mod token_budget;
 pub(crate) mod tool_filter;
 mod tool_loop;
 

src/openhuman/agent/harness/session/turn.rs

@@ -27,6 +27,7 @@ use crate::openhuman::agent::memory_loader::collect_recall_citations;
 use crate::openhuman::agent::progress::AgentProgress;
 use crate::openhuman::context::prompt::{LearnedContextData, PromptContext, PromptTool};
 use crate::openhuman::context::{ReductionOutcome, ARCHIVIST_EXTRACTION_PROMPT};
+use crate::openhuman::inference::model_context::context_window_for_model;
 use crate::openhuman::inference::provider::{
     ChatMessage, ChatRequest, ConversationMessage, ProviderDelta,
 };
@@ -34,6 +35,10 @@ use crate::openhuman::memory::MemoryCategory;
 use crate::openhuman::tools::traits::ToolCallOptions;
 use crate::openhuman::tools::Tool;
 use crate::openhuman::util::truncate_with_ellipsis;
+
+use crate::openhuman::agent::harness::token_budget::{
+    trim_chat_messages_to_budget, trim_conversation_history_to_budget,
+};
 use anyhow::Result;
 use std::hash::{Hash, Hasher};
 use std::sync::Arc;
@@ -489,6 +494,21 @@ impl Agent {
                     self.history.len()
                 );
 
+                if let Some(context_window) = context_window_for_model(&effective_model) {
+                    let budget_outcome =
+                        trim_conversation_history_to_budget(&mut self.history, context_window);
+                    if budget_outcome.trimmed {
+                        log::warn!(
+                            "[agent_loop] pre-dispatch history trimmed model={} context_window={} original_tokens={} final_tokens={} messages_removed={}",
+                            effective_model,
+                            context_window,
+                            budget_outcome.original_tokens,
+                            budget_outcome.final_tokens,
+                            budget_outcome.messages_removed
+                        );
+                    }
+                }
+
                 // Global context management: run the reduction chain
                 // before every provider hit. Cheap when the guard is
                 // healthy; executes the summarizer LLM call
@@ -558,7 +578,8 @@ impl Agent {
                 // a resumed session to provide a byte-identical prefix for
                 // KV cache reuse. After `.take()` the cache is consumed;
                 // subsequent iterations rebuild from history normally.
-                let messages = if let Some(mut cached) = self.cached_transcript_messages.take() {
+                let mut messages = if let Some(mut cached) = self.cached_transcript_messages.take()
+                {
                     // Append only the delta (new user message) from the
                     // end of the current history.
                     let new_tail = self.tool_dispatcher.to_provider_messages(
@@ -574,6 +595,21 @@ impl Agent {
                 } else {
                     self.tool_dispatcher.to_provider_messages(&self.history)
                 };
+                if let Some(context_window) = context_window_for_model(&effective_model) {
+                    let budget_outcome =
+                        trim_chat_messages_to_budget(&mut messages, context_window);
+                    if budget_outcome.trimmed {
+                        log::warn!(
+                            "[agent_loop] pre-dispatch provider messages trimmed model={} context_window={} original_tokens={} final_tokens={} messages_removed={}",
+                            effective_model,
+                            context_window,
+                            budget_outcome.original_tokens,
+                            budget_outcome.final_tokens,
+                            budget_outcome.messages_removed
+                        );
+                    }
+                }
+
                 last_provider_messages = Some(messages.clone());
 
                 log::info!(

src/openhuman/agent/harness/token_budget.rs

@@ -0,0 +1,246 @@
+//! Pre-dispatch token budgeting for agent conversation history.
+//!
+//! Estimates prompt size with the same ~4 chars/token heuristic used elsewhere
+//! in the codebase and drops the oldest non-system messages until the payload
+//! fits the target model's context window.
+
+use crate::openhuman::inference::provider::{ChatMessage, ConversationMessage};
+
+/// Tokens reserved for the model's completion, tool schemas, and provider overhead.
+pub const DEFAULT_OUTPUT_RESERVE_TOKENS: u64 = 8_192;
+
+/// Minimum reserve when the context window is very small.
+const MIN_OUTPUT_RESERVE_TOKENS: u64 = 512;
+
+/// Outcome of a pre-dispatch trim pass.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TokenBudgetOutcome {
+    pub original_tokens: usize,
+    pub final_tokens: usize,
+    pub messages_removed: usize,
+    pub trimmed: bool,
+}
+
+/// Rough token estimate: ~4 characters per token (matches tree summarizer).
+pub fn estimate_tokens(text: &str) -> usize {
+    text.len().saturating_add(3) / 4
+}
+
+pub fn estimate_chat_message_tokens(msg: &ChatMessage) -> usize {
+    estimate_tokens(&msg.content)
+}
+
+pub fn estimate_conversation_message_tokens(msg: &ConversationMessage) -> usize {
+    match msg {
+        ConversationMessage::Chat(chat) => estimate_chat_message_tokens(chat),
+        ConversationMessage::AssistantToolCalls { text, tool_calls } => {
+            let body = text.as_deref().unwrap_or_default();
+            let mut total = estimate_tokens(body);
+            for call in tool_calls {
+                total = total.saturating_add(estimate_tokens(&call.name));
+                total = total.saturating_add(estimate_tokens(&call.arguments));
+            }
+            total
+        }
+        ConversationMessage::ToolResults(results) => results
+            .iter()
+            .map(|r| estimate_tokens(&r.tool_call_id).saturating_add(estimate_tokens(&r.content)))
+            .sum(),
+    }
+}
+
+fn output_reserve_tokens(context_window: u64) -> u64 {
+    let pct = context_window / 10;
+    pct.max(MIN_OUTPUT_RESERVE_TOKENS)
+        .min(DEFAULT_OUTPUT_RESERVE_TOKENS.max(context_window / 4))
+}
+
+fn max_input_tokens(context_window: u64) -> u64 {
+    context_window.saturating_sub(output_reserve_tokens(context_window))
+}
+
+/// Trim `messages` oldest-first (never removing `system` role) until the
+/// estimated prompt fits `context_window`.
+pub fn trim_chat_messages_to_budget(
+    messages: &mut Vec<ChatMessage>,
+    context_window: u64,
+) -> TokenBudgetOutcome {
+    trim_messages_to_budget(
+        messages,
+        context_window,
+        estimate_chat_message_tokens,
+        |msg| msg.role == "system",
+    )
+}
+
+/// Trim conversation `history` oldest-first, preserving system chat messages.
+pub fn trim_conversation_history_to_budget(
+    history: &mut Vec<ConversationMessage>,
+    context_window: u64,
+) -> TokenBudgetOutcome {
+    trim_messages_to_budget(
+        history,
+        context_window,
+        estimate_conversation_message_tokens,
+        |msg| matches!(msg, ConversationMessage::Chat(c) if c.role == "system"),
+    )
+}
+
+fn trim_messages_to_budget<T, F, P>(
+    messages: &mut Vec<T>,
+    context_window: u64,
+    estimate: F,
+    is_system: P,
+) -> TokenBudgetOutcome
+where
+    F: Fn(&T) -> usize,
+    P: Fn(&T) -> bool,
+{
+    let max_tokens = max_input_tokens(context_window) as usize;
+    let original_tokens: usize = messages.iter().map(&estimate).sum();
+
+    if original_tokens <= max_tokens {
+        return TokenBudgetOutcome {
+            original_tokens,
+            final_tokens: original_tokens,
+            messages_removed: 0,
+            trimmed: false,
+        };
+    }
+
+    // Drop oldest non-system messages until the budget fits, preserving the
+    // original relative order of every retained message (system + non-system).
+    // Rebuilding as `system ++ other` would reorder history when a system
+    // message appears after non-system messages, which changes prompt
+    // semantics (see PR #2100 CodeRabbit review).
+    let mut removable_positions: Vec<usize> = messages
+        .iter()
+        .enumerate()
+        .filter_map(|(idx, msg)| (!is_system(msg)).then_some(idx))
+        .collect();
+
+    let mut removed = 0usize;
+    while !removable_positions.is_empty() {
+        let total: usize = messages.iter().map(&estimate).sum();
+        if total <= max_tokens {
+            break;
+        }
+        let absolute_idx = removable_positions.remove(0);
+        // Subsequent positions shift left by one for every prior removal.
+        let remove_at = absolute_idx - removed;
+        messages.remove(remove_at);
+        removed += 1;
+    }
+
+    let final_tokens: usize = messages.iter().map(&estimate).sum();
+
+    TokenBudgetOutcome {
+        original_tokens,
+        final_tokens,
+        messages_removed: removed,
+        trimmed: removed > 0,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::openhuman::inference::provider::ToolCall;
+
+    fn user_msg(content: &str) -> ChatMessage {
+        ChatMessage::user(content.to_string())
+    }
+
+    #[test]
+    fn under_limit_passes_through_unchanged() {
+        let mut messages = vec![
+            ChatMessage::system("sys"),
+            user_msg("hello"),
+            ChatMessage::assistant("hi"),
+        ];
+        let before_len = messages.len();
+        let outcome = trim_chat_messages_to_budget(&mut messages, 100_000);
+        assert!(!outcome.trimmed);
+        assert_eq!(outcome.original_tokens, outcome.final_tokens);
+        assert_eq!(messages.len(), before_len);
+    }
+
+    #[test]
+    fn over_limit_truncates_oldest_non_system_first() {
+        let mut messages = vec![
+            ChatMessage::system("system prompt"),
+            user_msg(&"x".repeat(400_000)),
+            user_msg("keep-me"),
+        ];
+        let outcome = trim_chat_messages_to_budget(&mut messages, 1_000);
+        assert!(outcome.trimmed);
+        assert!(outcome.final_tokens < outcome.original_tokens);
+        assert!(outcome.messages_removed >= 1);
+        assert_eq!(messages.first().unwrap().role, "system");
+        assert!(
+            messages.iter().any(|m| m.content.contains("keep-me")),
+            "newest user message should survive trimming"
+        );
+    }
+
+    #[test]
+    fn trim_conversation_history_drops_oldest_messages() {
+        let mut messages = vec![ConversationMessage::Chat(user_msg(&"y".repeat(80_000)))];
+        let outcome = trim_conversation_history_to_budget(&mut messages, 1_000);
+        assert!(outcome.trimmed);
+        assert!(outcome.original_tokens > outcome.final_tokens);
+    }
+
+    #[test]
+    fn conversation_tool_results_are_counted_in_estimate() {
+        let msg = ConversationMessage::ToolResults(vec![
+            crate::openhuman::inference::provider::ToolResultMessage {
+                tool_call_id: "c1".into(),
+                content: "z".repeat(8_000),
+            },
+        ]);
+        assert!(estimate_conversation_message_tokens(&msg) > 1_000);
+    }
+
+    #[test]
+    fn trim_preserves_relative_order_when_system_appears_late() {
+        // System message in the middle of history must not be moved to the
+        // front during trimming. Regression guard for PR #2100 review.
+        let mut messages = vec![
+            user_msg(&"a".repeat(40_000)), // oldest non-system, expected to drop
+            user_msg("first-user"),
+            ChatMessage::system("late-system"),
+            user_msg("last-user"),
+        ];
+        let outcome = trim_chat_messages_to_budget(&mut messages, 1_000);
+        assert!(outcome.trimmed);
+        // System position relative to surrounding messages is preserved.
+        let roles: Vec<&str> = messages.iter().map(|m| m.role.as_str()).collect();
+        let sys_idx = roles
+            .iter()
+            .position(|r| *r == "system")
+            .expect("system message must be retained");
+        // At least one user message should still precede the late system message.
+        assert!(
+            sys_idx > 0,
+            "late system message must remain after earlier surviving non-system messages"
+        );
+        assert!(
+            messages.iter().any(|m| m.content == "last-user"),
+            "newest user message must survive"
+        );
+    }
+
+    #[test]
+    fn assistant_tool_calls_estimate_includes_arguments() {
+        let msg = ConversationMessage::AssistantToolCalls {
+            text: Some("thinking".into()),
+            tool_calls: vec![ToolCall {
+                id: "1".into(),
+                name: "echo".into(),
+                arguments: "{\"value\":\"x\"}".into(),
+            }],
+        };
+        assert!(estimate_conversation_message_tokens(&msg) > 0);
+    }
+}

src/openhuman/agent/harness/tool_loop.rs

@@ -17,6 +17,9 @@ use super::credentials::scrub_credentials;
 use super::parse::{build_native_assistant_history, parse_structured_tool_calls, parse_tool_calls};
 use super::payload_summarizer::PayloadSummarizer;
 use crate::openhuman::context::guard::{ContextCheckResult, ContextGuard};
+use crate::openhuman::inference::model_context::context_window_for_model;
+
+use super::token_budget::trim_chat_messages_to_budget;
 
 /// Minimum characters per chunk when relaying LLM text to a streaming draft.
 const STREAM_CHUNK_MIN_CHARS: usize = 80;
@@ -153,7 +156,9 @@ pub(crate) async fn run_tool_call_loop(
             .join(", ")
     );
 
-    let mut context_guard = ContextGuard::new();
+    let mut context_guard = context_window_for_model(model)
+        .map(ContextGuard::with_context_window)
+        .unwrap_or_else(ContextGuard::new);
     let mut turn_cost = TurnCost::new();
 
     // Announce turn start to progress subscribers (if any). We use
@@ -236,6 +241,28 @@ pub(crate) async fn run_tool_call_loop(
             }
         }
 
+        if let Some(context_window) = context_window_for_model(model) {
+            let budget_outcome = trim_chat_messages_to_budget(history, context_window);
+            if budget_outcome.trimmed {
+                log::warn!(
+                    "[agent_loop] pre-dispatch history trimmed model={} context_window={} original_tokens={} final_tokens={} messages_removed={}",
+                    model,
+                    context_window,
+                    budget_outcome.original_tokens,
+                    budget_outcome.final_tokens,
+                    budget_outcome.messages_removed
+                );
+            } else {
+                tracing::debug!(
+                    iteration,
+                    model,
+                    context_window,
+                    estimated_tokens = budget_outcome.final_tokens,
+                    "[agent_loop] pre-dispatch token budget ok"
+                );
+            }
+        }
+
         tracing::debug!(iteration, "[agent_loop] sending LLM request");
         let image_marker_count = multimodal::count_image_markers(history);
         if image_marker_count > 0 && !provider.supports_vision() {

src/openhuman/inference/mod.rs

@@ -15,6 +15,7 @@
 pub mod device;
 pub mod http;
 pub mod local;
+pub mod model_context;
 pub mod model_ids;
 pub mod ops;
 pub mod parse;
@@ -36,6 +37,7 @@ pub use schemas::{
 pub use device::DeviceProfile;
 pub use local::all_local_ai_controller_schemas;
 pub use local::all_local_ai_registered_controllers;
+pub use model_context::context_window_for_model;
 pub use presets::{ModelPreset, ModelTier, VisionMode};
 pub use sentiment::SentimentResult;
 pub use types::{

src/openhuman/inference/model_context.rs

@@ -0,0 +1,155 @@
+//! Known model context-window sizes for pre-inference budgeting.
+//!
+//! Provider `/models` responses may include `context_length` / `context_window`,
+//! but the agent harness must enforce limits **before** the first dispatch —
+//! otherwise long histories produce upstream `400 Bad Request` errors when usage
+//! metadata is not yet available.
+
+use crate::openhuman::config::{
+    MODEL_AGENTIC_V1, MODEL_CODING_V1, MODEL_REASONING_QUICK_V1, MODEL_REASONING_V1,
+};
+
+/// Conservative default for OpenHuman abstract tier models (tokens).
+const TIER_LARGE_CONTEXT: u64 = 200_000;
+const TIER_STANDARD_CONTEXT: u64 = 128_000;
+const TIER_LOCAL_CONTEXT: u64 = 8_192;
+
+/// How a pattern in [`MODEL_CONTEXT_PATTERNS`] is matched against a model id.
+#[derive(Copy, Clone)]
+enum PatternMatch {
+    /// Pattern must appear anywhere as a substring (after lowercasing).
+    Substring,
+    /// Pattern must appear as a full `-`/`_`/`/`/`:`-delimited segment.
+    /// Prevents false positives like `"solo1-7b"` matching the `"o1"` pattern
+    /// or `"proto3-chat"` matching the `"o3"` pattern.
+    Segment,
+}
+
+/// `(pattern, match mode, context window in tokens)` — first match wins.
+const MODEL_CONTEXT_PATTERNS: &[(&str, PatternMatch, u64)] = &[
+    ("claude-haiku-4.5", PatternMatch::Substring, 200_000),
+    ("claude-haiku-4", PatternMatch::Substring, 200_000),
+    ("claude-haiku", PatternMatch::Substring, 200_000),
+    ("claude-sonnet-4", PatternMatch::Substring, 200_000),
+    ("claude-opus-4", PatternMatch::Substring, 200_000),
+    ("claude-3-5-sonnet", PatternMatch::Substring, 200_000),
+    ("claude-3-5-haiku", PatternMatch::Substring, 200_000),
+    ("claude-3-opus", PatternMatch::Substring, 200_000),
+    ("gpt-4.1", PatternMatch::Substring, 1_047_576),
+    ("gpt-4o", PatternMatch::Substring, 128_000),
+    ("gpt-4-turbo", PatternMatch::Substring, 128_000),
+    ("gpt-4", PatternMatch::Substring, 128_000),
+    ("gpt-3.5", PatternMatch::Substring, 16_385),
+    // `o1`/`o3` are short and collide with substrings of unrelated model ids
+    // (e.g. `solo1-7b`, `proto3-chat`). Require a segment-boundary match.
+    ("o1", PatternMatch::Segment, 200_000),
+    ("o3", PatternMatch::Segment, 200_000),
+    ("deepseek", PatternMatch::Substring, 128_000),
+    ("gemma3", PatternMatch::Substring, 8_192),
+    ("gemma", PatternMatch::Substring, 8_192),
+    ("llama-3", PatternMatch::Substring, 128_000),
+    ("llama3", PatternMatch::Substring, 128_000),
+];
+
+fn matches_pattern(lower: &str, pattern: &str, mode: PatternMatch) -> bool {
+    match mode {
+        PatternMatch::Substring => lower.contains(pattern),
+        PatternMatch::Segment => lower
+            .split(|c: char| matches!(c, '/' | '-' | '_' | ':' | '.'))
+            .any(|seg| seg == pattern),
+    }
+}
+
+/// Resolve the context window (in tokens) for a model id or OpenHuman tier alias.
+///
+/// Returns `None` when the model is unknown — callers should skip pre-dispatch
+/// trimming rather than guess.
+pub fn context_window_for_model(model: &str) -> Option<u64> {
+    let normalized = model.trim();
+    if normalized.is_empty() {
+        return None;
+    }
+
+    if let Some(window) = tier_context_window(normalized) {
+        return Some(window);
+    }
+
+    let lower = normalized.to_ascii_lowercase();
+    for (pattern, mode, window) in MODEL_CONTEXT_PATTERNS {
+        if matches_pattern(&lower, pattern, *mode) {
+            tracing::debug!(
+                model = normalized,
+                pattern,
+                context_window = window,
+                "[model_context] matched known model pattern"
+            );
+            return Some(*window);
+        }
+    }
+
+    None
+}
+
+fn tier_context_window(model: &str) -> Option<u64> {
+    match model {
+        MODEL_REASONING_V1 | MODEL_AGENTIC_V1 | MODEL_CODING_V1 => Some(TIER_LARGE_CONTEXT),
+        MODEL_REASONING_QUICK_V1 | "summarization-v1" | "chat" => Some(TIER_STANDARD_CONTEXT),
+        m if m.starts_with("gemma") || m.contains(":1b") || m.contains("270m") => {
+            Some(TIER_LOCAL_CONTEXT)
+        }
+        _ => None,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn tier_aliases_resolve() {
+        assert_eq!(context_window_for_model("reasoning-v1"), Some(200_000));
+        assert_eq!(context_window_for_model("agentic-v1"), Some(200_000));
+        assert_eq!(
+            context_window_for_model("reasoning-quick-v1"),
+            Some(128_000)
+        );
+    }
+
+    #[test]
+    fn copilot_haiku_resolves_to_200k() {
+        assert_eq!(
+            context_window_for_model("github_copilot/claude-haiku-4.5"),
+            Some(200_000)
+        );
+    }
+
+    #[test]
+    fn unknown_model_returns_none() {
+        assert_eq!(context_window_for_model("totally-unknown-model-xyz"), None);
+    }
+
+    #[test]
+    fn empty_model_returns_none() {
+        assert_eq!(context_window_for_model("   "), None);
+    }
+
+    #[test]
+    fn o1_o3_segment_match_does_not_overmatch() {
+        // Real OpenAI o1/o3 model ids must still resolve.
+        assert_eq!(context_window_for_model("o1"), Some(200_000));
+        assert_eq!(context_window_for_model("o1-mini"), Some(200_000));
+        assert_eq!(context_window_for_model("o3-mini"), Some(200_000));
+        assert_eq!(context_window_for_model("openai/o1-preview"), Some(200_000));
+
+        // Names that merely *contain* the substring "o1" / "o3" must NOT
+        // inherit the 200K window (regression guard for PR #2100 review).
+        assert_eq!(context_window_for_model("solo1-7b"), None);
+        assert_eq!(context_window_for_model("proto3-chat"), None);
+        assert_eq!(
+            context_window_for_model("ollama/mistral-for-o1-benchmark"),
+            Some(200_000),
+            "`-o1-` segment should still match"
+        );
+        assert_eq!(context_window_for_model("octo3thing"), None);
+    }
+}

src/openhuman/inference/provider/ops.rs

@@ -624,6 +624,10 @@ pub fn create_routed_provider_with_options(
                 router::Route {
                     provider_name: INFERENCE_BACKEND_ID.to_string(),
                     model: r.model.clone(),
+                    context_window:
+                        crate::openhuman::inference::model_context::context_window_for_model(
+                            &r.model,
+                        ),
                 },
             )
         })
@@ -705,6 +709,10 @@ pub fn create_intelligent_routing_provider(
                     router::Route {
                         provider_name: INFERENCE_BACKEND_ID.to_string(),
                         model: r.model.clone(),
+                        context_window:
+                            crate::openhuman::inference::model_context::context_window_for_model(
+                                &r.model,
+                            ),
                     },
                 )
             })

src/openhuman/inference/provider/router.rs

@@ -23,6 +23,8 @@ fn openhuman_tier_to_hint(model: &str) -> Option<&'static str> {
 pub struct Route {
     pub provider_name: String,
     pub model: String,
+    /// Known context window for `model` when discoverable (tokens).
+    pub context_window: Option<u64>,
 }
 
 /// Multi-model router — routes requests to different provider+model combos

src/openhuman/inference/provider/router_test.rs

@@ -69,6 +69,7 @@ fn make_router(
                 Route {
                     provider_name: provider_name.to_string(),
                     model: model.to_string(),
+                    context_window: None,
                 },
             )
         })