From afb33ac19ca7f0014f703ef86c4778aef1bdbaf9 Mon Sep 17 00:00:00 2001 From: oxoxDev <164490987+oxoxDev@users.noreply.github.com> Date: Fri, 3 Jul 2026 23:46:41 +0530 Subject: [PATCH] fix(cron): halt Sentry flood when a local LLM provider is offline (#4408) (#4415) --- src/core/observability.rs | 90 ++++++++++++++++- src/openhuman/cron/scheduler.rs | 73 +++++++++++++- src/openhuman/cron/scheduler_tests.rs | 138 ++++++++++++++++++++++++++ 3 files changed, 298 insertions(+), 3 deletions(-) diff --git a/src/core/observability.rs b/src/core/observability.rs index de9a617d0..816d37e30 100644 --- a/src/core/observability.rs +++ b/src/core/observability.rs @@ -1044,6 +1044,19 @@ pub fn is_api_key_unset_message(text: &str) -> bool { || lower.contains("no api key supplied") } +/// Does this error text describe a **local** LLM provider (loopback host, e.g. +/// LM Studio / Ollama / llama.cpp on `127.0.0.1:` or `localhost:`) +/// refusing the connection because its server isn't running? +/// +/// Single-source wrapper over [`is_loopback_unavailable`] so callers outside +/// this module (the cron scheduler's retry-halt guard) key off the exact same +/// matcher the [`expected_error_kind`] classifier uses — the phrasing cannot +/// drift between the source demotion and the cron suppression. Lowercases the +/// input to match the classifier's internal contract (TAURI-RUST-12K). +pub fn is_local_provider_unreachable_message(text: &str) -> bool { + is_loopback_unavailable(&text.to_ascii_lowercase()) +} + /// Detect the in-process-core boot-window shape: a sibling component /// (frontend RPC relay, agent-integrations / composio HTTP clients) tried to /// reach the embedded core's `127.0.0.1:` listener before it finished @@ -1065,6 +1078,19 @@ pub fn is_api_key_unset_message(text: &str) -> bool { /// swallowing higher-level wrappers that merely mention "connection /// refused" in prose. /// +/// 3. **Locale-independent fallback**: on non-English OS locales the kernel +/// renders the `ECONNREFUSED` / `WSAECONNREFUSED` text translated (e.g. a +/// zh-CN Windows host emits `由于目标计算机积极拒绝，无法连接。 (os error +/// 10061)`), so the English `connection refused` prefix in (2) is absent +/// and a genuine loopback-refused body would leak past this matcher into +/// the broad [`is_network_unreachable_message`] bucket (TAURI-RUST-12K). +/// Recover it by pairing the reqwest/hyper-stable `tcp connect error` +/// marker with the connect-refused errno alone. Scoped to the three +/// connect-refused errnos only — the distinct timeout errnos (`60` / `110` +/// / `10060`, `WSAETIMEDOUT`) are NOT matched, so a loopback *timeout* +/// still classifies as its own shape rather than being mislabelled +/// "refused". +/// /// Drops OPENHUMAN-TAURI-R5 (~2.5k events, `integrations.get` emit site) /// and OPENHUMAN-TAURI-R6 (~2.5k events, the `rpc.invoke_method` re-wrap of /// the same trace). Both share `trace_id=6ebf5b62748d5144e541e2cddeabbbd0` @@ -1085,9 +1111,21 @@ fn is_loopback_unavailable(lower: &str) -> bool { if !has_loopback_host { return false; } - lower.contains("connection refused (os error 61)") + // (2) English errno-prefixed form. + if lower.contains("connection refused (os error 61)") || lower.contains("connection refused (os error 111)") || lower.contains("connection refused (os error 10061)") + { + return true; + } + // (3) Locale-independent fallback: the OS-refused text may be translated, + // leaving only the errno. Require the transport-layer `tcp connect error` + // marker so a non-connect loopback failure cannot match, and pin to the + // connect-refused errnos (not the timeout errnos 60 / 110 / 10060). + lower.contains("tcp connect error") + && (lower.contains("(os error 61)") + || lower.contains("(os error 111)") + || lower.contains("(os error 10061)")) } /// Detect Ollama embed call sites that surface a user-config rejection from @@ -7120,6 +7158,56 @@ mod tests { ); } + /// Verbatim body from TAURI-RUST-12K (2802 events / 29 users): a cron + /// agent job hits a local LM Studio server (`localhost:1234`) that isn't + /// running, on a zh-CN Windows host — so the `WSAECONNREFUSED` text is + /// localized and the English "connection refused" prefix is absent, leaving + /// only `(os error 10061)` behind the transport-stable `tcp connect error` + /// marker. The locale-independent arm must still route it to the loopback + /// bucket rather than leaking to the broad `NetworkUnreachable`. + #[test] + fn classifies_localized_loopback_connect_refused_as_loopback_unavailable() { + let raw = "error sending request for url \ + (http://localhost:1234/v1/chat/completions): client error (Connect): \ + tcp connect error: 由于目标计算机积极拒绝，无法连接。 (os error 10061)"; + assert_eq!( + expected_error_kind(raw), + Some(ExpectedErrorKind::LoopbackUnavailable), + "localized WSAECONNREFUSED loopback body must classify as LoopbackUnavailable" + ); + } + + #[test] + fn does_not_classify_loopback_timeout_as_loopback() { + // A loopback *timeout* (WSAETIMEDOUT os error 10060, not the + // connect-refused 10061) shares the `tcp connect error` marker but is + // a distinct failure class — it must NOT be swallowed by the + // connect-refused loopback arm. + let raw = "error sending request for url \ + (http://localhost:1234/v1/chat/completions): \ + tcp connect error: connection timed out (os error 10060)"; + assert!( + !matches!( + expected_error_kind(raw), + Some(ExpectedErrorKind::LoopbackUnavailable) + ), + "loopback timeout must not classify as LoopbackUnavailable" + ); + } + + #[test] + fn is_local_provider_unreachable_message_wraps_loopback_matcher() { + let localized = "error sending request for url \ + (http://localhost:1234/v1/chat/completions): client error (Connect): \ + tcp connect error: 由于目标计算机积极拒绝，无法连接。 (os error 10061)"; + assert!(is_local_provider_unreachable_message(localized)); + // Remote refused must not match (loopback-only, keeps real outages visible). + assert!(!is_local_provider_unreachable_message( + "error sending request for url (https://api.tinyhumans.ai/x) \ + → tcp connect error → Connection refused (os error 61)" + )); + } + #[test] fn loopback_matcher_requires_both_host_and_errno_anchors() { // Defense against the matcher being too eager: bodies that satisfy diff --git a/src/openhuman/cron/scheduler.rs b/src/openhuman/cron/scheduler.rs index cecf1ff85..3e00229cd 100644 --- a/src/openhuman/cron/scheduler.rs +++ b/src/openhuman/cron/scheduler.rs @@ -432,6 +432,41 @@ fn is_api_key_unset_failure( crate::core::observability::is_api_key_unset_message(signal) } +/// TAURI-RUST-12K — a cron **agent** job pinned to a **local** LLM provider +/// (LM Studio / Ollama / llama.cpp on `localhost:`) fails at the TCP +/// layer with "connection refused" because the user's local server isn't +/// running. This is a genuinely unpreventable user-environment state: the app +/// has no lever to start a user's local model server, and retrying across the +/// backoff loop cannot bring the port up within one cron cycle. +/// +/// The provider / agent emit sites already demote this via +/// `report_error_or_expected` (the `expected_error_kind` classifier routes it +/// to `LoopbackUnavailable`), so it never reaches Sentry there. But the bare +/// cron `report_error` below bypasses that demotion and re-emitted the +/// `failure=retries_exhausted` capture on every cron cycle — 2802 events / 29 +/// users. So we halt on the first occurrence and skip the report, mirroring +/// the source demotion and the sibling billing / api-key guards +/// (TAURI-RUST-514 / -BMW / -HCK). +/// +/// Delegates to the single-source matcher +/// [`crate::core::observability::is_local_provider_unreachable_message`] so +/// the wording cannot drift from the classifier emit site. Narrow by design: +/// only **loopback** connection-refused matches, so a transient *remote* +/// provider / backend network error still retries and still reports. Routes on +/// `last_agent_error` first (the raw anyhow chain carrying the wire message), +/// falling back to `last_output`. Restricted to `JobType::Agent`. +fn is_local_provider_unreachable_failure( + job_type: &JobType, + last_agent_error: Option<&str>, + last_output: &str, +) -> bool { + if !matches!(job_type, JobType::Agent) { + return false; + } + let signal = last_agent_error.unwrap_or(last_output); + crate::core::observability::is_local_provider_unreachable_message(signal) +} + async fn execute_job_with_retry( config: &Config, security: &SecurityPolicy, @@ -445,6 +480,7 @@ async fn execute_job_with_retry( let mut credits_exhausted = false; let mut budget_exhausted = false; let mut key_unset = false; + let mut local_unreachable = false; for attempt in 0..=retries { let (success, output, agent_error) = match job.job_type { @@ -553,6 +589,30 @@ async fn execute_job_with_retry( break; } + if is_local_provider_unreachable_failure( + &job.job_type, + last_agent_error.as_deref(), + last_output.as_str(), + ) { + // Halt on the first occurrence — a local LLM provider refusing the + // loopback connection (LM Studio / Ollama not running) cannot + // recover across the backoff loop, and the provider/agent emit + // sites already demoted it from Sentry (`LoopbackUnavailable`). + // The bare cron `report_error` below bypasses that demotion, so + // suppressing here keeps the residual off Sentry at source + // (TAURI-RUST-12K). The failure stays visible via the run history + // + cron alert. See `is_local_provider_unreachable_failure`. + // Metadata-only log (no raw provider body — see CLAUDE.md). + log::debug!( + "[cron] action=halt_on_local_provider_unreachable job_id={} attempt={} retries={}", + job.id.as_str(), + attempt, + retries + ); + local_unreachable = true; + break; + } + if attempt < retries { let jitter_ms = u64::from(Utc::now().timestamp_subsec_millis() % 250); time::sleep(Duration::from_millis(backoff_ms + jitter_ms)).await; @@ -564,9 +624,18 @@ async fn execute_job_with_retry( // loop and skip the retries-exhausted report, independent of the tag-gated // before_send filters that the cron re-report does not match. Covers BYO // 402 out-of-credit + managed-backend 400 out-of-budget (TAURI-RUST-514 / - // -BMW) and a configured provider with no API key (TAURI-RUST-HCK). + // -BMW) and a configured provider with no API key (TAURI-RUST-HCK). The + // `session_expired` (TAURI-RUST-N) and `local_unreachable` (a local LLM + // server refusing the loopback connection, TAURI-RUST-12K) halts are the + // same shape — suppress the bypassing bare report — but carry no + // user-config remediation surface, so they gate the report directly rather + // than routing through `permanent_config_halt`'s UserErrorCenter swap. let permanent_config_halt = credits_exhausted || budget_exhausted || key_unset; - if matches!(job.job_type, JobType::Agent) && !session_expired && !permanent_config_halt { + if matches!(job.job_type, JobType::Agent) + && !session_expired + && !local_unreachable + && !permanent_config_halt + { let report_message = last_agent_error .as_deref() .unwrap_or_else(|| last_output.as_str()); diff --git a/src/openhuman/cron/scheduler_tests.rs b/src/openhuman/cron/scheduler_tests.rs index 98faf9c11..59daab260 100644 --- a/src/openhuman/cron/scheduler_tests.rs +++ b/src/openhuman/cron/scheduler_tests.rs @@ -705,6 +705,79 @@ fn is_api_key_unset_failure_does_not_halt_shell_jobs() { assert!(!is_api_key_unset_failure(&JobType::Shell, Some(wire), wire)); } +// TAURI-RUST-12K — a cron agent job pinned to a local LLM provider (LM Studio +// on localhost:1234) fails with a loopback connection-refused because the +// user's server isn't running. `is_local_provider_unreachable_failure` must +// consult the shared loopback matcher so the retry loop halts on the first +// occurrence (retries can't bring the port up) instead of re-emitting the +// `failure=retries_exhausted` bare `report_error` the classifier already +// demotes everywhere else. +#[test] +fn is_local_provider_unreachable_failure_matches_localized_loopback_in_agent_error() { + // Verbatim from the Sentry event: zh-CN Windows host, localized + // WSAECONNREFUSED text, only the errno + `tcp connect error` survive. + let wire = "error sending request for url \ + (http://localhost:1234/v1/chat/completions): client error (Connect): \ + tcp connect error: 由于目标计算机积极拒绝，无法连接。 (os error 10061)"; + assert!( + is_local_provider_unreachable_failure( + &JobType::Agent, + Some(wire), + AGENT_JOB_USER_FAILURE_MESSAGE + ), + "raw agent error carrying the localized loopback connect-refused must trip the halt" + ); +} + +// Defense-in-depth: classify even if a future path surfaces the raw error in +// `last_output` rather than `last_agent_error`. +#[test] +fn is_local_provider_unreachable_failure_matches_when_only_output_carries_signal() { + let wire = "error sending request for url (http://localhost:1234/v1/chat/completions) \ + → tcp connect error → Connection refused (os error 10061)"; + assert!(is_local_provider_unreachable_failure( + &JobType::Agent, + None, + wire + )); +} + +// Negative guard: a transient REMOTE provider / backend network error must NOT +// halt — it may recover on retry and stays actionable in Sentry. Narrowing to +// loopback is what keeps this guard from blinding real outages. +#[test] +fn is_local_provider_unreachable_failure_does_not_match_remote_network_errors() { + assert!(!is_local_provider_unreachable_failure( + &JobType::Agent, + Some(AGENT_JOB_USER_FAILURE_MESSAGE), + AGENT_JOB_USER_FAILURE_MESSAGE, + )); + let remote = "error sending request for url (https://api.tinyhumans.ai/v1/chat/completions) \ + → tcp connect error → Connection refused (os error 61)"; + assert!( + !is_local_provider_unreachable_failure(&JobType::Agent, Some(remote), ""), + "a remote-host connect-refused must retry + report, not halt as loopback" + ); +} + +// Scope guard: shell jobs that echo a loopback-refused string keep their retry +// semantics — only agent jobs route through the inference layer. +#[test] +fn is_local_provider_unreachable_failure_does_not_halt_shell_jobs() { + let wire = "error sending request for url (http://localhost:1234/v1/chat/completions) \ + → tcp connect error → Connection refused (os error 10061)"; + assert!(!is_local_provider_unreachable_failure( + &JobType::Shell, + None, + wire + )); + assert!(!is_local_provider_unreachable_failure( + &JobType::Shell, + Some(wire), + wire + )); +} + #[tokio::test] async fn run_agent_job_returns_error_without_provider_key() { let tmp = TempDir::new().unwrap(); @@ -1739,3 +1812,68 @@ fn publish_cron_user_error_broadcasts_metadata_only_for_each_kind() { assert!(ev.request_id.is_empty()); } } + +// TAURI-RUST-12K (end-to-end) — the predicate tests above key on hand-written +// wire strings; this test proves the REAL provider-generated error reaches and +// trips the guard. A cron agent job is routed to a keyless local provider +// (`AuthStyle::None`, LM Studio shape) whose server is offline: the chat +// workload skips the credential guard, attempts the loopback HTTP connect, and +// the OS refuses it. The resulting `last_agent_error` (the aggregated provider +// fallback chain carrying `…localhost:1234… tcp connect error … Connection +// refused (os error N)`) must classify as local-provider-unreachable so the +// cron loop halts and skips the bypassing `failure=retries_exhausted` report. +#[tokio::test] +async fn cron_agent_job_local_provider_offline_trips_halt_guard() { + use crate::openhuman::config::schema::cloud_providers::{AuthStyle, CloudProviderCreds}; + let tmp = TempDir::new().unwrap(); + let mut config = test_config(&tmp).await; + config.reliability.scheduler_retries = 5; + config.reliability.provider_backoff_ms = 1; + // Keyless local provider (`AuthStyle::None` → no credential requirement, so + // the request proceeds to the HTTP connect). `chat_provider` routes the + // chat workload to it; the slug resolves to LM Studio's default endpoint. + config.cloud_providers = vec![CloudProviderCreds { + id: "lmstudio-offline".into(), + slug: "lmstudio".into(), + label: "LM Studio".into(), + endpoint: "http://127.0.0.1:1".into(), + auth_style: AuthStyle::None, + ..Default::default() + }]; + config.default_model = Some("lmstudio:local-model".into()); + config.chat_provider = Some("lmstudio:local-model".into()); + let security = SecurityPolicy::from_config( + &config.autonomy, + &config.workspace_dir, + &config.workspace_dir, + ); + let mut job = test_job(""); + job.job_type = JobType::Agent; + job.prompt = Some("Say hello".into()); + + // Primary (deterministic): the real provider-generated failure trips the + // guard — the link the string-based predicate tests cannot prove. + let (success, output, raw) = run_agent_job(&config, &job).await; + assert!( + !success, + "a cron agent job against an offline local provider must fail" + ); + assert!( + is_local_provider_unreachable_failure(&JobType::Agent, raw.as_deref(), &output), + "provider-generated loopback connect-refused must trip the halt guard; got raw={raw:?}" + ); + + // Secondary (coarse): the full retry loop halts on the first occurrence + // rather than exhausting all 5 retries. Not halting would add the cron + // backoff sleeps (200ms floor, doubling → >6s total) on top of five extra + // agent runs; halting skips every cron-level sleep. A generous bound keeps + // this robust to agent-build jitter while still catching a regressed halt. + let start = std::time::Instant::now(); + let (loop_success, _out) = execute_job_with_retry(&config, &security, &job).await; + let elapsed = start.elapsed(); + assert!(!loop_success); + assert!( + elapsed < std::time::Duration::from_secs(5), + "loop must halt on the first loopback failure, not burn 5 retries with backoff (elapsed {elapsed:?})" + ); +}