fix(cron): halt Sentry flood when a local LLM provider is offline (#4408) (#4415)

src/core/observability.rs

@@ -1044,6 +1044,19 @@ pub fn is_api_key_unset_message(text: &str) -> bool {
         || lower.contains("no api key supplied")
 }
 
+/// Does this error text describe a **local** LLM provider (loopback host, e.g.
+/// LM Studio / Ollama / llama.cpp on `127.0.0.1:<port>` or `localhost:<port>`)
+/// refusing the connection because its server isn't running?
+///
+/// Single-source wrapper over [`is_loopback_unavailable`] so callers outside
+/// this module (the cron scheduler's retry-halt guard) key off the exact same
+/// matcher the [`expected_error_kind`] classifier uses — the phrasing cannot
+/// drift between the source demotion and the cron suppression. Lowercases the
+/// input to match the classifier's internal contract (TAURI-RUST-12K).
+pub fn is_local_provider_unreachable_message(text: &str) -> bool {
+    is_loopback_unavailable(&text.to_ascii_lowercase())
+}
+
 /// Detect the in-process-core boot-window shape: a sibling component
 /// (frontend RPC relay, agent-integrations / composio HTTP clients) tried to
 /// reach the embedded core's `127.0.0.1:<port>` listener before it finished
@@ -1065,6 +1078,19 @@ pub fn is_api_key_unset_message(text: &str) -> bool {
 ///    swallowing higher-level wrappers that merely mention "connection
 ///    refused" in prose.
 ///
+/// 3. **Locale-independent fallback**: on non-English OS locales the kernel
+///    renders the `ECONNREFUSED` / `WSAECONNREFUSED` text translated (e.g. a
+///    zh-CN Windows host emits `由于目标计算机积极拒绝，无法连接。 (os error
+///    10061)`), so the English `connection refused` prefix in (2) is absent
+///    and a genuine loopback-refused body would leak past this matcher into
+///    the broad [`is_network_unreachable_message`] bucket (TAURI-RUST-12K).
+///    Recover it by pairing the reqwest/hyper-stable `tcp connect error`
+///    marker with the connect-refused errno alone. Scoped to the three
+///    connect-refused errnos only — the distinct timeout errnos (`60` / `110`
+///    / `10060`, `WSAETIMEDOUT`) are NOT matched, so a loopback *timeout*
+///    still classifies as its own shape rather than being mislabelled
+///    "refused".
+///
 /// Drops OPENHUMAN-TAURI-R5 (~2.5k events, `integrations.get` emit site)
 /// and OPENHUMAN-TAURI-R6 (~2.5k events, the `rpc.invoke_method` re-wrap of
 /// the same trace). Both share `trace_id=6ebf5b62748d5144e541e2cddeabbbd0`
@@ -1085,9 +1111,21 @@ fn is_loopback_unavailable(lower: &str) -> bool {
     if !has_loopback_host {
         return false;
     }
-    lower.contains("connection refused (os error 61)")
+    // (2) English errno-prefixed form.
+    if lower.contains("connection refused (os error 61)")
         || lower.contains("connection refused (os error 111)")
         || lower.contains("connection refused (os error 10061)")
+    {
+        return true;
+    }
+    // (3) Locale-independent fallback: the OS-refused text may be translated,
+    // leaving only the errno. Require the transport-layer `tcp connect error`
+    // marker so a non-connect loopback failure cannot match, and pin to the
+    // connect-refused errnos (not the timeout errnos 60 / 110 / 10060).
+    lower.contains("tcp connect error")
+        && (lower.contains("(os error 61)")
+            || lower.contains("(os error 111)")
+            || lower.contains("(os error 10061)"))
 }
 
 /// Detect Ollama embed call sites that surface a user-config rejection from
@@ -7120,6 +7158,56 @@ mod tests {
         );
     }
 
+    /// Verbatim body from TAURI-RUST-12K (2802 events / 29 users): a cron
+    /// agent job hits a local LM Studio server (`localhost:1234`) that isn't
+    /// running, on a zh-CN Windows host — so the `WSAECONNREFUSED` text is
+    /// localized and the English "connection refused" prefix is absent, leaving
+    /// only `(os error 10061)` behind the transport-stable `tcp connect error`
+    /// marker. The locale-independent arm must still route it to the loopback
+    /// bucket rather than leaking to the broad `NetworkUnreachable`.
+    #[test]
+    fn classifies_localized_loopback_connect_refused_as_loopback_unavailable() {
+        let raw = "error sending request for url \
+                   (http://localhost:1234/v1/chat/completions): client error (Connect): \
+                   tcp connect error: 由于目标计算机积极拒绝，无法连接。 (os error 10061)";
+        assert_eq!(
+            expected_error_kind(raw),
+            Some(ExpectedErrorKind::LoopbackUnavailable),
+            "localized WSAECONNREFUSED loopback body must classify as LoopbackUnavailable"
+        );
+    }
+
+    #[test]
+    fn does_not_classify_loopback_timeout_as_loopback() {
+        // A loopback *timeout* (WSAETIMEDOUT os error 10060, not the
+        // connect-refused 10061) shares the `tcp connect error` marker but is
+        // a distinct failure class — it must NOT be swallowed by the
+        // connect-refused loopback arm.
+        let raw = "error sending request for url \
+                   (http://localhost:1234/v1/chat/completions): \
+                   tcp connect error: connection timed out (os error 10060)";
+        assert!(
+            !matches!(
+                expected_error_kind(raw),
+                Some(ExpectedErrorKind::LoopbackUnavailable)
+            ),
+            "loopback timeout must not classify as LoopbackUnavailable"
+        );
+    }
+
+    #[test]
+    fn is_local_provider_unreachable_message_wraps_loopback_matcher() {
+        let localized = "error sending request for url \
+                         (http://localhost:1234/v1/chat/completions): client error (Connect): \
+                         tcp connect error: 由于目标计算机积极拒绝，无法连接。 (os error 10061)";
+        assert!(is_local_provider_unreachable_message(localized));
+        // Remote refused must not match (loopback-only, keeps real outages visible).
+        assert!(!is_local_provider_unreachable_message(
+            "error sending request for url (https://api.tinyhumans.ai/x) \
+             → tcp connect error → Connection refused (os error 61)"
+        ));
+    }
+
     #[test]
     fn loopback_matcher_requires_both_host_and_errno_anchors() {
         // Defense against the matcher being too eager: bodies that satisfy

src/openhuman/cron/scheduler.rs

@@ -432,6 +432,41 @@ fn is_api_key_unset_failure(
     crate::core::observability::is_api_key_unset_message(signal)
 }
 
+/// TAURI-RUST-12K — a cron **agent** job pinned to a **local** LLM provider
+/// (LM Studio / Ollama / llama.cpp on `localhost:<port>`) fails at the TCP
+/// layer with "connection refused" because the user's local server isn't
+/// running. This is a genuinely unpreventable user-environment state: the app
+/// has no lever to start a user's local model server, and retrying across the
+/// backoff loop cannot bring the port up within one cron cycle.
+///
+/// The provider / agent emit sites already demote this via
+/// `report_error_or_expected` (the `expected_error_kind` classifier routes it
+/// to `LoopbackUnavailable`), so it never reaches Sentry there. But the bare
+/// cron `report_error` below bypasses that demotion and re-emitted the
+/// `failure=retries_exhausted` capture on every cron cycle — 2802 events / 29
+/// users. So we halt on the first occurrence and skip the report, mirroring
+/// the source demotion and the sibling billing / api-key guards
+/// (TAURI-RUST-514 / -BMW / -HCK).
+///
+/// Delegates to the single-source matcher
+/// [`crate::core::observability::is_local_provider_unreachable_message`] so
+/// the wording cannot drift from the classifier emit site. Narrow by design:
+/// only **loopback** connection-refused matches, so a transient *remote*
+/// provider / backend network error still retries and still reports. Routes on
+/// `last_agent_error` first (the raw anyhow chain carrying the wire message),
+/// falling back to `last_output`. Restricted to `JobType::Agent`.
+fn is_local_provider_unreachable_failure(
+    job_type: &JobType,
+    last_agent_error: Option<&str>,
+    last_output: &str,
+) -> bool {
+    if !matches!(job_type, JobType::Agent) {
+        return false;
+    }
+    let signal = last_agent_error.unwrap_or(last_output);
+    crate::core::observability::is_local_provider_unreachable_message(signal)
+}
+
 async fn execute_job_with_retry(
     config: &Config,
     security: &SecurityPolicy,
@@ -445,6 +480,7 @@ async fn execute_job_with_retry(
     let mut credits_exhausted = false;
     let mut budget_exhausted = false;
     let mut key_unset = false;
+    let mut local_unreachable = false;
 
     for attempt in 0..=retries {
         let (success, output, agent_error) = match job.job_type {
@@ -553,6 +589,30 @@ async fn execute_job_with_retry(
             break;
         }
 
+        if is_local_provider_unreachable_failure(
+            &job.job_type,
+            last_agent_error.as_deref(),
+            last_output.as_str(),
+        ) {
+            // Halt on the first occurrence — a local LLM provider refusing the
+            // loopback connection (LM Studio / Ollama not running) cannot
+            // recover across the backoff loop, and the provider/agent emit
+            // sites already demoted it from Sentry (`LoopbackUnavailable`).
+            // The bare cron `report_error` below bypasses that demotion, so
+            // suppressing here keeps the residual off Sentry at source
+            // (TAURI-RUST-12K). The failure stays visible via the run history
+            // + cron alert. See `is_local_provider_unreachable_failure`.
+            // Metadata-only log (no raw provider body — see CLAUDE.md).
+            log::debug!(
+                "[cron] action=halt_on_local_provider_unreachable job_id={} attempt={} retries={}",
+                job.id.as_str(),
+                attempt,
+                retries
+            );
+            local_unreachable = true;
+            break;
+        }
+
         if attempt < retries {
             let jitter_ms = u64::from(Utc::now().timestamp_subsec_millis() % 250);
             time::sleep(Duration::from_millis(backoff_ms + jitter_ms)).await;
@@ -564,9 +624,18 @@ async fn execute_job_with_retry(
     // loop and skip the retries-exhausted report, independent of the tag-gated
     // before_send filters that the cron re-report does not match. Covers BYO
     // 402 out-of-credit + managed-backend 400 out-of-budget (TAURI-RUST-514 /
-    // -BMW) and a configured provider with no API key (TAURI-RUST-HCK).
+    // -BMW) and a configured provider with no API key (TAURI-RUST-HCK). The
+    // `session_expired` (TAURI-RUST-N) and `local_unreachable` (a local LLM
+    // server refusing the loopback connection, TAURI-RUST-12K) halts are the
+    // same shape — suppress the bypassing bare report — but carry no
+    // user-config remediation surface, so they gate the report directly rather
+    // than routing through `permanent_config_halt`'s UserErrorCenter swap.
     let permanent_config_halt = credits_exhausted || budget_exhausted || key_unset;
-    if matches!(job.job_type, JobType::Agent) && !session_expired && !permanent_config_halt {
+    if matches!(job.job_type, JobType::Agent)
+        && !session_expired
+        && !local_unreachable
+        && !permanent_config_halt
+    {
         let report_message = last_agent_error
             .as_deref()
             .unwrap_or_else(|| last_output.as_str());

src/openhuman/cron/scheduler_tests.rs

@@ -705,6 +705,79 @@ fn is_api_key_unset_failure_does_not_halt_shell_jobs() {
     assert!(!is_api_key_unset_failure(&JobType::Shell, Some(wire), wire));
 }
 
+// TAURI-RUST-12K — a cron agent job pinned to a local LLM provider (LM Studio
+// on localhost:1234) fails with a loopback connection-refused because the
+// user's server isn't running. `is_local_provider_unreachable_failure` must
+// consult the shared loopback matcher so the retry loop halts on the first
+// occurrence (retries can't bring the port up) instead of re-emitting the
+// `failure=retries_exhausted` bare `report_error` the classifier already
+// demotes everywhere else.
+#[test]
+fn is_local_provider_unreachable_failure_matches_localized_loopback_in_agent_error() {
+    // Verbatim from the Sentry event: zh-CN Windows host, localized
+    // WSAECONNREFUSED text, only the errno + `tcp connect error` survive.
+    let wire = "error sending request for url \
+                (http://localhost:1234/v1/chat/completions): client error (Connect): \
+                tcp connect error: 由于目标计算机积极拒绝，无法连接。 (os error 10061)";
+    assert!(
+        is_local_provider_unreachable_failure(
+            &JobType::Agent,
+            Some(wire),
+            AGENT_JOB_USER_FAILURE_MESSAGE
+        ),
+        "raw agent error carrying the localized loopback connect-refused must trip the halt"
+    );
+}
+
+// Defense-in-depth: classify even if a future path surfaces the raw error in
+// `last_output` rather than `last_agent_error`.
+#[test]
+fn is_local_provider_unreachable_failure_matches_when_only_output_carries_signal() {
+    let wire = "error sending request for url (http://localhost:1234/v1/chat/completions) \
+                → tcp connect error → Connection refused (os error 10061)";
+    assert!(is_local_provider_unreachable_failure(
+        &JobType::Agent,
+        None,
+        wire
+    ));
+}
+
+// Negative guard: a transient REMOTE provider / backend network error must NOT
+// halt — it may recover on retry and stays actionable in Sentry. Narrowing to
+// loopback is what keeps this guard from blinding real outages.
+#[test]
+fn is_local_provider_unreachable_failure_does_not_match_remote_network_errors() {
+    assert!(!is_local_provider_unreachable_failure(
+        &JobType::Agent,
+        Some(AGENT_JOB_USER_FAILURE_MESSAGE),
+        AGENT_JOB_USER_FAILURE_MESSAGE,
+    ));
+    let remote = "error sending request for url (https://api.tinyhumans.ai/v1/chat/completions) \
+                  → tcp connect error → Connection refused (os error 61)";
+    assert!(
+        !is_local_provider_unreachable_failure(&JobType::Agent, Some(remote), ""),
+        "a remote-host connect-refused must retry + report, not halt as loopback"
+    );
+}
+
+// Scope guard: shell jobs that echo a loopback-refused string keep their retry
+// semantics — only agent jobs route through the inference layer.
+#[test]
+fn is_local_provider_unreachable_failure_does_not_halt_shell_jobs() {
+    let wire = "error sending request for url (http://localhost:1234/v1/chat/completions) \
+                → tcp connect error → Connection refused (os error 10061)";
+    assert!(!is_local_provider_unreachable_failure(
+        &JobType::Shell,
+        None,
+        wire
+    ));
+    assert!(!is_local_provider_unreachable_failure(
+        &JobType::Shell,
+        Some(wire),
+        wire
+    ));
+}
+
 #[tokio::test]
 async fn run_agent_job_returns_error_without_provider_key() {
     let tmp = TempDir::new().unwrap();
@@ -1739,3 +1812,68 @@ fn publish_cron_user_error_broadcasts_metadata_only_for_each_kind() {
         assert!(ev.request_id.is_empty());
     }
 }
+
+// TAURI-RUST-12K (end-to-end) — the predicate tests above key on hand-written
+// wire strings; this test proves the REAL provider-generated error reaches and
+// trips the guard. A cron agent job is routed to a keyless local provider
+// (`AuthStyle::None`, LM Studio shape) whose server is offline: the chat
+// workload skips the credential guard, attempts the loopback HTTP connect, and
+// the OS refuses it. The resulting `last_agent_error` (the aggregated provider
+// fallback chain carrying `…localhost:1234… tcp connect error … Connection
+// refused (os error N)`) must classify as local-provider-unreachable so the
+// cron loop halts and skips the bypassing `failure=retries_exhausted` report.
+#[tokio::test]
+async fn cron_agent_job_local_provider_offline_trips_halt_guard() {
+    use crate::openhuman::config::schema::cloud_providers::{AuthStyle, CloudProviderCreds};
+    let tmp = TempDir::new().unwrap();
+    let mut config = test_config(&tmp).await;
+    config.reliability.scheduler_retries = 5;
+    config.reliability.provider_backoff_ms = 1;
+    // Keyless local provider (`AuthStyle::None` → no credential requirement, so
+    // the request proceeds to the HTTP connect). `chat_provider` routes the
+    // chat workload to it; the slug resolves to LM Studio's default endpoint.
+    config.cloud_providers = vec![CloudProviderCreds {
+        id: "lmstudio-offline".into(),
+        slug: "lmstudio".into(),
+        label: "LM Studio".into(),
+        endpoint: "http://127.0.0.1:1".into(),
+        auth_style: AuthStyle::None,
+        ..Default::default()
+    }];
+    config.default_model = Some("lmstudio:local-model".into());
+    config.chat_provider = Some("lmstudio:local-model".into());
+    let security = SecurityPolicy::from_config(
+        &config.autonomy,
+        &config.workspace_dir,
+        &config.workspace_dir,
+    );
+    let mut job = test_job("");
+    job.job_type = JobType::Agent;
+    job.prompt = Some("Say hello".into());
+
+    // Primary (deterministic): the real provider-generated failure trips the
+    // guard — the link the string-based predicate tests cannot prove.
+    let (success, output, raw) = run_agent_job(&config, &job).await;
+    assert!(
+        !success,
+        "a cron agent job against an offline local provider must fail"
+    );
+    assert!(
+        is_local_provider_unreachable_failure(&JobType::Agent, raw.as_deref(), &output),
+        "provider-generated loopback connect-refused must trip the halt guard; got raw={raw:?}"
+    );
+
+    // Secondary (coarse): the full retry loop halts on the first occurrence
+    // rather than exhausting all 5 retries. Not halting would add the cron
+    // backoff sleeps (200ms floor, doubling → >6s total) on top of five extra
+    // agent runs; halting skips every cron-level sleep. A generous bound keeps
+    // this robust to agent-build jitter while still catching a regressed halt.
+    let start = std::time::Instant::now();
+    let (loop_success, _out) = execute_job_with_retry(&config, &security, &job).await;
+    let elapsed = start.elapsed();
+    assert!(!loop_success);
+    assert!(
+        elapsed < std::time::Duration::from_secs(5),
+        "loop must halt on the first loopback failure, not burn 5 retries with backoff (elapsed {elapsed:?})"
+    );
+}