diff --git a/src/core/observability.rs b/src/core/observability.rs index 43b349b68..37b7a2439 100644 --- a/src/core/observability.rs +++ b/src/core/observability.rs @@ -420,6 +420,20 @@ pub fn expected_error_kind(message: &str) -> Option { if crate::openhuman::inference::provider::is_openai_oauth_session_expired_message(message) { return Some(ExpectedErrorKind::ProviderUserState); } + // TAURI-RUST-5MV — ollama.com hosted-inference 500 (`Internal Server Error + // (ref: )`) for `*:cloud` models. The provider HTTP layer + // (`native_chat` / `streaming_chat` / `api_error`) already demotes its own + // per-attempt event and re-raises the actionable + // "Ollama cloud is temporarily unavailable …" string; this catches the + // re-report at the agent / RPC boundary (`provider_chat` → + // `report_error_or_expected`, the `domain=agent` half of the flood). Routed + // to `TransientUpstreamHttp` — it is an external upstream 5xx the + // reliable-provider layer retries + falls back over, with no client lever. + // Delegates to the single-source provider matcher so the phrasing can't + // drift. Distinct anchor from the matchers above, so it shadows nothing. + if crate::openhuman::inference::provider::is_ollama_cloud_internal_500_message(message) { + return Some(ExpectedErrorKind::TransientUpstreamHttp); + } if is_backend_user_error_message(&lower) { return Some(ExpectedErrorKind::BackendUserError); } @@ -2630,6 +2644,50 @@ pub fn is_quota_exhausted_event(event: &sentry::protocol::Event<'_>) -> bool { }) } +/// Whether a raw error / message string is an Ollama **Cloud** hosted-inference +/// `500` (`Internal Server Error (ref: )`). Matches either the raw emit +/// shape (`ollama API error (500 …): {"error":"Internal Server Error (ref: …)"}`) +/// or the actionable re-raise the emit sites swap in +/// (`is_ollama_cloud_internal_500_message`). The raw arm requires BOTH the +/// `ollama` provider name and the `internal server error (ref:` envelope, so a +/// generic 500 from another provider, or a local Ollama daemon crash (which +/// carries no `ref:` UUID), still reaches Sentry. +pub fn is_ollama_cloud_internal_500_message_any(text: &str) -> bool { + if crate::openhuman::inference::provider::is_ollama_cloud_internal_500_message(text) { + return true; + } + let lower = text.to_ascii_lowercase(); + lower.contains("ollama") && lower.contains("internal server error (ref:") +} + +/// Defense-in-depth `before_send` filter for **Ollama Cloud hosted-inference +/// 500s** (TAURI-RUST-5MV): ollama.com's `*:cloud` models intermittently +/// return an opaque `Internal Server Error (ref: )` with no client lever +/// (non-deterministic, byte-identical request succeeds when healthy), retried + +/// fallen-back by the reliable-provider layer. +/// +/// The primary demotion lives at the `native_chat` / `streaming_chat` / +/// `api_error` emit sites, and the agent re-report is demoted via +/// `expected_error_kind` → `TransientUpstreamHttp`. This is the single outermost +/// net for any other compatible-provider path (`chat_with_system`, +/// `chat_with_history`, the non-native cascades) that reports the same body, +/// keyed on the message rather than tags so it matches regardless of emitter. +pub fn is_ollama_cloud_internal_500_event(event: &sentry::protocol::Event<'_>) -> bool { + if event + .message + .as_deref() + .is_some_and(is_ollama_cloud_internal_500_message_any) + { + return true; + } + event.exception.values.iter().any(|exception| { + exception + .value + .as_deref() + .is_some_and(is_ollama_cloud_internal_500_message_any) + }) +} + /// 404 on PATCH/DELETE to a channel-message path is an expected backend state /// (user deleted the message provider-side, backend GC'd the relay row). The /// primary suppression lives in `authed_json` via `parse_message_path` + @@ -6089,6 +6147,51 @@ mod tests { ))); } + #[test] + fn ollama_cloud_internal_500_reraise_routes_through_expected_path() { + // TAURI-RUST-5MV — the actionable message the emit sites raise must + // demote to `TransientUpstreamHttp` when re-reported at the agent / RPC + // boundary (`provider_chat` → `report_error_or_expected`), so the + // `domain=agent` half of the flood is suppressed too. + let reraise = crate::openhuman::inference::provider::ollama_cloud_internal_500_user_message( + Some("minimax-m3:cloud"), + reqwest::StatusCode::INTERNAL_SERVER_ERROR, + ); + assert_eq!( + expected_error_kind(&reraise), + Some(ExpectedErrorKind::TransientUpstreamHttp) + ); + } + + #[test] + fn ollama_cloud_internal_500_before_send_matches_raw_and_reraised_shapes() { + // The outermost net catches BOTH the raw emit body (any compatible + // path that bypassed the cascade) and the actionable re-raise. + let raw = "ollama API error (500 Internal Server Error): \ + {\"error\":\"Internal Server Error (ref: df512dcb-d915-493b-8f2d-e8d3dfa640c1)\"}"; + assert!(is_ollama_cloud_internal_500_event(&event_with_message(raw))); + assert!(is_ollama_cloud_internal_500_event( + &event_with_exception_value(raw) + )); + + let reraise = crate::openhuman::inference::provider::ollama_cloud_internal_500_user_message( + None, + reqwest::StatusCode::INTERNAL_SERVER_ERROR, + ); + assert!(is_ollama_cloud_internal_500_event(&event_with_message( + &reraise + ))); + + // A local Ollama 500 without the `ref:` envelope, and a non-ollama 500, + // both stay reportable. + assert!(!is_ollama_cloud_internal_500_event(&event_with_message( + "ollama API error (500 Internal Server Error): {\"error\":\"out of memory\"}" + ))); + assert!(!is_ollama_cloud_internal_500_event(&event_with_message( + "openai API error (500): Internal Server Error (ref: abc)" + ))); + } + #[test] fn session_expired_before_send_matches_core_401_events() { let msg = "SESSION_EXPIRED: backend session not active — sign in to resume LLM work"; diff --git a/src/main.rs b/src/main.rs index b595e16f7..d381e5fb3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -106,6 +106,15 @@ fn main() { if openhuman_core::core::observability::is_quota_exhausted_event(&event) { return None; } + // Defense-in-depth for Ollama Cloud hosted-inference 500s. The + // native_chat / streaming_chat / api_error emit sites demote them and + // the agent re-report routes through `TransientUpstreamHttp`, but the + // compatible provider can report the same `Internal Server Error + // (ref: …)` body from other paths; this is the single net that + // catches every path (TAURI-RUST-5MV). + if openhuman_core::core::observability::is_ollama_cloud_internal_500_event(&event) { + return None; + } // Defense-in-depth: drop max-tool-iterations cap events that // slipped past the call-site filters in // `agent::harness::session::runtime::run_single`, diff --git a/src/openhuman/inference/provider/compatible_provider_impl.rs b/src/openhuman/inference/provider/compatible_provider_impl.rs index 2221d8e9f..2bf632dae 100644 --- a/src/openhuman/inference/provider/compatible_provider_impl.rs +++ b/src/openhuman/inference/provider/compatible_provider_impl.rs @@ -738,7 +738,7 @@ impl Provider for OpenAiCompatibleProvider { } let status_str = status.as_u16().to_string(); - let message = self.enrich_404_message( + let mut message = self.enrich_404_message( format!("{} API error ({status}): {sanitized}", self.name), status, ); @@ -811,6 +811,21 @@ impl Provider for OpenAiCompatibleProvider { Some(model), status, ); + } else if super::super::is_ollama_cloud_internal_500(self.name.as_str(), status, &error) + { + // ollama.com hosted-inference 500: opaque `Internal Server Error + // (ref: )` from the cloud backend for `*:cloud` models. + // Non-deterministic, byte-identical request succeeds when the + // cloud is healthy → no client lever; the reliable-provider layer + // retries + falls back. Demote to info and replace the ref body + // with actionable guidance (TAURI-RUST-5MV). + super::super::log_ollama_cloud_internal_500( + "native_chat", + self.name.as_str(), + Some(model), + status, + ); + message = super::super::ollama_cloud_internal_500_user_message(Some(model), status); } else if super::super::should_report_provider_http_failure(status) { crate::core::observability::report_error( message.as_str(), diff --git a/src/openhuman/inference/provider/compatible_stream_native.rs b/src/openhuman/inference/provider/compatible_stream_native.rs index 99f7eacde..ea4ef9d57 100644 --- a/src/openhuman/inference/provider/compatible_stream_native.rs +++ b/src/openhuman/inference/provider/compatible_stream_native.rs @@ -53,7 +53,7 @@ impl OpenAiCompatibleProvider { let status_str = status.as_u16().to_string(); let body = response.text().await.unwrap_or_default(); let sanitized = super::super::sanitize_api_error(&body); - let message = format!( + let mut message = format!( "{} streaming API error ({}): {}", self.name, status, sanitized ); @@ -147,6 +147,22 @@ impl OpenAiCompatibleProvider { Some(native_request.model.as_str()), status, ); + } else if super::super::is_ollama_cloud_internal_500(self.name.as_str(), status, &body) + { + // ollama.com hosted-inference 500 on the streaming path — same + // provider-internal, no-client-lever condition as the native + // cascade. Demote to info and swap the opaque ref body for + // actionable guidance (TAURI-RUST-5MV). + super::super::log_ollama_cloud_internal_500( + "streaming_chat", + self.name.as_str(), + Some(native_request.model.as_str()), + status, + ); + message = super::super::ollama_cloud_internal_500_user_message( + Some(native_request.model.as_str()), + status, + ); } else if super::super::should_report_provider_http_failure(status) { crate::core::observability::report_error( message.as_str(), diff --git a/src/openhuman/inference/provider/compatible_tests.rs b/src/openhuman/inference/provider/compatible_tests.rs index ea89ca667..3ca4a8114 100644 --- a/src/openhuman/inference/provider/compatible_tests.rs +++ b/src/openhuman/inference/provider/compatible_tests.rs @@ -372,6 +372,183 @@ async fn streaming_chat_frequency_penalty_rejection_not_reported_to_sentry() { ); } +/// Verbatim TAURI-RUST-5MV body — ollama.com hosted-inference 500 envelope. +const OLLAMA_CLOUD_500_WIRE_BODY: &str = + r#"{"error":"Internal Server Error (ref: df512dcb-d915-493b-8f2d-e8d3dfa640c1)"}"#; + +/// TAURI-RUST-5MV: ollama.com hosted-inference 500 on the non-streaming native +/// path must be demoted (no Sentry event) and surface the actionable message, +/// while still propagating as `Err` so reliable-provider retry/fallback runs. +#[tokio::test] +async fn native_chat_ollama_cloud_500_demoted_and_actionable() { + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with(ResponseTemplate::new(500).set_body_string(OLLAMA_CLOUD_500_WIRE_BODY)) + .mount(&mock_server) + .await; + + let transport = TestTransport::new(); + let sentry_options = sentry::ClientOptions { + dsn: Some("https://public@sentry.invalid/1".parse().unwrap()), + transport: Some(Arc::new(transport.clone())), + ..Default::default() + }; + let sentry_hub = Arc::new(sentry::Hub::new( + Some(Arc::new(sentry_options.into())), + Arc::new(Default::default()), + )); + let _sentry_guard = sentry::HubSwitchGuard::new(sentry_hub); + + let provider = + OpenAiCompatibleProvider::new("ollama", &mock_server.uri(), None, AuthStyle::None); + let messages = vec![ChatMessage { + id: None, + role: "user".to_string(), + content: "hello".to_string(), + extra_metadata: None, + }]; + let err = provider + .chat( + super::super::ChatRequest { + messages: &messages, + tools: None, + stream: None, + max_tokens: None, + }, + "minimax-m3:cloud", + 0.7, + ) + .await + .expect_err("ollama cloud 500 must still propagate as Err to drive retry/fallback"); + + let msg = err.to_string(); + assert!( + msg.contains("Ollama cloud is temporarily unavailable") && msg.contains("minimax-m3:cloud"), + "expected actionable message, got: {msg}" + ); + assert!( + !msg.contains("ref:"), + "opaque ref body must be replaced: {msg}" + ); + assert!( + transport.fetch_and_clear_events().is_empty(), + "ollama cloud 500 must be demoted, not reported to Sentry" + ); +} + +/// TAURI-RUST-5MV: same demotion on the streaming native path. +#[tokio::test] +async fn streaming_chat_ollama_cloud_500_demoted_and_actionable() { + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with(ResponseTemplate::new(500).set_body_string(OLLAMA_CLOUD_500_WIRE_BODY)) + .mount(&mock_server) + .await; + + let transport = TestTransport::new(); + let sentry_options = sentry::ClientOptions { + dsn: Some("https://public@sentry.invalid/1".parse().unwrap()), + transport: Some(Arc::new(transport.clone())), + ..Default::default() + }; + let sentry_hub = Arc::new(sentry::Hub::new( + Some(Arc::new(sentry_options.into())), + Arc::new(Default::default()), + )); + let _sentry_guard = sentry::HubSwitchGuard::new(sentry_hub); + + let provider = + OpenAiCompatibleProvider::new("ollama", &mock_server.uri(), None, AuthStyle::None); + let request = NativeChatRequest { + model: "minimax-m3:cloud".to_string(), + messages: vec![NativeMessage { + role: "user".to_string(), + content: Some("hello".into()), + tool_call_id: None, + tool_calls: None, + reasoning_content: None, + }], + temperature: Some(0.7), + stream: Some(true), + tools: None, + tool_choice: None, + thread_id: None, + stream_options: Some(super::compatible_types::OpenAiStreamOptions { + include_usage: true, + }), + options: None, + frequency_penalty: None, + max_tokens: None, + }; + let (delta_tx, _delta_rx) = tokio::sync::mpsc::channel(8); + + let err = provider + .stream_native_chat(None, &request, &delta_tx, 0) + .await + .expect_err("ollama cloud 500 must still propagate as Err to drive retry/fallback"); + + let msg = err.to_string(); + assert!( + msg.contains("Ollama cloud is temporarily unavailable") && msg.contains("minimax-m3:cloud"), + "expected actionable message, got: {msg}" + ); + assert!( + !msg.contains("ref:"), + "opaque ref body must be replaced: {msg}" + ); + assert!( + transport.fetch_and_clear_events().is_empty(), + "ollama cloud 500 must be demoted, not reported to Sentry" + ); +} + +/// TAURI-RUST-5MV: the shared `api_error` helper demotes the same body and +/// returns the actionable message (covers `chat_with_system` / `chat_with_history` +/// callers that funnel through it). +#[tokio::test] +async fn api_error_ollama_cloud_500_demoted_and_actionable() { + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(500).set_body_string(OLLAMA_CLOUD_500_WIRE_BODY)) + .mount(&mock_server) + .await; + + let transport = TestTransport::new(); + let sentry_options = sentry::ClientOptions { + dsn: Some("https://public@sentry.invalid/1".parse().unwrap()), + transport: Some(Arc::new(transport.clone())), + ..Default::default() + }; + let sentry_hub = Arc::new(sentry::Hub::new( + Some(Arc::new(sentry_options.into())), + Arc::new(Default::default()), + )); + let _sentry_guard = sentry::HubSwitchGuard::new(sentry_hub); + + let response = reqwest::Client::new() + .post(mock_server.uri()) + .send() + .await + .expect("mock request"); + let err = super::super::api_error("ollama", response).await; + + let msg = err.to_string(); + assert!( + msg.contains("Ollama cloud is temporarily unavailable"), + "expected actionable message, got: {msg}" + ); + assert!( + !msg.contains("ref:"), + "opaque ref body must be replaced: {msg}" + ); + assert!( + transport.fetch_and_clear_events().is_empty(), + "ollama cloud 500 must be demoted, not reported to Sentry" + ); +} + /// Streaming responses arrive without `usage` unless the request asks /// for `stream_options.include_usage = true` (OpenAI spec). Without it /// the OpenHuman backend's `openhuman.billing` block also never lands, diff --git a/src/openhuman/inference/provider/ops/http_error.rs b/src/openhuman/inference/provider/ops/http_error.rs index 734056bca..cdb44a19b 100644 --- a/src/openhuman/inference/provider/ops/http_error.rs +++ b/src/openhuman/inference/provider/ops/http_error.rs @@ -284,6 +284,98 @@ pub fn log_provider_quota_exhausted( ); } +/// Stable anchor phrase for the actionable Ollama-Cloud-500 user message, shared +/// by [`ollama_cloud_internal_500_user_message`] (which builds it) and +/// [`is_ollama_cloud_internal_500_message`] (which matches the re-raised string +/// at the RPC/agent boundary), so the two cannot drift. +const OLLAMA_CLOUD_INTERNAL_500_USER_PREFIX: &str = "Ollama cloud is temporarily unavailable"; + +/// Whether a provider non-2xx response is an Ollama **Cloud** hosted-inference +/// internal error: `500` + a `{"error":"Internal Server Error (ref: )"}` +/// body. +/// +/// ollama.com's hosted `*:cloud` models (minimax-m3 / qwen3.5 / gpt-oss …) +/// intermittently `500` with this opaque, server-generated envelope. The `ref:` +/// is a fresh UUID per event, the failure is non-deterministic, and the request +/// that 500s is byte-identical to the one that succeeds when the cloud backend +/// is healthy — so there is **no client lever** (nothing to validate, +/// reshape, or reconfigure). The reliable-provider layer already retries and +/// falls back across providers/models, so each per-attempt 500 is pure noise: +/// TAURI-RUST-5MV, 3,062 events from 5 users in a single window. Demote from +/// Sentry to an info log while the error still propagates so retry/fallback runs +/// unchanged. +/// +/// Anchored on the `internal server error (ref:` body shape, which is specific +/// to ollama.com's hosted envelope — a **local** Ollama daemon 500 (a genuine +/// model crash / OOM worth paging) does not carry a `ref:` UUID, so it still +/// reaches Sentry. The phrase is covered by a verbatim-body test so a provider +/// wording drift fails CI instead of silently leaking events. +pub fn is_ollama_cloud_internal_500( + provider: &str, + status: reqwest::StatusCode, + body: &str, +) -> bool { + provider == "ollama" + && status == reqwest::StatusCode::INTERNAL_SERVER_ERROR + && body + .to_ascii_lowercase() + .contains("internal server error (ref:") +} + +/// Message-level half of [`is_ollama_cloud_internal_500`]: matches the actionable +/// user message re-raised at the RPC/agent boundary +/// (`core::observability::expected_error_kind`), so the higher-layer re-report is +/// demoted too instead of leaking the event the emit-site already suppressed (the +/// `domain=agent` half of TAURI-RUST-5MV). Mirrors the +/// `is_provider_insufficient_credits_402` / `body_indicates_insufficient_credits` +/// split. Keyed on the [`OLLAMA_CLOUD_INTERNAL_500_USER_PREFIX`] anchor, which we +/// own, so it cannot collide with an unrelated provider body. +pub fn is_ollama_cloud_internal_500_message(message: &str) -> bool { + let needle = OLLAMA_CLOUD_INTERNAL_500_USER_PREFIX.to_ascii_lowercase(); + message.to_ascii_lowercase().contains(needle.as_str()) +} + +/// Build the actionable user-facing message for an Ollama-Cloud hosted-inference +/// 500, replacing the opaque `Internal Server Error (ref: )` body (which +/// carries no signal the user can act on) with retry/switch guidance. The model +/// is included when known (native/streaming chat); the `api_error` path has no +/// model in scope and omits it. +pub fn ollama_cloud_internal_500_user_message( + model: Option<&str>, + status: reqwest::StatusCode, +) -> String { + let code = status.as_u16(); + match model { + Some(model) => format!( + "{OLLAMA_CLOUD_INTERNAL_500_USER_PREFIX} for model `{model}` (Ollama returned HTTP \ + {code}); the hosted model failed on Ollama's side — retry shortly or switch models." + ), + None => format!( + "{OLLAMA_CLOUD_INTERNAL_500_USER_PREFIX} (Ollama returned HTTP {code}); the hosted \ + model failed on Ollama's side — retry shortly or switch models." + ), + } +} + +pub fn log_ollama_cloud_internal_500( + operation: &str, + provider: &str, + model: Option<&str>, + status: reqwest::StatusCode, +) { + tracing::info!( + domain = "llm_provider", + operation = operation, + provider = provider, + model = model.unwrap_or(""), + status = status.as_u16(), + failure = "non_2xx", + kind = "ollama_cloud_internal_500", + "[llm_provider] {operation} Ollama Cloud hosted-inference 500 — provider-internal \ + (no client lever), not reporting to Sentry" + ); +} + /// Whether a provider non-2xx response is a deterministic /// **configuration-rejection** user-state error (unknown model id, /// abstract tier leaked to a custom provider, model-specific temperature @@ -802,6 +894,10 @@ pub async fn api_error(provider: &str, response: reqwest::Response) -> anyhow::E // Balance"). This shared helper backs the two methods that delegate here // (chat_via_responses fallback and the non-streaming completion path). let is_insufficient_credits_402 = is_provider_insufficient_credits_402(status, &body); + // Ollama Cloud hosted-inference 500 (`Internal Server Error (ref: )`): + // provider-internal, non-deterministic, no client lever. Demote from Sentry + // and replace the opaque ref body with actionable guidance (TAURI-RUST-5MV). + let is_ollama_cloud_internal_500 = is_ollama_cloud_internal_500(provider, status, &body); if is_auth_failure && is_backend { // Single source of truth for backend session-expiry handling (warn + @@ -828,6 +924,8 @@ pub async fn api_error(provider: &str, response: reqwest::Response) -> anyhow::E log_openai_oauth_session_expired("api_error", provider, None, status); } else if is_insufficient_credits_402 { log_provider_insufficient_credits_402("api_error", provider, None, status); + } else if is_ollama_cloud_internal_500 { + log_ollama_cloud_internal_500("api_error", provider, None, status); } else if should_report_provider_http_failure(status) { crate::core::observability::report_error( message.as_str(), @@ -840,6 +938,12 @@ pub async fn api_error(provider: &str, response: reqwest::Response) -> anyhow::E ], ); } + // Replace the opaque `Internal Server Error (ref: )` body with + // actionable guidance; the prefix anchors the higher-layer re-report + // demotion (`is_ollama_cloud_internal_500_message`). + if is_ollama_cloud_internal_500 { + return anyhow::anyhow!(ollama_cloud_internal_500_user_message(None, status)); + } anyhow::anyhow!(message) } @@ -1032,6 +1136,89 @@ mod tests { )); } + /// Verbatim TAURI-RUST-5MV provider body. The matcher keys on the + /// `Internal Server Error (ref:` envelope, so coupling the test to the exact + /// wire shape makes an Ollama-Cloud wording drift fail CI rather than + /// silently leak events back to Sentry. + const OLLAMA_CLOUD_500_BODY: &str = + "{\"error\":\"Internal Server Error (ref: df512dcb-d915-493b-8f2d-e8d3dfa640c1)\"}"; + + #[test] + fn ollama_cloud_internal_500_matches_verbatim_5mv_body() { + assert!(is_ollama_cloud_internal_500( + "ollama", + StatusCode::INTERNAL_SERVER_ERROR, + OLLAMA_CLOUD_500_BODY + )); + } + + #[test] + fn ollama_cloud_internal_500_ignores_non_500_status() { + // Same body on a non-500 status is not this provider-internal flood — + // keep it reportable so a genuine bug elsewhere isn't masked. + assert!(!is_ollama_cloud_internal_500( + "ollama", + StatusCode::BAD_REQUEST, + OLLAMA_CLOUD_500_BODY + )); + assert!(!is_ollama_cloud_internal_500( + "ollama", + StatusCode::SERVICE_UNAVAILABLE, + OLLAMA_CLOUD_500_BODY + )); + } + + #[test] + fn ollama_cloud_internal_500_ignores_other_providers() { + // A 500 with the same envelope from a non-ollama provider stays + // reportable — this gate is scoped to ollama.com hosted inference. + assert!(!is_ollama_cloud_internal_500( + "openai", + StatusCode::INTERNAL_SERVER_ERROR, + OLLAMA_CLOUD_500_BODY + )); + } + + #[test] + fn ollama_cloud_internal_500_ignores_local_ollama_500_without_ref() { + // A local Ollama daemon 500 (genuine model crash / OOM, worth paging) + // does not carry the `ref:` UUID, so it must NOT be swallowed. + assert!(!is_ollama_cloud_internal_500( + "ollama", + StatusCode::INTERNAL_SERVER_ERROR, + "{\"error\":\"llama runner process has terminated: exit status 0xc0000409\"}" + )); + } + + #[test] + fn ollama_cloud_internal_500_user_message_is_matched_by_message_matcher() { + // Couple the prose builder to the re-report matcher so the + // `expected_error_kind` / before_send demotion can't drift from the + // string the emit sites actually raise. + let with_model = ollama_cloud_internal_500_user_message( + Some("minimax-m3:cloud"), + StatusCode::INTERNAL_SERVER_ERROR, + ); + assert!(with_model.contains("minimax-m3:cloud")); + assert!(!with_model.contains("ref:")); + assert!(is_ollama_cloud_internal_500_message(&with_model)); + + let without_model = + ollama_cloud_internal_500_user_message(None, StatusCode::INTERNAL_SERVER_ERROR); + assert!(is_ollama_cloud_internal_500_message(&without_model)); + } + + #[test] + fn log_ollama_cloud_internal_500_smoke() { + // The helper only emits a demotion info log; calling it covers that path. + log_ollama_cloud_internal_500( + "native_chat", + "ollama", + Some("minimax-m3:cloud"), + StatusCode::INTERNAL_SERVER_ERROR, + ); + } + #[test] fn openai_oauth_session_expired_excludes_backend_provider() { // The OpenHuman backend owns app-session expiry via diff --git a/src/openhuman/inference/provider/ops/mod.rs b/src/openhuman/inference/provider/ops/mod.rs index 64928c8ef..5f87c4b2a 100644 --- a/src/openhuman/inference/provider/ops/mod.rs +++ b/src/openhuman/inference/provider/ops/mod.rs @@ -22,14 +22,16 @@ pub use http_error::{ api_error, body_indicates_insufficient_credits, body_indicates_quota_exhausted, is_backend_auth_failure, is_backend_error_code_owned, is_budget_exhausted_http_400, is_byo_provider_auth_failure_http, is_context_window_exceeded_message, - is_custom_openai_upstream_bad_request_http_400, is_openai_oauth_session_expired_http, + is_custom_openai_upstream_bad_request_http_400, is_ollama_cloud_internal_500, + is_ollama_cloud_internal_500_message, is_openai_oauth_session_expired_http, is_openai_oauth_session_expired_message, is_provider_access_policy_denied_http_403, is_provider_config_rejection_http, is_provider_insufficient_credits_402, is_provider_quota_exhausted, log_backend_error_code_owned, log_budget_exhausted_http_400, log_byo_provider_auth_failure, log_context_window_exceeded, - log_custom_openai_upstream_bad_request_http_400, log_openai_oauth_session_expired, - log_provider_access_policy_denied_http_403, log_provider_config_rejection, - log_provider_insufficient_credits_402, log_provider_quota_exhausted, + log_custom_openai_upstream_bad_request_http_400, log_ollama_cloud_internal_500, + log_openai_oauth_session_expired, log_provider_access_policy_denied_http_403, + log_provider_config_rejection, log_provider_insufficient_credits_402, + log_provider_quota_exhausted, ollama_cloud_internal_500_user_message, publish_backend_session_expired, should_report_provider_http_failure, };