mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
fix(observability): demote Ollama Cloud hosted-inference 500 flood (TAURI-RUST-5MV) (#4069)
This commit is contained in:
@@ -420,6 +420,20 @@ pub fn expected_error_kind(message: &str) -> Option<ExpectedErrorKind> {
|
||||
if crate::openhuman::inference::provider::is_openai_oauth_session_expired_message(message) {
|
||||
return Some(ExpectedErrorKind::ProviderUserState);
|
||||
}
|
||||
// TAURI-RUST-5MV — ollama.com hosted-inference 500 (`Internal Server Error
|
||||
// (ref: <uuid>)`) for `*:cloud` models. The provider HTTP layer
|
||||
// (`native_chat` / `streaming_chat` / `api_error`) already demotes its own
|
||||
// per-attempt event and re-raises the actionable
|
||||
// "Ollama cloud is temporarily unavailable …" string; this catches the
|
||||
// re-report at the agent / RPC boundary (`provider_chat` →
|
||||
// `report_error_or_expected`, the `domain=agent` half of the flood). Routed
|
||||
// to `TransientUpstreamHttp` — it is an external upstream 5xx the
|
||||
// reliable-provider layer retries + falls back over, with no client lever.
|
||||
// Delegates to the single-source provider matcher so the phrasing can't
|
||||
// drift. Distinct anchor from the matchers above, so it shadows nothing.
|
||||
if crate::openhuman::inference::provider::is_ollama_cloud_internal_500_message(message) {
|
||||
return Some(ExpectedErrorKind::TransientUpstreamHttp);
|
||||
}
|
||||
if is_backend_user_error_message(&lower) {
|
||||
return Some(ExpectedErrorKind::BackendUserError);
|
||||
}
|
||||
@@ -2630,6 +2644,50 @@ pub fn is_quota_exhausted_event(event: &sentry::protocol::Event<'_>) -> bool {
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether a raw error / message string is an Ollama **Cloud** hosted-inference
|
||||
/// `500` (`Internal Server Error (ref: <uuid>)`). Matches either the raw emit
|
||||
/// shape (`ollama API error (500 …): {"error":"Internal Server Error (ref: …)"}`)
|
||||
/// or the actionable re-raise the emit sites swap in
|
||||
/// (`is_ollama_cloud_internal_500_message`). The raw arm requires BOTH the
|
||||
/// `ollama` provider name and the `internal server error (ref:` envelope, so a
|
||||
/// generic 500 from another provider, or a local Ollama daemon crash (which
|
||||
/// carries no `ref:` UUID), still reaches Sentry.
|
||||
pub fn is_ollama_cloud_internal_500_message_any(text: &str) -> bool {
|
||||
if crate::openhuman::inference::provider::is_ollama_cloud_internal_500_message(text) {
|
||||
return true;
|
||||
}
|
||||
let lower = text.to_ascii_lowercase();
|
||||
lower.contains("ollama") && lower.contains("internal server error (ref:")
|
||||
}
|
||||
|
||||
/// Defense-in-depth `before_send` filter for **Ollama Cloud hosted-inference
|
||||
/// 500s** (TAURI-RUST-5MV): ollama.com's `*:cloud` models intermittently
|
||||
/// return an opaque `Internal Server Error (ref: <uuid>)` with no client lever
|
||||
/// (non-deterministic, byte-identical request succeeds when healthy), retried +
|
||||
/// fallen-back by the reliable-provider layer.
|
||||
///
|
||||
/// The primary demotion lives at the `native_chat` / `streaming_chat` /
|
||||
/// `api_error` emit sites, and the agent re-report is demoted via
|
||||
/// `expected_error_kind` → `TransientUpstreamHttp`. This is the single outermost
|
||||
/// net for any other compatible-provider path (`chat_with_system`,
|
||||
/// `chat_with_history`, the non-native cascades) that reports the same body,
|
||||
/// keyed on the message rather than tags so it matches regardless of emitter.
|
||||
pub fn is_ollama_cloud_internal_500_event(event: &sentry::protocol::Event<'_>) -> bool {
|
||||
if event
|
||||
.message
|
||||
.as_deref()
|
||||
.is_some_and(is_ollama_cloud_internal_500_message_any)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
event.exception.values.iter().any(|exception| {
|
||||
exception
|
||||
.value
|
||||
.as_deref()
|
||||
.is_some_and(is_ollama_cloud_internal_500_message_any)
|
||||
})
|
||||
}
|
||||
|
||||
/// 404 on PATCH/DELETE to a channel-message path is an expected backend state
|
||||
/// (user deleted the message provider-side, backend GC'd the relay row). The
|
||||
/// primary suppression lives in `authed_json` via `parse_message_path` +
|
||||
@@ -6089,6 +6147,51 @@ mod tests {
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ollama_cloud_internal_500_reraise_routes_through_expected_path() {
|
||||
// TAURI-RUST-5MV — the actionable message the emit sites raise must
|
||||
// demote to `TransientUpstreamHttp` when re-reported at the agent / RPC
|
||||
// boundary (`provider_chat` → `report_error_or_expected`), so the
|
||||
// `domain=agent` half of the flood is suppressed too.
|
||||
let reraise = crate::openhuman::inference::provider::ollama_cloud_internal_500_user_message(
|
||||
Some("minimax-m3:cloud"),
|
||||
reqwest::StatusCode::INTERNAL_SERVER_ERROR,
|
||||
);
|
||||
assert_eq!(
|
||||
expected_error_kind(&reraise),
|
||||
Some(ExpectedErrorKind::TransientUpstreamHttp)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ollama_cloud_internal_500_before_send_matches_raw_and_reraised_shapes() {
|
||||
// The outermost net catches BOTH the raw emit body (any compatible
|
||||
// path that bypassed the cascade) and the actionable re-raise.
|
||||
let raw = "ollama API error (500 Internal Server Error): \
|
||||
{\"error\":\"Internal Server Error (ref: df512dcb-d915-493b-8f2d-e8d3dfa640c1)\"}";
|
||||
assert!(is_ollama_cloud_internal_500_event(&event_with_message(raw)));
|
||||
assert!(is_ollama_cloud_internal_500_event(
|
||||
&event_with_exception_value(raw)
|
||||
));
|
||||
|
||||
let reraise = crate::openhuman::inference::provider::ollama_cloud_internal_500_user_message(
|
||||
None,
|
||||
reqwest::StatusCode::INTERNAL_SERVER_ERROR,
|
||||
);
|
||||
assert!(is_ollama_cloud_internal_500_event(&event_with_message(
|
||||
&reraise
|
||||
)));
|
||||
|
||||
// A local Ollama 500 without the `ref:` envelope, and a non-ollama 500,
|
||||
// both stay reportable.
|
||||
assert!(!is_ollama_cloud_internal_500_event(&event_with_message(
|
||||
"ollama API error (500 Internal Server Error): {\"error\":\"out of memory\"}"
|
||||
)));
|
||||
assert!(!is_ollama_cloud_internal_500_event(&event_with_message(
|
||||
"openai API error (500): Internal Server Error (ref: abc)"
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_expired_before_send_matches_core_401_events() {
|
||||
let msg = "SESSION_EXPIRED: backend session not active — sign in to resume LLM work";
|
||||
|
||||
@@ -106,6 +106,15 @@ fn main() {
|
||||
if openhuman_core::core::observability::is_quota_exhausted_event(&event) {
|
||||
return None;
|
||||
}
|
||||
// Defense-in-depth for Ollama Cloud hosted-inference 500s. The
|
||||
// native_chat / streaming_chat / api_error emit sites demote them and
|
||||
// the agent re-report routes through `TransientUpstreamHttp`, but the
|
||||
// compatible provider can report the same `Internal Server Error
|
||||
// (ref: …)` body from other paths; this is the single net that
|
||||
// catches every path (TAURI-RUST-5MV).
|
||||
if openhuman_core::core::observability::is_ollama_cloud_internal_500_event(&event) {
|
||||
return None;
|
||||
}
|
||||
// Defense-in-depth: drop max-tool-iterations cap events that
|
||||
// slipped past the call-site filters in
|
||||
// `agent::harness::session::runtime::run_single`,
|
||||
|
||||
@@ -738,7 +738,7 @@ impl Provider for OpenAiCompatibleProvider {
|
||||
}
|
||||
|
||||
let status_str = status.as_u16().to_string();
|
||||
let message = self.enrich_404_message(
|
||||
let mut message = self.enrich_404_message(
|
||||
format!("{} API error ({status}): {sanitized}", self.name),
|
||||
status,
|
||||
);
|
||||
@@ -811,6 +811,21 @@ impl Provider for OpenAiCompatibleProvider {
|
||||
Some(model),
|
||||
status,
|
||||
);
|
||||
} else if super::super::is_ollama_cloud_internal_500(self.name.as_str(), status, &error)
|
||||
{
|
||||
// ollama.com hosted-inference 500: opaque `Internal Server Error
|
||||
// (ref: <uuid>)` from the cloud backend for `*:cloud` models.
|
||||
// Non-deterministic, byte-identical request succeeds when the
|
||||
// cloud is healthy → no client lever; the reliable-provider layer
|
||||
// retries + falls back. Demote to info and replace the ref body
|
||||
// with actionable guidance (TAURI-RUST-5MV).
|
||||
super::super::log_ollama_cloud_internal_500(
|
||||
"native_chat",
|
||||
self.name.as_str(),
|
||||
Some(model),
|
||||
status,
|
||||
);
|
||||
message = super::super::ollama_cloud_internal_500_user_message(Some(model), status);
|
||||
} else if super::super::should_report_provider_http_failure(status) {
|
||||
crate::core::observability::report_error(
|
||||
message.as_str(),
|
||||
|
||||
@@ -53,7 +53,7 @@ impl OpenAiCompatibleProvider {
|
||||
let status_str = status.as_u16().to_string();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
let sanitized = super::super::sanitize_api_error(&body);
|
||||
let message = format!(
|
||||
let mut message = format!(
|
||||
"{} streaming API error ({}): {}",
|
||||
self.name, status, sanitized
|
||||
);
|
||||
@@ -147,6 +147,22 @@ impl OpenAiCompatibleProvider {
|
||||
Some(native_request.model.as_str()),
|
||||
status,
|
||||
);
|
||||
} else if super::super::is_ollama_cloud_internal_500(self.name.as_str(), status, &body)
|
||||
{
|
||||
// ollama.com hosted-inference 500 on the streaming path — same
|
||||
// provider-internal, no-client-lever condition as the native
|
||||
// cascade. Demote to info and swap the opaque ref body for
|
||||
// actionable guidance (TAURI-RUST-5MV).
|
||||
super::super::log_ollama_cloud_internal_500(
|
||||
"streaming_chat",
|
||||
self.name.as_str(),
|
||||
Some(native_request.model.as_str()),
|
||||
status,
|
||||
);
|
||||
message = super::super::ollama_cloud_internal_500_user_message(
|
||||
Some(native_request.model.as_str()),
|
||||
status,
|
||||
);
|
||||
} else if super::super::should_report_provider_http_failure(status) {
|
||||
crate::core::observability::report_error(
|
||||
message.as_str(),
|
||||
|
||||
@@ -372,6 +372,183 @@ async fn streaming_chat_frequency_penalty_rejection_not_reported_to_sentry() {
|
||||
);
|
||||
}
|
||||
|
||||
/// Verbatim TAURI-RUST-5MV body — ollama.com hosted-inference 500 envelope.
|
||||
const OLLAMA_CLOUD_500_WIRE_BODY: &str =
|
||||
r#"{"error":"Internal Server Error (ref: df512dcb-d915-493b-8f2d-e8d3dfa640c1)"}"#;
|
||||
|
||||
/// TAURI-RUST-5MV: ollama.com hosted-inference 500 on the non-streaming native
|
||||
/// path must be demoted (no Sentry event) and surface the actionable message,
|
||||
/// while still propagating as `Err` so reliable-provider retry/fallback runs.
|
||||
#[tokio::test]
|
||||
async fn native_chat_ollama_cloud_500_demoted_and_actionable() {
|
||||
let mock_server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/chat/completions"))
|
||||
.respond_with(ResponseTemplate::new(500).set_body_string(OLLAMA_CLOUD_500_WIRE_BODY))
|
||||
.mount(&mock_server)
|
||||
.await;
|
||||
|
||||
let transport = TestTransport::new();
|
||||
let sentry_options = sentry::ClientOptions {
|
||||
dsn: Some("https://public@sentry.invalid/1".parse().unwrap()),
|
||||
transport: Some(Arc::new(transport.clone())),
|
||||
..Default::default()
|
||||
};
|
||||
let sentry_hub = Arc::new(sentry::Hub::new(
|
||||
Some(Arc::new(sentry_options.into())),
|
||||
Arc::new(Default::default()),
|
||||
));
|
||||
let _sentry_guard = sentry::HubSwitchGuard::new(sentry_hub);
|
||||
|
||||
let provider =
|
||||
OpenAiCompatibleProvider::new("ollama", &mock_server.uri(), None, AuthStyle::None);
|
||||
let messages = vec![ChatMessage {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: "hello".to_string(),
|
||||
extra_metadata: None,
|
||||
}];
|
||||
let err = provider
|
||||
.chat(
|
||||
super::super::ChatRequest {
|
||||
messages: &messages,
|
||||
tools: None,
|
||||
stream: None,
|
||||
max_tokens: None,
|
||||
},
|
||||
"minimax-m3:cloud",
|
||||
0.7,
|
||||
)
|
||||
.await
|
||||
.expect_err("ollama cloud 500 must still propagate as Err to drive retry/fallback");
|
||||
|
||||
let msg = err.to_string();
|
||||
assert!(
|
||||
msg.contains("Ollama cloud is temporarily unavailable") && msg.contains("minimax-m3:cloud"),
|
||||
"expected actionable message, got: {msg}"
|
||||
);
|
||||
assert!(
|
||||
!msg.contains("ref:"),
|
||||
"opaque ref body must be replaced: {msg}"
|
||||
);
|
||||
assert!(
|
||||
transport.fetch_and_clear_events().is_empty(),
|
||||
"ollama cloud 500 must be demoted, not reported to Sentry"
|
||||
);
|
||||
}
|
||||
|
||||
/// TAURI-RUST-5MV: same demotion on the streaming native path.
|
||||
#[tokio::test]
|
||||
async fn streaming_chat_ollama_cloud_500_demoted_and_actionable() {
|
||||
let mock_server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/chat/completions"))
|
||||
.respond_with(ResponseTemplate::new(500).set_body_string(OLLAMA_CLOUD_500_WIRE_BODY))
|
||||
.mount(&mock_server)
|
||||
.await;
|
||||
|
||||
let transport = TestTransport::new();
|
||||
let sentry_options = sentry::ClientOptions {
|
||||
dsn: Some("https://public@sentry.invalid/1".parse().unwrap()),
|
||||
transport: Some(Arc::new(transport.clone())),
|
||||
..Default::default()
|
||||
};
|
||||
let sentry_hub = Arc::new(sentry::Hub::new(
|
||||
Some(Arc::new(sentry_options.into())),
|
||||
Arc::new(Default::default()),
|
||||
));
|
||||
let _sentry_guard = sentry::HubSwitchGuard::new(sentry_hub);
|
||||
|
||||
let provider =
|
||||
OpenAiCompatibleProvider::new("ollama", &mock_server.uri(), None, AuthStyle::None);
|
||||
let request = NativeChatRequest {
|
||||
model: "minimax-m3:cloud".to_string(),
|
||||
messages: vec![NativeMessage {
|
||||
role: "user".to_string(),
|
||||
content: Some("hello".into()),
|
||||
tool_call_id: None,
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
}],
|
||||
temperature: Some(0.7),
|
||||
stream: Some(true),
|
||||
tools: None,
|
||||
tool_choice: None,
|
||||
thread_id: None,
|
||||
stream_options: Some(super::compatible_types::OpenAiStreamOptions {
|
||||
include_usage: true,
|
||||
}),
|
||||
options: None,
|
||||
frequency_penalty: None,
|
||||
max_tokens: None,
|
||||
};
|
||||
let (delta_tx, _delta_rx) = tokio::sync::mpsc::channel(8);
|
||||
|
||||
let err = provider
|
||||
.stream_native_chat(None, &request, &delta_tx, 0)
|
||||
.await
|
||||
.expect_err("ollama cloud 500 must still propagate as Err to drive retry/fallback");
|
||||
|
||||
let msg = err.to_string();
|
||||
assert!(
|
||||
msg.contains("Ollama cloud is temporarily unavailable") && msg.contains("minimax-m3:cloud"),
|
||||
"expected actionable message, got: {msg}"
|
||||
);
|
||||
assert!(
|
||||
!msg.contains("ref:"),
|
||||
"opaque ref body must be replaced: {msg}"
|
||||
);
|
||||
assert!(
|
||||
transport.fetch_and_clear_events().is_empty(),
|
||||
"ollama cloud 500 must be demoted, not reported to Sentry"
|
||||
);
|
||||
}
|
||||
|
||||
/// TAURI-RUST-5MV: the shared `api_error` helper demotes the same body and
|
||||
/// returns the actionable message (covers `chat_with_system` / `chat_with_history`
|
||||
/// callers that funnel through it).
|
||||
#[tokio::test]
|
||||
async fn api_error_ollama_cloud_500_demoted_and_actionable() {
|
||||
let mock_server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.respond_with(ResponseTemplate::new(500).set_body_string(OLLAMA_CLOUD_500_WIRE_BODY))
|
||||
.mount(&mock_server)
|
||||
.await;
|
||||
|
||||
let transport = TestTransport::new();
|
||||
let sentry_options = sentry::ClientOptions {
|
||||
dsn: Some("https://public@sentry.invalid/1".parse().unwrap()),
|
||||
transport: Some(Arc::new(transport.clone())),
|
||||
..Default::default()
|
||||
};
|
||||
let sentry_hub = Arc::new(sentry::Hub::new(
|
||||
Some(Arc::new(sentry_options.into())),
|
||||
Arc::new(Default::default()),
|
||||
));
|
||||
let _sentry_guard = sentry::HubSwitchGuard::new(sentry_hub);
|
||||
|
||||
let response = reqwest::Client::new()
|
||||
.post(mock_server.uri())
|
||||
.send()
|
||||
.await
|
||||
.expect("mock request");
|
||||
let err = super::super::api_error("ollama", response).await;
|
||||
|
||||
let msg = err.to_string();
|
||||
assert!(
|
||||
msg.contains("Ollama cloud is temporarily unavailable"),
|
||||
"expected actionable message, got: {msg}"
|
||||
);
|
||||
assert!(
|
||||
!msg.contains("ref:"),
|
||||
"opaque ref body must be replaced: {msg}"
|
||||
);
|
||||
assert!(
|
||||
transport.fetch_and_clear_events().is_empty(),
|
||||
"ollama cloud 500 must be demoted, not reported to Sentry"
|
||||
);
|
||||
}
|
||||
|
||||
/// Streaming responses arrive without `usage` unless the request asks
|
||||
/// for `stream_options.include_usage = true` (OpenAI spec). Without it
|
||||
/// the OpenHuman backend's `openhuman.billing` block also never lands,
|
||||
|
||||
@@ -284,6 +284,98 @@ pub fn log_provider_quota_exhausted(
|
||||
);
|
||||
}
|
||||
|
||||
/// Stable anchor phrase for the actionable Ollama-Cloud-500 user message, shared
|
||||
/// by [`ollama_cloud_internal_500_user_message`] (which builds it) and
|
||||
/// [`is_ollama_cloud_internal_500_message`] (which matches the re-raised string
|
||||
/// at the RPC/agent boundary), so the two cannot drift.
|
||||
const OLLAMA_CLOUD_INTERNAL_500_USER_PREFIX: &str = "Ollama cloud is temporarily unavailable";
|
||||
|
||||
/// Whether a provider non-2xx response is an Ollama **Cloud** hosted-inference
|
||||
/// internal error: `500` + a `{"error":"Internal Server Error (ref: <uuid>)"}`
|
||||
/// body.
|
||||
///
|
||||
/// ollama.com's hosted `*:cloud` models (minimax-m3 / qwen3.5 / gpt-oss …)
|
||||
/// intermittently `500` with this opaque, server-generated envelope. The `ref:`
|
||||
/// is a fresh UUID per event, the failure is non-deterministic, and the request
|
||||
/// that 500s is byte-identical to the one that succeeds when the cloud backend
|
||||
/// is healthy — so there is **no client lever** (nothing to validate,
|
||||
/// reshape, or reconfigure). The reliable-provider layer already retries and
|
||||
/// falls back across providers/models, so each per-attempt 500 is pure noise:
|
||||
/// TAURI-RUST-5MV, 3,062 events from 5 users in a single window. Demote from
|
||||
/// Sentry to an info log while the error still propagates so retry/fallback runs
|
||||
/// unchanged.
|
||||
///
|
||||
/// Anchored on the `internal server error (ref:` body shape, which is specific
|
||||
/// to ollama.com's hosted envelope — a **local** Ollama daemon 500 (a genuine
|
||||
/// model crash / OOM worth paging) does not carry a `ref:` UUID, so it still
|
||||
/// reaches Sentry. The phrase is covered by a verbatim-body test so a provider
|
||||
/// wording drift fails CI instead of silently leaking events.
|
||||
pub fn is_ollama_cloud_internal_500(
|
||||
provider: &str,
|
||||
status: reqwest::StatusCode,
|
||||
body: &str,
|
||||
) -> bool {
|
||||
provider == "ollama"
|
||||
&& status == reqwest::StatusCode::INTERNAL_SERVER_ERROR
|
||||
&& body
|
||||
.to_ascii_lowercase()
|
||||
.contains("internal server error (ref:")
|
||||
}
|
||||
|
||||
/// Message-level half of [`is_ollama_cloud_internal_500`]: matches the actionable
|
||||
/// user message re-raised at the RPC/agent boundary
|
||||
/// (`core::observability::expected_error_kind`), so the higher-layer re-report is
|
||||
/// demoted too instead of leaking the event the emit-site already suppressed (the
|
||||
/// `domain=agent` half of TAURI-RUST-5MV). Mirrors the
|
||||
/// `is_provider_insufficient_credits_402` / `body_indicates_insufficient_credits`
|
||||
/// split. Keyed on the [`OLLAMA_CLOUD_INTERNAL_500_USER_PREFIX`] anchor, which we
|
||||
/// own, so it cannot collide with an unrelated provider body.
|
||||
pub fn is_ollama_cloud_internal_500_message(message: &str) -> bool {
|
||||
let needle = OLLAMA_CLOUD_INTERNAL_500_USER_PREFIX.to_ascii_lowercase();
|
||||
message.to_ascii_lowercase().contains(needle.as_str())
|
||||
}
|
||||
|
||||
/// Build the actionable user-facing message for an Ollama-Cloud hosted-inference
|
||||
/// 500, replacing the opaque `Internal Server Error (ref: <uuid>)` body (which
|
||||
/// carries no signal the user can act on) with retry/switch guidance. The model
|
||||
/// is included when known (native/streaming chat); the `api_error` path has no
|
||||
/// model in scope and omits it.
|
||||
pub fn ollama_cloud_internal_500_user_message(
|
||||
model: Option<&str>,
|
||||
status: reqwest::StatusCode,
|
||||
) -> String {
|
||||
let code = status.as_u16();
|
||||
match model {
|
||||
Some(model) => format!(
|
||||
"{OLLAMA_CLOUD_INTERNAL_500_USER_PREFIX} for model `{model}` (Ollama returned HTTP \
|
||||
{code}); the hosted model failed on Ollama's side — retry shortly or switch models."
|
||||
),
|
||||
None => format!(
|
||||
"{OLLAMA_CLOUD_INTERNAL_500_USER_PREFIX} (Ollama returned HTTP {code}); the hosted \
|
||||
model failed on Ollama's side — retry shortly or switch models."
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn log_ollama_cloud_internal_500(
|
||||
operation: &str,
|
||||
provider: &str,
|
||||
model: Option<&str>,
|
||||
status: reqwest::StatusCode,
|
||||
) {
|
||||
tracing::info!(
|
||||
domain = "llm_provider",
|
||||
operation = operation,
|
||||
provider = provider,
|
||||
model = model.unwrap_or(""),
|
||||
status = status.as_u16(),
|
||||
failure = "non_2xx",
|
||||
kind = "ollama_cloud_internal_500",
|
||||
"[llm_provider] {operation} Ollama Cloud hosted-inference 500 — provider-internal \
|
||||
(no client lever), not reporting to Sentry"
|
||||
);
|
||||
}
|
||||
|
||||
/// Whether a provider non-2xx response is a deterministic
|
||||
/// **configuration-rejection** user-state error (unknown model id,
|
||||
/// abstract tier leaked to a custom provider, model-specific temperature
|
||||
@@ -802,6 +894,10 @@ pub async fn api_error(provider: &str, response: reqwest::Response) -> anyhow::E
|
||||
// Balance"). This shared helper backs the two methods that delegate here
|
||||
// (chat_via_responses fallback and the non-streaming completion path).
|
||||
let is_insufficient_credits_402 = is_provider_insufficient_credits_402(status, &body);
|
||||
// Ollama Cloud hosted-inference 500 (`Internal Server Error (ref: <uuid>)`):
|
||||
// provider-internal, non-deterministic, no client lever. Demote from Sentry
|
||||
// and replace the opaque ref body with actionable guidance (TAURI-RUST-5MV).
|
||||
let is_ollama_cloud_internal_500 = is_ollama_cloud_internal_500(provider, status, &body);
|
||||
|
||||
if is_auth_failure && is_backend {
|
||||
// Single source of truth for backend session-expiry handling (warn +
|
||||
@@ -828,6 +924,8 @@ pub async fn api_error(provider: &str, response: reqwest::Response) -> anyhow::E
|
||||
log_openai_oauth_session_expired("api_error", provider, None, status);
|
||||
} else if is_insufficient_credits_402 {
|
||||
log_provider_insufficient_credits_402("api_error", provider, None, status);
|
||||
} else if is_ollama_cloud_internal_500 {
|
||||
log_ollama_cloud_internal_500("api_error", provider, None, status);
|
||||
} else if should_report_provider_http_failure(status) {
|
||||
crate::core::observability::report_error(
|
||||
message.as_str(),
|
||||
@@ -840,6 +938,12 @@ pub async fn api_error(provider: &str, response: reqwest::Response) -> anyhow::E
|
||||
],
|
||||
);
|
||||
}
|
||||
// Replace the opaque `Internal Server Error (ref: <uuid>)` body with
|
||||
// actionable guidance; the prefix anchors the higher-layer re-report
|
||||
// demotion (`is_ollama_cloud_internal_500_message`).
|
||||
if is_ollama_cloud_internal_500 {
|
||||
return anyhow::anyhow!(ollama_cloud_internal_500_user_message(None, status));
|
||||
}
|
||||
anyhow::anyhow!(message)
|
||||
}
|
||||
|
||||
@@ -1032,6 +1136,89 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
/// Verbatim TAURI-RUST-5MV provider body. The matcher keys on the
|
||||
/// `Internal Server Error (ref:` envelope, so coupling the test to the exact
|
||||
/// wire shape makes an Ollama-Cloud wording drift fail CI rather than
|
||||
/// silently leak events back to Sentry.
|
||||
const OLLAMA_CLOUD_500_BODY: &str =
|
||||
"{\"error\":\"Internal Server Error (ref: df512dcb-d915-493b-8f2d-e8d3dfa640c1)\"}";
|
||||
|
||||
#[test]
|
||||
fn ollama_cloud_internal_500_matches_verbatim_5mv_body() {
|
||||
assert!(is_ollama_cloud_internal_500(
|
||||
"ollama",
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
OLLAMA_CLOUD_500_BODY
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ollama_cloud_internal_500_ignores_non_500_status() {
|
||||
// Same body on a non-500 status is not this provider-internal flood —
|
||||
// keep it reportable so a genuine bug elsewhere isn't masked.
|
||||
assert!(!is_ollama_cloud_internal_500(
|
||||
"ollama",
|
||||
StatusCode::BAD_REQUEST,
|
||||
OLLAMA_CLOUD_500_BODY
|
||||
));
|
||||
assert!(!is_ollama_cloud_internal_500(
|
||||
"ollama",
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
OLLAMA_CLOUD_500_BODY
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ollama_cloud_internal_500_ignores_other_providers() {
|
||||
// A 500 with the same envelope from a non-ollama provider stays
|
||||
// reportable — this gate is scoped to ollama.com hosted inference.
|
||||
assert!(!is_ollama_cloud_internal_500(
|
||||
"openai",
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
OLLAMA_CLOUD_500_BODY
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ollama_cloud_internal_500_ignores_local_ollama_500_without_ref() {
|
||||
// A local Ollama daemon 500 (genuine model crash / OOM, worth paging)
|
||||
// does not carry the `ref:` UUID, so it must NOT be swallowed.
|
||||
assert!(!is_ollama_cloud_internal_500(
|
||||
"ollama",
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"{\"error\":\"llama runner process has terminated: exit status 0xc0000409\"}"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ollama_cloud_internal_500_user_message_is_matched_by_message_matcher() {
|
||||
// Couple the prose builder to the re-report matcher so the
|
||||
// `expected_error_kind` / before_send demotion can't drift from the
|
||||
// string the emit sites actually raise.
|
||||
let with_model = ollama_cloud_internal_500_user_message(
|
||||
Some("minimax-m3:cloud"),
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
);
|
||||
assert!(with_model.contains("minimax-m3:cloud"));
|
||||
assert!(!with_model.contains("ref:"));
|
||||
assert!(is_ollama_cloud_internal_500_message(&with_model));
|
||||
|
||||
let without_model =
|
||||
ollama_cloud_internal_500_user_message(None, StatusCode::INTERNAL_SERVER_ERROR);
|
||||
assert!(is_ollama_cloud_internal_500_message(&without_model));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn log_ollama_cloud_internal_500_smoke() {
|
||||
// The helper only emits a demotion info log; calling it covers that path.
|
||||
log_ollama_cloud_internal_500(
|
||||
"native_chat",
|
||||
"ollama",
|
||||
Some("minimax-m3:cloud"),
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_oauth_session_expired_excludes_backend_provider() {
|
||||
// The OpenHuman backend owns app-session expiry via
|
||||
|
||||
@@ -22,14 +22,16 @@ pub use http_error::{
|
||||
api_error, body_indicates_insufficient_credits, body_indicates_quota_exhausted,
|
||||
is_backend_auth_failure, is_backend_error_code_owned, is_budget_exhausted_http_400,
|
||||
is_byo_provider_auth_failure_http, is_context_window_exceeded_message,
|
||||
is_custom_openai_upstream_bad_request_http_400, is_openai_oauth_session_expired_http,
|
||||
is_custom_openai_upstream_bad_request_http_400, is_ollama_cloud_internal_500,
|
||||
is_ollama_cloud_internal_500_message, is_openai_oauth_session_expired_http,
|
||||
is_openai_oauth_session_expired_message, is_provider_access_policy_denied_http_403,
|
||||
is_provider_config_rejection_http, is_provider_insufficient_credits_402,
|
||||
is_provider_quota_exhausted, log_backend_error_code_owned, log_budget_exhausted_http_400,
|
||||
log_byo_provider_auth_failure, log_context_window_exceeded,
|
||||
log_custom_openai_upstream_bad_request_http_400, log_openai_oauth_session_expired,
|
||||
log_provider_access_policy_denied_http_403, log_provider_config_rejection,
|
||||
log_provider_insufficient_credits_402, log_provider_quota_exhausted,
|
||||
log_custom_openai_upstream_bad_request_http_400, log_ollama_cloud_internal_500,
|
||||
log_openai_oauth_session_expired, log_provider_access_policy_denied_http_403,
|
||||
log_provider_config_rejection, log_provider_insufficient_credits_402,
|
||||
log_provider_quota_exhausted, ollama_cloud_internal_500_user_message,
|
||||
publish_backend_session_expired, should_report_provider_http_failure,
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user