diff --git a/src/core/observability.rs b/src/core/observability.rs index 6c3c7d6b0..17c85fedd 100644 --- a/src/core/observability.rs +++ b/src/core/observability.rs @@ -2016,6 +2016,46 @@ pub fn is_transient_provider_transport_failure(event: &sentry::protocol::Event<' event_has_transient_transport_phrase(event) } +/// Defense-in-depth filter for aggregate provider exhaustion events where the +/// aggregate only restates transient attempt failures. +/// +/// Keep ordinary `failure=all_exhausted` events: they are the useful "every +/// fallback failed" signal. Drop only the narrow shape observed in #3542, +/// where the aggregate body starts with the reliable-provider exhaustion +/// prefix and contains transient HTTP/transport wording already classified by +/// [`is_transient_message_failure`]. +pub fn is_all_transient_provider_exhaustion_event(event: &sentry::protocol::Event<'_>) -> bool { + let tags = &event.tags; + if tags.get("domain").map(String::as_str) != Some("llm_provider") { + return false; + } + if tags.get("failure").map(String::as_str) != Some("all_exhausted") { + return false; + } + + let direct = event.message.as_deref(); + let from_logentry = event.logentry.as_ref().map(|log| log.message.as_str()); + let from_exception = event.exception.last().and_then(|e| e.value.as_deref()); + [direct, from_logentry, from_exception] + .into_iter() + .flatten() + .any(all_provider_attempts_are_transient) +} + +fn all_provider_attempts_are_transient(message: &str) -> bool { + let Some(attempts) = message.strip_prefix("All providers/models failed. Attempts:") else { + return false; + }; + let mut saw_attempt = false; + for attempt in attempts.split(';').map(str::trim).filter(|s| !s.is_empty()) { + saw_attempt = true; + if !is_transient_message_failure(attempt) { + return false; + } + } + saw_attempt +} + /// Returns true when a Sentry event's message/exception text contains the /// canonical max-tool-iterations cap phrase (see /// `openhuman::agent::error::MAX_ITERATIONS_ERROR_PREFIX`). diff --git a/src/main.rs b/src/main.rs index 81d963c3a..c3d4c4fd6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -61,6 +61,11 @@ fn main() { if openhuman_core::core::observability::is_transient_provider_http_failure(&event) { return None; } + if openhuman_core::core::observability::is_all_transient_provider_exhaustion_event( + &event, + ) { + return None; + } // Defense-in-depth: drop managed-backend `errorCode` events (#870) // the backend owns (F2/F4) — primary suppression lives in // `api_error` / the streaming gates and the `web_channel` diff --git a/tests/observability_smoke.rs b/tests/observability_smoke.rs index 810985f4d..304694f4e 100644 --- a/tests/observability_smoke.rs +++ b/tests/observability_smoke.rs @@ -9,9 +9,9 @@ //! and aggregate `all_exhausted` events still surface. use openhuman_core::core::observability::{ - is_budget_event, is_session_expired_event, is_transient_backend_api_failure, - is_transient_integrations_failure, is_transient_provider_http_failure, - is_updater_transient_event, + is_all_transient_provider_exhaustion_event, is_budget_event, is_session_expired_event, + is_transient_backend_api_failure, is_transient_integrations_failure, + is_transient_provider_http_failure, is_updater_transient_event, }; use sentry::protocol::Event; use std::collections::BTreeMap; @@ -58,6 +58,7 @@ fn count_captured(events: Vec>) -> usize { // Same filter shape the real binary installs in main.rs. before_send: Some(Arc::new(|event| { if is_transient_provider_http_failure(&event) + || is_all_transient_provider_exhaustion_event(&event) || is_transient_backend_api_failure(&event) || is_transient_integrations_failure(&event) || is_budget_event(&event) @@ -243,6 +244,40 @@ fn keeps_aggregate_all_exhausted_event() { ); } +#[test] +fn drops_aggregate_all_exhausted_when_attempts_are_transient() { + let event = event_with_tags_and_message( + &[ + ("domain", "llm_provider"), + ("failure", "all_exhausted"), + ("attempts", "2"), + ], + "All providers/models failed. Attempts: openai API error (503 Service Unavailable); custom_openai API error (502 Bad Gateway)", + ); + assert_eq!( + count_captured(vec![event]), + 0, + "all-transient aggregate should not recreate per-attempt Sentry noise" + ); +} + +#[test] +fn keeps_aggregate_all_exhausted_with_permanent_attempt() { + let event = event_with_tags_and_message( + &[ + ("domain", "llm_provider"), + ("failure", "all_exhausted"), + ("attempts", "2"), + ], + "All providers/models failed. Attempts: openai API error (401 Unauthorized); custom_openai API error (503 Service Unavailable)", + ); + assert_eq!( + count_captured(vec![event]), + 1, + "mixed/permanent aggregate should remain actionable" + ); +} + #[test] fn keeps_event_missing_status_tag() { // Belt-and-suspenders: an event with `failure=non_2xx` but no `status`