//! Runtime smoke for the Sentry `before_send` filters that drop per-attempt //! transient-upstream provider, backend_api, integrations, and updater //! failures plus budget-exhausted user-state 400s (OPENHUMAN-TAURI-3M / 12 / 13). //! //! Unit tests in `src/core/observability.rs` exercise the pure filter //! function. This integration test wires the actual `sentry::init` → //! `before_send` → transport chain so we have proof the runtime path //! behaves as designed: transient events are dropped, permanent events //! and aggregate `all_exhausted` events still surface. use openhuman_core::core::observability::{ is_all_transient_provider_exhaustion_event, is_budget_event, is_session_expired_event, is_transient_backend_api_failure, is_transient_integrations_failure, is_transient_provider_http_failure, is_updater_transient_event, }; use sentry::protocol::Event; use std::collections::BTreeMap; use std::sync::Arc; fn event_with_tags(tags: &[(&str, &str)]) -> Event<'static> { let mut event = Event::default(); let mut t: BTreeMap = BTreeMap::new(); for (k, v) in tags { t.insert((*k).to_string(), (*v).to_string()); } event.tags = t; event } fn event_with_tags_and_message(tags: &[(&str, &str)], message: &str) -> Event<'static> { let mut event = event_with_tags(tags); event.message = Some(message.to_string()); event } /// Drive an envelope-capturing Sentry client through a sequence of events /// and return how many made it past `before_send`. /// /// `sentry::init` mutates the process-global Sentry hub; Cargo runs integration /// test functions in parallel threads by default, so two `count_captured` calls /// would otherwise race on the global hub and one test's `capture_event` could /// land in another test's transport. Serialize the critical section here rather /// than imposing `--test-threads=1` on the whole binary. fn count_captured(events: Vec>) -> usize { static SENTRY_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); let _guard = SENTRY_TEST_LOCK .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); let transport = sentry::test::TestTransport::new(); let transport_for_factory = transport.clone(); let options = sentry::ClientOptions { dsn: Some( "https://public@sentry.example.com/1" .parse() .expect("dsn parses"), ), // Same filter shape the real binary installs in main.rs. before_send: Some(Arc::new(|event| { if is_transient_provider_http_failure(&event) || is_all_transient_provider_exhaustion_event(&event) || is_transient_backend_api_failure(&event) || is_transient_integrations_failure(&event) || is_budget_event(&event) || is_updater_transient_event(&event) || is_session_expired_event(&event) { None } else { Some(event) } })), transport: Some(Arc::new(move |_opts: &sentry::ClientOptions| { transport_for_factory.clone() as Arc })), sample_rate: 1.0, ..sentry::ClientOptions::default() }; let _sentry_guard = sentry::init(options); for event in events { sentry::capture_event(event); } sentry::Hub::current() .client() .map(|c| c.flush(Some(std::time::Duration::from_secs(2)))); transport.fetch_and_clear_envelopes().len() } #[test] fn drops_updater_transient_check_failure() { let event = event_with_tags_and_message( &[], "failed to check for updates: error sending request for url \ (https://github.com/tinyhumansai/openhuman/releases/latest/download/latest.json)", ); assert_eq!( count_captured(vec![event]), 0, "transient updater check failures must be filtered in before_send" ); } #[test] fn drops_backend_api_transient_statuses() { let events = ["408", "429", "502", "503", "504", "520"] .into_iter() .map(|status| { event_with_tags(&[ ("domain", "backend_api"), ("failure", "non_2xx"), ("status", status), ]) }) .collect(); assert_eq!( count_captured(events), 0, "transient backend_api statuses must be filtered in before_send" ); } #[test] fn drops_integrations_transient_transport_timeout() { let event = event_with_tags_and_message( &[("domain", "integrations"), ("failure", "transport")], "GET /agent-integrations/tools failed: operation timed out", ); assert_eq!( count_captured(vec![event]), 0, "transient integrations timeouts must be filtered in before_send" ); } #[test] fn drops_budget_exhausted_400() { let event = event_with_tags_and_message( &[ ("domain", "llm_provider"), ("failure", "non_2xx"), ("status", "400"), ], r#"OpenHuman API error (400 Bad Request): {"success":false,"error":"Insufficient budget"}"#, ); assert_eq!( count_captured(vec![event]), 0, "budget-exhausted 400s must be filtered in before_send" ); } #[test] fn keeps_non_budget_400() { let event = event_with_tags_and_message( &[ ("domain", "llm_provider"), ("failure", "non_2xx"), ("status", "400"), ], "Bad request: missing field", ); assert_eq!( count_captured(vec![event]), 1, "non-budget 400s must still reach Sentry" ); } #[test] fn drops_per_attempt_429_503_504_408_502() { // Each of these matches the tag shape `ops::api_error` sets when a // transient upstream status returns. With the filter installed in // before_send, none should leak through to the transport. let events = ["429", "503", "504", "408", "502"] .into_iter() .map(|status| { event_with_tags(&[ ("domain", "llm_provider"), ("failure", "non_2xx"), ("status", status), ]) }) .collect(); assert_eq!( count_captured(events), 0, "transient per-attempt failures must be filtered in before_send" ); } #[test] fn keeps_permanent_failures() { // 4xx auth / not-found / etc. and 500 internal errors are actionable — // they must reach Sentry exactly as before. let events = ["400", "401", "403", "404", "500"] .into_iter() .map(|status| { event_with_tags(&[ ("domain", "llm_provider"), ("failure", "non_2xx"), ("status", status), ]) }) .collect(); assert_eq!( count_captured(events), 5, "permanent provider failures must reach Sentry" ); } #[test] fn keeps_backend_api_404_failure() { let event = event_with_tags(&[ ("domain", "backend_api"), ("failure", "non_2xx"), ("status", "404"), ]); assert_eq!( count_captured(vec![event]), 1, "non-transient backend_api 404 failures must reach Sentry" ); } #[test] fn keeps_aggregate_all_exhausted_event() { // The reliable_chat layer fires a single aggregate // `failure=all_exhausted` event when every provider/model has been // tried. That's the cascade signal we want — only the per-attempt // noise gets dropped. let event = event_with_tags(&[ ("domain", "llm_provider"), ("failure", "all_exhausted"), ("model", "claude-haiku-4-5-20251001"), ("attempts", "12"), ]); assert_eq!( count_captured(vec![event]), 1, "aggregate all_exhausted event must surface for genuine outages" ); } #[test] fn drops_aggregate_all_exhausted_when_attempts_are_transient() { let event = event_with_tags_and_message( &[ ("domain", "llm_provider"), ("failure", "all_exhausted"), ("attempts", "2"), ], "All providers/models failed. Attempts: openai API error (503 Service Unavailable); custom_openai API error (502 Bad Gateway)", ); assert_eq!( count_captured(vec![event]), 0, "all-transient aggregate should not recreate per-attempt Sentry noise" ); } #[test] fn keeps_aggregate_all_exhausted_with_permanent_attempt() { let event = event_with_tags_and_message( &[ ("domain", "llm_provider"), ("failure", "all_exhausted"), ("attempts", "2"), ], "All providers/models failed. Attempts: openai API error (401 Unauthorized); custom_openai API error (503 Service Unavailable)", ); assert_eq!( count_captured(vec![event]), 1, "mixed/permanent aggregate should remain actionable" ); } #[test] fn keeps_event_missing_status_tag() { // Belt-and-suspenders: an event with `failure=non_2xx` but no `status` // tag (e.g. a future call site forgets to attach one) must NOT be // silently dropped — we'd rather see it and fix the tag emission. let event = event_with_tags(&[("domain", "llm_provider"), ("failure", "non_2xx")]); assert_eq!( count_captured(vec![event]), 1, "event without status tag must not be silently dropped" ); }