fix(socket): route sustained-outage escalation through observability classifier (OPENHUMAN-TAURI-BH) (#1672)

This commit is contained in:
CodeGhost21
2026-05-13 14:25:06 -07:00
committed by GitHub
parent b7032e65bf
commit ca0920f168
2 changed files with 68 additions and 8 deletions
+24 -8
View File
@@ -161,8 +161,13 @@ pub(super) async fn ws_loop(
///
/// - Below `FAIL_ESCALATE_THRESHOLD`: `warn` — transient blips (DNS, gateway
/// 5xx, TLS resets) stay out of Sentry.
/// - Exactly at the threshold: `error` — fires the one-shot Sentry event that
/// signals a sustained outage.
/// - Exactly at the threshold: routed through
/// [`crate::core::observability::report_error_or_expected`] so transport-
/// level user-environment shapes (`network is unreachable`, `dns error`,
/// `connection refused/reset`, `tls handshake`) demote to a `warn`
/// breadcrumb while genuine outages (gateway 5xx, server-side WebSocket
/// close, malformed handshake) fire exactly one Sentry event per affected
/// client.
/// - Above the threshold: `warn` — already paged once; avoid unbounded events
/// during a long outage.
///
@@ -170,12 +175,23 @@ pub(super) async fn ws_loop(
/// async event loop or touching the WS stack.
fn log_connection_failure(consecutive: u32, reason: &str) {
if consecutive == FAIL_ESCALATE_THRESHOLD {
// One-shot escalation: fire exactly one Sentry-visible `error` event so
// sustained outages surface without generating unbounded events.
log::error!(
"[socket] Connection failed (sustained outage after {} attempts): {}",
consecutive,
reason
// Route the one-shot sustained-outage escalation through the
// observability classifier so an offline user (no wifi / airplane mode
// / `Network is unreachable (os error 51)` — see OPENHUMAN-TAURI-BH)
// does not page on every affected client. Sentry has no signal to act
// on a user being offline — no status, no trace, no payload — so the
// event was pure noise. Genuine outage shapes (gateway 5xx, malformed
// handshake, …) don't match the classifier and still fire one Sentry
// event per affected client, preserving the OPENHUMAN-TAURI-8M intent.
let detailed = format!(
"[socket] Connection failed (sustained outage after {consecutive} attempts): {reason}"
);
let attempts = consecutive.to_string();
crate::core::observability::report_error_or_expected(
detailed.as_str(),
"socket",
"ws_connect",
&[("attempts", attempts.as_str())],
);
} else {
// Below threshold (transient blips) or above threshold (already fired
+44
View File
@@ -278,6 +278,50 @@ fn fail_escalate_threshold_is_five() {
);
}
/// Regression guard for OPENHUMAN-TAURI-BH: the exact wire shape the
/// sustained-outage escalation builds for an offline user
/// (`Network is unreachable (os error 51)`) must classify as a
/// network-unreachable expected error so the observability layer routes
/// it to a warn breadcrumb rather than a Sentry event. If the format
/// string in `log_connection_failure` drifts away from the substrings
/// `is_network_unreachable_message` matches on, an offline Mac will
/// start spamming Sentry again — exactly the regression this guards.
#[test]
fn sustained_outage_for_network_unreachable_classifies_as_expected() {
use crate::core::observability::{expected_error_kind, ExpectedErrorKind};
let reason = "WebSocket connect: IO error: Network is unreachable (os error 51)";
let detailed = format!(
"[socket] Connection failed (sustained outage after {FAIL_ESCALATE_THRESHOLD} attempts): {reason}"
);
assert_eq!(
expected_error_kind(&detailed),
Some(ExpectedErrorKind::NetworkUnreachable),
"offline-user shape must classify as expected; got message: {detailed}"
);
}
/// Counterpart: a genuine outage that lacks any of the transport-level
/// markers (e.g. a server-side HTTP 500 wrapped by tungstenite) must
/// still surface as an actionable Sentry event — i.e. not classify as
/// any expected kind. Pins the OPENHUMAN-TAURI-8M invariant ("one event
/// per sustained outage") so the BH fix doesn't accidentally silence
/// real outages.
#[test]
fn sustained_outage_for_actionable_server_error_does_not_classify() {
use crate::core::observability::expected_error_kind;
let reason = "SIO CONNECT: Socket.IO connect error: internal server error";
let detailed = format!(
"[socket] Connection failed (sustained outage after {FAIL_ESCALATE_THRESHOLD} attempts): {reason}"
);
assert_eq!(
expected_error_kind(&detailed),
None,
"actionable outage must not be silenced; got message: {detailed}"
);
}
// ── End-to-end handshake tests against a local WS server ───────
//
// These tests drive the real `ws_loop` / `run_connection` code path