From 4764493b7d9bdab54d36d0d267bbd62691bed3dd Mon Sep 17 00:00:00 2001 From: oxoxDev <164490987+oxoxDev@users.noreply.github.com> Date: Wed, 24 Jun 2026 22:38:33 +0530 Subject: [PATCH] fix(inference): demote BYO insufficient-credits 402 across all compatible-provider paths (#3961) (#3976) --- .../inference/provider/compatible_helpers.rs | 14 + .../provider/compatible_provider_impl.rs | 48 ++++ .../provider/compatible_stream_native.rs | 14 + .../inference/provider/compatible_tests.rs | 248 ++++++++++++++++++ .../inference/provider/ops/http_error.rs | 10 + 5 files changed, 334 insertions(+) diff --git a/src/openhuman/inference/provider/compatible_helpers.rs b/src/openhuman/inference/provider/compatible_helpers.rs index f01603215..3c3b4d49f 100644 --- a/src/openhuman/inference/provider/compatible_helpers.rs +++ b/src/openhuman/inference/provider/compatible_helpers.rs @@ -133,6 +133,20 @@ impl OpenAiCompatibleProvider { Some(model), status, ); + } else if super::super::is_provider_insufficient_credits_402(status, &error) { + // Insufficient-credits 402: the user's own BYO provider account + // is out of balance — a flat billing fact, not a reservation- + // window error, so there is NO local max_tokens lever to apply. + // Demote to info instead of paging on every retry; this is the + // complete classification for a genuinely-unpreventable + // BYO-balance condition + // (TAURI-RUST-4QF — DeepSeek "Insufficient Balance"). + super::super::log_provider_insufficient_credits_402( + "responses_api", + self.name.as_str(), + Some(model), + status, + ); } else if super::super::should_report_provider_http_failure(status) { crate::core::observability::report_error( message.as_str(), diff --git a/src/openhuman/inference/provider/compatible_provider_impl.rs b/src/openhuman/inference/provider/compatible_provider_impl.rs index 023248c4d..37359ce34 100644 --- a/src/openhuman/inference/provider/compatible_provider_impl.rs +++ b/src/openhuman/inference/provider/compatible_provider_impl.rs @@ -203,6 +203,20 @@ impl Provider for OpenAiCompatibleProvider { Some(model), status, ); + } else if super::super::is_provider_insufficient_credits_402(status, &error) { + // Insufficient-credits 402: the user's own BYO provider account + // is out of balance — a flat billing fact, not a reservation- + // window error, so there is NO local max_tokens lever to apply. + // Demote to info instead of paging on every retry; this is the + // complete classification for a genuinely-unpreventable + // BYO-balance condition (TAURI-RUST-4QF — DeepSeek "Insufficient + // Balance"; same class as the native_chat arm added for -C62). + super::super::log_provider_insufficient_credits_402( + "chat_completions", + self.name.as_str(), + Some(model), + status, + ); } else if super::super::should_report_provider_http_failure(status) { crate::core::observability::report_error( message.as_str(), @@ -1005,6 +1019,23 @@ impl Provider for OpenAiCompatibleProvider { Some(model_owned.as_str()), status, ); + } else if crate::openhuman::inference::provider::is_provider_insufficient_credits_402( + status, + &raw_error, + ) { + // Insufficient-credits 402: the user's own BYO provider + // account is out of balance — a flat billing fact, not a + // reservation-window error, so there is NO local max_tokens + // lever to apply. Demote to info instead of paging on every + // retry; complete classification for a genuinely- + // unpreventable BYO-balance condition + // (TAURI-RUST-4QF — DeepSeek "Insufficient Balance"). + crate::openhuman::inference::provider::log_provider_insufficient_credits_402( + "stream_chat", + provider_name.as_str(), + Some(model_owned.as_str()), + status, + ); } else if crate::openhuman::inference::provider::should_report_provider_http_failure( status, ) { @@ -1222,6 +1253,23 @@ impl Provider for OpenAiCompatibleProvider { Some(model_owned.as_str()), status, ); + } else if crate::openhuman::inference::provider::is_provider_insufficient_credits_402( + status, + &raw_error, + ) { + // Insufficient-credits 402: the user's own BYO provider + // account is out of balance — a flat billing fact, not a + // reservation-window error, so there is NO local max_tokens + // lever to apply. Demote to info instead of paging on every + // retry; complete classification for a genuinely- + // unpreventable BYO-balance condition + // (TAURI-RUST-4QF — DeepSeek "Insufficient Balance"). + crate::openhuman::inference::provider::log_provider_insufficient_credits_402( + "stream_chat_history", + provider_name.as_str(), + Some(model_owned.as_str()), + status, + ); } else if crate::openhuman::inference::provider::should_report_provider_http_failure( status, ) { diff --git a/src/openhuman/inference/provider/compatible_stream_native.rs b/src/openhuman/inference/provider/compatible_stream_native.rs index 062140531..99f7eacde 100644 --- a/src/openhuman/inference/provider/compatible_stream_native.rs +++ b/src/openhuman/inference/provider/compatible_stream_native.rs @@ -133,6 +133,20 @@ impl OpenAiCompatibleProvider { Some(native_request.model.as_str()), status, ); + } else if super::super::is_provider_insufficient_credits_402(status, &body) { + // Insufficient-credits 402: the user's own BYO provider account + // is out of balance — a flat billing fact, not a reservation- + // window error, so there is NO local max_tokens lever to apply. + // Demote to info instead of paging on every retry; this is the + // complete classification for a genuinely-unpreventable + // BYO-balance condition + // (TAURI-RUST-4QF — DeepSeek "Insufficient Balance"). + super::super::log_provider_insufficient_credits_402( + "streaming_chat", + self.name.as_str(), + Some(native_request.model.as_str()), + status, + ); } else if super::super::should_report_provider_http_failure(status) { crate::core::observability::report_error( message.as_str(), diff --git a/src/openhuman/inference/provider/compatible_tests.rs b/src/openhuman/inference/provider/compatible_tests.rs index cf74b56e1..176cf159d 100644 --- a/src/openhuman/inference/provider/compatible_tests.rs +++ b/src/openhuman/inference/provider/compatible_tests.rs @@ -3709,6 +3709,254 @@ async fn effective_context_window_lmstudio_falls_back_when_native_unavailable() ); } +/// Verbatim TAURI-RUST-4QF DeepSeek 402 body. The demote arms key on the +/// "insufficient balance" phrase via `body_indicates_insufficient_credits`, so +/// pinning the exact wire shape makes a provider wording drift fail CI rather +/// than silently re-flood Sentry (23,970 events from 17 BYO users). +const DEEPSEEK_4QF_402_BODY: &str = r#"{"error":{"message":"Insufficient Balance","type":"unknown_error","param":null,"code":"invalid_request_error"}}"#; + +/// Build a Sentry hub backed by a `TestTransport` and install it for the +/// current scope, returning the transport so the test can assert no events +/// were captured. Mirrors the inline setup used by the other +/// `*_not_reported_to_sentry` guards above. +fn install_test_sentry_hub() -> (Arc, sentry::HubSwitchGuard) { + let transport = TestTransport::new(); + let options = sentry::ClientOptions { + dsn: Some("https://public@sentry.invalid/1".parse().unwrap()), + transport: Some(Arc::new(transport.clone())), + ..Default::default() + }; + let hub = Arc::new(sentry::Hub::new( + Some(Arc::new(options.into())), + Arc::new(Default::default()), + )); + let guard = sentry::HubSwitchGuard::new(hub); + (transport, guard) +} + +#[tokio::test] +async fn chat_with_system_insufficient_balance_402_not_reported_to_sentry() { + // TAURI-RUST-4QF: a BYO DeepSeek account out of balance 402s on every agent + // turn. The non-streaming `chat_with_system` path (operation=chat_completions + // in the Sentry tags) must still propagate the error, but must NOT page + // Sentry — the user's own provider balance is exhausted, no local lever. + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with(ResponseTemplate::new(402).set_body_string(DEEPSEEK_4QF_402_BODY)) + .mount(&mock_server) + .await; + + let (transport, _guard) = install_test_sentry_hub(); + + let provider = make_provider("deepseek", &mock_server.uri(), Some("byo-key")); + let err = provider + .chat_with_system(None, "hello", "deepseek-v4-flash", 0.7) + .await + .expect_err("a 402 insufficient-balance must still propagate as Err"); + assert!(err.to_string().contains("402"), "err: {err}"); + assert!( + transport.fetch_and_clear_events().is_empty(), + "an exhausted BYO balance 402 must not be reported to Sentry" + ); +} + +#[tokio::test] +async fn chat_with_history_insufficient_balance_402_not_reported_to_sentry() { + // Same 402 user-state reached through `chat_with_history`, which routes its + // non-2xx through the shared `api_error` helper. Guards the api_error arm. + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with(ResponseTemplate::new(402).set_body_string(DEEPSEEK_4QF_402_BODY)) + .mount(&mock_server) + .await; + + let (transport, _guard) = install_test_sentry_hub(); + + let provider = make_provider("deepseek", &mock_server.uri(), Some("byo-key")); + let err = provider + .chat_with_history(&[ChatMessage::user("hello")], "deepseek-v4-flash", 0.0) + .await + .expect_err("a 402 insufficient-balance must still propagate as Err"); + assert!(err.to_string().contains("402"), "err: {err}"); + assert!( + transport.fetch_and_clear_events().is_empty(), + "an exhausted BYO balance 402 (api_error path) must not be reported to Sentry" + ); +} + +#[tokio::test] +async fn streaming_chat_insufficient_balance_402_not_reported_to_sentry() { + // The streaming entry (`stream_native_chat`, operation=streaming_chat) has + // its own non-2xx cascade. A 402 insufficient-balance there must demote too. + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with(ResponseTemplate::new(402).set_body_string(DEEPSEEK_4QF_402_BODY)) + .mount(&mock_server) + .await; + + let (transport, _guard) = install_test_sentry_hub(); + + let provider = + OpenAiCompatibleProvider::new("deepseek", &mock_server.uri(), None, AuthStyle::None); + let request = NativeChatRequest { + model: "deepseek-v4-flash".to_string(), + messages: vec![NativeMessage { + role: "user".to_string(), + content: Some("hello".into()), + tool_call_id: None, + tool_calls: None, + reasoning_content: None, + }], + temperature: Some(0.7), + stream: Some(true), + tools: None, + tool_choice: None, + thread_id: None, + stream_options: Some(super::compatible_types::OpenAiStreamOptions { + include_usage: true, + }), + options: None, + frequency_penalty: None, + max_tokens: None, + }; + let (delta_tx, _delta_rx) = tokio::sync::mpsc::channel(8); + + let err = provider + .stream_native_chat(None, &request, &delta_tx, 0) + .await + .expect_err("a 402 insufficient-balance must still propagate as Err"); + assert!( + err.to_string().contains("streaming API error"), + "err: {err}" + ); + assert!( + transport.fetch_and_clear_events().is_empty(), + "an exhausted BYO balance 402 (streaming path) must not be reported to Sentry" + ); +} + +#[tokio::test] +async fn responses_api_insufficient_balance_402_not_reported_to_sentry() { + // The responses-API path (`chat_via_responses`, operation=responses_api) is + // reached when the provider is configured responses-primary. A 402 + // insufficient-balance there must demote, not page Sentry. This path is + // awaited inline, so the TestTransport assertion is authoritative. + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(402).set_body_string(DEEPSEEK_4QF_402_BODY)) + .mount(&mock_server) + .await; + + let (transport, _guard) = install_test_sentry_hub(); + + let provider = OpenAiCompatibleProvider::new( + "deepseek", + &mock_server.uri(), + Some("byo-key"), + AuthStyle::Bearer, + ) + .with_responses_api_primary(); + let err = provider + .chat_with_history(&[ChatMessage::user("hello")], "deepseek-v4-flash", 0.0) + .await + .expect_err("a 402 insufficient-balance must still propagate as Err"); + // The responses-API error message is ` Responses API error: ` + // — note it carries NO "(402" status token, so the message-keyed before_send + // net (#3913, `is_insufficient_credits_message`) would MISS it. Only the + // status-based source arm here catches it, which the empty-transport + // assertion below proves. + assert!( + err.to_string().contains("Insufficient Balance"), + "err: {err}" + ); + assert!( + transport.fetch_and_clear_events().is_empty(), + "an exhausted BYO balance 402 (responses_api path) must not be reported to Sentry" + ); +} + +#[tokio::test] +async fn stream_chat_with_system_insufficient_balance_402_propagates_error() { + // Exercises the `stream_chat_with_system` (operation=stream_chat) 402 + // insufficient-balance demote arm. The streaming work runs in a detached + // tokio task whose Sentry hub is NOT the test hub, so a TestTransport + // assertion would be vacuous here; instead assert the error still + // propagates as a terminal Provider chunk (the demote arm and the report + // fallback bail the same message, so this guards line execution + error + // shape). The Sentry-suppression behavior for this exact body is proven by + // the inline-awaited native/api tests above and the + // `detects_insufficient_balance_402_family` predicate test. + use crate::openhuman::inference::provider::traits::{StreamError, StreamOptions}; + use futures_util::StreamExt; + + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with(ResponseTemplate::new(402).set_body_string(DEEPSEEK_4QF_402_BODY)) + .mount(&mock_server) + .await; + + let provider = + OpenAiCompatibleProvider::new("deepseek", &mock_server.uri(), None, AuthStyle::None); + let mut stream = provider.stream_chat_with_system( + None, + "hello", + "deepseek-v4-flash", + 0.7, + StreamOptions::new(true), + ); + let mut terminal: Option = None; + while let Some(item) = stream.next().await { + if let Err(StreamError::Provider(msg)) = item { + terminal = Some(msg); + } + } + let msg = terminal.expect("a 402 must yield a terminal Provider error chunk"); + assert!( + msg.contains("402") && msg.contains("Insufficient Balance"), + "msg: {msg}" + ); +} + +#[tokio::test] +async fn stream_chat_with_history_insufficient_balance_402_propagates_error() { + // Sibling of the above for `stream_chat_with_history` + // (operation=stream_chat_history). Same detached-task caveat — asserts the + // 402 path executes and propagates as a terminal Provider error chunk. + use crate::openhuman::inference::provider::traits::{StreamError, StreamOptions}; + use futures_util::StreamExt; + + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with(ResponseTemplate::new(402).set_body_string(DEEPSEEK_4QF_402_BODY)) + .mount(&mock_server) + .await; + + let provider = + OpenAiCompatibleProvider::new("deepseek", &mock_server.uri(), None, AuthStyle::None); + let mut stream = provider.stream_chat_with_history( + &[ChatMessage::user("hello")], + "deepseek-v4-flash", + 0.7, + StreamOptions::new(true), + ); + let mut terminal: Option = None; + while let Some(item) = stream.next().await { + if let Err(StreamError::Provider(msg)) = item { + terminal = Some(msg); + } + } + let msg = terminal.expect("a 402 must yield a terminal Provider error chunk"); + assert!( + msg.contains("402") && msg.contains("Insufficient Balance"), + "msg: {msg}" + ); +} + // ---------------------------------------------------------- // Prompt-cache capability model (#3939) // ---------------------------------------------------------- diff --git a/src/openhuman/inference/provider/ops/http_error.rs b/src/openhuman/inference/provider/ops/http_error.rs index 0d3f0bc51..95108d0cc 100644 --- a/src/openhuman/inference/provider/ops/http_error.rs +++ b/src/openhuman/inference/provider/ops/http_error.rs @@ -719,6 +719,14 @@ pub async fn api_error(provider: &str, response: reqwest::Response) -> anyhow::E // from Sentry (TAURI-RUST-8FQ flood). let is_openai_oauth_session_expired = is_openai_oauth_session_expired_http(provider, status, &body); + // Insufficient-credits 402: the user's own BYO provider account is out of + // balance — a flat billing fact, not a reservation-window error, so there is + // NO local max_tokens lever to apply. Demote from Sentry like the per-method + // compatible-provider arms; the complete classification for a genuinely- + // unpreventable BYO-balance condition (TAURI-RUST-4QF DeepSeek "Insufficient + // Balance"). This shared helper backs the two methods that delegate here + // (chat_via_responses fallback and the non-streaming completion path). + let is_insufficient_credits_402 = is_provider_insufficient_credits_402(status, &body); if is_auth_failure && is_backend { // Single source of truth for backend session-expiry handling (warn + @@ -741,6 +749,8 @@ pub async fn api_error(provider: &str, response: reqwest::Response) -> anyhow::E log_byo_provider_auth_failure("api_error", provider, None, status); } else if is_openai_oauth_session_expired { log_openai_oauth_session_expired("api_error", provider, None, status); + } else if is_insufficient_credits_402 { + log_provider_insufficient_credits_402("api_error", provider, None, status); } else if should_report_provider_http_failure(status) { crate::core::observability::report_error( message.as_str(),