mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
fix(inference): demote BYO insufficient-credits 402 across all compatible-provider paths (#3961) (#3976)
This commit is contained in:
@@ -133,6 +133,20 @@ impl OpenAiCompatibleProvider {
|
||||
Some(model),
|
||||
status,
|
||||
);
|
||||
} else if super::super::is_provider_insufficient_credits_402(status, &error) {
|
||||
// Insufficient-credits 402: the user's own BYO provider account
|
||||
// is out of balance — a flat billing fact, not a reservation-
|
||||
// window error, so there is NO local max_tokens lever to apply.
|
||||
// Demote to info instead of paging on every retry; this is the
|
||||
// complete classification for a genuinely-unpreventable
|
||||
// BYO-balance condition
|
||||
// (TAURI-RUST-4QF — DeepSeek "Insufficient Balance").
|
||||
super::super::log_provider_insufficient_credits_402(
|
||||
"responses_api",
|
||||
self.name.as_str(),
|
||||
Some(model),
|
||||
status,
|
||||
);
|
||||
} else if super::super::should_report_provider_http_failure(status) {
|
||||
crate::core::observability::report_error(
|
||||
message.as_str(),
|
||||
|
||||
@@ -203,6 +203,20 @@ impl Provider for OpenAiCompatibleProvider {
|
||||
Some(model),
|
||||
status,
|
||||
);
|
||||
} else if super::super::is_provider_insufficient_credits_402(status, &error) {
|
||||
// Insufficient-credits 402: the user's own BYO provider account
|
||||
// is out of balance — a flat billing fact, not a reservation-
|
||||
// window error, so there is NO local max_tokens lever to apply.
|
||||
// Demote to info instead of paging on every retry; this is the
|
||||
// complete classification for a genuinely-unpreventable
|
||||
// BYO-balance condition (TAURI-RUST-4QF — DeepSeek "Insufficient
|
||||
// Balance"; same class as the native_chat arm added for -C62).
|
||||
super::super::log_provider_insufficient_credits_402(
|
||||
"chat_completions",
|
||||
self.name.as_str(),
|
||||
Some(model),
|
||||
status,
|
||||
);
|
||||
} else if super::super::should_report_provider_http_failure(status) {
|
||||
crate::core::observability::report_error(
|
||||
message.as_str(),
|
||||
@@ -1005,6 +1019,23 @@ impl Provider for OpenAiCompatibleProvider {
|
||||
Some(model_owned.as_str()),
|
||||
status,
|
||||
);
|
||||
} else if crate::openhuman::inference::provider::is_provider_insufficient_credits_402(
|
||||
status,
|
||||
&raw_error,
|
||||
) {
|
||||
// Insufficient-credits 402: the user's own BYO provider
|
||||
// account is out of balance — a flat billing fact, not a
|
||||
// reservation-window error, so there is NO local max_tokens
|
||||
// lever to apply. Demote to info instead of paging on every
|
||||
// retry; complete classification for a genuinely-
|
||||
// unpreventable BYO-balance condition
|
||||
// (TAURI-RUST-4QF — DeepSeek "Insufficient Balance").
|
||||
crate::openhuman::inference::provider::log_provider_insufficient_credits_402(
|
||||
"stream_chat",
|
||||
provider_name.as_str(),
|
||||
Some(model_owned.as_str()),
|
||||
status,
|
||||
);
|
||||
} else if crate::openhuman::inference::provider::should_report_provider_http_failure(
|
||||
status,
|
||||
) {
|
||||
@@ -1222,6 +1253,23 @@ impl Provider for OpenAiCompatibleProvider {
|
||||
Some(model_owned.as_str()),
|
||||
status,
|
||||
);
|
||||
} else if crate::openhuman::inference::provider::is_provider_insufficient_credits_402(
|
||||
status,
|
||||
&raw_error,
|
||||
) {
|
||||
// Insufficient-credits 402: the user's own BYO provider
|
||||
// account is out of balance — a flat billing fact, not a
|
||||
// reservation-window error, so there is NO local max_tokens
|
||||
// lever to apply. Demote to info instead of paging on every
|
||||
// retry; complete classification for a genuinely-
|
||||
// unpreventable BYO-balance condition
|
||||
// (TAURI-RUST-4QF — DeepSeek "Insufficient Balance").
|
||||
crate::openhuman::inference::provider::log_provider_insufficient_credits_402(
|
||||
"stream_chat_history",
|
||||
provider_name.as_str(),
|
||||
Some(model_owned.as_str()),
|
||||
status,
|
||||
);
|
||||
} else if crate::openhuman::inference::provider::should_report_provider_http_failure(
|
||||
status,
|
||||
) {
|
||||
|
||||
@@ -133,6 +133,20 @@ impl OpenAiCompatibleProvider {
|
||||
Some(native_request.model.as_str()),
|
||||
status,
|
||||
);
|
||||
} else if super::super::is_provider_insufficient_credits_402(status, &body) {
|
||||
// Insufficient-credits 402: the user's own BYO provider account
|
||||
// is out of balance — a flat billing fact, not a reservation-
|
||||
// window error, so there is NO local max_tokens lever to apply.
|
||||
// Demote to info instead of paging on every retry; this is the
|
||||
// complete classification for a genuinely-unpreventable
|
||||
// BYO-balance condition
|
||||
// (TAURI-RUST-4QF — DeepSeek "Insufficient Balance").
|
||||
super::super::log_provider_insufficient_credits_402(
|
||||
"streaming_chat",
|
||||
self.name.as_str(),
|
||||
Some(native_request.model.as_str()),
|
||||
status,
|
||||
);
|
||||
} else if super::super::should_report_provider_http_failure(status) {
|
||||
crate::core::observability::report_error(
|
||||
message.as_str(),
|
||||
|
||||
@@ -3709,6 +3709,254 @@ async fn effective_context_window_lmstudio_falls_back_when_native_unavailable()
|
||||
);
|
||||
}
|
||||
|
||||
/// Verbatim TAURI-RUST-4QF DeepSeek 402 body. The demote arms key on the
|
||||
/// "insufficient balance" phrase via `body_indicates_insufficient_credits`, so
|
||||
/// pinning the exact wire shape makes a provider wording drift fail CI rather
|
||||
/// than silently re-flood Sentry (23,970 events from 17 BYO users).
|
||||
const DEEPSEEK_4QF_402_BODY: &str = r#"{"error":{"message":"Insufficient Balance","type":"unknown_error","param":null,"code":"invalid_request_error"}}"#;
|
||||
|
||||
/// Build a Sentry hub backed by a `TestTransport` and install it for the
|
||||
/// current scope, returning the transport so the test can assert no events
|
||||
/// were captured. Mirrors the inline setup used by the other
|
||||
/// `*_not_reported_to_sentry` guards above.
|
||||
fn install_test_sentry_hub() -> (Arc<TestTransport>, sentry::HubSwitchGuard) {
|
||||
let transport = TestTransport::new();
|
||||
let options = sentry::ClientOptions {
|
||||
dsn: Some("https://public@sentry.invalid/1".parse().unwrap()),
|
||||
transport: Some(Arc::new(transport.clone())),
|
||||
..Default::default()
|
||||
};
|
||||
let hub = Arc::new(sentry::Hub::new(
|
||||
Some(Arc::new(options.into())),
|
||||
Arc::new(Default::default()),
|
||||
));
|
||||
let guard = sentry::HubSwitchGuard::new(hub);
|
||||
(transport, guard)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chat_with_system_insufficient_balance_402_not_reported_to_sentry() {
|
||||
// TAURI-RUST-4QF: a BYO DeepSeek account out of balance 402s on every agent
|
||||
// turn. The non-streaming `chat_with_system` path (operation=chat_completions
|
||||
// in the Sentry tags) must still propagate the error, but must NOT page
|
||||
// Sentry — the user's own provider balance is exhausted, no local lever.
|
||||
let mock_server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/chat/completions"))
|
||||
.respond_with(ResponseTemplate::new(402).set_body_string(DEEPSEEK_4QF_402_BODY))
|
||||
.mount(&mock_server)
|
||||
.await;
|
||||
|
||||
let (transport, _guard) = install_test_sentry_hub();
|
||||
|
||||
let provider = make_provider("deepseek", &mock_server.uri(), Some("byo-key"));
|
||||
let err = provider
|
||||
.chat_with_system(None, "hello", "deepseek-v4-flash", 0.7)
|
||||
.await
|
||||
.expect_err("a 402 insufficient-balance must still propagate as Err");
|
||||
assert!(err.to_string().contains("402"), "err: {err}");
|
||||
assert!(
|
||||
transport.fetch_and_clear_events().is_empty(),
|
||||
"an exhausted BYO balance 402 must not be reported to Sentry"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chat_with_history_insufficient_balance_402_not_reported_to_sentry() {
|
||||
// Same 402 user-state reached through `chat_with_history`, which routes its
|
||||
// non-2xx through the shared `api_error` helper. Guards the api_error arm.
|
||||
let mock_server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/chat/completions"))
|
||||
.respond_with(ResponseTemplate::new(402).set_body_string(DEEPSEEK_4QF_402_BODY))
|
||||
.mount(&mock_server)
|
||||
.await;
|
||||
|
||||
let (transport, _guard) = install_test_sentry_hub();
|
||||
|
||||
let provider = make_provider("deepseek", &mock_server.uri(), Some("byo-key"));
|
||||
let err = provider
|
||||
.chat_with_history(&[ChatMessage::user("hello")], "deepseek-v4-flash", 0.0)
|
||||
.await
|
||||
.expect_err("a 402 insufficient-balance must still propagate as Err");
|
||||
assert!(err.to_string().contains("402"), "err: {err}");
|
||||
assert!(
|
||||
transport.fetch_and_clear_events().is_empty(),
|
||||
"an exhausted BYO balance 402 (api_error path) must not be reported to Sentry"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_chat_insufficient_balance_402_not_reported_to_sentry() {
|
||||
// The streaming entry (`stream_native_chat`, operation=streaming_chat) has
|
||||
// its own non-2xx cascade. A 402 insufficient-balance there must demote too.
|
||||
let mock_server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/chat/completions"))
|
||||
.respond_with(ResponseTemplate::new(402).set_body_string(DEEPSEEK_4QF_402_BODY))
|
||||
.mount(&mock_server)
|
||||
.await;
|
||||
|
||||
let (transport, _guard) = install_test_sentry_hub();
|
||||
|
||||
let provider =
|
||||
OpenAiCompatibleProvider::new("deepseek", &mock_server.uri(), None, AuthStyle::None);
|
||||
let request = NativeChatRequest {
|
||||
model: "deepseek-v4-flash".to_string(),
|
||||
messages: vec![NativeMessage {
|
||||
role: "user".to_string(),
|
||||
content: Some("hello".into()),
|
||||
tool_call_id: None,
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
}],
|
||||
temperature: Some(0.7),
|
||||
stream: Some(true),
|
||||
tools: None,
|
||||
tool_choice: None,
|
||||
thread_id: None,
|
||||
stream_options: Some(super::compatible_types::OpenAiStreamOptions {
|
||||
include_usage: true,
|
||||
}),
|
||||
options: None,
|
||||
frequency_penalty: None,
|
||||
max_tokens: None,
|
||||
};
|
||||
let (delta_tx, _delta_rx) = tokio::sync::mpsc::channel(8);
|
||||
|
||||
let err = provider
|
||||
.stream_native_chat(None, &request, &delta_tx, 0)
|
||||
.await
|
||||
.expect_err("a 402 insufficient-balance must still propagate as Err");
|
||||
assert!(
|
||||
err.to_string().contains("streaming API error"),
|
||||
"err: {err}"
|
||||
);
|
||||
assert!(
|
||||
transport.fetch_and_clear_events().is_empty(),
|
||||
"an exhausted BYO balance 402 (streaming path) must not be reported to Sentry"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_api_insufficient_balance_402_not_reported_to_sentry() {
|
||||
// The responses-API path (`chat_via_responses`, operation=responses_api) is
|
||||
// reached when the provider is configured responses-primary. A 402
|
||||
// insufficient-balance there must demote, not page Sentry. This path is
|
||||
// awaited inline, so the TestTransport assertion is authoritative.
|
||||
let mock_server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.respond_with(ResponseTemplate::new(402).set_body_string(DEEPSEEK_4QF_402_BODY))
|
||||
.mount(&mock_server)
|
||||
.await;
|
||||
|
||||
let (transport, _guard) = install_test_sentry_hub();
|
||||
|
||||
let provider = OpenAiCompatibleProvider::new(
|
||||
"deepseek",
|
||||
&mock_server.uri(),
|
||||
Some("byo-key"),
|
||||
AuthStyle::Bearer,
|
||||
)
|
||||
.with_responses_api_primary();
|
||||
let err = provider
|
||||
.chat_with_history(&[ChatMessage::user("hello")], "deepseek-v4-flash", 0.0)
|
||||
.await
|
||||
.expect_err("a 402 insufficient-balance must still propagate as Err");
|
||||
// The responses-API error message is `<provider> Responses API error: <body>`
|
||||
// — note it carries NO "(402" status token, so the message-keyed before_send
|
||||
// net (#3913, `is_insufficient_credits_message`) would MISS it. Only the
|
||||
// status-based source arm here catches it, which the empty-transport
|
||||
// assertion below proves.
|
||||
assert!(
|
||||
err.to_string().contains("Insufficient Balance"),
|
||||
"err: {err}"
|
||||
);
|
||||
assert!(
|
||||
transport.fetch_and_clear_events().is_empty(),
|
||||
"an exhausted BYO balance 402 (responses_api path) must not be reported to Sentry"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_chat_with_system_insufficient_balance_402_propagates_error() {
|
||||
// Exercises the `stream_chat_with_system` (operation=stream_chat) 402
|
||||
// insufficient-balance demote arm. The streaming work runs in a detached
|
||||
// tokio task whose Sentry hub is NOT the test hub, so a TestTransport
|
||||
// assertion would be vacuous here; instead assert the error still
|
||||
// propagates as a terminal Provider chunk (the demote arm and the report
|
||||
// fallback bail the same message, so this guards line execution + error
|
||||
// shape). The Sentry-suppression behavior for this exact body is proven by
|
||||
// the inline-awaited native/api tests above and the
|
||||
// `detects_insufficient_balance_402_family` predicate test.
|
||||
use crate::openhuman::inference::provider::traits::{StreamError, StreamOptions};
|
||||
use futures_util::StreamExt;
|
||||
|
||||
let mock_server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/chat/completions"))
|
||||
.respond_with(ResponseTemplate::new(402).set_body_string(DEEPSEEK_4QF_402_BODY))
|
||||
.mount(&mock_server)
|
||||
.await;
|
||||
|
||||
let provider =
|
||||
OpenAiCompatibleProvider::new("deepseek", &mock_server.uri(), None, AuthStyle::None);
|
||||
let mut stream = provider.stream_chat_with_system(
|
||||
None,
|
||||
"hello",
|
||||
"deepseek-v4-flash",
|
||||
0.7,
|
||||
StreamOptions::new(true),
|
||||
);
|
||||
let mut terminal: Option<String> = None;
|
||||
while let Some(item) = stream.next().await {
|
||||
if let Err(StreamError::Provider(msg)) = item {
|
||||
terminal = Some(msg);
|
||||
}
|
||||
}
|
||||
let msg = terminal.expect("a 402 must yield a terminal Provider error chunk");
|
||||
assert!(
|
||||
msg.contains("402") && msg.contains("Insufficient Balance"),
|
||||
"msg: {msg}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_chat_with_history_insufficient_balance_402_propagates_error() {
|
||||
// Sibling of the above for `stream_chat_with_history`
|
||||
// (operation=stream_chat_history). Same detached-task caveat — asserts the
|
||||
// 402 path executes and propagates as a terminal Provider error chunk.
|
||||
use crate::openhuman::inference::provider::traits::{StreamError, StreamOptions};
|
||||
use futures_util::StreamExt;
|
||||
|
||||
let mock_server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/chat/completions"))
|
||||
.respond_with(ResponseTemplate::new(402).set_body_string(DEEPSEEK_4QF_402_BODY))
|
||||
.mount(&mock_server)
|
||||
.await;
|
||||
|
||||
let provider =
|
||||
OpenAiCompatibleProvider::new("deepseek", &mock_server.uri(), None, AuthStyle::None);
|
||||
let mut stream = provider.stream_chat_with_history(
|
||||
&[ChatMessage::user("hello")],
|
||||
"deepseek-v4-flash",
|
||||
0.7,
|
||||
StreamOptions::new(true),
|
||||
);
|
||||
let mut terminal: Option<String> = None;
|
||||
while let Some(item) = stream.next().await {
|
||||
if let Err(StreamError::Provider(msg)) = item {
|
||||
terminal = Some(msg);
|
||||
}
|
||||
}
|
||||
let msg = terminal.expect("a 402 must yield a terminal Provider error chunk");
|
||||
assert!(
|
||||
msg.contains("402") && msg.contains("Insufficient Balance"),
|
||||
"msg: {msg}"
|
||||
);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------
|
||||
// Prompt-cache capability model (#3939)
|
||||
// ----------------------------------------------------------
|
||||
|
||||
@@ -719,6 +719,14 @@ pub async fn api_error(provider: &str, response: reqwest::Response) -> anyhow::E
|
||||
// from Sentry (TAURI-RUST-8FQ flood).
|
||||
let is_openai_oauth_session_expired =
|
||||
is_openai_oauth_session_expired_http(provider, status, &body);
|
||||
// Insufficient-credits 402: the user's own BYO provider account is out of
|
||||
// balance — a flat billing fact, not a reservation-window error, so there is
|
||||
// NO local max_tokens lever to apply. Demote from Sentry like the per-method
|
||||
// compatible-provider arms; the complete classification for a genuinely-
|
||||
// unpreventable BYO-balance condition (TAURI-RUST-4QF DeepSeek "Insufficient
|
||||
// Balance"). This shared helper backs the two methods that delegate here
|
||||
// (chat_via_responses fallback and the non-streaming completion path).
|
||||
let is_insufficient_credits_402 = is_provider_insufficient_credits_402(status, &body);
|
||||
|
||||
if is_auth_failure && is_backend {
|
||||
// Single source of truth for backend session-expiry handling (warn +
|
||||
@@ -741,6 +749,8 @@ pub async fn api_error(provider: &str, response: reqwest::Response) -> anyhow::E
|
||||
log_byo_provider_auth_failure("api_error", provider, None, status);
|
||||
} else if is_openai_oauth_session_expired {
|
||||
log_openai_oauth_session_expired("api_error", provider, None, status);
|
||||
} else if is_insufficient_credits_402 {
|
||||
log_provider_insufficient_credits_402("api_error", provider, None, status);
|
||||
} else if should_report_provider_http_failure(status) {
|
||||
crate::core::observability::report_error(
|
||||
message.as_str(),
|
||||
|
||||
Reference in New Issue
Block a user