fix(embeddings): validate model id + demote chat-model-as-embeddings 400 (TAURI-RUST-9SK) (#4070)

This commit is contained in:
Mega Mind
2026-06-24 10:25:13 -07:00
committed by GitHub
parent 7025548f7e
commit ab8ecca5e1
5 changed files with 172 additions and 0 deletions
+81
View File
@@ -449,6 +449,21 @@ pub fn expected_error_kind(message: &str) -> Option<ExpectedErrorKind> {
if is_embedding_endpoint_absent(&lower) {
return Some(ExpectedErrorKind::ProviderConfigRejection);
}
// TAURI-RUST-9SK — the user entered a non-embedding (chat) model id as the
// embeddings model (e.g. an OpenRouter `…:free` chat model), so the
// embeddings endpoint 400s `Model <id> does not exist` on every memory
// re-embed (2205 events / 1 user). OpenRouter's bare `"does not exist"` +
// integer `"code":400` body matches none of the chat-side phrases in
// `is_provider_config_rejection_message` (which key on the OpenAI-native
// `"does not exist or you do not have access"` / `model_not_found`), so
// without this it reaches Sentry. Deterministic user-config state; the
// settings UI surfaces an actionable "pick an embeddings-capable model"
// remediation. Scoped to the 400 model-rejection body so a real 400
// (oversized input, server fault) stays visible — same polarity contract as
// `is_embedding_endpoint_absent`.
if is_embedding_model_rejected(&lower) {
return Some(ExpectedErrorKind::ProviderConfigRejection);
}
// Provider config-rejection (unknown model / abstract tier leaked to a
// custom provider / model-specific temperature). Body-shape based and
// intrinsically scoped to third-party providers — the OpenHuman
@@ -733,6 +748,34 @@ pub(crate) fn is_embedding_endpoint_absent(lower: &str) -> bool {
lower.contains("embedding api error") && (lower.contains("(404") || lower.contains("(405"))
}
/// Detect a custom/cloud embeddings endpoint that IS an embeddings API but
/// **rejected the configured model id** — the user pasted a non-embedding
/// (chat/reasoning) model into the embeddings model field. Canonical wire shape
/// from `src/openhuman/embeddings/openai.rs` (TAURI-RUST-9SK, ~2205 events):
///
/// ```text
/// Embedding API error (400 Bad Request): {"error":{"message":"Model nvidia/nemotron-3-super-120b-a12b does not exist","code":400}}
/// ```
///
/// Deterministic user-config state, re-emitted on every memory re-embed; the
/// embeddings settings UI surfaces an actionable "pick an embeddings-capable
/// model" remediation (appended to the message at the emit site). The
/// OpenRouter body — bare `"does not exist"` with an integer `"code":400` —
/// matches none of the chat-side phrases in
/// `inference::provider::is_provider_config_rejection_message` (those key on the
/// OpenAI-native `"does not exist or you do not have access"` /
/// `model_not_found`), so this dedicated matcher is what demotes it.
///
/// Polarity (important): scoped to **400** + a model-rejection body. A bare
/// 400 (oversized input — prevented at source by the chunk cap #3598) or a 500
/// from a valid embeddings endpoint is a real fault and must keep reaching
/// Sentry, so this never fires on them.
fn is_embedding_model_rejected(lower: &str) -> bool {
lower.contains("embedding api error")
&& lower.contains("(400")
&& (lower.contains("does not exist") || lower.contains("does not support embeddings"))
}
/// Detect the memory-store chunk DB's circuit-breaker-open message that
/// `memory_store::chunks::store::get_or_init_connection` emits via
/// `anyhow::bail!` when the per-path breaker rejects new init attempts.
@@ -2885,6 +2928,44 @@ mod tests {
}
}
#[test]
fn classifies_embedding_model_does_not_exist_400_as_config_rejection() {
// TAURI-RUST-9SK (~2205 events / 1 user) — a chat model id pasted as the
// embeddings model. Verbatim OpenRouter wire body (bare "does not exist"
// + integer "code":400), plus the enriched form after the emit site
// appends the actionable remediation. Both must demote so the per-embed
// flood stays out of Sentry.
for raw in [
r#"Embedding API error (400 Bad Request): {"error":{"message":"Model nvidia/nemotron-3-super-120b-a12b does not exist","code":400}}"#,
"Embedding API error (400 Bad Request): {\"error\":{\"message\":\"Model nvidia/nemotron-3-super-120b-a12b does not exist\",\"code\":400}} — this model isn't an embeddings model; pick an embeddings-capable model in Settings → Memory",
r#"Embedding API error (400 Bad Request): {"error":{"message":"this model does not support embeddings"}}"#,
] {
assert_eq!(
expected_error_kind(raw),
Some(ExpectedErrorKind::ProviderConfigRejection),
"should classify embedding model-rejection 400 as config rejection: {raw}"
);
}
}
#[test]
fn does_not_classify_unrelated_embedding_400s() {
// Polarity: a 400 that is NOT a model-rejection (e.g. oversized input)
// and any non-400 must keep reaching Sentry.
assert_eq!(
expected_error_kind(
r#"Embedding API error (400 Bad Request): {"error":{"message":"input exceeds the maximum number of tokens"}}"#
),
None
);
assert_eq!(
expected_error_kind(
r#"Embedding API error (500 Internal Server Error): {"error":"model does not exist"}"#
),
None
);
}
#[test]
fn does_not_classify_unrelated_invalid_token_messages() {
// Provider 401s with "invalid token" in the body but no
+10
View File
@@ -281,6 +281,16 @@ pub async fn apply_memory_settings(
config.memory.embedding_provider = provider;
}
if let Some(model) = update.embedding_model {
// Source-gate (TAURI-RUST-9SK): reject an unmistakably non-embedding
// model id before persisting it. This save path has no live verify
// probe (unlike the Custom-provider setup flow in
// `embeddings::rpc::update_settings`), so a chat model id pasted here
// would otherwise be stored unchecked and 400 "does not exist" on every
// memory re-embed (2205 events from one user). Conservative check — see
// `embeddings::non_embedding_model_reason`.
if let Some(reason) = crate::openhuman::embeddings::non_embedding_model_reason(&model) {
return Err(format!("invalid embeddings model `{model}`: {reason}"));
}
config.memory.embedding_model = model;
}
if let Some(dimensions) = update.embedding_dimensions {
+61
View File
@@ -172,6 +172,32 @@ pub fn default_model_for(provider_slug: &str) -> Option<&'static EmbeddingModelP
find_provider(provider_slug).and_then(|p| p.models.first())
}
/// Returns `Some(reason)` when `model` is unmistakably **not** an embeddings
/// model and must be rejected before it is persisted as the embeddings model.
///
/// Conservative by design (zero false positives on real embedding ids): the
/// only hard signal is the OpenRouter `:free` chat/reasoning tier suffix — no
/// embeddings model is served under it, and a chat model id pasted into the
/// free-text custom-model field is exactly how TAURI-RUST-9SK happened
/// (`nvidia/nemotron-3-super-120b-a12b:free` saved as the embeddings model,
/// then 400 "does not exist" on every memory re-embed — 2205 events).
///
/// This is a source-gate for the persist paths that have **no** live verify
/// probe (`config::ops::model::apply_memory_settings`); the Custom-provider
/// setup flow already runs a save-time test embed (`embeddings::rpc::
/// update_settings`), and the broad long tail of chat ids is caught at runtime
/// by the 400 "does not exist" classifier in `core::observability`. Kept
/// deliberately tight: a false positive blocks a user's valid embeddings model.
pub fn non_embedding_model_reason(model: &str) -> Option<&'static str> {
if model.trim().to_ascii_lowercase().ends_with(":free") {
return Some(
"`:free` denotes an OpenRouter chat/reasoning tier, not an embeddings model — \
pick an embeddings-capable model in Settings → Memory",
);
}
None
}
#[cfg(test)]
mod tests {
use super::*;
@@ -238,6 +264,41 @@ mod tests {
}
}
#[test]
fn non_embedding_model_reason_rejects_openrouter_free_tier() {
// TAURI-RUST-9SK — the exact incident id and case/whitespace variants.
for id in [
"nvidia/nemotron-3-super-120b-a12b:free",
"meta-llama/llama-3-70b-instruct:FREE",
" some-chat-model:free ",
] {
assert!(
non_embedding_model_reason(id).is_some(),
"{id:?} (`:free` chat tier) must be rejected as an embeddings model"
);
}
}
#[test]
fn non_embedding_model_reason_accepts_real_embedding_ids() {
// Must NOT false-positive on genuine embedding model ids across providers.
for id in [
"text-embedding-3-small",
"text-embedding-3-large",
"voyage-3-large",
"embed-english-v3.0",
"nomic-embed-text:latest",
"bge-m3",
"mxbai-embed-large",
"",
] {
assert!(
non_embedding_model_reason(id).is_none(),
"{id:?} is a valid embeddings model and must not be rejected"
);
}
}
#[test]
fn default_model_for_all_providers_with_models() {
for entry in all_providers() {
+1
View File
@@ -32,6 +32,7 @@ pub use crate::openhuman::memory_store::vectors::store;
pub use crate::openhuman::memory_store::vectors::{
bytes_to_vec, cosine_similarity, vec_to_bytes, SearchResult, VectorStore,
};
pub use catalog::non_embedding_model_reason;
pub use cloud::{
OpenHumanCloudEmbedding, DEFAULT_CLOUD_EMBEDDING_DIMENSIONS, DEFAULT_CLOUD_EMBEDDING_MODEL,
};
+19
View File
@@ -294,6 +294,25 @@ impl EmbeddingProvider for OpenAiEmbedding {
embeddings-capable provider in Settings → Memory",
);
}
// A 400 "… does not exist" / "does not support embeddings" body
// means the endpoint IS an embeddings API but the configured
// model id is not an embeddings model — the user pasted a chat
// model (e.g. an OpenRouter `…:free` id) into the embeddings
// model field. Append an actionable remediation while PRESERVING
// the `(400` + body text that `observability::
// is_embedding_model_rejected` keys on, so the event is demoted
// from a per-embed Sentry flood to a breadcrumb. Scoped to the
// model-rejection body shape so a genuine 400 (oversized input,
// real server fault) still reaches Sentry. TAURI-RUST-9SK.
else if status.as_u16() == 400
&& (text.contains("does not exist")
|| text.contains("does not support embeddings"))
{
message.push_str(
" — this model isn't an embeddings model; pick an \
embeddings-capable model in Settings → Memory",
);
}
// Use `report_error_or_expected` so transient upstream HTTP
// failures (e.g. 429 Too Many Requests after retry cap) log a
// warning breadcrumb instead of firing a Sentry error event.