fix(memory): classify cloud-embedding "No backend session" as AuthMissing (#4699)

Co-authored-by: Steven Enamakel <enamakel@tinyhumans.ai>
This commit is contained in:
YellowSnnowmann
2026-07-08 19:15:12 -07:00
committed by GitHub
co-authored by Steven Enamakel
parent 9175603a9c
commit 531119d825
9 changed files with 290 additions and 69 deletions
+7
View File
@@ -473,6 +473,13 @@ async fn store_session_inner(
// in place. Workers that were sleeping in the paused poll loop will
// pick this up at their next iteration and resume LLM-bound work.
crate::openhuman::scheduler_gate::set_signed_out(false);
tracing::debug!(
domain = "credentials",
operation = "store_session",
"[credentials][auth-store] scheduler gate cleared; ensuring re-embed backfill after login"
);
crate::openhuman::memory_queue::ensure_reembed_backfill(&effective_config);
logs.push("memory re-embed backfill checked after login".to_string());
// Bind the Sentry scope to this user so background events that fire
// before the frontend's `app_state_snapshot` warms the user cache still
+89
View File
@@ -47,6 +47,17 @@ fn jwt_with_payload(payload: serde_json::Value) -> String {
format!("eyJhbGciOiJIUzI1NiJ9.{payload}.sig")
}
fn count_reembed_backfill_jobs(config: &Config) -> i64 {
crate::openhuman::memory_store::chunks::store::with_connection(config, |conn| {
Ok(conn.query_row(
"SELECT COUNT(*) FROM mem_tree_jobs WHERE kind = 'reembed_backfill'",
[],
|row| row.get(0),
)?)
})
.unwrap()
}
async fn spawn_auth_me_status(status: StatusCode) -> String {
let app = Router::new().route("/auth/me", get(move || async move { status }));
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -275,6 +286,84 @@ async fn store_session_defers_minimal_live_jwt_when_auth_me_transient_and_allowe
);
}
#[tokio::test]
async fn store_session_requeues_reembed_backfill_after_login() {
use crate::openhuman::memory_store::chunks::store::{upsert_chunks, upsert_staged_chunks_tx};
use crate::openhuman::memory_store::chunks::types::{
chunk_id, Chunk, Metadata, SourceKind, SourceRef,
};
use crate::openhuman::memory_store::content as content_store;
use chrono::TimeZone;
let _env_guard = crate::openhuman::config::TEST_ENV_LOCK
.lock()
.unwrap_or_else(|e| e.into_inner());
let tmp = TempDir::new().unwrap();
std::fs::create_dir_all(tmp.path().join("workspace")).unwrap();
let _home = EnvVarGuard::set_to_path("HOME", tmp.path());
let mut config = test_config(&tmp);
config.api_url = Some(spawn_auth_me_status(StatusCode::SERVICE_UNAVAILABLE).await);
let ts = chrono::Utc.timestamp_millis_opt(1_700_000_000_000).unwrap();
let chunk = Chunk {
id: chunk_id(SourceKind::Chat, "slack:#eng", 0, "login-reembed-seed"),
content: "memory content that needs embedding after the user logs in".into(),
metadata: Metadata {
source_kind: SourceKind::Chat,
source_id: "slack:#eng".into(),
owner: "alice".into(),
timestamp: ts,
time_range: (ts, ts),
tags: vec![],
source_ref: Some(SourceRef::new("slack://x")),
path_scope: None,
},
token_count: 12,
seq_in_source: 0,
created_at: ts,
partial_message: false,
};
upsert_chunks(&config, &[chunk.clone()]).unwrap();
let content_root = config.memory_tree_content_root();
std::fs::create_dir_all(&content_root).unwrap();
let staged = content_store::stage_chunks(&content_root, &[chunk]).unwrap();
crate::openhuman::memory_store::chunks::store::with_connection(&config, |conn| {
let tx = conn.unchecked_transaction()?;
upsert_staged_chunks_tx(&tx, &staged)?;
tx.commit()?;
Ok(())
})
.unwrap();
assert_eq!(
count_reembed_backfill_jobs(&config),
0,
"precondition: no active reembed backfill job exists before login"
);
let token = jwt_with_payload(json!({
"sub": "unverified-jwt-user",
"email": "jwt@example.test",
"name": "Unverified JWT User",
"exp": (chrono::Utc::now() + chrono::Duration::hours(1)).timestamp()
}));
let result = store_session_with_deferred_validation(&config, &token, None, Some(json!({})))
.await
.unwrap();
let log_text = result.logs.join(" ");
assert!(
log_text.contains("memory re-embed backfill checked after login"),
"store_session should report the post-login backfill probe, got: {log_text}"
);
assert_eq!(
count_reembed_backfill_jobs(&config),
1,
"login must enqueue exactly one reembed_backfill chain for uncovered rows"
);
}
#[tokio::test]
async fn deferred_session_without_user_id_does_not_replace_active_user_profile() {
let _env_guard = crate::openhuman::config::TEST_ENV_LOCK
+24 -1
View File
@@ -738,7 +738,9 @@ fn try_mark_summary_reembed_skipped(
/// Failure semantics are preserved verbatim from #1574 §6:
/// - body read failure → log + persistent tombstone (`"body read failed: {e}"`)
/// - embed wrong dim → log + tombstone (`"embed wrong dim"`)
/// - embed error → log + tombstone (`"embed failed: {e}"`)
/// - per-row unrecoverable embed error → log + tombstone (`"embed failed: {e}"`)
/// - global cloud-auth absence (`AuthMissing`, #4359) → fail the backfill
/// **without** tombstoning, so the same rows re-embed once the user logs in.
///
/// The single difference vs the old code is the embed call shape: one
/// `embed_batch` (which collapses N provider round-trips into one for batch-
@@ -809,6 +811,27 @@ async fn reembed_collect(
}
Err(e) => {
let failure = crate::openhuman::memory_tree::health::classify_embed_error(&e);
// #4359: a missing cloud session is a GLOBAL, recoverable-by-
// login condition, not a per-row defect — the row is perfectly
// embeddable once the user signs in. Tombstoning it (the
// unrecoverable branch below) would exclude it from the
// worklist forever, so it would never re-embed even after login.
// Fail the backfill fast with the typed `AuthMissing` failure
// instead: the health banner surfaces the "log in" remediation
// and every row stays re-embeddable.
if matches!(
failure.code,
crate::openhuman::memory_tree::health::FailureCode::AuthMissing
) {
log::warn!(
"[memory::jobs] reembed_backfill: {label} {id} embed failed — no cloud \
session; failing backfill without tombstoning so rows stay re-embeddable \
after login (sig={active_sig}): {e:#}"
);
return Err(anyhow::Error::new(failure).context(format!(
"reembed_backfill: {label} {id} cloud auth missing (sig={active_sig}): {e:#}"
)));
}
if !failure.is_unrecoverable() {
return Err(anyhow::Error::new(failure).context(format!(
"reembed_backfill: {label} {id} transient embed failed (sig={active_sig}): {e:#}"
@@ -680,3 +680,65 @@ async fn prepare_extract_scores_full_body_but_returns_preview() {
"returned chunk must NOT retain the full on-disk body"
);
}
/// #4359: a cloud-embedding "No backend session" bail is a global, login-
/// recoverable condition — `reembed_collect` must fail the backfill fast with a
/// typed `AuthMissing` failure and must **not** tombstone any row, so the same
/// rows re-embed once the user signs in. (A per-row tombstone here would
/// exclude the rows from the worklist forever — the regression this guards.)
struct MissingSessionEmbedder;
#[async_trait::async_trait]
impl crate::openhuman::memory_tree::score::embed::Embedder for MissingSessionEmbedder {
fn name(&self) -> &'static str {
"missing-session"
}
async fn embed(&self, _text: &str) -> anyhow::Result<Vec<f32>> {
anyhow::bail!(
"No backend session for cloud embeddings: log in to OpenHuman, or set \
memory.embedding_provider to \"ollama\" / \"none\" in config.toml"
)
}
}
#[tokio::test]
async fn reembed_backfill_auth_missing_fails_fast_without_tombstone() {
use crate::openhuman::memory_tree::health::{FailureCode, PipelineFailure};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
let (_tmp, cfg) = test_config();
let ids = vec![
"chunk-a".to_string(),
"chunk-b".to_string(),
"chunk-c".to_string(),
];
let tombstoned = Arc::new(AtomicUsize::new(0));
let tombstoned_for_mark = Arc::clone(&tombstoned);
let err = reembed_collect(
&cfg,
&MissingSessionEmbedder,
"provider=cloud;model=embedding-v1;dims=1024",
&ids,
"chunk",
|_cfg, id| Ok(format!("body for {id}")),
move |_cfg, _id, _sig, _reason| {
tombstoned_for_mark.fetch_add(1, Ordering::SeqCst);
},
)
.await
.expect_err("missing cloud session must fail the backfill, not return Ok");
let failure = err
.downcast_ref::<PipelineFailure>()
.expect("auth-missing backfill error must carry a typed PipelineFailure");
assert_eq!(failure.code, FailureCode::AuthMissing);
assert!(failure.is_unrecoverable());
assert_eq!(
tombstoned.load(Ordering::SeqCst),
0,
"auth-missing must NOT tombstone any row — they must stay re-embeddable after login"
);
}
+52
View File
@@ -242,6 +242,20 @@ pub fn classify_embed_error_str(msg: &str) -> PipelineFailure {
.with_detail(truncate_detail(msg));
}
// Sibling of the #13021 case above: `OpenHumanCloudEmbedding::resolve_bearer`
// bails *before any HTTP round-trip* when the desktop/backend session
// bearer is absent (user signed out), with the literal phrase
// "No backend session for cloud embeddings ..." (see
// `src/openhuman/embeddings/cloud.rs`). Being a client-side bail it carries
// no `Embedding API error (<status>)` shape, so without this match it falls
// through to `Transient` — the Memory Tree then shows "temporary error…
// will retry automatically" and the worker retries an auth failure that a
// retry can never fix. Classify as `AuthMissing` so the health banner
// surfaces the "log in to OpenHuman" remediation and the job fails fast.
if lower.contains("no backend session") {
return PipelineFailure::new(FailureCode::AuthMissing).with_detail(truncate_detail(msg));
}
// Dimension mismatch — the trait validator / CloudEmbedder rejects a
// vector whose length isn't EMBEDDING_DIM. Check before status parsing:
// it's a 2xx-but-wrong-shape case with no HTTP status to match.
@@ -723,6 +737,44 @@ mod tests {
assert!(f.is_unrecoverable());
}
/// #4359: `OpenHumanCloudEmbedding::resolve_bearer` bails with "No backend
/// session for cloud embeddings ..." *before any HTTP call* when the user
/// is signed out. This must classify as `AuthMissing` (unrecoverable, "log
/// in to OpenHuman" remediation) rather than falling through to `Transient`
/// ("will retry automatically" — a loop that an auth failure can never win).
#[test]
fn classify_no_backend_session_as_auth_missing() {
let msg = "No backend session for cloud embeddings: log in to OpenHuman, or set \
memory.embedding_provider to \"ollama\" / \"none\" in config.toml";
let f = classify_embed_error_str(msg);
assert_eq!(
f.code,
FailureCode::AuthMissing,
"expected AuthMissing for {msg:?}"
);
assert_eq!(f.class, FailureClass::Unrecoverable);
assert_eq!(f.remediation_key, "memory.health.remediation.auth_missing");
assert!(f.is_unrecoverable());
}
/// The match must be case-insensitive and survive `anyhow` context wrapping:
/// the bail is `.context()`-wrapped on its way up through the embed pipeline
/// (e.g. `embed_each_via_provider` adds "cloud embeddings failed"), and
/// `classify_embed_error` flattens the chain via `{err:#}`.
#[test]
fn classify_no_backend_session_through_anyhow_context_chain() {
let base = anyhow::anyhow!(
"No backend session for cloud embeddings: log in to OpenHuman, or set \
memory.embedding_provider to \"ollama\" / \"none\" in config.toml"
);
let wrapped = base
.context("cloud embeddings failed")
.context("reembed_backfill chunk_id=c");
let f = classify_embed_error(&wrapped);
assert_eq!(f.code, FailureCode::AuthMissing);
assert!(f.is_unrecoverable());
}
#[test]
fn classify_5xx_is_transient() {
let f = classify_embed_error_str("Embedding API error (503 Service Unavailable): retry");
+1
View File
@@ -1333,6 +1333,7 @@ fn composio_types_roundtrip_connection_tool_trigger_and_history_shapes() {
name: "GMAIL_SEND_EMAIL".into(),
description: Some("Send mail".into()),
parameters: Some(json!({ "type": "object" })),
output_parameters: None,
},
}],
};
@@ -7,7 +7,6 @@ use std::ffi::OsString;
use std::path::Path;
use std::sync::{Arc, Mutex, OnceLock};
use anyhow::Result;
use async_trait::async_trait;
use chrono::{TimeZone, Utc};
use serde_json::{json, Map, Value};
@@ -15,7 +14,6 @@ use tempfile::TempDir;
use openhuman_core::openhuman::config::Config;
use openhuman_core::openhuman::embeddings::NoopEmbedding;
use openhuman_core::openhuman::inference::provider::traits::Provider;
use openhuman_core::openhuman::memory::{
ExtractionMode, MemoryIngestionConfig, MemoryIngestionRequest,
};
@@ -24,6 +22,7 @@ use openhuman_core::openhuman::memory_tree::tree_runtime::{
all_tree_summarizer_registered_controllers, engine, rpc as tree_runtime_rpc,
store as tree_runtime_store,
};
use tinyagents::harness::model::{ChatModel, ModelRequest, ModelResponse};
struct EnvVarGuard {
key: &'static str,
@@ -72,21 +71,18 @@ struct ScriptedProvider {
}
#[async_trait]
impl Provider for ScriptedProvider {
async fn chat_with_system(
impl ChatModel<()> for ScriptedProvider {
async fn invoke(
&self,
system_prompt: Option<&str>,
message: &str,
model: &str,
temperature: f64,
) -> Result<String> {
let system_prompt = system_prompt.expect("tree runtime should pass a system prompt");
assert!(system_prompt.contains("hierarchical summarizer"));
assert!(system_prompt.contains("under"));
assert!(!message.trim().is_empty());
assert!(!model.trim().is_empty());
assert!(temperature > 0.0);
Ok(self.response.clone())
_state: &(),
request: ModelRequest,
) -> tinyagents::Result<ModelResponse> {
let prompt = format!("{:?}", request.messages);
assert!(prompt.contains("hierarchical summarizer"));
assert!(prompt.contains("under"));
assert!(!request.messages.is_empty());
assert!(request.temperature.unwrap_or_default() > 0.0);
Ok(ModelResponse::assistant(self.response.clone()))
}
}
@@ -222,7 +218,7 @@ async fn tree_runtime_engine_summarizes_preserves_buffer_and_rebuilds() {
)
.expect("buffer second entry");
let hour = engine::run_summarization(&config, &provider, "test-model", namespace, ts)
let hour = engine::run_summarization(&config, &provider, namespace, ts)
.await
.expect("run summarization")
.expect("hour node");
@@ -240,12 +236,10 @@ async fn tree_runtime_engine_summarizes_preserves_buffer_and_rebuilds() {
assert!(node.summary.contains("round23 summary") || node.summary.contains("##"));
}
assert!(
engine::run_summarization(&config, &provider, "test-model", namespace, ts)
.await
.expect("empty run")
.is_none()
);
assert!(engine::run_summarization(&config, &provider, namespace, ts)
.await
.expect("empty run")
.is_none());
tree_runtime_store::buffer_write(
&config,
@@ -256,7 +250,7 @@ async fn tree_runtime_engine_summarizes_preserves_buffer_and_rebuilds() {
)
.expect("buffer pending entry");
let status = engine::rebuild_tree(&config, &provider, "test-model", namespace)
let status = engine::rebuild_tree(&config, &provider, namespace)
.await
.expect("rebuild tree");
assert!(status.total_nodes >= 5);
+26 -21
View File
@@ -31,8 +31,9 @@ use chrono::{DateTime, TimeZone, Utc};
use tempfile::tempdir;
use openhuman_core::openhuman::config::Config;
use openhuman_core::openhuman::inference::provider::traits::Provider;
use openhuman_core::openhuman::memory_tree::tree_runtime::{engine, store};
use tinyagents::harness::model::{ChatModel, ModelRequest, ModelResponse};
use tinyagents::TinyAgentsError;
// ── Env isolation ─────────────────────────────────────────────────────────
@@ -73,7 +74,7 @@ fn env_lock() -> std::sync::MutexGuard<'static, ()> {
// ── Mock provider helpers ─────────────────────────────────────────────────
/// A provider whose `chat_with_system` returns scripted responses in order.
/// A provider whose `invoke` returns scripted responses in order.
/// Thread-safe via a `Mutex<VecDeque>`. Each pop returns the next scripted
/// response; once the queue is exhausted, every subsequent call returns an
/// error so missing a setup step is caught immediately.
@@ -100,24 +101,28 @@ impl ScriptedProvider {
}
#[async_trait]
impl Provider for ScriptedProvider {
async fn chat_with_system(
impl ChatModel<()> for ScriptedProvider {
async fn invoke(
&self,
system_prompt: Option<&str>,
message: &str,
model: &str,
_temperature: f64,
) -> anyhow::Result<String> {
_state: &(),
request: ModelRequest,
) -> tinyagents::Result<ModelResponse> {
let mut count = self.call_count.lock().expect("call_count lock");
*count += 1;
let call_n = *count;
drop(count);
let message_len: usize = request
.messages
.iter()
.map(|message| format!("{message:?}").len())
.sum();
log::debug!(
"[memory_tree_summarizer_e2e] ScriptedProvider.chat_with_system call #{call_n}: \
model={model} system_prompt_len={} msg_len={}",
system_prompt.map(|s| s.len()).unwrap_or(0),
message.len()
"[memory_tree_summarizer_e2e] ScriptedProvider.invoke call #{call_n}: \
model={:?} message_count={} msg_len={}",
request.model,
request.messages.len(),
message_len
);
let mut q = self.responses.lock().expect("responses lock");
@@ -127,19 +132,19 @@ impl Provider for ScriptedProvider {
"[memory_tree_summarizer_e2e] call #{call_n} → scripted Ok ({} chars)",
text.len()
);
Ok(text)
Ok(ModelResponse::assistant(text))
}
Some(Err(msg)) => {
log::debug!("[memory_tree_summarizer_e2e] call #{call_n} → scripted Err: {msg}");
Err(anyhow::anyhow!("{msg}"))
Err(TinyAgentsError::Model(msg))
}
None => {
log::debug!(
"[memory_tree_summarizer_e2e] call #{call_n} → queue exhausted (fallback error)"
);
Err(anyhow::anyhow!(
Err(TinyAgentsError::Model(format!(
"ScriptedProvider queue exhausted at call #{call_n}"
))
)))
}
}
}
@@ -233,7 +238,7 @@ async fn builds_hour_day_month_year_chain() {
]);
log::debug!("[memory_tree_summarizer_e2e] running summarization");
let result = engine::run_summarization(&config, &provider, "test-model", NS, Utc::now()).await;
let result = engine::run_summarization(&config, &provider, NS, Utc::now()).await;
log::debug!(
"[memory_tree_summarizer_e2e] run_summarization returned: {:?}",
@@ -379,7 +384,7 @@ async fn merges_into_existing_hour_node() {
)]);
log::debug!("[memory_tree_summarizer_e2e] first run");
let r1 = engine::run_summarization(&config, &provider1, "test-model", NS, Utc::now())
let r1 = engine::run_summarization(&config, &provider1, NS, Utc::now())
.await
.expect("first run_summarization");
assert!(r1.is_some(), "first run should yield a node");
@@ -415,7 +420,7 @@ async fn merges_into_existing_hour_node() {
)]);
log::debug!("[memory_tree_summarizer_e2e] second run (same hour)");
let r2 = engine::run_summarization(&config, &provider2, "test-model", NS, Utc::now())
let r2 = engine::run_summarization(&config, &provider2, NS, Utc::now())
.await
.expect("second run_summarization");
assert!(r2.is_some(), "second run should yield a node");
@@ -504,7 +509,7 @@ async fn survives_llm_error_with_partial_progress() {
]);
log::debug!("[memory_tree_summarizer_e2e] running summarization expecting partial failure");
let result = engine::run_summarization(&config, &provider, "test-model", NS, Utc::now()).await;
let result = engine::run_summarization(&config, &provider, NS, Utc::now()).await;
log::debug!(
"[memory_tree_summarizer_e2e] run_summarization result: is_ok={}",
+11 -23
View File
@@ -15,7 +15,6 @@ use tempfile::TempDir;
use openhuman_core::core::event_bus::{DomainEvent, EventHandler};
use openhuman_core::openhuman::config::Config;
use openhuman_core::openhuman::inference::provider::traits::{ChatMessage, Provider};
use openhuman_core::openhuman::memory_store::chunks::store::upsert_chunks;
use openhuman_core::openhuman::memory_store::chunks::types::{
approx_token_count, chunk_id, Chunk, Metadata, SourceKind as ChunkSourceKind, SourceRef,
@@ -44,6 +43,7 @@ use openhuman_core::openhuman::memory_tree::tree::{
use openhuman_core::openhuman::memory_tree::tree_runtime::{
engine, rpc as tree_runtime_rpc, store as runtime_store,
};
use tinyagents::harness::model::{ChatModel, ModelRequest, ModelResponse};
struct EnvVarGuard {
key: &'static str,
@@ -159,31 +159,19 @@ impl ScriptedProvider {
}
#[async_trait]
impl Provider for ScriptedProvider {
async fn chat_with_system(
impl ChatModel<()> for ScriptedProvider {
async fn invoke(
&self,
system_prompt: Option<&str>,
message: &str,
model: &str,
temperature: f64,
) -> anyhow::Result<String> {
let _ = (system_prompt, message, model, temperature);
Ok(self
_state: &(),
_request: ModelRequest,
) -> tinyagents::Result<ModelResponse> {
let response = self
.responses
.lock()
.unwrap()
.pop()
.unwrap_or_else(|| "fallback scripted summary".to_string()))
}
async fn chat_with_history(
&self,
messages: &[ChatMessage],
model: &str,
temperature: f64,
) -> anyhow::Result<String> {
let _ = (messages, model, temperature);
self.chat_with_system(None, "", "", 0.0).await
.unwrap_or_else(|| "fallback scripted summary".to_string());
Ok(ModelResponse::assistant(response))
}
}
@@ -220,7 +208,7 @@ async fn tree_runtime_engine_rpc_and_walk_cover_success_and_edge_paths() {
"rebuilt hour 10",
"rebuilt hour 11",
]);
let last = engine::run_summarization(&cfg, &provider, "test-model", ns, Utc::now())
let last = engine::run_summarization(&cfg, &provider, ns, Utc::now())
.await
.expect("run summarization")
.expect("last hour node");
@@ -254,7 +242,7 @@ async fn tree_runtime_engine_rpc_and_walk_cover_success_and_edge_paths() {
"rebuilt year summary",
"rebuilt root summary",
]);
let rebuilt = engine::rebuild_tree(&cfg, &rebuild_provider, "test-model", ns)
let rebuilt = engine::rebuild_tree(&cfg, &rebuild_provider, ns)
.await
.expect("rebuild tree");
assert_eq!(rebuilt.total_nodes, 6);