From 9cbf023750ad4b8e690bb1dac4c569d7e2181857 Mon Sep 17 00:00:00 2001 From: oxoxDev <164490987+oxoxDev@users.noreply.github.com> Date: Mon, 22 Jun 2026 23:39:52 +0530 Subject: [PATCH] fix(memory): stop SQLITE_FULL flood from mem_tree_jobs worker (#3909) (#3911) --- src/core/observability.rs | 84 +++++++++++++++---- src/openhuman/memory_queue/worker.rs | 119 +++++++++++++++++++++++++++ 2 files changed, 188 insertions(+), 15 deletions(-) diff --git a/src/core/observability.rs b/src/core/observability.rs index 0758f48d0..cdb284080 100644 --- a/src/core/observability.rs +++ b/src/core/observability.rs @@ -518,15 +518,29 @@ pub fn expected_error_kind(message: &str) -> Option { /// burst always produces an os-error-28/112 sibling event) or a /// `max_page_count` PRAGMA cap (we set none). /// -/// The rusqlite `Display` for `SQLITE_FULL` is exactly the five words -/// `"database or disk is full"` — no preamble, no trailing context. Our local -/// memory-store write call-sites wrap it with `format!(": {e}")` -/// (e.g. `"commit tx: ..."` / `"clear_namespace commit tx: ..."` in -/// `memory_store::unified::documents`), so the phrase always lands as the -/// **suffix** of the local emit. Anchor on suffix, not `contains`, so the -/// silencer does not match a non-2xx backend response body whose payload -/// happens to mention the same phrase (e.g. an `api.tinyhumans.ai` 5xx whose -/// server-side SQLite is full). Non-2xx backend bodies are framed by +/// rusqlite renders `SQLITE_FULL` in one of two shapes. The **bare** shape is +/// the five words `"database or disk is full"` — Our local memory-store write +/// call-sites wrap it with `format!(": {e}")` (e.g. `"commit tx: ..."` / +/// `"clear_namespace commit tx: ..."` in `memory_store::unified::documents`), +/// so the phrase lands as the **suffix** of the local emit. The **extended** +/// shape carries the full error-code envelope, `"database or disk is full: +/// Error code 13: Insertion failed because database is full"` (Sentry +/// TAURI-RUST-4R8, `memory_queue::store::claim_next` on `mem_tree_jobs`); here +/// the canonical phrase sits mid-string, so the suffix anchor can't catch it. +/// We detect this shape by requiring **both** local fragments together — the +/// `"database or disk is full"` phrase AND the libsqlite3-sys `code_to_str` +/// token `"insertion failed because database is full"` — which only rusqlite's +/// own `SQLITE_FULL` Display emits as a pair. Requiring both (rather than the +/// `code_to_str` token alone) keeps the silencer from matching a remote +/// provider body that merely quotes the token half — e.g. an OpenAI-compatible +/// `OpenAiEmbedding::embed` failure framed as `"Embedding API error (… Error +/// code 13: Insertion failed because database is full)"` whose *server-side* +/// SQLite is full is operator-actionable and must still surface to Sentry +/// (codex CR on #3911). Both arms anchor on local-emit fragments rather than a +/// bare `contains("database or disk is full")` so the silencer does not match a +/// non-2xx backend response body whose payload happens to mention the phrase +/// (e.g. an `api.tinyhumans.ai` 5xx whose server-side SQLite is full). Non-2xx +/// backend bodies are framed by /// `integrations::client::post` / `composio::client` as `"Backend returned /// for : "` — an operator-actionable /// server/storage failure that must still surface to Sentry. As @@ -546,15 +560,22 @@ fn is_disk_full_message(lower: &str) -> bool { { return true; } - // SQLITE_FULL — see the fifth-shape section above for the scoping - // rationale. Suffix anchor (after trimming trailing whitespace and - // punctuation that closures / JSON wrappers commonly append) pins to the - // local-emit shape; the negative `"backend returned "` guard rejects the - // remote envelope as a second line of defense. + // Two SQLITE_FULL renderings — see the fifth-shape section above. The + // **bare** shape lands the phrase as a suffix (after trimming trailing + // whitespace / punctuation that closures + JSON wrappers append). The + // **extended** shape (TAURI-RUST-4R8) puts the phrase mid-string followed + // by `: Error code 13: Insertion failed because database is full`; require + // BOTH local fragments so we match only rusqlite's own `SQLITE_FULL` Display + // and never a remote provider body that merely quotes the `code_to_str` + // half (codex CR on #3911). The negative `"backend returned "` guard + // rejects the remote 5xx envelope as a further line of defense. let trimmed = lower.trim_end_matches(|c: char| { c.is_ascii_whitespace() || matches!(c, '.' | ',' | ';' | ':' | '"' | '\'') }); - trimmed.ends_with("database or disk is full") && !lower.contains("backend returned ") + let bare_suffix = trimmed.ends_with("database or disk is full"); + let extended_local = lower.contains("database or disk is full") + && lower.contains("insertion failed because database is full"); + (bare_suffix || extended_local) && !lower.contains("backend returned ") } /// Detect the literal `"Config loading timed out"` string produced by @@ -3000,6 +3021,12 @@ mod tests { // `openhuman.memory_doc_ingest`, in the same burst that emits // os-error-112 siblings (Sentry TAURI-RUST-B6N). "commit tx: database or disk is full", + // SQLITE_FULL **extended** rendering — the full error-code envelope + // where the canonical phrase is mid-string, not a suffix (Sentry + // TAURI-RUST-4R8, `memory_queue::store::claim_next` on + // `mem_tree_jobs`). Caught via the `code_to_str` token arm. + "Failed to claim next mem_tree_jobs row: database or disk is full: \ + Error code 13: Insertion failed because database is full", ] { assert_eq!( expected_error_kind(raw), @@ -3060,6 +3087,33 @@ mod tests { None, "remote-backend body must surface even when the body itself ends with the phrase" ); + // Same guard for the extended-code token: a backend body that quotes + // the SQLITE_FULL `code_to_str` string is still operator-actionable + // and must surface (TAURI-RUST-4R8 token arm + `"backend returned "` + // exclusion). + assert_eq!( + expected_error_kind( + "Backend returned 507 Insufficient Storage for POST \ + https://api.tinyhumans.ai/agent-integrations/composio/list: \ + Error code 13: Insertion failed because database is full" + ), + None, + "remote-backend body carrying the extended SQLITE_FULL token must surface" + ); + // A remote OpenAI-compatible embeddings 500 whose server-side SQLite is + // full is wrapped by `OpenAiEmbedding::embed` as `"Embedding API error + // (…)"` — no `"backend returned "` prefix and no local `"database or + // disk is full"` phrase, just the `code_to_str` half. Requiring BOTH + // local fragments for the extended shape keeps this operator-actionable + // server fault reportable (codex CR on #3911). + assert_eq!( + expected_error_kind( + "Embedding API error (status 500): Error code 13: \ + Insertion failed because database is full" + ), + None, + "remote embedding-API body quoting only the code_to_str token must surface" + ); // Non-suffix occurrences in other body framings (no `"Backend // returned"` prefix) are also excluded by the suffix anchor — locks // in the primary defense layer. diff --git a/src/openhuman/memory_queue/worker.rs b/src/openhuman/memory_queue/worker.rs index 642e8eac7..be73e5c89 100644 --- a/src/openhuman/memory_queue/worker.rs +++ b/src/openhuman/memory_queue/worker.rs @@ -143,6 +143,22 @@ pub fn start(config: Config) { backing off 30s: {err:#}" ); tokio::time::sleep(Duration::from_secs(30)).await; + } else if is_sqlite_disk_full(&err) { + // SQLITE_FULL (code 13): the host disk is full. + // A claim UPDATE cannot succeed until the user + // frees space — this is persistent, not + // transient, so re-polling every second and + // paging Sentry on each failure floods the + // dashboard (TAURI-RUST-4R8: ~95k events, one + // user) for a condition only the user can + // clear. Back off long and stay silent; the + // `ready` rows resume when space returns and + // `notify` still wakes us on new enqueues. + log::warn!( + "[memory::jobs] worker {idx} hit SQLITE_FULL (disk full), \ + backing off 300s without reporting: {err:#}" + ); + tokio::time::sleep(Duration::from_secs(300)).await; } else { crate::core::observability::report_error( &err, @@ -367,6 +383,34 @@ fn is_sqlite_busy(err: &anyhow::Error) -> bool { msg.contains("database is locked") || msg.contains("database table is locked") } +/// Classify whether an error from `claim_next` is a `SQLITE_FULL` disk-full +/// condition (primary code `DiskFull`, extended 13). +/// +/// Unlike `SQLITE_BUSY`/`LOCKED` or the transient I/O family, a full disk is a +/// **persistent** host condition: the claim `UPDATE` cannot succeed until the +/// user frees space. Re-polling every second and paging Sentry on each failure +/// turns one unrecoverable condition into a flood (Sentry TAURI-RUST-4R8: +/// ~95k events from a single user). The worker backs off long and stays +/// silent; the rows stay `ready` and resume when space returns. +/// +/// Matching on the `DiskFull` error code is rusqlite-version-stable. The text +/// fallback covers the case where the error was flattened to a plain `anyhow!` +/// string across `.context()` layers — rusqlite renders `SQLITE_FULL` as +/// `"database or disk is full: Error code 13: Insertion failed because +/// database is full"`, so anchor on either canonical fragment. +fn is_sqlite_disk_full(err: &anyhow::Error) -> bool { + if let Some(rusqlite::Error::SqliteFailure(sqlite_err, _)) = + err.downcast_ref::() + { + if sqlite_err.code == rusqlite::ErrorCode::DiskFull { + return true; + } + } + let msg = format!("{err:#}").to_ascii_lowercase(); + msg.contains("database or disk is full") + || msg.contains("insertion failed because database is full") +} + #[cfg(test)] mod tests { use super::*; @@ -547,6 +591,81 @@ mod tests { assert!(!is_sqlite_io_transient(&anyhow::Error::from(raw))); } + // ── is_sqlite_disk_full tests (#3909 / Sentry TAURI-RUST-4R8) ───────── + + /// `SQLITE_FULL` (primary code `DiskFull`, extended 13) is the disk-full + /// signal from `claim_next`; it must classify so the worker backs off + /// long instead of paging Sentry every second. + #[test] + fn is_sqlite_disk_full_matches_disk_full_code() { + let raw = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::DiskFull, + extended_code: 13, + }, + Some("database or disk is full".into()), + ); + assert!(is_sqlite_disk_full(&anyhow::Error::from(raw))); + } + + /// The rusqlite error sits a few `.context()` layers deep when it bubbles + /// out of `claim_next` → `with_connection`; the downcast must still find + /// the `DiskFull` code. + #[test] + fn is_sqlite_disk_full_matches_through_context_layers() { + let raw = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::DiskFull, + extended_code: 13, + }, + Some("database or disk is full".into()), + ); + let wrapped = anyhow::Error::from(raw) + .context("Failed to claim next mem_tree_jobs row") + .context("with_connection closure failed"); + assert!(is_sqlite_disk_full(&wrapped)); + } + + /// Text fallback: the exact flattened Sentry string (TAURI-RUST-4R8) is + /// classified even when no rusqlite error is available to downcast (the + /// canonical phrase is mid-string, not a suffix). + #[test] + fn is_sqlite_disk_full_text_fallback() { + let err = anyhow::anyhow!( + "Failed to claim next mem_tree_jobs row: database or disk is full: \ + Error code 13: Insertion failed because database is full" + ); + assert!(is_sqlite_disk_full(&err)); + } + + /// Busy/locked, constraint violations, and unrelated errors must NOT be + /// swallowed as disk-full — those still warrant their own handling / + /// Sentry escalation. + #[test] + fn is_sqlite_disk_full_does_not_match_other_errors() { + let busy = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::DatabaseBusy, + extended_code: 5, + }, + Some("database is locked".into()), + ); + assert!(!is_sqlite_disk_full(&anyhow::Error::from(busy))); + + let constraint = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::ConstraintViolation, + extended_code: 19, + }, + Some("UNIQUE constraint failed: mem_tree_jobs.dedupe_key".into()), + ); + assert!(!is_sqlite_disk_full(&anyhow::Error::from(constraint))); + + assert!(!is_sqlite_disk_full(&anyhow::anyhow!( + "upstream returned 500: internal server error" + ))); + } + #[tokio::test] async fn wake_workers_is_noop_before_start() { wake_workers();