From 1aaa59f17d085a4e1c03fb890350c54c1a79991d Mon Sep 17 00:00:00 2001 From: Mega Mind <146339422+M3gA-Mind@users.noreply.github.com> Date: Mon, 4 May 2026 23:20:42 +0530 Subject: [PATCH] fix(composio): cap Gmail HTML body before strip (crash mitigation) (#1191) Co-authored-by: Cursor --- .../composio/providers/gmail/post_process.rs | 59 +++++++++++++++---- .../providers/gmail/post_process_tests.rs | 50 +++++++++++++--- .../composio/providers/gmail/provider.rs | 6 +- 3 files changed, 92 insertions(+), 23 deletions(-) diff --git a/src/openhuman/composio/providers/gmail/post_process.rs b/src/openhuman/composio/providers/gmail/post_process.rs index 20f9f4712..935dda6e9 100644 --- a/src/openhuman/composio/providers/gmail/post_process.rs +++ b/src/openhuman/composio/providers/gmail/post_process.rs @@ -12,7 +12,9 @@ //! //! Feeding all of that back to the LLM burns context on presentational //! markup. By default this module rewrites the payload into a slim -//! envelope per message: +//! envelope per message. HTML bodies are converted with a bounded +//! linear stripper (never `html2md`), because nested-table HTML can +//! trigger catastrophic allocator growth in third-party parsers. //! //! ```json //! { @@ -129,8 +131,8 @@ fn reshape_fetch_emails(data: &mut Value) { /// Map one raw Composio message object to its slim counterpart. /// /// Preference order for the body: -/// 1. A `text/html` MIME part's base64url-decoded body → html2md. -/// 2. A `text/plain` MIME part's base64url-decoded body. +/// 1. A `text/plain` MIME part's base64url-decoded body (preferred). +/// 2. A `text/html` MIME part's base64url-decoded body → bounded HTML strip. /// 3. The top-level `messageText` (Composio's decoded plain text). /// 4. Empty string. fn reshape_message(raw: Value) -> Value { @@ -192,14 +194,13 @@ fn pick_header(msg: &Map, name: &str) -> Option { /// /// Preference order: /// 1. `text/plain` — author-provided plaintext fallback. Standard MIME -/// multipart/alternative. Bypasses html2md entirely. For LLM -/// extraction + retrieval embedding, plaintext is generally the -/// better input: less noise (no tracking URLs, inline CSS, table -/// formatting artifacts), zero conversion cost, no allocator -/// pressure from `html2md`'s pathological walk over nested HTML -/// tables (verified GB-scale heap peaks via dhat-rs profiling). -/// 2. `text/html` → html2md — only used when the email shipped no -/// plaintext part (rare; some HTML-only marketing senders). +/// multipart/alternative. For LLM extraction + retrieval embedding, +/// plaintext is generally the better input: less noise (no tracking +/// URLs, inline CSS, table formatting artifacts), zero conversion cost, +/// and no pathological nested-table allocator blowups seen with legacy +/// HTML-to-markdown crates (dhat-rs profiling on real inboxes). +/// 2. `text/html` → fast linear strip (see [`html_email_to_markdown`]) — +/// only when the email shipped no plaintext part (rare). /// 3. Top-level `messageText` (Composio convenience field) — html or /// plaintext depending on what the source had. fn extract_markdown_body(msg: &Map) -> String { @@ -252,13 +253,45 @@ fn extract_markdown_body(msg: &Map) -> String { /// For multipart emails, `extract_markdown_body` prefers `text/plain` /// before this function is reached, so this path only runs for /// HTML-only emails (rare). +// UTF-8 cap: multi‑MB HTML MIME would otherwise amplify memory in strip passes. +pub(super) const MAX_GMAIL_HTML_BODY_BYTES: usize = 512 * 1024; + +fn truncate_utf8_to_bytes(s: &str, max_bytes: usize) -> (&str, bool) { + if s.len() <= max_bytes { + return (s, false); + } + let mut end = max_bytes; + while end > 0 && !s.is_char_boundary(end) { + end -= 1; + } + (&s[..end], true) +} + fn html_email_to_markdown(html: &str) -> String { + let input_bytes = html.len(); + if input_bytes > MAX_GMAIL_HTML_BODY_BYTES { + tracing::warn!( + original_bytes = input_bytes, + max_bytes = MAX_GMAIL_HTML_BODY_BYTES, + "[composio:gmail][post-process] HTML body truncated before strip (size cap)" + ); + } + let (html, truncated) = truncate_utf8_to_bytes(html, MAX_GMAIL_HTML_BODY_BYTES); + let cleaned = strip_html_noise_blocks(html); let cleaned = cleaned.trim(); if cleaned.is_empty() { - return String::new(); + return if truncated { + "[Email HTML body truncated for processing]".to_string() + } else { + String::new() + }; } - normalize_markdownish_text(&fast_html_to_text(cleaned)) + let mut out = normalize_markdownish_text(&fast_html_to_text(cleaned)); + if truncated { + out.push_str("\n\n[Email HTML body truncated for processing]"); + } + out } fn strip_html_noise_blocks(html: &str) -> String { diff --git a/src/openhuman/composio/providers/gmail/post_process_tests.rs b/src/openhuman/composio/providers/gmail/post_process_tests.rs index 96bd80030..74b920e34 100644 --- a/src/openhuman/composio/providers/gmail/post_process_tests.rs +++ b/src/openhuman/composio/providers/gmail/post_process_tests.rs @@ -170,8 +170,8 @@ fn non_fetch_slug_is_noop() { fn nested_multipart_prefers_plaintext_over_html() { // When a multipart/alternative ships BOTH text/plain and text/html, the // plaintext part wins. text/plain is the author's intended fallback, - // bypasses html2md (which has GB-scale heap peaks on rich HTML — see - // post_process.rs::extract_markdown_body docstring), and is generally + // bypasses HTML stripping on the sibling `text/html` part (see + // post_process.rs::extract_markdown_body), and is generally // cleaner input for downstream LLM extraction. let html = "

Deep body

"; let mut v = json!({ @@ -203,7 +203,7 @@ fn nested_multipart_prefers_plaintext_over_html() { md.contains("plain fallback"), "plaintext should win, got: {md:?}" ); - // The HTML body should NOT appear — html2md was bypassed entirely. + // The HTML body should NOT appear — the text/html branch was bypassed. assert!( !md.contains("Deep"), "html should not have been used: {md:?}" @@ -217,8 +217,7 @@ fn nested_multipart_prefers_plaintext_over_html() { #[test] fn nested_multipart_falls_back_to_html_when_no_plaintext() { // For html-only emails (rare — some poorly-built marketing senders), - // we still need html2md → markdown. The 24 KB threshold and noise-block - // stripper guard against pathological cases. + // we run the bounded linear HTML strip (see `html_email_to_markdown`). let html = "

Deep body

"; let mut v = json!({ "messages": [{ @@ -244,8 +243,14 @@ fn nested_multipart_falls_back_to_html_when_no_plaintext() { }); post_process("GMAIL_FETCH_EMAILS", None, &mut v); let md = v["messages"][0]["markdown"].as_str().unwrap(); - assert!(md.contains("Deep"), "html2md should have run: {md:?}"); - assert!(md.contains("body"), "html2md should have run: {md:?}"); + assert!( + md.contains("Deep"), + "HTML strip should preserve text: {md:?}" + ); + assert!( + md.contains("body"), + "HTML strip should preserve text: {md:?}" + ); assert!( !md.contains("

"), "raw html should not leak through: {md:?}" @@ -279,6 +284,37 @@ fn large_html_uses_fast_strip_fallback() { ); } +#[test] +fn oversized_html_is_truncated_before_processing() { + let cap = super::MAX_GMAIL_HTML_BODY_BYTES; + let filler = "x".repeat(600 * 1024); + let html = + format!("

HEAD_MARKER

{filler}

TAIL_NEVER_SEEN

"); + assert!(html.len() > cap); + let md = html_email_to_markdown(&html); + assert!(md.contains("HEAD_MARKER"), "{md:?}"); + assert!( + !md.contains("TAIL_NEVER_SEEN"), + "tail past cap must not be processed: {md:?}" + ); + assert!( + md.contains("[Email HTML body truncated for processing]"), + "expected truncation note: {md:?}" + ); +} + +#[test] +fn truncated_all_whitespace_html_still_emits_truncation_note() { + let cap = super::MAX_GMAIL_HTML_BODY_BYTES; + let html = " ".repeat(cap + 10_000); + assert!(html.len() > cap); + let md = html_email_to_markdown(&html); + assert_eq!( + md, "[Email HTML body truncated for processing]", + "empty body after strip must still signal truncation: {md:?}" + ); +} + #[test] fn normalize_markdownish_text_removes_invisible_and_extra_spaces() { let input = " Hello\u{200b} world \n\n line\u{00a0}two "; diff --git a/src/openhuman/composio/providers/gmail/provider.rs b/src/openhuman/composio/providers/gmail/provider.rs index c74f040c2..d4c68c11e 100644 --- a/src/openhuman/composio/providers/gmail/provider.rs +++ b/src/openhuman/composio/providers/gmail/provider.rs @@ -6,9 +6,9 @@ //! 2. Check the daily request budget — bail early if exhausted. //! 3. Fetch a page of recent messages via `GMAIL_FETCH_EMAILS`, adding //! a date filter when a cursor exists so only newer mail is returned. -//! 4. Run [`ComposioProvider::post_process_action_result`] (HTML→md, -//! normalise, sanitise) on the page so the LLM-facing chunk content -//! is cleaned, not raw. +//! 4. Run [`ComposioProvider::post_process_action_result`] (bounded +//! HTML→text, normalise, sanitise) on the page so the LLM-facing chunk +//! content is cleaned, not raw. //! 5. Filter against `synced_ids` for an early-stop optimisation, //! then ingest the new messages into the memory tree via //! [`super::ingest::ingest_page_into_memory_tree`] — same pipeline