From b90dad121d7232de9cf7a8f1eb51460aa2d7806f Mon Sep 17 00:00:00 2001
From: sanil-23 <sanil@vezures.xyz>
Date: Mon, 4 May 2026 12:41:04 +0530
Subject: [PATCH] fix(composio/gmail): phase out html2md, prefer text/plain
 MIME part (#1170)

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 Cargo.lock                                    |  92 +-----------
 Cargo.toml                                    |  17 ++-
 .../composio/providers/gmail/post_process.rs  | 106 ++++++++------
 .../providers/gmail/post_process_tests.rs     | 133 ++++++++++++++++--
 4 files changed, 194 insertions(+), 154 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index ef1724a91..99acf669b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1163,7 +1163,7 @@ dependencies = [
  "core-foundation-sys 0.8.7",
  "coreaudio-rs",
  "dasp_sample",
- "jni 0.21.1",
+ "jni",
  "js-sys",
  "libc",
  "mach2 0.4.3",
@@ -2566,34 +2566,6 @@ version = "3.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f"
 
-[[package]]
-name = "html2md"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cff9891f2e0d9048927fbdfc28b11bf378f6a93c7ba70b23d0fbee9af6071b4"
-dependencies = [
- "html5ever 0.27.0",
- "jni 0.19.0",
- "lazy_static",
- "markup5ever_rcdom",
- "percent-encoding",
- "regex",
-]
-
-[[package]]
-name = "html5ever"
-version = "0.27.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c13771afe0e6e846f1e67d038d4cb29998a6779f93c809212e4e9c32efd244d4"
-dependencies = [
- "log",
- "mac",
- "markup5ever 0.12.1",
- "proc-macro2",
- "quote",
- "syn 2.0.117",
-]
-
 [[package]]
 name = "html5ever"
 version = "0.35.0"
@@ -2601,7 +2573,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "55d958c2f74b664487a2035fe1dadb032c48718a03b63f3ab0b8537db8549ed4"
 dependencies = [
  "log",
- "markup5ever 0.35.0",
+ "markup5ever",
  "match_token",
 ]
 
@@ -3151,20 +3123,6 @@ dependencies = [
  "syn 2.0.117",
 ]
 
-[[package]]
-name = "jni"
-version = "0.19.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6df18c2e3db7e453d3c6ac5b3e9d5182664d28788126d39b91f2d1e22b017ec"
-dependencies = [
- "cesu8",
- "combine",
- "jni-sys 0.3.1",
- "log",
- "thiserror 1.0.69",
- "walkdir",
-]
-
 [[package]]
 name = "jni"
 version = "0.21.1"
@@ -3552,20 +3510,6 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
 
-[[package]]
-name = "markup5ever"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16ce3abbeba692c8b8441d036ef91aea6df8da2c6b6e21c7e14d3c18e526be45"
-dependencies = [
- "log",
- "phf 0.11.3",
- "phf_codegen 0.11.3",
- "string_cache",
- "string_cache_codegen",
- "tendril",
-]
-
 [[package]]
 name = "markup5ever"
 version = "0.35.0"
@@ -3577,18 +3521,6 @@ dependencies = [
  "web_atoms",
 ]
 
-[[package]]
-name = "markup5ever_rcdom"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edaa21ab3701bfee5099ade5f7e1f84553fd19228cf332f13cd6e964bf59be18"
-dependencies = [
- "html5ever 0.27.0",
- "markup5ever 0.12.1",
- "tendril",
- "xml5ever",
-]
-
 [[package]]
 name = "match_token"
 version = "0.35.0"
@@ -4531,7 +4463,7 @@ version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb"
 dependencies = [
- "jni 0.21.1",
+ "jni",
  "ndk",
  "ndk-context",
  "num-derive",
@@ -4568,7 +4500,7 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
 
 [[package]]
 name = "openhuman"
-version = "0.53.12"
+version = "0.53.13"
 dependencies = [
  "aes-gcm",
  "anyhow",
@@ -4600,7 +4532,6 @@ dependencies = [
  "hmac 0.12.1",
  "hostname",
  "hound",
- "html2md",
  "iana-time-zone",
  "image",
  "landlock",
@@ -5949,7 +5880,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7a6dcd6e9823e177d15460d3cd3a413f38a2beea381f26aca1001c05cd6954ff"
 dependencies = [
  "as_variant",
- "html5ever 0.35.0",
+ "html5ever",
  "tracing",
  "wildmatch",
 ]
@@ -6091,7 +6022,7 @@ checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784"
 dependencies = [
  "core-foundation 0.10.1",
  "core-foundation-sys 0.8.7",
- "jni 0.21.1",
+ "jni",
  "log",
  "once_cell",
  "rustls",
@@ -8956,17 +8887,6 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56"
 
-[[package]]
-name = "xml5ever"
-version = "0.18.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9bbb26405d8e919bc1547a5aa9abc95cbfa438f04844f5fdd9dc7596b748bf69"
-dependencies = [
- "log",
- "mac",
- "markup5ever 0.12.1",
-]
-
 [[package]]
 name = "xxhash-rust"
 version = "0.8.15"
diff --git a/Cargo.toml b/Cargo.toml
index 3b9f56d39..00a7ff05e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -25,13 +25,16 @@ crate-type = ["rlib"]
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 serde_yaml = "0.9"
-# Used by composio post-processing to convert Gmail HTML bodies to
-# markdown. `html2md` is the canonical pure-Rust crate (no system deps);
-# `htmd` (>0.5) is more actively maintained but our usage is one
-# function call (`parse_html`) and the migration cost outweighs the
-# upside today. Re-evaluate if html2md goes unmaintained or we need
-# GFM/table fidelity beyond what html2md emits.
-html2md = "0.2"
+# (Removed `html2md` dep. dhat-rs profiling on real Gmail inboxes
+# showed `html2md::walk` and `html2md::tables::handle` allocating
+# ~894 MB peak heap on a 10 KB HTML input from Otter.ai-style emails
+# (deeply-nested table-as-layout HTML). Cause: recursive walker holding
+# per-frame Vec state across nesting layers + 5 sequential
+# `regex::replace_all` passes in `clean_markdown` each producing a
+# fresh full-size String. We now use a linear-time tag-and-entity
+# stripper (`fast_html_to_text` in
+# providers/gmail/post_process.rs) and prefer the email's
+# `text/plain` MIME part when available.)
 reqwest = { version = "0.12", default-features = false, features = ["json", "blocking", "rustls-tls", "native-tls", "stream", "http2", "multipart", "socks"] }
 tokio = { version = "1", features = ["full", "sync"] }
 once_cell = "1.19"
diff --git a/src/openhuman/composio/providers/gmail/post_process.rs b/src/openhuman/composio/providers/gmail/post_process.rs
index 9e0b9b7a8..20f9f4712 100644
--- a/src/openhuman/composio/providers/gmail/post_process.rs
+++ b/src/openhuman/composio/providers/gmail/post_process.rs
@@ -49,11 +49,6 @@ use once_cell::sync::Lazy;
 use regex::Regex;
 use serde_json::{json, Map, Value};
 
-/// `html2md` is fine for normal transactional emails, but large marketing
-/// HTML can explode CPU / latency. Above this size we switch to a bounded
-/// fast-strip path that preserves readable text and link labels.
-const MAX_HTML2MD_INPUT_BYTES: usize = 24_000;
-
 static HTML_NOISE_BLOCK_RE: Lazy<Regex> =
     Lazy::new(|| Regex::new(r"(?is)<!--.*?-->").expect("valid html comment regex"));
 
@@ -194,14 +189,27 @@ fn pick_header(msg: &Map<String, Value>, name: &str) -> Option<Value> {
 /// Extract the best body representation and return it as markdown.
 /// Walks `payload.parts[]` recursively — Gmail nests multipart/alternative
 /// inside multipart/mixed when attachments are present.
+///
+/// Preference order:
+///   1. `text/plain` — author-provided plaintext fallback. Standard MIME
+///      multipart/alternative. Bypasses html2md entirely. For LLM
+///      extraction + retrieval embedding, plaintext is generally the
+///      better input: less noise (no tracking URLs, inline CSS, table
+///      formatting artifacts), zero conversion cost, no allocator
+///      pressure from `html2md`'s pathological walk over nested HTML
+///      tables (verified GB-scale heap peaks via dhat-rs profiling).
+///   2. `text/html` → html2md — only used when the email shipped no
+///      plaintext part (rare; some HTML-only marketing senders).
+///   3. Top-level `messageText` (Composio convenience field) — html or
+///      plaintext depending on what the source had.
 fn extract_markdown_body(msg: &Map<String, Value>) -> String {
     if let Some(parts) = msg.get("payload").and_then(|p| p.get("parts")) {
-        if let Some(html) = find_decoded_part(parts, "text/html") {
-            return html_email_to_markdown(&html);
-        }
         if let Some(text) = find_decoded_part(parts, "text/plain") {
             return normalize_markdownish_text(&text);
         }
+        if let Some(html) = find_decoded_part(parts, "text/html") {
+            return html_email_to_markdown(&html);
+        }
     }
     // Fallback: top-level decoded plain text (Composio convenience field).
     if let Some(text) = msg.get("messageText").and_then(|v| v.as_str()) {
@@ -210,51 +218,47 @@ fn extract_markdown_body(msg: &Map<String, Value>) -> String {
                 text_bytes = text.len(),
                 "[composio:gmail][post-process] messageText looked like html, using fast html strip"
             );
-            return fast_html_email_to_markdown(text);
+            return html_email_to_markdown(text);
         }
         return normalize_markdownish_text(text);
     }
     String::new()
 }
 
-/// Convert raw HTML email into markdown-ish text that is safe and cheap for
-/// LLM consumption. Small / normal HTML uses `html2md`; oversized HTML falls
-/// back to a linear-time stripper so one pathological newsletter cannot stall
-/// the whole tool call.
+/// Convert raw HTML email into markdown-ish text suitable for LLM
+/// consumption.
+///
+/// Pipeline:
+///   1. `strip_html_noise_blocks` — remove `<script>`, `<style>`, `<head>`,
+///      `<title>`, `<svg>`, `<noscript>` plus the `<!-- ... -->` /
+///      `<!--[if]-->` MSO conditional comments.
+///   2. `fast_html_to_text` — linear-time tag-and-entity stripper.
+///   3. `normalize_markdownish_text` — collapse whitespace, decode
+///      remaining entities, drop invisible characters.
+///
+/// **Why not `html2md`**: the previous implementation used
+/// `html2md::parse_html` for "small" inputs (under 24 KB) and fell back
+/// to the fast stripper above the threshold. dhat-rs heap profiling on
+/// real Gmail inboxes (Otter.ai meeting summaries with deeply-nested
+/// `<table>` layouts) showed `html2md::walk` and `html2md::tables::handle`
+/// allocating **894 MB peak** on a 10 KB HTML input — roughly 37,000×
+/// input size. Cause: recursive walker holding per-frame Vec state
+/// across nested-table layers, plus 5 sequential `regex::replace_all`
+/// passes in `clean_markdown` each producing a fresh full-size String.
+/// We removed the dependency entirely; the fast stripper produces
+/// somewhat plainer output (no `**bold**` or `[label](url)` markdown
+/// affordances) but preserves all content and runs in O(n) memory.
+///
+/// For multipart emails, `extract_markdown_body` prefers `text/plain`
+/// before this function is reached, so this path only runs for
+/// HTML-only emails (rare).
 fn html_email_to_markdown(html: &str) -> String {
     let cleaned = strip_html_noise_blocks(html);
     let cleaned = cleaned.trim();
     if cleaned.is_empty() {
         return String::new();
     }
-
-    if cleaned.len() > MAX_HTML2MD_INPUT_BYTES {
-        tracing::debug!(
-            html_bytes = cleaned.len(),
-            threshold = MAX_HTML2MD_INPUT_BYTES,
-            "[composio:gmail][post-process] large html body, using fast strip fallback"
-        );
-        return normalize_markdownish_text(&fast_html_to_text(cleaned));
-    }
-
-    let md = html2md::parse_html(cleaned);
-    let normalized = normalize_markdownish_text(&md);
-    if normalized.is_empty()
-        || looks_like_raw_html(&normalized)
-        || suspiciously_short_markdown(cleaned, &normalized)
-    {
-        tracing::debug!(
-            html_bytes = cleaned.len(),
-            "[composio:gmail][post-process] html2md output still looked like html, using fast strip fallback"
-        );
-        return normalize_markdownish_text(&fast_html_to_text(cleaned));
-    }
-    normalized
-}
-
-fn fast_html_email_to_markdown(html: &str) -> String {
-    let cleaned = strip_html_noise_blocks(html);
-    normalize_markdownish_text(&fast_html_to_text(cleaned.trim()))
+    normalize_markdownish_text(&fast_html_to_text(cleaned))
 }
 
 fn strip_html_noise_blocks(html: &str) -> String {
@@ -308,13 +312,18 @@ fn looks_like_raw_html(s: &str) -> bool {
     .any(|needle| lower.contains(needle))
 }
 
-fn suspiciously_short_markdown(source_html: &str, markdown: &str) -> bool {
-    source_html.len() >= 2_000 && markdown.len().saturating_mul(20) < source_html.len()
-}
-
 /// Recursively search a `parts` array for the first MIME part whose
 /// `mimeType` starts with `prefix` (e.g. `"text/html"`), and return its
 /// base64url-decoded UTF-8 body.
+///
+/// **Skips attachment parts.** A `multipart/mixed` email may contain
+/// the real body (in a nested `multipart/alternative`) plus attached
+/// `.txt` / `.ics` / `.html` files. Each attachment also has a
+/// `text/plain` or `text/html` `mimeType` and a populated `body.data`,
+/// so a naive walk would return the attachment instead of the body.
+/// We treat any part with a non-empty `filename` (top-level or nested
+/// in `body.attachmentId`) as an attachment and skip its body — but
+/// still recurse into its `parts` for symmetry.
 fn find_decoded_part(parts: &Value, prefix: &str) -> Option<String> {
     let arr = parts.as_array()?;
     for part in arr {
@@ -322,7 +331,12 @@ fn find_decoded_part(parts: &Value, prefix: &str) -> Option<String> {
             .get("mimeType")
             .and_then(|v| v.as_str())
             .unwrap_or_default();
-        if mime.starts_with(prefix) {
+        let is_attachment = part
+            .get("filename")
+            .and_then(|v| v.as_str())
+            .map(|s| !s.is_empty())
+            .unwrap_or(false);
+        if mime.starts_with(prefix) && !is_attachment {
             if let Some(b64) = part.pointer("/body/data").and_then(|v| v.as_str()) {
                 if let Ok(bytes) = URL_SAFE_NO_PAD.decode(b64) {
                     if let Ok(s) = String::from_utf8(bytes) {
diff --git a/src/openhuman/composio/providers/gmail/post_process_tests.rs b/src/openhuman/composio/providers/gmail/post_process_tests.rs
index 4b77cce01..96bd80030 100644
--- a/src/openhuman/composio/providers/gmail/post_process_tests.rs
+++ b/src/openhuman/composio/providers/gmail/post_process_tests.rs
@@ -28,9 +28,13 @@ fn fixture() -> Value {
                 "payload": {
                     "headers": [ { "name": "Date", "value": "Fri, 17 Apr 2026 12:00:00 +0000" } ],
                     "parts": [
+                        // Realistic MIME multipart/alternative: text/plain
+                        // mirrors text/html content. Author-provided
+                        // plaintext is what we now prefer for downstream
+                        // ingestion (see extract_markdown_body docstring).
                         {
                             "mimeType": "text/plain",
-                            "body": { "data": b64("Hi plain text") }
+                            "body": { "data": b64("Title\n\nHello world") }
                         },
                         {
                             "mimeType": "text/html",
@@ -163,7 +167,12 @@ fn non_fetch_slug_is_noop() {
 }
 
 #[test]
-fn nested_multipart_html_is_found() {
+fn nested_multipart_prefers_plaintext_over_html() {
+    // When a multipart/alternative ships BOTH text/plain and text/html, the
+    // plaintext part wins. text/plain is the author's intended fallback,
+    // bypasses html2md (which has GB-scale heap peaks on rich HTML — see
+    // post_process.rs::extract_markdown_body docstring), and is generally
+    // cleaner input for downstream LLM extraction.
     let html = "<p>Deep <b>body</b></p>";
     let mut v = json!({
         "messages": [{
@@ -190,9 +199,57 @@ fn nested_multipart_html_is_found() {
     });
     post_process("GMAIL_FETCH_EMAILS", None, &mut v);
     let md = v["messages"][0]["markdown"].as_str().unwrap();
-    assert!(md.contains("Deep"));
-    assert!(md.contains("body"));
-    assert!(!md.contains("<p>"));
+    assert!(
+        md.contains("plain fallback"),
+        "plaintext should win, got: {md:?}"
+    );
+    // The HTML body should NOT appear — html2md was bypassed entirely.
+    assert!(
+        !md.contains("Deep"),
+        "html should not have been used: {md:?}"
+    );
+    assert!(
+        !md.contains("<p>"),
+        "raw html should never leak through: {md:?}"
+    );
+}
+
+#[test]
+fn nested_multipart_falls_back_to_html_when_no_plaintext() {
+    // For html-only emails (rare — some poorly-built marketing senders),
+    // we still need html2md → markdown. The 24 KB threshold and noise-block
+    // stripper guard against pathological cases.
+    let html = "<p>Deep <b>body</b></p>";
+    let mut v = json!({
+        "messages": [{
+            "messageId": "m1",
+            "threadId": "t1",
+            "subject": "s",
+            "sender": "a@x.com",
+            "to": "b@y.com",
+            "messageTimestamp": "2026-04-17",
+            "labelIds": [],
+            "messageText": "",
+            "payload": {
+                "parts": [
+                    {
+                        "mimeType": "multipart/alternative",
+                        "parts": [
+                            { "mimeType": "text/html", "body": { "data": b64(html) } }
+                        ]
+                    }
+                ]
+            }
+        }]
+    });
+    post_process("GMAIL_FETCH_EMAILS", None, &mut v);
+    let md = v["messages"][0]["markdown"].as_str().unwrap();
+    assert!(md.contains("Deep"), "html2md should have run: {md:?}");
+    assert!(md.contains("body"), "html2md should have run: {md:?}");
+    assert!(
+        !md.contains("<p>"),
+        "raw html should not leak through: {md:?}"
+    );
 }
 
 #[test]
@@ -316,22 +373,68 @@ fn html_in_message_text_is_converted() {
     assert!(!md.contains("<html>"));
 }
 
-#[test]
-fn suspiciously_short_markdown_detects_large_collapse() {
-    assert!(suspiciously_short_markdown(&"x".repeat(4000), "tiny"));
-    assert!(!suspiciously_short_markdown(
-        &"x".repeat(4000),
-        &"y".repeat(400)
-    ));
-}
-
 #[test]
 fn fast_html_strip_handles_long_tags() {
     let long_href = format!(
         "<a href=\"https://example.com/{}\">Click me</a><p>After link</p>",
         "x".repeat(600)
     );
-    let md = fast_html_email_to_markdown(&long_href);
+    let md = html_email_to_markdown(&long_href);
     assert!(md.contains("Click me"));
     assert!(md.contains("After link"));
 }
+
+#[test]
+fn text_plain_attachment_does_not_outrank_html_body() {
+    // multipart/mixed email with:
+    //   - multipart/alternative (real body: text/plain + text/html)
+    //   - text/plain attachment ("notes.txt")
+    //
+    // Without filtering attachments, find_decoded_part(_, "text/plain")
+    // could pick up the attachment's content and return it instead of
+    // the body. This test pins the attachment-skip behaviour.
+    let mut v = json!({
+        "messages": [{
+            "messageId": "m1",
+            "threadId": "t1",
+            "subject": "s",
+            "sender": "a@x.com",
+            "to": "b@y.com",
+            "messageTimestamp": "2026-04-17",
+            "labelIds": [],
+            "messageText": "",
+            "payload": {
+                "parts": [
+                    {
+                        "mimeType": "multipart/alternative",
+                        "parts": [
+                            {
+                                "mimeType": "text/plain",
+                                "body": { "data": b64("real body content") }
+                            },
+                            {
+                                "mimeType": "text/html",
+                                "body": { "data": b64("<p>real body content</p>") }
+                            }
+                        ]
+                    },
+                    {
+                        "mimeType": "text/plain",
+                        "filename": "notes.txt",
+                        "body": { "data": b64("attachment content — not the body") }
+                    }
+                ]
+            }
+        }]
+    });
+    post_process("GMAIL_FETCH_EMAILS", None, &mut v);
+    let md = v["messages"][0]["markdown"].as_str().unwrap();
+    assert!(
+        md.contains("real body content"),
+        "real body should win, got: {md:?}"
+    );
+    assert!(
+        !md.contains("attachment content"),
+        "attachment must not leak into markdown body: {md:?}"
+    );
+}