mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
fix(composio/gmail): phase out html2md, prefer text/plain MIME part (#1170)
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
sanil-23
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
e0ff95ae79
commit
b90dad121d
Generated
+6
-86
@@ -1163,7 +1163,7 @@ dependencies = [
|
||||
"core-foundation-sys 0.8.7",
|
||||
"coreaudio-rs",
|
||||
"dasp_sample",
|
||||
"jni 0.21.1",
|
||||
"jni",
|
||||
"js-sys",
|
||||
"libc",
|
||||
"mach2 0.4.3",
|
||||
@@ -2566,34 +2566,6 @@ version = "3.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f"
|
||||
|
||||
[[package]]
|
||||
name = "html2md"
|
||||
version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8cff9891f2e0d9048927fbdfc28b11bf378f6a93c7ba70b23d0fbee9af6071b4"
|
||||
dependencies = [
|
||||
"html5ever 0.27.0",
|
||||
"jni 0.19.0",
|
||||
"lazy_static",
|
||||
"markup5ever_rcdom",
|
||||
"percent-encoding",
|
||||
"regex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html5ever"
|
||||
version = "0.27.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c13771afe0e6e846f1e67d038d4cb29998a6779f93c809212e4e9c32efd244d4"
|
||||
dependencies = [
|
||||
"log",
|
||||
"mac",
|
||||
"markup5ever 0.12.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html5ever"
|
||||
version = "0.35.0"
|
||||
@@ -2601,7 +2573,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "55d958c2f74b664487a2035fe1dadb032c48718a03b63f3ab0b8537db8549ed4"
|
||||
dependencies = [
|
||||
"log",
|
||||
"markup5ever 0.35.0",
|
||||
"markup5ever",
|
||||
"match_token",
|
||||
]
|
||||
|
||||
@@ -3151,20 +3123,6 @@ dependencies = [
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jni"
|
||||
version = "0.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6df18c2e3db7e453d3c6ac5b3e9d5182664d28788126d39b91f2d1e22b017ec"
|
||||
dependencies = [
|
||||
"cesu8",
|
||||
"combine",
|
||||
"jni-sys 0.3.1",
|
||||
"log",
|
||||
"thiserror 1.0.69",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jni"
|
||||
version = "0.21.1"
|
||||
@@ -3552,20 +3510,6 @@ version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
|
||||
|
||||
[[package]]
|
||||
name = "markup5ever"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "16ce3abbeba692c8b8441d036ef91aea6df8da2c6b6e21c7e14d3c18e526be45"
|
||||
dependencies = [
|
||||
"log",
|
||||
"phf 0.11.3",
|
||||
"phf_codegen 0.11.3",
|
||||
"string_cache",
|
||||
"string_cache_codegen",
|
||||
"tendril",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "markup5ever"
|
||||
version = "0.35.0"
|
||||
@@ -3577,18 +3521,6 @@ dependencies = [
|
||||
"web_atoms",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "markup5ever_rcdom"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "edaa21ab3701bfee5099ade5f7e1f84553fd19228cf332f13cd6e964bf59be18"
|
||||
dependencies = [
|
||||
"html5ever 0.27.0",
|
||||
"markup5ever 0.12.1",
|
||||
"tendril",
|
||||
"xml5ever",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "match_token"
|
||||
version = "0.35.0"
|
||||
@@ -4531,7 +4463,7 @@ version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb"
|
||||
dependencies = [
|
||||
"jni 0.21.1",
|
||||
"jni",
|
||||
"ndk",
|
||||
"ndk-context",
|
||||
"num-derive",
|
||||
@@ -4568,7 +4500,7 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
|
||||
|
||||
[[package]]
|
||||
name = "openhuman"
|
||||
version = "0.53.12"
|
||||
version = "0.53.13"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -4600,7 +4532,6 @@ dependencies = [
|
||||
"hmac 0.12.1",
|
||||
"hostname",
|
||||
"hound",
|
||||
"html2md",
|
||||
"iana-time-zone",
|
||||
"image",
|
||||
"landlock",
|
||||
@@ -5949,7 +5880,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a6dcd6e9823e177d15460d3cd3a413f38a2beea381f26aca1001c05cd6954ff"
|
||||
dependencies = [
|
||||
"as_variant",
|
||||
"html5ever 0.35.0",
|
||||
"html5ever",
|
||||
"tracing",
|
||||
"wildmatch",
|
||||
]
|
||||
@@ -6091,7 +6022,7 @@ checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784"
|
||||
dependencies = [
|
||||
"core-foundation 0.10.1",
|
||||
"core-foundation-sys 0.8.7",
|
||||
"jni 0.21.1",
|
||||
"jni",
|
||||
"log",
|
||||
"once_cell",
|
||||
"rustls",
|
||||
@@ -8956,17 +8887,6 @@ version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56"
|
||||
|
||||
[[package]]
|
||||
name = "xml5ever"
|
||||
version = "0.18.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9bbb26405d8e919bc1547a5aa9abc95cbfa438f04844f5fdd9dc7596b748bf69"
|
||||
dependencies = [
|
||||
"log",
|
||||
"mac",
|
||||
"markup5ever 0.12.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "xxhash-rust"
|
||||
version = "0.8.15"
|
||||
|
||||
+10
-7
@@ -25,13 +25,16 @@ crate-type = ["rlib"]
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
serde_yaml = "0.9"
|
||||
# Used by composio post-processing to convert Gmail HTML bodies to
|
||||
# markdown. `html2md` is the canonical pure-Rust crate (no system deps);
|
||||
# `htmd` (>0.5) is more actively maintained but our usage is one
|
||||
# function call (`parse_html`) and the migration cost outweighs the
|
||||
# upside today. Re-evaluate if html2md goes unmaintained or we need
|
||||
# GFM/table fidelity beyond what html2md emits.
|
||||
html2md = "0.2"
|
||||
# (Removed `html2md` dep. dhat-rs profiling on real Gmail inboxes
|
||||
# showed `html2md::walk` and `html2md::tables::handle` allocating
|
||||
# ~894 MB peak heap on a 10 KB HTML input from Otter.ai-style emails
|
||||
# (deeply-nested table-as-layout HTML). Cause: recursive walker holding
|
||||
# per-frame Vec state across nesting layers + 5 sequential
|
||||
# `regex::replace_all` passes in `clean_markdown` each producing a
|
||||
# fresh full-size String. We now use a linear-time tag-and-entity
|
||||
# stripper (`fast_html_to_text` in
|
||||
# providers/gmail/post_process.rs) and prefer the email's
|
||||
# `text/plain` MIME part when available.)
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "blocking", "rustls-tls", "native-tls", "stream", "http2", "multipart", "socks"] }
|
||||
tokio = { version = "1", features = ["full", "sync"] }
|
||||
once_cell = "1.19"
|
||||
|
||||
@@ -49,11 +49,6 @@ use once_cell::sync::Lazy;
|
||||
use regex::Regex;
|
||||
use serde_json::{json, Map, Value};
|
||||
|
||||
/// `html2md` is fine for normal transactional emails, but large marketing
|
||||
/// HTML can explode CPU / latency. Above this size we switch to a bounded
|
||||
/// fast-strip path that preserves readable text and link labels.
|
||||
const MAX_HTML2MD_INPUT_BYTES: usize = 24_000;
|
||||
|
||||
static HTML_NOISE_BLOCK_RE: Lazy<Regex> =
|
||||
Lazy::new(|| Regex::new(r"(?is)<!--.*?-->").expect("valid html comment regex"));
|
||||
|
||||
@@ -194,14 +189,27 @@ fn pick_header(msg: &Map<String, Value>, name: &str) -> Option<Value> {
|
||||
/// Extract the best body representation and return it as markdown.
|
||||
/// Walks `payload.parts[]` recursively — Gmail nests multipart/alternative
|
||||
/// inside multipart/mixed when attachments are present.
|
||||
///
|
||||
/// Preference order:
|
||||
/// 1. `text/plain` — author-provided plaintext fallback. Standard MIME
|
||||
/// multipart/alternative. Bypasses html2md entirely. For LLM
|
||||
/// extraction + retrieval embedding, plaintext is generally the
|
||||
/// better input: less noise (no tracking URLs, inline CSS, table
|
||||
/// formatting artifacts), zero conversion cost, no allocator
|
||||
/// pressure from `html2md`'s pathological walk over nested HTML
|
||||
/// tables (verified GB-scale heap peaks via dhat-rs profiling).
|
||||
/// 2. `text/html` → html2md — only used when the email shipped no
|
||||
/// plaintext part (rare; some HTML-only marketing senders).
|
||||
/// 3. Top-level `messageText` (Composio convenience field) — html or
|
||||
/// plaintext depending on what the source had.
|
||||
fn extract_markdown_body(msg: &Map<String, Value>) -> String {
|
||||
if let Some(parts) = msg.get("payload").and_then(|p| p.get("parts")) {
|
||||
if let Some(html) = find_decoded_part(parts, "text/html") {
|
||||
return html_email_to_markdown(&html);
|
||||
}
|
||||
if let Some(text) = find_decoded_part(parts, "text/plain") {
|
||||
return normalize_markdownish_text(&text);
|
||||
}
|
||||
if let Some(html) = find_decoded_part(parts, "text/html") {
|
||||
return html_email_to_markdown(&html);
|
||||
}
|
||||
}
|
||||
// Fallback: top-level decoded plain text (Composio convenience field).
|
||||
if let Some(text) = msg.get("messageText").and_then(|v| v.as_str()) {
|
||||
@@ -210,51 +218,47 @@ fn extract_markdown_body(msg: &Map<String, Value>) -> String {
|
||||
text_bytes = text.len(),
|
||||
"[composio:gmail][post-process] messageText looked like html, using fast html strip"
|
||||
);
|
||||
return fast_html_email_to_markdown(text);
|
||||
return html_email_to_markdown(text);
|
||||
}
|
||||
return normalize_markdownish_text(text);
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
|
||||
/// Convert raw HTML email into markdown-ish text that is safe and cheap for
|
||||
/// LLM consumption. Small / normal HTML uses `html2md`; oversized HTML falls
|
||||
/// back to a linear-time stripper so one pathological newsletter cannot stall
|
||||
/// the whole tool call.
|
||||
/// Convert raw HTML email into markdown-ish text suitable for LLM
|
||||
/// consumption.
|
||||
///
|
||||
/// Pipeline:
|
||||
/// 1. `strip_html_noise_blocks` — remove `<script>`, `<style>`, `<head>`,
|
||||
/// `<title>`, `<svg>`, `<noscript>` plus the `<!-- ... -->` /
|
||||
/// `<!--[if]-->` MSO conditional comments.
|
||||
/// 2. `fast_html_to_text` — linear-time tag-and-entity stripper.
|
||||
/// 3. `normalize_markdownish_text` — collapse whitespace, decode
|
||||
/// remaining entities, drop invisible characters.
|
||||
///
|
||||
/// **Why not `html2md`**: the previous implementation used
|
||||
/// `html2md::parse_html` for "small" inputs (under 24 KB) and fell back
|
||||
/// to the fast stripper above the threshold. dhat-rs heap profiling on
|
||||
/// real Gmail inboxes (Otter.ai meeting summaries with deeply-nested
|
||||
/// `<table>` layouts) showed `html2md::walk` and `html2md::tables::handle`
|
||||
/// allocating **894 MB peak** on a 10 KB HTML input — roughly 37,000×
|
||||
/// input size. Cause: recursive walker holding per-frame Vec state
|
||||
/// across nested-table layers, plus 5 sequential `regex::replace_all`
|
||||
/// passes in `clean_markdown` each producing a fresh full-size String.
|
||||
/// We removed the dependency entirely; the fast stripper produces
|
||||
/// somewhat plainer output (no `**bold**` or `[label](url)` markdown
|
||||
/// affordances) but preserves all content and runs in O(n) memory.
|
||||
///
|
||||
/// For multipart emails, `extract_markdown_body` prefers `text/plain`
|
||||
/// before this function is reached, so this path only runs for
|
||||
/// HTML-only emails (rare).
|
||||
fn html_email_to_markdown(html: &str) -> String {
|
||||
let cleaned = strip_html_noise_blocks(html);
|
||||
let cleaned = cleaned.trim();
|
||||
if cleaned.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
if cleaned.len() > MAX_HTML2MD_INPUT_BYTES {
|
||||
tracing::debug!(
|
||||
html_bytes = cleaned.len(),
|
||||
threshold = MAX_HTML2MD_INPUT_BYTES,
|
||||
"[composio:gmail][post-process] large html body, using fast strip fallback"
|
||||
);
|
||||
return normalize_markdownish_text(&fast_html_to_text(cleaned));
|
||||
}
|
||||
|
||||
let md = html2md::parse_html(cleaned);
|
||||
let normalized = normalize_markdownish_text(&md);
|
||||
if normalized.is_empty()
|
||||
|| looks_like_raw_html(&normalized)
|
||||
|| suspiciously_short_markdown(cleaned, &normalized)
|
||||
{
|
||||
tracing::debug!(
|
||||
html_bytes = cleaned.len(),
|
||||
"[composio:gmail][post-process] html2md output still looked like html, using fast strip fallback"
|
||||
);
|
||||
return normalize_markdownish_text(&fast_html_to_text(cleaned));
|
||||
}
|
||||
normalized
|
||||
}
|
||||
|
||||
fn fast_html_email_to_markdown(html: &str) -> String {
|
||||
let cleaned = strip_html_noise_blocks(html);
|
||||
normalize_markdownish_text(&fast_html_to_text(cleaned.trim()))
|
||||
normalize_markdownish_text(&fast_html_to_text(cleaned))
|
||||
}
|
||||
|
||||
fn strip_html_noise_blocks(html: &str) -> String {
|
||||
@@ -308,13 +312,18 @@ fn looks_like_raw_html(s: &str) -> bool {
|
||||
.any(|needle| lower.contains(needle))
|
||||
}
|
||||
|
||||
fn suspiciously_short_markdown(source_html: &str, markdown: &str) -> bool {
|
||||
source_html.len() >= 2_000 && markdown.len().saturating_mul(20) < source_html.len()
|
||||
}
|
||||
|
||||
/// Recursively search a `parts` array for the first MIME part whose
|
||||
/// `mimeType` starts with `prefix` (e.g. `"text/html"`), and return its
|
||||
/// base64url-decoded UTF-8 body.
|
||||
///
|
||||
/// **Skips attachment parts.** A `multipart/mixed` email may contain
|
||||
/// the real body (in a nested `multipart/alternative`) plus attached
|
||||
/// `.txt` / `.ics` / `.html` files. Each attachment also has a
|
||||
/// `text/plain` or `text/html` `mimeType` and a populated `body.data`,
|
||||
/// so a naive walk would return the attachment instead of the body.
|
||||
/// We treat any part with a non-empty `filename` (top-level or nested
|
||||
/// in `body.attachmentId`) as an attachment and skip its body — but
|
||||
/// still recurse into its `parts` for symmetry.
|
||||
fn find_decoded_part(parts: &Value, prefix: &str) -> Option<String> {
|
||||
let arr = parts.as_array()?;
|
||||
for part in arr {
|
||||
@@ -322,7 +331,12 @@ fn find_decoded_part(parts: &Value, prefix: &str) -> Option<String> {
|
||||
.get("mimeType")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or_default();
|
||||
if mime.starts_with(prefix) {
|
||||
let is_attachment = part
|
||||
.get("filename")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| !s.is_empty())
|
||||
.unwrap_or(false);
|
||||
if mime.starts_with(prefix) && !is_attachment {
|
||||
if let Some(b64) = part.pointer("/body/data").and_then(|v| v.as_str()) {
|
||||
if let Ok(bytes) = URL_SAFE_NO_PAD.decode(b64) {
|
||||
if let Ok(s) = String::from_utf8(bytes) {
|
||||
|
||||
@@ -28,9 +28,13 @@ fn fixture() -> Value {
|
||||
"payload": {
|
||||
"headers": [ { "name": "Date", "value": "Fri, 17 Apr 2026 12:00:00 +0000" } ],
|
||||
"parts": [
|
||||
// Realistic MIME multipart/alternative: text/plain
|
||||
// mirrors text/html content. Author-provided
|
||||
// plaintext is what we now prefer for downstream
|
||||
// ingestion (see extract_markdown_body docstring).
|
||||
{
|
||||
"mimeType": "text/plain",
|
||||
"body": { "data": b64("Hi plain text") }
|
||||
"body": { "data": b64("Title\n\nHello world") }
|
||||
},
|
||||
{
|
||||
"mimeType": "text/html",
|
||||
@@ -163,7 +167,12 @@ fn non_fetch_slug_is_noop() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_multipart_html_is_found() {
|
||||
fn nested_multipart_prefers_plaintext_over_html() {
|
||||
// When a multipart/alternative ships BOTH text/plain and text/html, the
|
||||
// plaintext part wins. text/plain is the author's intended fallback,
|
||||
// bypasses html2md (which has GB-scale heap peaks on rich HTML — see
|
||||
// post_process.rs::extract_markdown_body docstring), and is generally
|
||||
// cleaner input for downstream LLM extraction.
|
||||
let html = "<p>Deep <b>body</b></p>";
|
||||
let mut v = json!({
|
||||
"messages": [{
|
||||
@@ -190,9 +199,57 @@ fn nested_multipart_html_is_found() {
|
||||
});
|
||||
post_process("GMAIL_FETCH_EMAILS", None, &mut v);
|
||||
let md = v["messages"][0]["markdown"].as_str().unwrap();
|
||||
assert!(md.contains("Deep"));
|
||||
assert!(md.contains("body"));
|
||||
assert!(!md.contains("<p>"));
|
||||
assert!(
|
||||
md.contains("plain fallback"),
|
||||
"plaintext should win, got: {md:?}"
|
||||
);
|
||||
// The HTML body should NOT appear — html2md was bypassed entirely.
|
||||
assert!(
|
||||
!md.contains("Deep"),
|
||||
"html should not have been used: {md:?}"
|
||||
);
|
||||
assert!(
|
||||
!md.contains("<p>"),
|
||||
"raw html should never leak through: {md:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_multipart_falls_back_to_html_when_no_plaintext() {
|
||||
// For html-only emails (rare — some poorly-built marketing senders),
|
||||
// we still need html2md → markdown. The 24 KB threshold and noise-block
|
||||
// stripper guard against pathological cases.
|
||||
let html = "<p>Deep <b>body</b></p>";
|
||||
let mut v = json!({
|
||||
"messages": [{
|
||||
"messageId": "m1",
|
||||
"threadId": "t1",
|
||||
"subject": "s",
|
||||
"sender": "a@x.com",
|
||||
"to": "b@y.com",
|
||||
"messageTimestamp": "2026-04-17",
|
||||
"labelIds": [],
|
||||
"messageText": "",
|
||||
"payload": {
|
||||
"parts": [
|
||||
{
|
||||
"mimeType": "multipart/alternative",
|
||||
"parts": [
|
||||
{ "mimeType": "text/html", "body": { "data": b64(html) } }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}]
|
||||
});
|
||||
post_process("GMAIL_FETCH_EMAILS", None, &mut v);
|
||||
let md = v["messages"][0]["markdown"].as_str().unwrap();
|
||||
assert!(md.contains("Deep"), "html2md should have run: {md:?}");
|
||||
assert!(md.contains("body"), "html2md should have run: {md:?}");
|
||||
assert!(
|
||||
!md.contains("<p>"),
|
||||
"raw html should not leak through: {md:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -316,22 +373,68 @@ fn html_in_message_text_is_converted() {
|
||||
assert!(!md.contains("<html>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suspiciously_short_markdown_detects_large_collapse() {
|
||||
assert!(suspiciously_short_markdown(&"x".repeat(4000), "tiny"));
|
||||
assert!(!suspiciously_short_markdown(
|
||||
&"x".repeat(4000),
|
||||
&"y".repeat(400)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fast_html_strip_handles_long_tags() {
|
||||
let long_href = format!(
|
||||
"<a href=\"https://example.com/{}\">Click me</a><p>After link</p>",
|
||||
"x".repeat(600)
|
||||
);
|
||||
let md = fast_html_email_to_markdown(&long_href);
|
||||
let md = html_email_to_markdown(&long_href);
|
||||
assert!(md.contains("Click me"));
|
||||
assert!(md.contains("After link"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_plain_attachment_does_not_outrank_html_body() {
|
||||
// multipart/mixed email with:
|
||||
// - multipart/alternative (real body: text/plain + text/html)
|
||||
// - text/plain attachment ("notes.txt")
|
||||
//
|
||||
// Without filtering attachments, find_decoded_part(_, "text/plain")
|
||||
// could pick up the attachment's content and return it instead of
|
||||
// the body. This test pins the attachment-skip behaviour.
|
||||
let mut v = json!({
|
||||
"messages": [{
|
||||
"messageId": "m1",
|
||||
"threadId": "t1",
|
||||
"subject": "s",
|
||||
"sender": "a@x.com",
|
||||
"to": "b@y.com",
|
||||
"messageTimestamp": "2026-04-17",
|
||||
"labelIds": [],
|
||||
"messageText": "",
|
||||
"payload": {
|
||||
"parts": [
|
||||
{
|
||||
"mimeType": "multipart/alternative",
|
||||
"parts": [
|
||||
{
|
||||
"mimeType": "text/plain",
|
||||
"body": { "data": b64("real body content") }
|
||||
},
|
||||
{
|
||||
"mimeType": "text/html",
|
||||
"body": { "data": b64("<p>real body content</p>") }
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mimeType": "text/plain",
|
||||
"filename": "notes.txt",
|
||||
"body": { "data": b64("attachment content — not the body") }
|
||||
}
|
||||
]
|
||||
}
|
||||
}]
|
||||
});
|
||||
post_process("GMAIL_FETCH_EMAILS", None, &mut v);
|
||||
let md = v["messages"][0]["markdown"].as_str().unwrap();
|
||||
assert!(
|
||||
md.contains("real body content"),
|
||||
"real body should win, got: {md:?}"
|
||||
);
|
||||
assert!(
|
||||
!md.contains("attachment content"),
|
||||
"attachment must not leak into markdown body: {md:?}"
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user