From 4d73bf869a41caff4e840b22a739d8f43284733b Mon Sep 17 00:00:00 2001 From: oxoxDev <164490987+oxoxDev@users.noreply.github.com> Date: Sat, 16 May 2026 08:59:29 +0530 Subject: [PATCH] =?UTF-8?q?fix(whatsapp):=20recover=20DOM=20message=20bodi?= =?UTF-8?q?es=20=E2=80=94=20telemetry,=20tier-3=20fallback,=20source=20tag?= =?UTF-8?q?,=20synthetic=20chat=5Fid=20(#1376)=20(#1804)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Claude Opus 4.7 Co-authored-by: Steven Enamakel --- .../src/whatsapp_scanner/dom_snapshot.rs | 342 ++++++++++++++++-- .../src/whatsapp_scanner/dom_snapshot_test.rs | 131 +++++++ app/src-tauri/src/whatsapp_scanner/mod.rs | 223 ++++++++++-- .../test_fixtures/dom_snapshot_2026_05.json | 53 +++ 4 files changed, 691 insertions(+), 58 deletions(-) create mode 100644 app/src-tauri/src/whatsapp_scanner/dom_snapshot_test.rs create mode 100644 app/src-tauri/src/whatsapp_scanner/test_fixtures/dom_snapshot_2026_05.json diff --git a/app/src-tauri/src/whatsapp_scanner/dom_snapshot.rs b/app/src-tauri/src/whatsapp_scanner/dom_snapshot.rs index e930969d7..82aba2c85 100644 --- a/app/src-tauri/src/whatsapp_scanner/dom_snapshot.rs +++ b/app/src-tauri/src/whatsapp_scanner/dom_snapshot.rs @@ -56,6 +56,34 @@ impl DomMessage { } } +/// Per-stage telemetry produced by [`capture_messages`]. The counters +/// disambiguate the three failure modes that `dom=0` used to collapse into +/// a single number (issue #1376): rows never matched, rows matched but +/// body extraction returned empty, or active chat name failed to resolve +/// (forcing rows to be filtered out downstream by the merge step). +/// +/// Field invariants: +/// * `rows_seen` — `[data-id]` elements that parsed as message rows BEFORE +/// any body filter. Counts every accepted `data-id` shape (legacy +/// compound + bare-msgId). +/// * `rows_with_body` — subset where [`find_body`] returned a non-empty +/// string. `rows_seen - rows_with_body` is bodies that vanished. +/// * `rows_dropped_no_body` — convenience, equals +/// `rows_seen - rows_with_body`. +/// * `active_chat_resolved` — true when +/// `header[data-testid="conversation-header"]` produced a display name +/// (precondition for the chat-id reverse lookup in `mod.rs`). +#[derive(Debug, Clone, Default)] +pub struct CaptureReport { + pub rows: Vec, + pub hash: u64, + pub active_chat_name: Option, + pub rows_seen: usize, + pub rows_with_body: usize, + pub rows_dropped_no_body: usize, + pub active_chat_resolved: bool, +} + /// Run `DOMSnapshot.captureSnapshot` against an attached page session and /// return parsed message rows, a FNV-1a hash over (dataId, body), and the /// active conversation's display name (from @@ -64,10 +92,10 @@ impl DomMessage { /// modern WhatsApp Web omits the chat JID from the URL, from `data-id`, and /// from any DOM attribute, so the merge step in `mod.rs` reverse-looks-up /// `chats[*].name → chats[*].jid` to stamp `chatId` onto DOM rows. -pub async fn capture_messages( - cdp: &mut CdpConn, - session: &str, -) -> Result<(Vec, u64, Option), String> { +/// +/// Returns a [`CaptureReport`] with per-stage counters. See the type doc +/// for invariants. +pub async fn capture_messages(cdp: &mut CdpConn, session: &str) -> Result { // `computedStyles` is a required array — empty is fine, we don't need // any CSS. The other flags default sensibly; explicitly disable the // heavy paint/rect output to keep payloads small. @@ -84,16 +112,32 @@ pub async fn capture_messages( .await?; let snap: CaptureSnapshot = serde_json::from_value(raw).map_err(|e| format!("decode DOMSnapshot: {e}"))?; - let rows = parse_rows(&snap); - let hash = fnv_hash(&rows); - let active_chat_name = parse_active_chat_name(&snap); - Ok((rows, hash, active_chat_name)) + Ok(report_from_snapshot(&snap)) +} + +/// Synthesize a [`CaptureReport`] from a parsed `CaptureSnapshot`. Split +/// out from [`capture_messages`] so unit tests can drive the body-finder +/// + counters off a JSON fixture without mocking CDP. +pub(crate) fn report_from_snapshot(snap: &CaptureSnapshot) -> CaptureReport { + let stats = parse_rows(snap); + let hash = fnv_hash(&stats.rows); + let active_chat_name = parse_active_chat_name(snap); + let active_chat_resolved = active_chat_name.is_some(); + CaptureReport { + rows: stats.rows, + hash, + active_chat_name, + rows_seen: stats.rows_seen, + rows_with_body: stats.rows_with_body, + rows_dropped_no_body: stats.rows_seen.saturating_sub(stats.rows_with_body), + active_chat_resolved, + } } // ─── CDP response shape ───────────────────────────────────────────── #[derive(Deserialize, Debug, Default)] -struct CaptureSnapshot { +pub(crate) struct CaptureSnapshot { #[serde(default)] documents: Vec, #[serde(default)] @@ -123,6 +167,18 @@ struct NodeTreeSnap { attributes: Vec>, } +/// Output of [`parse_rows`] including per-stage counters used to populate +/// [`CaptureReport`]. `rows` is the filtered keep-set (rows with body OR +/// `data-pre-plain-text`); `rows_seen` is the count of accepted `data-id` +/// shapes BEFORE any body/chrome filter; `rows_with_body` is the count +/// where [`find_body`] returned non-empty for a row in `rows_seen`. +#[derive(Debug, Default)] +pub(crate) struct ParseStats { + pub rows: Vec, + pub rows_seen: usize, + pub rows_with_body: usize, +} + const NODE_TYPE_ELEMENT: i32 = 1; const NODE_TYPE_TEXT: i32 = 3; /// Hard cap on body length to mirror `dom_scan.js` (which sliced at 4000). @@ -130,17 +186,17 @@ const MAX_BODY_CHARS: usize = 4000; // ─── parsing ──────────────────────────────────────────────────────── -fn parse_rows(snap: &CaptureSnapshot) -> Vec { +fn parse_rows(snap: &CaptureSnapshot) -> ParseStats { // Main frame only — iframes aren't used by WhatsApp's message list. let doc = match snap.documents.first() { Some(d) => d, - None => return Vec::new(), + None => return ParseStats::default(), }; let nodes = &doc.nodes; let strings = &snap.strings; let count = nodes.node_type.len(); if count == 0 { - return Vec::new(); + return ParseStats::default(); } // Precompute children map so descendant walks are O(subtree) instead of @@ -154,6 +210,8 @@ fn parse_rows(snap: &CaptureSnapshot) -> Vec { let mut out = Vec::new(); let mut seen: HashSet = HashSet::new(); + let mut rows_seen = 0usize; + let mut rows_with_body = 0usize; for i in 0..count { if nodes.node_type.get(i).copied().unwrap_or(0) != NODE_TYPE_ELEMENT { @@ -174,10 +232,20 @@ fn parse_rows(snap: &CaptureSnapshot) -> Vec { continue; } + // Telemetry: count every accepted data-id BEFORE the body filter so + // the per-stage counts in `CaptureReport` distinguish "no rows + // matched at all" from "rows matched but body extraction failed". + rows_seen += 1; + let (pre_ts, author) = find_pre_plain(nodes, strings, &children, i); let body = find_body(nodes, strings, &children, i); + if !body.is_empty() { + rows_with_body += 1; + } // A row with neither a body nor a pre-plain-text tag is just chrome - // (avatar wrapper, reaction chip, etc) — skip it. + // (avatar wrapper, reaction chip, etc) — skip it. Note: this still + // contributes to `rows_seen` because the goal of that counter is + // "did we find rows at all", not "did we keep them". if body.is_empty() && pre_ts.is_none() { continue; } @@ -193,7 +261,11 @@ fn parse_rows(snap: &CaptureSnapshot) -> Vec { }); } - out + ParseStats { + rows: out, + rows_seen, + rows_with_body, + } } /// Find the `header[data-testid="conversation-header"]` element and return @@ -276,16 +348,24 @@ fn parse_active_chat_name(snap: &CaptureSnapshot) -> Option { /// `arrow_forward`). These appear as the first SPAN inside icon wrappers /// and would otherwise win the chat-title race in `parse_active_chat_name`. /// -/// Heuristic: starts with `wds-ic-` / `wds-icon` (WhatsApp Design System -/// icon prefix), or is a single token with no whitespace whose chars are -/// all `[a-z0-9_-]` (Material Icon ligature shape). +/// **Two-pass heuristic (issue #1376 fix):** +/// 1. Explicit WDS prefix: `wds-ic-*` / `wds-icon*` — always icon. +/// 2. Token-shape check: no whitespace, all chars `[a-z0-9_-]`, AND the +/// token must contain at least one `-` or `_` delimiter. This distinguishes +/// icon names like `arrow_forward` / `material-icons` from plain one-word +/// message bodies like "ok", "hello", "yes" — real words never contain +/// hyphens or underscores in this context, but icon ligature names always do. fn looks_like_icon_ligature(s: &str) -> bool { - if s.starts_with("wds-ic-") || s.starts_with("wds-icon") { + let t = s.trim(); + if t.starts_with("wds-ic-") || t.starts_with("wds-icon") { return true; } - !s.is_empty() - && !s.contains(char::is_whitespace) - && s.chars() + // Require at least one delimiter so plain lowercase words (e.g. "ok", + // "hello") are NOT treated as ligatures. + !t.is_empty() + && !t.contains(char::is_whitespace) + && (t.contains('-') || t.contains('_')) + && t.chars() .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_' || c == '-') } @@ -399,17 +479,35 @@ fn find_pre_plain( (None, None) } -/// Pick the longest rendered body text inside the row. WhatsApp puts each -/// message's text in a descendant `span.selectable-text` or a -/// `span[dir="ltr|rtl"]`; walking every such span and keeping the longest -/// one matches `dom_scan.js`. Falls back to the row's full text with the -/// "[HH:MM, D/M/YYYY] Author:" prefix stripped. +/// Pick the longest rendered body text inside the row. +/// +/// Three tiers, tried in order — each tier only runs when the previous +/// returned empty: +/// +/// **Tier 1** — descendant `span.selectable-text` (legacy WhatsApp Web). +/// **Tier 2** — descendant `span[dir="ltr"|"rtl"]` (current WhatsApp Web +/// drops the `selectable-text` class but keeps the `dir` hint on text +/// spans). Both tiers walk every match and keep the longest, mirroring +/// the original `dom_scan.js` behavior. +/// +/// **Tier 3 fallback (issue #1376)** — when WhatsApp Web layout drift +/// strips both class+dir hints from the message body, walk every +/// descendant TEXT node, skip icon ligatures (`wds-ic-*`, `wds-icon`) +/// and chrome strings (timestamps `H:MM`, status indicators ✓ ✓✓ 🔇), +/// concatenate the remainder. This is the broadest recovery path and +/// catches rows that render their body via a plain `
` or unhinted +/// `` wrapper. Capped at [`MAX_BODY_CHARS`]. +/// +/// Final fallback (unchanged): everything under the row with the +/// `"[HH:MM, D/M/YYYY] Author:"` prefix stripped — handles rows rendered +/// without any dedicated text span at all. fn find_body( nodes: &NodeTreeSnap, strings: &[String], children: &[Vec], root: usize, ) -> String { + // Tiers 1 + 2 — span-attribute-driven discovery (longest wins). let mut best = String::new(); let mut stack = vec![root]; while let Some(idx) = stack.pop() { @@ -440,12 +538,130 @@ fn find_body( if !best.is_empty() { return best; } - // Fallback: everything under the row, with the "[HH:MM, ...] Author:" - // prefix stripped — handles rows rendered without a dedicated text span. + + // Tier 3 fallback (issue #1376): walk every descendant text node, + // filter out icon ligatures + chrome strings, concatenate. + let tier3 = collect_descendant_text_filtered(nodes, strings, children, root); + if !tier3.is_empty() { + return truncate_chars(&tier3, MAX_BODY_CHARS); + } + + // Last-resort: everything under the row, with the + // "[HH:MM, ...] Author:" prefix stripped — handles rows rendered + // without a dedicated text span. let full = collect_text(nodes, strings, children, root); strip_pre_prefix(full.trim()).to_string() } +/// Tier-3 helper for [`find_body`] (issue #1376). Walks every TEXT node +/// under `root` whose nearest element ancestor is NOT an icon wrapper +/// (`wds-ic-*` / `wds-icon` class) and whose trimmed value is NOT a +/// chrome string (timestamp `H:MM[ AM/PM]`, single status indicator). +/// Joins surviving snippets with spaces. +/// +/// Skip rules (in the order they're checked): +/// 1. Element ancestor's `class` contains `wds-icon` or any `wds-ic-*` +/// token — entire icon subtree is ignored. +/// 2. Trimmed text matches the timestamp regex shape `H:MM` / +/// `HH:MM` / `H:MM AM` / `H:MM PM` (case-insensitive). Captures +/// WhatsApp's per-bubble timestamp + delivery clock chip. +/// 3. Trimmed text is a single delivery-status glyph (✓, ✓✓, ✓✓ tinted +/// blue, 🔇, 📌, 📷, 🎤, 🎥, 📎, 📄). The first two are the only +/// common ones today; the rest are defensive against future glyph +/// drift. +fn collect_descendant_text_filtered( + nodes: &NodeTreeSnap, + strings: &[String], + children: &[Vec], + root: usize, +) -> String { + // Build "is this node inside an icon wrapper?" lookup as we walk — + // cheaper than recomputing per text node. + let mut out_parts: Vec = Vec::new(); + // Stack carries (node_idx, ancestor_is_icon). + let mut stack: Vec<(usize, bool)> = vec![(root, false)]; + while let Some((idx, ancestor_is_icon)) = stack.pop() { + let node_type = nodes.node_type.get(idx).copied().unwrap_or(0); + let mut now_is_icon = ancestor_is_icon; + if node_type == NODE_TYPE_ELEMENT { + let attrs = attrs_map(nodes, idx, strings); + if let Some(class) = attrs.get("class") { + if class + .split_whitespace() + .any(|w| w == "wds-icon" || w.starts_with("wds-ic-")) + { + now_is_icon = true; + } + } + } else if node_type == NODE_TYPE_TEXT && !ancestor_is_icon { + let raw = str_at(strings, *nodes.node_value.get(idx).unwrap_or(&-1)); + let trimmed = raw.trim(); + if !trimmed.is_empty() + && !looks_like_timestamp(trimmed) + && !looks_like_status_glyph(trimmed) + && !looks_like_icon_ligature(trimmed) + { + out_parts.push(trimmed.to_string()); + } + } + if let Some(kids) = children.get(idx) { + for &k in kids.iter().rev() { + stack.push((k, now_is_icon)); + } + } + } + out_parts.join(" ").trim().to_string() +} + +/// Returns true when `s` looks like a WhatsApp Web message-bubble +/// timestamp: `H:MM`, `HH:MM`, optionally followed by ` AM` / ` PM` +/// (any case). Reject anything else so real bodies that happen to +/// include digits aren't dropped. +fn looks_like_timestamp(s: &str) -> bool { + let t = s.trim(); + if t.is_empty() { + return false; + } + // Optional trailing AM/PM after a single space — accept and strip. + let upper = t.to_ascii_uppercase(); + let core = if let Some(stripped) = upper + .strip_suffix(" AM") + .or_else(|| upper.strip_suffix(" PM")) + { + stripped + } else { + upper.as_str() + }; + // Now `core` must be exactly H:MM or HH:MM with both halves digits. + let mut parts = core.split(':'); + let (Some(h), Some(m), None) = (parts.next(), parts.next(), parts.next()) else { + return false; + }; + if h.is_empty() || h.len() > 2 || m.len() != 2 { + return false; + } + h.bytes().all(|b| b.is_ascii_digit()) && m.bytes().all(|b| b.is_ascii_digit()) +} + +/// Returns true when `s` is a single delivery-status glyph WhatsApp +/// renders next to message bubbles (e.g. ✓, ✓✓, 🔇). The check is +/// intentionally conservative: short string AND every char is in a +/// small allow-list of known glyph code points. Real one-char message +/// bodies (e.g. emoji-only "👍" reactions still render as a separate +/// bubble) are NOT in the list and survive the filter. +fn looks_like_status_glyph(s: &str) -> bool { + let t = s.trim(); + if t.is_empty() || t.chars().count() > 2 { + return false; + } + t.chars().all(|c| { + matches!( + c, + '\u{2713}' | '\u{2714}' | '\u{1F507}' | '\u{1F508}' | '\u{1F509}' + ) + }) +} + /// Concatenate every TEXT_NODE nodeValue under `root` in document order. fn collect_text( nodes: &NodeTreeSnap, @@ -513,6 +729,34 @@ fn truncate_chars(s: &str, max: usize) -> String { s.chars().take(max).collect() } +/// Render a TRACE-friendly preview of a `DomMessage` row for the +/// per-row debug dump in `mod.rs::scan_once`. Each entry is a +/// `(attribute key, snippet)` pair drawn from the row's identifying +/// attributes and the first few non-icon child text nodes (≤ `cap` chars +/// each, no PII beyond what already lives in the row). +/// +/// Designed for `log::trace!` only — keep payloads small so the lines fit +/// in stdout without scrolling. The icon filter reuses +/// [`looks_like_icon_ligature`] so `wds-ic-*` ligatures don't dominate +/// previews of rows that render mostly chrome. +pub(crate) fn text_snippet_preview(row: &DomMessage, cap: usize) -> Vec<(String, String)> { + let mut out: Vec<(String, String)> = Vec::new(); + out.push(("dataId".to_string(), truncate_chars(&row.data_id, cap))); + out.push(("msgId".to_string(), truncate_chars(&row.msg_id, cap))); + if !row.chat_id.is_empty() { + out.push(("chatId".to_string(), truncate_chars(&row.chat_id, cap))); + } + if let Some(author) = &row.author { + out.push(("author".to_string(), truncate_chars(author, cap))); + } + if let Some(pre) = &row.pre_timestamp { + out.push(("preTs".to_string(), truncate_chars(pre, cap))); + } + // Body preview always last so the visually heavy line wraps at the end. + out.push(("body".to_string(), truncate_chars(&row.body, cap))); + out +} + /// FNV-1a 32-bit rolling hash over `(dataId + 0x01 + body)` per row. Used /// purely for change detection on the Rust side — no persistence, no wire /// format. Byte-based (JS version was UTF-16 code units; ASCII-equivalent). @@ -614,4 +858,46 @@ mod tests { assert_eq!(truncate_chars(s, 3), "💬💬💬"); assert_eq!(truncate_chars(s, 10), s); } + + // ── Issue #1376 — looks_like_icon_ligature must not drop real words ── + + #[test] + fn icon_ligature_matches_wds_prefix() { + assert!(looks_like_icon_ligature("wds-ic-search")); + assert!(looks_like_icon_ligature("wds-ic-disappearing-messages")); + assert!(looks_like_icon_ligature("wds-icon")); + assert!(looks_like_icon_ligature("wds-icon-foo")); + } + + #[test] + fn icon_ligature_matches_delimiter_tokens() { + // Material icon ligature names always contain a delimiter. + assert!(looks_like_icon_ligature("arrow_forward")); + assert!(looks_like_icon_ligature("material-icons")); + assert!(looks_like_icon_ligature("search-icon")); + } + + #[test] + fn icon_ligature_does_not_match_plain_words() { + // These are ordinary one-word message bodies — must NOT be filtered. + assert!(!looks_like_icon_ligature("ok")); + assert!(!looks_like_icon_ligature("hello")); + assert!(!looks_like_icon_ligature("yes")); + assert!(!looks_like_icon_ligature("no")); + assert!(!looks_like_icon_ligature("thanks")); + assert!(!looks_like_icon_ligature("lol")); + } + + #[test] + fn icon_ligature_does_not_match_multi_word() { + // Multi-word text is never a ligature (whitespace present). + assert!(!looks_like_icon_ligature("hello tier 3")); + assert!(!looks_like_icon_ligature("hello world")); + } + + #[test] + fn icon_ligature_does_not_match_empty() { + assert!(!looks_like_icon_ligature("")); + assert!(!looks_like_icon_ligature(" ")); + } } diff --git a/app/src-tauri/src/whatsapp_scanner/dom_snapshot_test.rs b/app/src-tauri/src/whatsapp_scanner/dom_snapshot_test.rs new file mode 100644 index 000000000..5bd934401 --- /dev/null +++ b/app/src-tauri/src/whatsapp_scanner/dom_snapshot_test.rs @@ -0,0 +1,131 @@ +//! Fixture-driven tests for `dom_snapshot::report_from_snapshot` (issue +//! #1376). The fixture lives at `test_fixtures/dom_snapshot_2026_05.json` +//! and exercises four body-extraction tiers in `find_body`: +//! +//! * **Tier 1** — `` (legacy WhatsApp Web +//! shape; some bubbles still render this). +//! * **Tier 2** — `` (current WhatsApp Web shape after +//! the `selectable-text` class was dropped). +//! * **Tier 3 multi-word (issue #1376 fallback)** — body wrapper with +//! neither `selectable-text` class nor `dir` hint; only the descendant +//! text walk + chrome filter recovers the body. +//! * **Tier 3 single-word (regression guard)** — same as tier 3 but with +//! a one-word body like "ok"; guards against +//! `looks_like_icon_ligature` false-positives that would silently drop +//! short plain-text bodies (CodeRabbit issue, fix in #1804). +//! +//! Each row in the fixture stresses exactly one tier so a regression in +//! any tier surfaces as a single failed assertion. The fixture is +//! intentionally synthetic — replace with a captured live snapshot once +//! one is available. +//! +//! Tests use the `pub(crate)` exports `CaptureSnapshot` + +//! `report_from_snapshot` from the parent module so they exercise the +//! full `parse_rows` → `find_body` pipeline. + +use super::dom_snapshot::{report_from_snapshot, CaptureSnapshot}; + +const FIXTURE_2026_05: &str = include_str!("test_fixtures/dom_snapshot_2026_05.json"); + +fn load_fixture() -> CaptureSnapshot { + serde_json::from_str(FIXTURE_2026_05) + .expect("dom_snapshot_2026_05.json must be valid CaptureSnapshot JSON") +} + +#[test] +fn parse_rows_finds_four_data_id_rows() { + let snap = load_fixture(); + let report = report_from_snapshot(&snap); + assert_eq!( + report.rows_seen, 4, + "fixture has four [data-id] rows (tiers 1/2/3-multi/3-single), all should be counted in rows_seen" + ); +} + +#[test] +fn capture_pipeline_resolves_active_chat_name() { + let snap = load_fixture(); + let report = report_from_snapshot(&snap); + assert!( + report.active_chat_resolved, + "fixture has header[data-testid=conversation-header] with text \"Test Chat\"" + ); + assert_eq!(report.active_chat_name.as_deref(), Some("Test Chat")); +} + +#[test] +fn find_body_extracts_via_selectable_text_tier1() { + let snap = load_fixture(); + let report = report_from_snapshot(&snap); + let row = report + .rows + .iter() + .find(|r| r.msg_id == "msgABC123") + .expect("tier 1 row (msgABC123) must survive"); + assert_eq!( + row.body, "hello tier 1", + "tier 1 row body comes from " + ); +} + +#[test] +fn find_body_extracts_via_dir_attr_tier2() { + let snap = load_fixture(); + let report = report_from_snapshot(&snap); + let row = report + .rows + .iter() + .find(|r| r.msg_id == "msgDEF456") + .expect("tier 2 row (msgDEF456) must survive"); + assert_eq!( + row.body, "hello tier 2", + "tier 2 row body comes from " + ); +} + +#[test] +fn find_body_extracts_via_descendant_text_tier3_fallback() { + let snap = load_fixture(); + let report = report_from_snapshot(&snap); + let row = report.rows.iter().find(|r| r.msg_id == "msgGHI789").expect( + "tier 3 row (msgGHI789) must survive — body comes from \ + descendant text walk fallback (issue #1376)", + ); + assert_eq!( + row.body, "hello tier 3", + "tier 3 fallback recovers body when neither class nor dir hint is present" + ); +} + +#[test] +fn find_body_tier3_does_not_drop_single_word_body() { + // Regression guard for CodeRabbit finding (PR #1804): the old + // `looks_like_icon_ligature` treated any lowercase single-token as a + // ligature, silently dropping "ok", "yes", "hello" etc. via tier-3. + // This test fails if that regression reappears. + let snap = load_fixture(); + let report = report_from_snapshot(&snap); + let row = report.rows.iter().find(|r| r.msg_id == "msgJKL012").expect( + "tier 3 single-word row (msgJKL012) must survive — \ + looks_like_icon_ligature must not filter plain words like 'ok'", + ); + assert_eq!( + row.body, "ok", + "single-word body 'ok' must not be dropped by looks_like_icon_ligature" + ); +} + +#[test] +fn capture_pipeline_extracts_all_four_bodies() { + let snap = load_fixture(); + let report = report_from_snapshot(&snap); + assert!( + report.rows_with_body >= 4, + "all four tiers should produce non-empty bodies; got rows_with_body={}", + report.rows_with_body + ); + assert_eq!( + report.rows_dropped_no_body, 0, + "no rows should be dropped when fixture contains body text in every row" + ); +} diff --git a/app/src-tauri/src/whatsapp_scanner/mod.rs b/app/src-tauri/src/whatsapp_scanner/mod.rs index 18417abb5..240e52de0 100644 --- a/app/src-tauri/src/whatsapp_scanner/mod.rs +++ b/app/src-tauri/src/whatsapp_scanner/mod.rs @@ -32,6 +32,8 @@ use tokio::time::sleep; use tokio_tungstenite::{connect_async, tungstenite::Message}; mod dom_snapshot; +#[cfg(test)] +mod dom_snapshot_test; mod idb; const CDP_HOST: &str = "127.0.0.1"; @@ -74,6 +76,11 @@ pub struct ScanSnapshot { /// to reverse-look-up `chatId` for DOM rows that lack one (modern /// WhatsApp Web doesn't expose chat JID anywhere on the message rows). pub active_chat_name: Option, + /// Per-stage telemetry from the DOM-capture pass (issue #1376). `None` + /// when the DOM call failed entirely (e.g. CDP disconnect mid-scan); + /// otherwise carries the same `rows` as `dom_messages` plus counters + /// disambiguating "0 rows" from "rows with no body". + pub capture_report: Option, } /// Spawn a per-account CDP poller. Idempotent at call site (caller tracks @@ -141,12 +148,29 @@ pub fn spawn_scanner( last_full = Instant::now(); match scan_once(&app, &account_id, &url_prefix, &fragment).await { Ok(snap) => { + // Per-stage telemetry from the DOM capture (#1376) — + // disambiguates "no rows seen" from "rows seen but body + // extraction returned empty" from "active chat header + // unresolved (downstream filter will drop the rows)". + let (seen, with_body, no_body, chat_resolved) = match &snap.capture_report { + Some(r) => ( + r.rows_seen, + r.rows_with_body, + r.rows_dropped_no_body, + r.active_chat_resolved, + ), + None => (0, 0, 0, false), + }; log::info!( - "[wa][{}] full scan ok messages={} chats={} dom={}", + "[wa][{}] full scan ok messages={} chats={} dom={} (seen={} with_body={} no_body={} chat_resolved={})", account_id, snap.messages.len(), snap.chats.len(), snap.dom_messages.len(), + seen, + with_body, + no_body, + chat_resolved, ); // Preview a few DOM-scraped rows so it's obvious from the // log whether the active chat produced fresh bodies. @@ -173,6 +197,26 @@ pub fn spawn_scanner( preview ); } + // TRACE-level structured row dump (first 3 rows, ≤120 char + // snippets) so a developer chasing #1376-style "dom=0 but + // bodies exist" can see exactly what the parser produced + // without re-instrumenting. Truncation lives in + // `dom_snapshot::text_snippet_preview` to honor the + // CLAUDE.md "no PII in trace dumps" rule. + if let Some(report) = snap.capture_report.as_ref() { + for (i, row) in report.rows.iter().take(3).enumerate() { + let pairs = dom_snapshot::text_snippet_preview(row, 120); + for (k, v) in &pairs { + log::trace!( + "[wa][{}] dom-trace#{} {}={:?}", + account_id, + i + 1, + k, + v + ); + } + } + } emit_snapshot(&app, &account_id, &snap); } Err(e) => { @@ -229,6 +273,24 @@ fn chrono_now_millis() -> i64 { .unwrap_or(0) } +/// Normalize a chat display name for active-chat → JID matching (issue #1376). +/// Lowercases, strips every non-ASCII-alphanumeric code point (drops emoji, +/// punctuation, dashes, separators), and collapses internal whitespace runs +/// to nothing — leaves `[a-z0-9]*`. Lets the matcher find a match when the +/// DOM-parsed header text drifts slightly from the IDB-stored chat name +/// (extra spaces, trailing emoji, hyphenation differences). +pub(crate) fn normalize_chat_name(s: &str) -> String { + s.chars() + .filter_map(|c| { + if c.is_ascii_alphanumeric() { + Some(c.to_ascii_lowercase()) + } else { + None + } + }) + .collect() +} + async fn scan_once( app: &AppHandle, account_id: &str, @@ -286,10 +348,16 @@ async fn scan_once( log::warn!("[wa][{}] idb walk failed: {}", account_id, e); } } + let mut capture_report: Option = None; match dom_snapshot::capture_messages(&mut cdp, &page_session).await { - Ok((rows, _hash, active_chat_name)) => { - snap.dom_messages = rows.iter().map(dom_snapshot::DomMessage::to_json).collect(); - snap.active_chat_name = active_chat_name; + Ok(report) => { + snap.dom_messages = report + .rows + .iter() + .map(dom_snapshot::DomMessage::to_json) + .collect(); + snap.active_chat_name = report.active_chat_name.clone(); + capture_report = Some(report); } Err(e) => { // Fast-tick DOM scans will retry every 2s, so degrade gracefully. @@ -305,6 +373,7 @@ async fn scan_once( ) .await; let _ = app; + snap.capture_report = capture_report; Ok(snap) } @@ -357,15 +426,26 @@ async fn scan_dom_once( None, ) .await; - let (rows, hash, _active_chat_name) = captured?; - let dom_messages: Vec = rows.iter().map(dom_snapshot::DomMessage::to_json).collect(); + let report = captured?; + let dom_messages: Vec = report + .rows + .iter() + .map(dom_snapshot::DomMessage::to_json) + .collect(); log::debug!( - "[wa][{}] fast dom-scan rows={} hash={}", + "[wa][{}] fast dom-scan rows={} hash={} (seen={} with_body={} no_body={} chat_resolved={})", account_id, dom_messages.len(), - hash + report.hash, + report.rows_seen, + report.rows_with_body, + report.rows_dropped_no_body, + report.active_chat_resolved, ); - Ok(DomScanResult { dom_messages, hash }) + Ok(DomScanResult { + dom_messages, + hash: report.hash, + }) } fn parse_targets(v: &Value) -> Vec { @@ -506,8 +586,10 @@ fn emit_snapshot(app: &AppHandle, account_id: &str, snap: &ScanSn // don't mis-attribute messages. let active_chat_jid: Option = snap.active_chat_name.as_deref().and_then(|name| { let name_lc = name.to_ascii_lowercase(); + let name_norm = normalize_chat_name(name); let mut exact: Vec<&str> = Vec::new(); let mut ci: Vec<&str> = Vec::new(); + let mut normalized: Vec<&str> = Vec::new(); let mut substring: Vec<&str> = Vec::new(); for (jid, chat) in snap.chats.iter() { let chat_name = chat.get("name").and_then(|v| v.as_str()).unwrap_or(""); @@ -515,29 +597,57 @@ fn emit_snapshot(app: &AppHandle, account_id: &str, snap: &ScanSn exact.push(jid); } else if !chat_name.is_empty() && chat_name.to_ascii_lowercase() == name_lc { ci.push(jid); - } else if !chat_name.is_empty() - && (chat_name.to_ascii_lowercase().contains(&name_lc) - || name_lc.contains(&chat_name.to_ascii_lowercase())) - { - substring.push(jid); + } else if !chat_name.is_empty() { + // Normalized tier (issue #1376): strip non-alphanumeric + + // collapse whitespace + lowercase, then compare equality. + // Catches drift introduced by emoji, punctuation, or + // double-spaces between the DOM header text and the IDB + // chat name (e.g. group titles like "17-18-19 July samagam" + // vs "17 18 19 July samagam ✨"). Substring stays as a final + // fallback so loose matches are tried last. + let chat_norm = normalize_chat_name(chat_name); + if !name_norm.is_empty() && chat_norm == name_norm { + normalized.push(jid); + } else if chat_name.to_ascii_lowercase().contains(&name_lc) + || name_lc.contains(&chat_name.to_ascii_lowercase()) + { + substring.push(jid); + } } } - // Prefer exact > case-insensitive > substring. Substring only wins - // when there's exactly one candidate (avoids cross-attribution when - // many chats share a token like a common first name). - match (exact.len(), ci.len(), substring.len()) { - (1, _, _) => Some(exact[0].to_string()), - (0, 1, _) => Some(ci[0].to_string()), - (0, 0, 1) => Some(substring[0].to_string()), - (e, c, s) => { - let count = e + c + s; - if count > 1 { - log::warn!( - "[whatsapp_scanner] ambiguous active-chat resolution: {} candidates for '{}' — skipping backfill", - count, - name - ); + // Prefer exact > case-insensitive > normalized > substring. Each + // tier only wins when it has exactly one candidate (avoids + // cross-attribution when many chats share a token like a common + // first name). Tier counts feed into the warn log so ambiguous + // resolutions are visible in the scanner output. + match (exact.len(), ci.len(), normalized.len(), substring.len()) { + (1, _, _, _) => Some(exact[0].to_string()), + (0, 1, _, _) => Some(ci[0].to_string()), + (0, 0, 1, _) => Some(normalized[0].to_string()), + (0, 0, 0, 1) => Some(substring[0].to_string()), + (0, 0, 0, 0) => { + // No IDB chat matches the active-chat header — happens for + // 1:1 chats where IDB stores the peer JID but the + // human-readable contact name lives in the device address + // book (never reaches IDB). Synthesize a stable + // `dom:` backfill key so DOM rows still + // persist instead of being filtered at the structured-store + // empty-`chat_id` guard. Distinct from real WhatsApp JIDs + // (which always contain `@`), so downstream consumers can + // tell DOM-only chat ids apart. + let synth = normalize_chat_name(name); + if synth.is_empty() { + None + } else { + Some(format!("dom:{synth}")) } + } + (e, c, n, s) => { + log::warn!( + "[whatsapp_scanner] ambiguous active-chat resolution: {} candidates for '{}' — skipping backfill", + e + c + n + s, + name + ); None } } @@ -821,6 +931,20 @@ fn emit_grouped_whatsapp( .and_then(|v| v.as_i64()) .or_else(|| m.get("preTimestamp").and_then(|v| v.as_i64())) .unwrap_or(0); + // Per-row source tag (issue #1376): rows that picked up their + // body from the DOM scrape get tagged `cdp-dom` so the + // structured store can distinguish DOM-sourced text from + // IDB-sourced metadata. `merge_dom_into_snapshot` stamps + // `bodySource = "dom"` (IDB row patched with DOM body) and + // `"dom-only"` (DOM row with no IDB peer); both cases fall + // back to `cdp-dom`. Everything else inherits the caller's + // tag (full-scan = `cdp-indexeddb`, fast-tick = `cdp-dom`). + let row_source = m + .get("bodySource") + .and_then(|v| v.as_str()) + .filter(|s| matches!(*s, "dom" | "dom-only")) + .map(|_| "cdp-dom") + .unwrap_or(source); Some(json!({ "message_id": msg_id, "chat_id": chat_id, @@ -830,7 +954,7 @@ fn emit_grouped_whatsapp( "body": body, "timestamp": timestamp, "message_type": m.get("type").cloned().unwrap_or(Value::Null), - "source": source, + "source": row_source, })) }) .collect(); @@ -1358,6 +1482,45 @@ impl ScannerRegistry { mod tests { use super::*; + // ── Issue #1376 — chat-name normalization for active-chat → JID lookup ── + + #[test] + fn normalize_chat_name_strips_punctuation_and_emoji() { + // Group titles routinely pick up emoji + punctuation drift between + // the DOM-parsed conversation header and the IDB-stored chat name. + // Normalization should collapse both sides to the same key so the + // lookup at scan_once succeeds. + assert_eq!( + normalize_chat_name("17-18-19 July samagam"), + "171819julysamagam" + ); + assert_eq!( + normalize_chat_name("17 18 19 July samagam"), + "171819julysamagam" + ); + assert_eq!( + normalize_chat_name("17-18-19 July samagam ✨"), + "171819julysamagam" + ); + assert_eq!( + normalize_chat_name("17.18.19 July, samagam!"), + "171819julysamagam" + ); + // Identity property — already-normal strings round-trip unchanged. + assert_eq!(normalize_chat_name("foo123"), "foo123"); + // Empty input → empty output (caller guards against this). + assert_eq!(normalize_chat_name(""), ""); + assert_eq!(normalize_chat_name(" "), ""); + assert_eq!(normalize_chat_name("✨"), ""); + } + + #[test] + fn normalize_chat_name_lowercases() { + assert_eq!(normalize_chat_name("Hello World"), "helloworld"); + assert_eq!(normalize_chat_name("HELLO"), "hello"); + assert_eq!(normalize_chat_name("hElLo"), "hello"); + } + fn insert_pending_tasks( registry: &ScannerRegistry, account_id: &str, diff --git a/app/src-tauri/src/whatsapp_scanner/test_fixtures/dom_snapshot_2026_05.json b/app/src-tauri/src/whatsapp_scanner/test_fixtures/dom_snapshot_2026_05.json new file mode 100644 index 000000000..db8bc22a0 --- /dev/null +++ b/app/src-tauri/src/whatsapp_scanner/test_fixtures/dom_snapshot_2026_05.json @@ -0,0 +1,53 @@ +{ + "_comment": "Synthetic CDP DOMSnapshot.captureSnapshot fixture for issue #1376 regression tests. Replace with a real WA Web capture once available. Four message rows: tier 1 (selectable-text class), tier 2 (dir=ltr), tier 3 multi-word body (descendant text fallback), and tier 3 single-word body (regression guard: must not be filtered by looks_like_icon_ligature). One conversation-header gives parse_active_chat_name something to resolve.", + "documents": [ + { + "nodes": { + "parentIndex": [-1, 0, 1, 2, 0, 4, 5, 0, 7, 8, 0, 10, 11, 0, 13, 14], + "nodeType": [ 1, 1, 1, 3, 1, 1, 3, 1, 1, 3, 1, 1, 3, 1, 1, 3], + "nodeName": [ 0, 1, 4, -1, 6, 4, -1, 6, 4, -1, 6, 6, -1, 6, 6, -1], + "nodeValue": [-1, -1, -1, 5, -1, -1, 11, -1, -1, 15, -1, -1, 17, -1, -1, 19], + "attributes": [ + [], + [2, 3], + [], + [], + [7, 8], + [9, 10], + [], + [7, 12], + [13, 14], + [], + [7, 16], + [], + [], + [7, 18], + [], + [] + ] + } + } + ], + "strings": [ + "html", + "header", + "data-testid", + "conversation-header", + "span", + "Test Chat", + "div", + "data-id", + "false_chat1@c.us_msgABC123", + "class", + "selectable-text", + "hello tier 1", + "false_chat1@c.us_msgDEF456", + "dir", + "ltr", + "hello tier 2", + "false_chat1@c.us_msgGHI789", + "hello tier 3", + "false_chat1@c.us_msgJKL012", + "ok" + ] +}