fix(whatsapp): recover DOM message bodies — telemetry, tier-3 fallback, source tag, synthetic chat_id (#1376) (#1804)

Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
Co-authored-by: Steven Enamakel <enamakel@tinyhumans.ai>
This commit is contained in:
oxoxDev
2026-05-15 20:29:29 -07:00
committed by GitHub
co-authored by Claude Opus 4.7 Steven Enamakel
parent 6872eb0139
commit 4d73bf869a
4 changed files with 691 additions and 58 deletions
@@ -56,6 +56,34 @@ impl DomMessage {
}
}
/// Per-stage telemetry produced by [`capture_messages`]. The counters
/// disambiguate the three failure modes that `dom=0` used to collapse into
/// a single number (issue #1376): rows never matched, rows matched but
/// body extraction returned empty, or active chat name failed to resolve
/// (forcing rows to be filtered out downstream by the merge step).
///
/// Field invariants:
/// * `rows_seen` — `[data-id]` elements that parsed as message rows BEFORE
/// any body filter. Counts every accepted `data-id` shape (legacy
/// compound + bare-msgId).
/// * `rows_with_body` — subset where [`find_body`] returned a non-empty
/// string. `rows_seen - rows_with_body` is bodies that vanished.
/// * `rows_dropped_no_body` — convenience, equals
/// `rows_seen - rows_with_body`.
/// * `active_chat_resolved` — true when
/// `header[data-testid="conversation-header"]` produced a display name
/// (precondition for the chat-id reverse lookup in `mod.rs`).
#[derive(Debug, Clone, Default)]
pub struct CaptureReport {
pub rows: Vec<DomMessage>,
pub hash: u64,
pub active_chat_name: Option<String>,
pub rows_seen: usize,
pub rows_with_body: usize,
pub rows_dropped_no_body: usize,
pub active_chat_resolved: bool,
}
/// Run `DOMSnapshot.captureSnapshot` against an attached page session and
/// return parsed message rows, a FNV-1a hash over (dataId, body), and the
/// active conversation's display name (from
@@ -64,10 +92,10 @@ impl DomMessage {
/// modern WhatsApp Web omits the chat JID from the URL, from `data-id`, and
/// from any DOM attribute, so the merge step in `mod.rs` reverse-looks-up
/// `chats[*].name → chats[*].jid` to stamp `chatId` onto DOM rows.
pub async fn capture_messages(
cdp: &mut CdpConn,
session: &str,
) -> Result<(Vec<DomMessage>, u64, Option<String>), String> {
///
/// Returns a [`CaptureReport`] with per-stage counters. See the type doc
/// for invariants.
pub async fn capture_messages(cdp: &mut CdpConn, session: &str) -> Result<CaptureReport, String> {
// `computedStyles` is a required array — empty is fine, we don't need
// any CSS. The other flags default sensibly; explicitly disable the
// heavy paint/rect output to keep payloads small.
@@ -84,16 +112,32 @@ pub async fn capture_messages(
.await?;
let snap: CaptureSnapshot =
serde_json::from_value(raw).map_err(|e| format!("decode DOMSnapshot: {e}"))?;
let rows = parse_rows(&snap);
let hash = fnv_hash(&rows);
let active_chat_name = parse_active_chat_name(&snap);
Ok((rows, hash, active_chat_name))
Ok(report_from_snapshot(&snap))
}
/// Synthesize a [`CaptureReport`] from a parsed `CaptureSnapshot`. Split
/// out from [`capture_messages`] so unit tests can drive the body-finder
/// + counters off a JSON fixture without mocking CDP.
pub(crate) fn report_from_snapshot(snap: &CaptureSnapshot) -> CaptureReport {
let stats = parse_rows(snap);
let hash = fnv_hash(&stats.rows);
let active_chat_name = parse_active_chat_name(snap);
let active_chat_resolved = active_chat_name.is_some();
CaptureReport {
rows: stats.rows,
hash,
active_chat_name,
rows_seen: stats.rows_seen,
rows_with_body: stats.rows_with_body,
rows_dropped_no_body: stats.rows_seen.saturating_sub(stats.rows_with_body),
active_chat_resolved,
}
}
// ─── CDP response shape ─────────────────────────────────────────────
#[derive(Deserialize, Debug, Default)]
struct CaptureSnapshot {
pub(crate) struct CaptureSnapshot {
#[serde(default)]
documents: Vec<DocumentSnap>,
#[serde(default)]
@@ -123,6 +167,18 @@ struct NodeTreeSnap {
attributes: Vec<Vec<i32>>,
}
/// Output of [`parse_rows`] including per-stage counters used to populate
/// [`CaptureReport`]. `rows` is the filtered keep-set (rows with body OR
/// `data-pre-plain-text`); `rows_seen` is the count of accepted `data-id`
/// shapes BEFORE any body/chrome filter; `rows_with_body` is the count
/// where [`find_body`] returned non-empty for a row in `rows_seen`.
#[derive(Debug, Default)]
pub(crate) struct ParseStats {
pub rows: Vec<DomMessage>,
pub rows_seen: usize,
pub rows_with_body: usize,
}
const NODE_TYPE_ELEMENT: i32 = 1;
const NODE_TYPE_TEXT: i32 = 3;
/// Hard cap on body length to mirror `dom_scan.js` (which sliced at 4000).
@@ -130,17 +186,17 @@ const MAX_BODY_CHARS: usize = 4000;
// ─── parsing ────────────────────────────────────────────────────────
fn parse_rows(snap: &CaptureSnapshot) -> Vec<DomMessage> {
fn parse_rows(snap: &CaptureSnapshot) -> ParseStats {
// Main frame only — iframes aren't used by WhatsApp's message list.
let doc = match snap.documents.first() {
Some(d) => d,
None => return Vec::new(),
None => return ParseStats::default(),
};
let nodes = &doc.nodes;
let strings = &snap.strings;
let count = nodes.node_type.len();
if count == 0 {
return Vec::new();
return ParseStats::default();
}
// Precompute children map so descendant walks are O(subtree) instead of
@@ -154,6 +210,8 @@ fn parse_rows(snap: &CaptureSnapshot) -> Vec<DomMessage> {
let mut out = Vec::new();
let mut seen: HashSet<String> = HashSet::new();
let mut rows_seen = 0usize;
let mut rows_with_body = 0usize;
for i in 0..count {
if nodes.node_type.get(i).copied().unwrap_or(0) != NODE_TYPE_ELEMENT {
@@ -174,10 +232,20 @@ fn parse_rows(snap: &CaptureSnapshot) -> Vec<DomMessage> {
continue;
}
// Telemetry: count every accepted data-id BEFORE the body filter so
// the per-stage counts in `CaptureReport` distinguish "no rows
// matched at all" from "rows matched but body extraction failed".
rows_seen += 1;
let (pre_ts, author) = find_pre_plain(nodes, strings, &children, i);
let body = find_body(nodes, strings, &children, i);
if !body.is_empty() {
rows_with_body += 1;
}
// A row with neither a body nor a pre-plain-text tag is just chrome
// (avatar wrapper, reaction chip, etc) — skip it.
// (avatar wrapper, reaction chip, etc) — skip it. Note: this still
// contributes to `rows_seen` because the goal of that counter is
// "did we find rows at all", not "did we keep them".
if body.is_empty() && pre_ts.is_none() {
continue;
}
@@ -193,7 +261,11 @@ fn parse_rows(snap: &CaptureSnapshot) -> Vec<DomMessage> {
});
}
out
ParseStats {
rows: out,
rows_seen,
rows_with_body,
}
}
/// Find the `header[data-testid="conversation-header"]` element and return
@@ -276,16 +348,24 @@ fn parse_active_chat_name(snap: &CaptureSnapshot) -> Option<String> {
/// `arrow_forward`). These appear as the first SPAN inside icon wrappers
/// and would otherwise win the chat-title race in `parse_active_chat_name`.
///
/// Heuristic: starts with `wds-ic-` / `wds-icon` (WhatsApp Design System
/// icon prefix), or is a single token with no whitespace whose chars are
/// all `[a-z0-9_-]` (Material Icon ligature shape).
/// **Two-pass heuristic (issue #1376 fix):**
/// 1. Explicit WDS prefix: `wds-ic-*` / `wds-icon*` — always icon.
/// 2. Token-shape check: no whitespace, all chars `[a-z0-9_-]`, AND the
/// token must contain at least one `-` or `_` delimiter. This distinguishes
/// icon names like `arrow_forward` / `material-icons` from plain one-word
/// message bodies like "ok", "hello", "yes" — real words never contain
/// hyphens or underscores in this context, but icon ligature names always do.
fn looks_like_icon_ligature(s: &str) -> bool {
if s.starts_with("wds-ic-") || s.starts_with("wds-icon") {
let t = s.trim();
if t.starts_with("wds-ic-") || t.starts_with("wds-icon") {
return true;
}
!s.is_empty()
&& !s.contains(char::is_whitespace)
&& s.chars()
// Require at least one delimiter so plain lowercase words (e.g. "ok",
// "hello") are NOT treated as ligatures.
!t.is_empty()
&& !t.contains(char::is_whitespace)
&& (t.contains('-') || t.contains('_'))
&& t.chars()
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_' || c == '-')
}
@@ -399,17 +479,35 @@ fn find_pre_plain(
(None, None)
}
/// Pick the longest rendered body text inside the row. WhatsApp puts each
/// message's text in a descendant `span.selectable-text` or a
/// `span[dir="ltr|rtl"]`; walking every such span and keeping the longest
/// one matches `dom_scan.js`. Falls back to the row's full text with the
/// "[HH:MM, D/M/YYYY] Author:" prefix stripped.
/// Pick the longest rendered body text inside the row.
///
/// Three tiers, tried in order — each tier only runs when the previous
/// returned empty:
///
/// **Tier 1** — descendant `span.selectable-text` (legacy WhatsApp Web).
/// **Tier 2** — descendant `span[dir="ltr"|"rtl"]` (current WhatsApp Web
/// drops the `selectable-text` class but keeps the `dir` hint on text
/// spans). Both tiers walk every match and keep the longest, mirroring
/// the original `dom_scan.js` behavior.
///
/// **Tier 3 fallback (issue #1376)** — when WhatsApp Web layout drift
/// strips both class+dir hints from the message body, walk every
/// descendant TEXT node, skip icon ligatures (`wds-ic-*`, `wds-icon`)
/// and chrome strings (timestamps `H:MM`, status indicators ✓ ✓✓ 🔇),
/// concatenate the remainder. This is the broadest recovery path and
/// catches rows that render their body via a plain `<div>` or unhinted
/// `<span>` wrapper. Capped at [`MAX_BODY_CHARS`].
///
/// Final fallback (unchanged): everything under the row with the
/// `"[HH:MM, D/M/YYYY] Author:"` prefix stripped — handles rows rendered
/// without any dedicated text span at all.
fn find_body(
nodes: &NodeTreeSnap,
strings: &[String],
children: &[Vec<usize>],
root: usize,
) -> String {
// Tiers 1 + 2 — span-attribute-driven discovery (longest wins).
let mut best = String::new();
let mut stack = vec![root];
while let Some(idx) = stack.pop() {
@@ -440,12 +538,130 @@ fn find_body(
if !best.is_empty() {
return best;
}
// Fallback: everything under the row, with the "[HH:MM, ...] Author:"
// prefix stripped — handles rows rendered without a dedicated text span.
// Tier 3 fallback (issue #1376): walk every descendant text node,
// filter out icon ligatures + chrome strings, concatenate.
let tier3 = collect_descendant_text_filtered(nodes, strings, children, root);
if !tier3.is_empty() {
return truncate_chars(&tier3, MAX_BODY_CHARS);
}
// Last-resort: everything under the row, with the
// "[HH:MM, ...] Author:" prefix stripped — handles rows rendered
// without a dedicated text span.
let full = collect_text(nodes, strings, children, root);
strip_pre_prefix(full.trim()).to_string()
}
/// Tier-3 helper for [`find_body`] (issue #1376). Walks every TEXT node
/// under `root` whose nearest element ancestor is NOT an icon wrapper
/// (`wds-ic-*` / `wds-icon` class) and whose trimmed value is NOT a
/// chrome string (timestamp `H:MM[ AM/PM]`, single status indicator).
/// Joins surviving snippets with spaces.
///
/// Skip rules (in the order they're checked):
/// 1. Element ancestor's `class` contains `wds-icon` or any `wds-ic-*`
/// token — entire icon subtree is ignored.
/// 2. Trimmed text matches the timestamp regex shape `H:MM` /
/// `HH:MM` / `H:MM AM` / `H:MM PM` (case-insensitive). Captures
/// WhatsApp's per-bubble timestamp + delivery clock chip.
/// 3. Trimmed text is a single delivery-status glyph (✓, ✓✓, ✓✓ tinted
/// blue, 🔇, 📌, 📷, 🎤, 🎥, 📎, 📄). The first two are the only
/// common ones today; the rest are defensive against future glyph
/// drift.
fn collect_descendant_text_filtered(
nodes: &NodeTreeSnap,
strings: &[String],
children: &[Vec<usize>],
root: usize,
) -> String {
// Build "is this node inside an icon wrapper?" lookup as we walk —
// cheaper than recomputing per text node.
let mut out_parts: Vec<String> = Vec::new();
// Stack carries (node_idx, ancestor_is_icon).
let mut stack: Vec<(usize, bool)> = vec![(root, false)];
while let Some((idx, ancestor_is_icon)) = stack.pop() {
let node_type = nodes.node_type.get(idx).copied().unwrap_or(0);
let mut now_is_icon = ancestor_is_icon;
if node_type == NODE_TYPE_ELEMENT {
let attrs = attrs_map(nodes, idx, strings);
if let Some(class) = attrs.get("class") {
if class
.split_whitespace()
.any(|w| w == "wds-icon" || w.starts_with("wds-ic-"))
{
now_is_icon = true;
}
}
} else if node_type == NODE_TYPE_TEXT && !ancestor_is_icon {
let raw = str_at(strings, *nodes.node_value.get(idx).unwrap_or(&-1));
let trimmed = raw.trim();
if !trimmed.is_empty()
&& !looks_like_timestamp(trimmed)
&& !looks_like_status_glyph(trimmed)
&& !looks_like_icon_ligature(trimmed)
{
out_parts.push(trimmed.to_string());
}
}
if let Some(kids) = children.get(idx) {
for &k in kids.iter().rev() {
stack.push((k, now_is_icon));
}
}
}
out_parts.join(" ").trim().to_string()
}
/// Returns true when `s` looks like a WhatsApp Web message-bubble
/// timestamp: `H:MM`, `HH:MM`, optionally followed by ` AM` / ` PM`
/// (any case). Reject anything else so real bodies that happen to
/// include digits aren't dropped.
fn looks_like_timestamp(s: &str) -> bool {
let t = s.trim();
if t.is_empty() {
return false;
}
// Optional trailing AM/PM after a single space — accept and strip.
let upper = t.to_ascii_uppercase();
let core = if let Some(stripped) = upper
.strip_suffix(" AM")
.or_else(|| upper.strip_suffix(" PM"))
{
stripped
} else {
upper.as_str()
};
// Now `core` must be exactly H:MM or HH:MM with both halves digits.
let mut parts = core.split(':');
let (Some(h), Some(m), None) = (parts.next(), parts.next(), parts.next()) else {
return false;
};
if h.is_empty() || h.len() > 2 || m.len() != 2 {
return false;
}
h.bytes().all(|b| b.is_ascii_digit()) && m.bytes().all(|b| b.is_ascii_digit())
}
/// Returns true when `s` is a single delivery-status glyph WhatsApp
/// renders next to message bubbles (e.g. ✓, ✓✓, 🔇). The check is
/// intentionally conservative: short string AND every char is in a
/// small allow-list of known glyph code points. Real one-char message
/// bodies (e.g. emoji-only "👍" reactions still render as a separate
/// bubble) are NOT in the list and survive the filter.
fn looks_like_status_glyph(s: &str) -> bool {
let t = s.trim();
if t.is_empty() || t.chars().count() > 2 {
return false;
}
t.chars().all(|c| {
matches!(
c,
'\u{2713}' | '\u{2714}' | '\u{1F507}' | '\u{1F508}' | '\u{1F509}'
)
})
}
/// Concatenate every TEXT_NODE nodeValue under `root` in document order.
fn collect_text(
nodes: &NodeTreeSnap,
@@ -513,6 +729,34 @@ fn truncate_chars(s: &str, max: usize) -> String {
s.chars().take(max).collect()
}
/// Render a TRACE-friendly preview of a `DomMessage` row for the
/// per-row debug dump in `mod.rs::scan_once`. Each entry is a
/// `(attribute key, snippet)` pair drawn from the row's identifying
/// attributes and the first few non-icon child text nodes (≤ `cap` chars
/// each, no PII beyond what already lives in the row).
///
/// Designed for `log::trace!` only — keep payloads small so the lines fit
/// in stdout without scrolling. The icon filter reuses
/// [`looks_like_icon_ligature`] so `wds-ic-*` ligatures don't dominate
/// previews of rows that render mostly chrome.
pub(crate) fn text_snippet_preview(row: &DomMessage, cap: usize) -> Vec<(String, String)> {
let mut out: Vec<(String, String)> = Vec::new();
out.push(("dataId".to_string(), truncate_chars(&row.data_id, cap)));
out.push(("msgId".to_string(), truncate_chars(&row.msg_id, cap)));
if !row.chat_id.is_empty() {
out.push(("chatId".to_string(), truncate_chars(&row.chat_id, cap)));
}
if let Some(author) = &row.author {
out.push(("author".to_string(), truncate_chars(author, cap)));
}
if let Some(pre) = &row.pre_timestamp {
out.push(("preTs".to_string(), truncate_chars(pre, cap)));
}
// Body preview always last so the visually heavy line wraps at the end.
out.push(("body".to_string(), truncate_chars(&row.body, cap)));
out
}
/// FNV-1a 32-bit rolling hash over `(dataId + 0x01 + body)` per row. Used
/// purely for change detection on the Rust side — no persistence, no wire
/// format. Byte-based (JS version was UTF-16 code units; ASCII-equivalent).
@@ -614,4 +858,46 @@ mod tests {
assert_eq!(truncate_chars(s, 3), "💬💬💬");
assert_eq!(truncate_chars(s, 10), s);
}
// ── Issue #1376 — looks_like_icon_ligature must not drop real words ──
#[test]
fn icon_ligature_matches_wds_prefix() {
assert!(looks_like_icon_ligature("wds-ic-search"));
assert!(looks_like_icon_ligature("wds-ic-disappearing-messages"));
assert!(looks_like_icon_ligature("wds-icon"));
assert!(looks_like_icon_ligature("wds-icon-foo"));
}
#[test]
fn icon_ligature_matches_delimiter_tokens() {
// Material icon ligature names always contain a delimiter.
assert!(looks_like_icon_ligature("arrow_forward"));
assert!(looks_like_icon_ligature("material-icons"));
assert!(looks_like_icon_ligature("search-icon"));
}
#[test]
fn icon_ligature_does_not_match_plain_words() {
// These are ordinary one-word message bodies — must NOT be filtered.
assert!(!looks_like_icon_ligature("ok"));
assert!(!looks_like_icon_ligature("hello"));
assert!(!looks_like_icon_ligature("yes"));
assert!(!looks_like_icon_ligature("no"));
assert!(!looks_like_icon_ligature("thanks"));
assert!(!looks_like_icon_ligature("lol"));
}
#[test]
fn icon_ligature_does_not_match_multi_word() {
// Multi-word text is never a ligature (whitespace present).
assert!(!looks_like_icon_ligature("hello tier 3"));
assert!(!looks_like_icon_ligature("hello world"));
}
#[test]
fn icon_ligature_does_not_match_empty() {
assert!(!looks_like_icon_ligature(""));
assert!(!looks_like_icon_ligature(" "));
}
}
@@ -0,0 +1,131 @@
//! Fixture-driven tests for `dom_snapshot::report_from_snapshot` (issue
//! #1376). The fixture lives at `test_fixtures/dom_snapshot_2026_05.json`
//! and exercises four body-extraction tiers in `find_body`:
//!
//! * **Tier 1** — `<span class="selectable-text">` (legacy WhatsApp Web
//! shape; some bubbles still render this).
//! * **Tier 2** — `<span dir="ltr">` (current WhatsApp Web shape after
//! the `selectable-text` class was dropped).
//! * **Tier 3 multi-word (issue #1376 fallback)** — body wrapper with
//! neither `selectable-text` class nor `dir` hint; only the descendant
//! text walk + chrome filter recovers the body.
//! * **Tier 3 single-word (regression guard)** — same as tier 3 but with
//! a one-word body like "ok"; guards against
//! `looks_like_icon_ligature` false-positives that would silently drop
//! short plain-text bodies (CodeRabbit issue, fix in #1804).
//!
//! Each row in the fixture stresses exactly one tier so a regression in
//! any tier surfaces as a single failed assertion. The fixture is
//! intentionally synthetic — replace with a captured live snapshot once
//! one is available.
//!
//! Tests use the `pub(crate)` exports `CaptureSnapshot` +
//! `report_from_snapshot` from the parent module so they exercise the
//! full `parse_rows` → `find_body` pipeline.
use super::dom_snapshot::{report_from_snapshot, CaptureSnapshot};
const FIXTURE_2026_05: &str = include_str!("test_fixtures/dom_snapshot_2026_05.json");
fn load_fixture() -> CaptureSnapshot {
serde_json::from_str(FIXTURE_2026_05)
.expect("dom_snapshot_2026_05.json must be valid CaptureSnapshot JSON")
}
#[test]
fn parse_rows_finds_four_data_id_rows() {
let snap = load_fixture();
let report = report_from_snapshot(&snap);
assert_eq!(
report.rows_seen, 4,
"fixture has four [data-id] rows (tiers 1/2/3-multi/3-single), all should be counted in rows_seen"
);
}
#[test]
fn capture_pipeline_resolves_active_chat_name() {
let snap = load_fixture();
let report = report_from_snapshot(&snap);
assert!(
report.active_chat_resolved,
"fixture has header[data-testid=conversation-header] with text \"Test Chat\""
);
assert_eq!(report.active_chat_name.as_deref(), Some("Test Chat"));
}
#[test]
fn find_body_extracts_via_selectable_text_tier1() {
let snap = load_fixture();
let report = report_from_snapshot(&snap);
let row = report
.rows
.iter()
.find(|r| r.msg_id == "msgABC123")
.expect("tier 1 row (msgABC123) must survive");
assert_eq!(
row.body, "hello tier 1",
"tier 1 row body comes from <span class=selectable-text>"
);
}
#[test]
fn find_body_extracts_via_dir_attr_tier2() {
let snap = load_fixture();
let report = report_from_snapshot(&snap);
let row = report
.rows
.iter()
.find(|r| r.msg_id == "msgDEF456")
.expect("tier 2 row (msgDEF456) must survive");
assert_eq!(
row.body, "hello tier 2",
"tier 2 row body comes from <span dir=ltr>"
);
}
#[test]
fn find_body_extracts_via_descendant_text_tier3_fallback() {
let snap = load_fixture();
let report = report_from_snapshot(&snap);
let row = report.rows.iter().find(|r| r.msg_id == "msgGHI789").expect(
"tier 3 row (msgGHI789) must survive — body comes from \
descendant text walk fallback (issue #1376)",
);
assert_eq!(
row.body, "hello tier 3",
"tier 3 fallback recovers body when neither class nor dir hint is present"
);
}
#[test]
fn find_body_tier3_does_not_drop_single_word_body() {
// Regression guard for CodeRabbit finding (PR #1804): the old
// `looks_like_icon_ligature` treated any lowercase single-token as a
// ligature, silently dropping "ok", "yes", "hello" etc. via tier-3.
// This test fails if that regression reappears.
let snap = load_fixture();
let report = report_from_snapshot(&snap);
let row = report.rows.iter().find(|r| r.msg_id == "msgJKL012").expect(
"tier 3 single-word row (msgJKL012) must survive — \
looks_like_icon_ligature must not filter plain words like 'ok'",
);
assert_eq!(
row.body, "ok",
"single-word body 'ok' must not be dropped by looks_like_icon_ligature"
);
}
#[test]
fn capture_pipeline_extracts_all_four_bodies() {
let snap = load_fixture();
let report = report_from_snapshot(&snap);
assert!(
report.rows_with_body >= 4,
"all four tiers should produce non-empty bodies; got rows_with_body={}",
report.rows_with_body
);
assert_eq!(
report.rows_dropped_no_body, 0,
"no rows should be dropped when fixture contains body text in every row"
);
}
+193 -30
View File
@@ -32,6 +32,8 @@ use tokio::time::sleep;
use tokio_tungstenite::{connect_async, tungstenite::Message};
mod dom_snapshot;
#[cfg(test)]
mod dom_snapshot_test;
mod idb;
const CDP_HOST: &str = "127.0.0.1";
@@ -74,6 +76,11 @@ pub struct ScanSnapshot {
/// to reverse-look-up `chatId` for DOM rows that lack one (modern
/// WhatsApp Web doesn't expose chat JID anywhere on the message rows).
pub active_chat_name: Option<String>,
/// Per-stage telemetry from the DOM-capture pass (issue #1376). `None`
/// when the DOM call failed entirely (e.g. CDP disconnect mid-scan);
/// otherwise carries the same `rows` as `dom_messages` plus counters
/// disambiguating "0 rows" from "rows with no body".
pub capture_report: Option<dom_snapshot::CaptureReport>,
}
/// Spawn a per-account CDP poller. Idempotent at call site (caller tracks
@@ -141,12 +148,29 @@ pub fn spawn_scanner<R: Runtime>(
last_full = Instant::now();
match scan_once(&app, &account_id, &url_prefix, &fragment).await {
Ok(snap) => {
// Per-stage telemetry from the DOM capture (#1376) —
// disambiguates "no rows seen" from "rows seen but body
// extraction returned empty" from "active chat header
// unresolved (downstream filter will drop the rows)".
let (seen, with_body, no_body, chat_resolved) = match &snap.capture_report {
Some(r) => (
r.rows_seen,
r.rows_with_body,
r.rows_dropped_no_body,
r.active_chat_resolved,
),
None => (0, 0, 0, false),
};
log::info!(
"[wa][{}] full scan ok messages={} chats={} dom={}",
"[wa][{}] full scan ok messages={} chats={} dom={} (seen={} with_body={} no_body={} chat_resolved={})",
account_id,
snap.messages.len(),
snap.chats.len(),
snap.dom_messages.len(),
seen,
with_body,
no_body,
chat_resolved,
);
// Preview a few DOM-scraped rows so it's obvious from the
// log whether the active chat produced fresh bodies.
@@ -173,6 +197,26 @@ pub fn spawn_scanner<R: Runtime>(
preview
);
}
// TRACE-level structured row dump (first 3 rows, ≤120 char
// snippets) so a developer chasing #1376-style "dom=0 but
// bodies exist" can see exactly what the parser produced
// without re-instrumenting. Truncation lives in
// `dom_snapshot::text_snippet_preview` to honor the
// CLAUDE.md "no PII in trace dumps" rule.
if let Some(report) = snap.capture_report.as_ref() {
for (i, row) in report.rows.iter().take(3).enumerate() {
let pairs = dom_snapshot::text_snippet_preview(row, 120);
for (k, v) in &pairs {
log::trace!(
"[wa][{}] dom-trace#{} {}={:?}",
account_id,
i + 1,
k,
v
);
}
}
}
emit_snapshot(&app, &account_id, &snap);
}
Err(e) => {
@@ -229,6 +273,24 @@ fn chrono_now_millis() -> i64 {
.unwrap_or(0)
}
/// Normalize a chat display name for active-chat → JID matching (issue #1376).
/// Lowercases, strips every non-ASCII-alphanumeric code point (drops emoji,
/// punctuation, dashes, separators), and collapses internal whitespace runs
/// to nothing — leaves `[a-z0-9]*`. Lets the matcher find a match when the
/// DOM-parsed header text drifts slightly from the IDB-stored chat name
/// (extra spaces, trailing emoji, hyphenation differences).
pub(crate) fn normalize_chat_name(s: &str) -> String {
s.chars()
.filter_map(|c| {
if c.is_ascii_alphanumeric() {
Some(c.to_ascii_lowercase())
} else {
None
}
})
.collect()
}
async fn scan_once<R: Runtime>(
app: &AppHandle<R>,
account_id: &str,
@@ -286,10 +348,16 @@ async fn scan_once<R: Runtime>(
log::warn!("[wa][{}] idb walk failed: {}", account_id, e);
}
}
let mut capture_report: Option<dom_snapshot::CaptureReport> = None;
match dom_snapshot::capture_messages(&mut cdp, &page_session).await {
Ok((rows, _hash, active_chat_name)) => {
snap.dom_messages = rows.iter().map(dom_snapshot::DomMessage::to_json).collect();
snap.active_chat_name = active_chat_name;
Ok(report) => {
snap.dom_messages = report
.rows
.iter()
.map(dom_snapshot::DomMessage::to_json)
.collect();
snap.active_chat_name = report.active_chat_name.clone();
capture_report = Some(report);
}
Err(e) => {
// Fast-tick DOM scans will retry every 2s, so degrade gracefully.
@@ -305,6 +373,7 @@ async fn scan_once<R: Runtime>(
)
.await;
let _ = app;
snap.capture_report = capture_report;
Ok(snap)
}
@@ -357,15 +426,26 @@ async fn scan_dom_once(
None,
)
.await;
let (rows, hash, _active_chat_name) = captured?;
let dom_messages: Vec<Value> = rows.iter().map(dom_snapshot::DomMessage::to_json).collect();
let report = captured?;
let dom_messages: Vec<Value> = report
.rows
.iter()
.map(dom_snapshot::DomMessage::to_json)
.collect();
log::debug!(
"[wa][{}] fast dom-scan rows={} hash={}",
"[wa][{}] fast dom-scan rows={} hash={} (seen={} with_body={} no_body={} chat_resolved={})",
account_id,
dom_messages.len(),
hash
report.hash,
report.rows_seen,
report.rows_with_body,
report.rows_dropped_no_body,
report.active_chat_resolved,
);
Ok(DomScanResult { dom_messages, hash })
Ok(DomScanResult {
dom_messages,
hash: report.hash,
})
}
fn parse_targets(v: &Value) -> Vec<CdpTarget> {
@@ -506,8 +586,10 @@ fn emit_snapshot<R: Runtime>(app: &AppHandle<R>, account_id: &str, snap: &ScanSn
// don't mis-attribute messages.
let active_chat_jid: Option<String> = snap.active_chat_name.as_deref().and_then(|name| {
let name_lc = name.to_ascii_lowercase();
let name_norm = normalize_chat_name(name);
let mut exact: Vec<&str> = Vec::new();
let mut ci: Vec<&str> = Vec::new();
let mut normalized: Vec<&str> = Vec::new();
let mut substring: Vec<&str> = Vec::new();
for (jid, chat) in snap.chats.iter() {
let chat_name = chat.get("name").and_then(|v| v.as_str()).unwrap_or("");
@@ -515,29 +597,57 @@ fn emit_snapshot<R: Runtime>(app: &AppHandle<R>, account_id: &str, snap: &ScanSn
exact.push(jid);
} else if !chat_name.is_empty() && chat_name.to_ascii_lowercase() == name_lc {
ci.push(jid);
} else if !chat_name.is_empty()
&& (chat_name.to_ascii_lowercase().contains(&name_lc)
|| name_lc.contains(&chat_name.to_ascii_lowercase()))
{
substring.push(jid);
} else if !chat_name.is_empty() {
// Normalized tier (issue #1376): strip non-alphanumeric +
// collapse whitespace + lowercase, then compare equality.
// Catches drift introduced by emoji, punctuation, or
// double-spaces between the DOM header text and the IDB
// chat name (e.g. group titles like "17-18-19 July samagam"
// vs "17 18 19 July samagam ✨"). Substring stays as a final
// fallback so loose matches are tried last.
let chat_norm = normalize_chat_name(chat_name);
if !name_norm.is_empty() && chat_norm == name_norm {
normalized.push(jid);
} else if chat_name.to_ascii_lowercase().contains(&name_lc)
|| name_lc.contains(&chat_name.to_ascii_lowercase())
{
substring.push(jid);
}
}
}
// Prefer exact > case-insensitive > substring. Substring only wins
// when there's exactly one candidate (avoids cross-attribution when
// many chats share a token like a common first name).
match (exact.len(), ci.len(), substring.len()) {
(1, _, _) => Some(exact[0].to_string()),
(0, 1, _) => Some(ci[0].to_string()),
(0, 0, 1) => Some(substring[0].to_string()),
(e, c, s) => {
let count = e + c + s;
if count > 1 {
log::warn!(
"[whatsapp_scanner] ambiguous active-chat resolution: {} candidates for '{}' — skipping backfill",
count,
name
);
// Prefer exact > case-insensitive > normalized > substring. Each
// tier only wins when it has exactly one candidate (avoids
// cross-attribution when many chats share a token like a common
// first name). Tier counts feed into the warn log so ambiguous
// resolutions are visible in the scanner output.
match (exact.len(), ci.len(), normalized.len(), substring.len()) {
(1, _, _, _) => Some(exact[0].to_string()),
(0, 1, _, _) => Some(ci[0].to_string()),
(0, 0, 1, _) => Some(normalized[0].to_string()),
(0, 0, 0, 1) => Some(substring[0].to_string()),
(0, 0, 0, 0) => {
// No IDB chat matches the active-chat header — happens for
// 1:1 chats where IDB stores the peer JID but the
// human-readable contact name lives in the device address
// book (never reaches IDB). Synthesize a stable
// `dom:<normalized-name>` backfill key so DOM rows still
// persist instead of being filtered at the structured-store
// empty-`chat_id` guard. Distinct from real WhatsApp JIDs
// (which always contain `@`), so downstream consumers can
// tell DOM-only chat ids apart.
let synth = normalize_chat_name(name);
if synth.is_empty() {
None
} else {
Some(format!("dom:{synth}"))
}
}
(e, c, n, s) => {
log::warn!(
"[whatsapp_scanner] ambiguous active-chat resolution: {} candidates for '{}' — skipping backfill",
e + c + n + s,
name
);
None
}
}
@@ -821,6 +931,20 @@ fn emit_grouped_whatsapp<R: Runtime>(
.and_then(|v| v.as_i64())
.or_else(|| m.get("preTimestamp").and_then(|v| v.as_i64()))
.unwrap_or(0);
// Per-row source tag (issue #1376): rows that picked up their
// body from the DOM scrape get tagged `cdp-dom` so the
// structured store can distinguish DOM-sourced text from
// IDB-sourced metadata. `merge_dom_into_snapshot` stamps
// `bodySource = "dom"` (IDB row patched with DOM body) and
// `"dom-only"` (DOM row with no IDB peer); both cases fall
// back to `cdp-dom`. Everything else inherits the caller's
// tag (full-scan = `cdp-indexeddb`, fast-tick = `cdp-dom`).
let row_source = m
.get("bodySource")
.and_then(|v| v.as_str())
.filter(|s| matches!(*s, "dom" | "dom-only"))
.map(|_| "cdp-dom")
.unwrap_or(source);
Some(json!({
"message_id": msg_id,
"chat_id": chat_id,
@@ -830,7 +954,7 @@ fn emit_grouped_whatsapp<R: Runtime>(
"body": body,
"timestamp": timestamp,
"message_type": m.get("type").cloned().unwrap_or(Value::Null),
"source": source,
"source": row_source,
}))
})
.collect();
@@ -1358,6 +1482,45 @@ impl ScannerRegistry {
mod tests {
use super::*;
// ── Issue #1376 — chat-name normalization for active-chat → JID lookup ──
#[test]
fn normalize_chat_name_strips_punctuation_and_emoji() {
// Group titles routinely pick up emoji + punctuation drift between
// the DOM-parsed conversation header and the IDB-stored chat name.
// Normalization should collapse both sides to the same key so the
// lookup at scan_once succeeds.
assert_eq!(
normalize_chat_name("17-18-19 July samagam"),
"171819julysamagam"
);
assert_eq!(
normalize_chat_name("17 18 19 July samagam"),
"171819julysamagam"
);
assert_eq!(
normalize_chat_name("17-18-19 July samagam ✨"),
"171819julysamagam"
);
assert_eq!(
normalize_chat_name("17.18.19 July, samagam!"),
"171819julysamagam"
);
// Identity property — already-normal strings round-trip unchanged.
assert_eq!(normalize_chat_name("foo123"), "foo123");
// Empty input → empty output (caller guards against this).
assert_eq!(normalize_chat_name(""), "");
assert_eq!(normalize_chat_name(" "), "");
assert_eq!(normalize_chat_name(""), "");
}
#[test]
fn normalize_chat_name_lowercases() {
assert_eq!(normalize_chat_name("Hello World"), "helloworld");
assert_eq!(normalize_chat_name("HELLO"), "hello");
assert_eq!(normalize_chat_name("hElLo"), "hello");
}
fn insert_pending_tasks(
registry: &ScannerRegistry,
account_id: &str,
@@ -0,0 +1,53 @@
{
"_comment": "Synthetic CDP DOMSnapshot.captureSnapshot fixture for issue #1376 regression tests. Replace with a real WA Web capture once available. Four message rows: tier 1 (selectable-text class), tier 2 (dir=ltr), tier 3 multi-word body (descendant text fallback), and tier 3 single-word body (regression guard: must not be filtered by looks_like_icon_ligature). One conversation-header gives parse_active_chat_name something to resolve.",
"documents": [
{
"nodes": {
"parentIndex": [-1, 0, 1, 2, 0, 4, 5, 0, 7, 8, 0, 10, 11, 0, 13, 14],
"nodeType": [ 1, 1, 1, 3, 1, 1, 3, 1, 1, 3, 1, 1, 3, 1, 1, 3],
"nodeName": [ 0, 1, 4, -1, 6, 4, -1, 6, 4, -1, 6, 6, -1, 6, 6, -1],
"nodeValue": [-1, -1, -1, 5, -1, -1, 11, -1, -1, 15, -1, -1, 17, -1, -1, 19],
"attributes": [
[],
[2, 3],
[],
[],
[7, 8],
[9, 10],
[],
[7, 12],
[13, 14],
[],
[7, 16],
[],
[],
[7, 18],
[],
[]
]
}
}
],
"strings": [
"html",
"header",
"data-testid",
"conversation-header",
"span",
"Test Chat",
"div",
"data-id",
"false_chat1@c.us_msgABC123",
"class",
"selectable-text",
"hello tier 1",
"false_chat1@c.us_msgDEF456",
"dir",
"ltr",
"hello tier 2",
"false_chat1@c.us_msgGHI789",
"hello tier 3",
"false_chat1@c.us_msgJKL012",
"ok"
]
}