From 744da567db13d1d65f5ffb588db991da63758a4f Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Thu, 26 Mar 2026 16:15:58 -0700 Subject: [PATCH] fix: Apple Notes protobuf extraction handles HTML too, fix ZTITLE1 column Co-Authored-By: Claude Sonnet 4.6 --- src/openjarvis/connectors/apple_notes.py | 35 ++++++++++++++++++------ tests/connectors/test_apple_notes.py | 5 ++-- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/openjarvis/connectors/apple_notes.py b/src/openjarvis/connectors/apple_notes.py index 1755a786..c197f83c 100644 --- a/src/openjarvis/connectors/apple_notes.py +++ b/src/openjarvis/connectors/apple_notes.py @@ -16,8 +16,9 @@ epoch of 2001-01-01 00:00:00 UTC. Conversion formula:: Content extraction ------------------ -The ``ZDATA`` column in ``ZICNOTEDATA`` contains gzip-compressed HTML. Plain -text is obtained by decompressing the bytes and stripping HTML tags. +The ``ZDATA`` column in ``ZICNOTEDATA`` contains gzip-compressed protobuf +(``com.apple.notes.ICNote``). Plain text is obtained by decompressing the +bytes, decoding to UTF-8, and stripping protobuf control bytes. """ from __future__ import annotations @@ -70,24 +71,38 @@ def _apple_ts_to_datetime(apple_seconds: float) -> datetime: def _extract_text_from_zdata(zdata: bytes) -> str: - """Decompress gzip bytes and strip HTML tags to return plain text. + """Decompress gzip bytes and extract plain text from the protobuf payload. Parameters ---------- zdata: - Raw bytes from the ``ZDATA`` column (gzip-compressed HTML). + Raw bytes from the ``ZDATA`` column — gzip-compressed protobuf + (``com.apple.notes.ICNote``). Returns ------- str - Plain text with all HTML tags removed. Returns an empty string if - decompression fails. + Plain text with protobuf control bytes stripped. Returns an empty + string if decompression fails. """ try: - html = gzip.decompress(zdata).decode("utf-8", errors="replace") + raw = gzip.decompress(zdata) except Exception: # noqa: BLE001 return "" - return re.sub(r"<[^>]+>", "", html) + + text = raw.decode("utf-8", errors="replace") + # Strip HTML tags first (handles test fixtures and any HTML-formatted notes) + text = re.sub(r"<[^>]+>", "", text) + # Strip non-printable control bytes and U+FFFD replacement chars that + # come from the protobuf wire format. + cleaned = re.sub(r"[\x00-\x09\x0b\x0c\x0e-\x1f\x7f-\x9f\ufffd]+", " ", text) + # Collapse whitespace runs + cleaned = re.sub(r" {2,}", " ", cleaned) + cleaned = re.sub(r"\n{3,}", "\n\n", cleaned) + cleaned = cleaned.strip() + # Strip short leading protobuf varint artifacts (e.g. "3 3 " or "b b ") + cleaned = re.sub(r"^(?:[a-z0-9] ){1,4}", "", cleaned) + return cleaned.strip() # --------------------------------------------------------------------------- @@ -160,7 +175,9 @@ class AppleNotesConnector(BaseConnector): try: rows = conn.execute( - "SELECT n.ZIDENTIFIER, n.ZTITLE, n.ZMODIFICATIONDATE, d.ZDATA " + "SELECT n.ZIDENTIFIER, " + " COALESCE(n.ZTITLE1, n.ZTITLE, '') AS title, " + " n.ZMODIFICATIONDATE, d.ZDATA " "FROM ZICCLOUDSYNCINGOBJECT n " "JOIN ZICNOTEDATA d ON d.ZNOTE = n.Z_PK " "ORDER BY n.ZMODIFICATIONDATE ASC" diff --git a/tests/connectors/test_apple_notes.py b/tests/connectors/test_apple_notes.py index c3ee4ed5..e98a26e2 100644 --- a/tests/connectors/test_apple_notes.py +++ b/tests/connectors/test_apple_notes.py @@ -28,6 +28,7 @@ def _create_fake_notes_db(db_path: Path) -> None: CREATE TABLE ZICCLOUDSYNCINGOBJECT ( Z_PK INTEGER PRIMARY KEY, ZTITLE TEXT, + ZTITLE1 TEXT, ZMODIFICATIONDATE REAL, ZIDENTIFIER TEXT, ZNOTE INTEGER @@ -44,7 +45,7 @@ def _create_fake_notes_db(db_path: Path) -> None: compressed1 = gzip.compress(html1.encode()) conn.execute( "INSERT INTO ZICCLOUDSYNCINGOBJECT VALUES " - "(1, 'Shopping List', 694310400.0, 'note-001', 1)" + "(1, NULL, 'Shopping List', 694310400.0, 'note-001', 1)" ) conn.execute("INSERT INTO ZICNOTEDATA VALUES (1, ?, 1)", (compressed1,)) @@ -53,7 +54,7 @@ def _create_fake_notes_db(db_path: Path) -> None: compressed2 = gzip.compress(html2.encode()) conn.execute( "INSERT INTO ZICCLOUDSYNCINGOBJECT VALUES " - "(2, 'Meeting Notes', 694396800.0, 'note-002', 2)" + "(2, NULL, 'Meeting Notes', 694396800.0, 'note-002', 2)" ) conn.execute("INSERT INTO ZICNOTEDATA VALUES (2, ?, 2)", (compressed2,))