Personal AI, On Personal Devices
Python
fix: Apple Notes protobuf extraction handles HTML too, fix ZTITLE1 column
Jon Saad-Falcon744da567db13
diff --git a/src/openjarvis/connectors/apple_notes.py b/src/openjarvis/connectors/apple_notes.pyindex 1755a786..c197f83c 100644--- a/src/openjarvis/connectors/apple_notes.py+++ b/src/openjarvis/connectors/apple_notes.py@@ -16,8 +16,9 @@ epoch of 2001-01-01 00:00:00 UTC. Conversion formula::Content extraction-------------------The ``ZDATA`` column in ``ZICNOTEDATA`` contains gzip-compressed HTML. Plain-text is obtained by decompressing the bytes and stripping HTML tags.+The ``ZDATA`` column in ``ZICNOTEDATA`` contains gzip-compressed protobuf+(``com.apple.notes.ICNote``). Plain text is obtained by decompressing the+bytes, decoding to UTF-8, and stripping protobuf control bytes."""from __future__ import annotations@@ -70,24 +71,38 @@ def _apple_ts_to_datetime(apple_seconds: float) -> datetime:def _extract_text_from_zdata(zdata: bytes) -> str:- """Decompress gzip bytes and strip HTML tags to return plain text.+ """Decompress gzip bytes and extract plain text from the protobuf payload.Parameters----------zdata:- Raw bytes from the ``ZDATA`` column (gzip-compressed HTML).+ Raw bytes from the ``ZDATA`` column — gzip-compressed protobuf+ (``com.apple.notes.ICNote``).Returns-------str- Plain text with all HTML tags removed. Returns an empty string if- decompression fails.+ Plain text with protobuf control bytes stripped. Returns an empty+ string if decompression fails."""try:- html = gzip.decompress(zdata).decode("utf-8", errors="replace")+ raw = gzip.decompress(zdata)except Exception: # noqa: BLE001return ""- return re.sub(r"<[^>]+>", "", html)++ text = raw.decode("utf-8", errors="replace")+ # Strip HTML tags first (handles test fixtures and any HTML-formatted notes)+ text = re.sub(r"<[^>]+>", "", text)+ # Strip non-printable control bytes and U+FFFD replacement chars that+ # come from the protobuf wire format.+ cleaned = re.sub(r"[\x00-\x09\x0b\x0c\x0e-\x1f\x7f-\x9f\ufffd]+", " ", text)+ # Collapse whitespace runs+ cleaned = re.sub(r" {2,}", " ", cleaned)+ cleaned = re.sub(r"\n{3,}", "\n\n", cleaned)+ cleaned = cleaned.strip()+ # Strip short leading protobuf varint artifacts (e.g. "3 3 " or "b b ")+ cleaned = re.sub(r"^(?:[a-z0-9] ){1,4}", "", cleaned)+ return cleaned.strip()# ---------------------------------------------------------------------------@@ -160,7 +175,9 @@ class AppleNotesConnector(BaseConnector):try:rows = conn.execute(- "SELECT n.ZIDENTIFIER, n.ZTITLE, n.ZMODIFICATIONDATE, d.ZDATA "+ "SELECT n.ZIDENTIFIER, "+ " COALESCE(n.ZTITLE1, n.ZTITLE, '') AS title, "+ " n.ZMODIFICATIONDATE, d.ZDATA ""FROM ZICCLOUDSYNCINGOBJECT n ""JOIN ZICNOTEDATA d ON d.ZNOTE = n.Z_PK ""ORDER BY n.ZMODIFICATIONDATE ASC"diff --git a/tests/connectors/test_apple_notes.py b/tests/connectors/test_apple_notes.pyindex c3ee4ed5..e98a26e2 100644--- a/tests/connectors/test_apple_notes.py+++ b/tests/connectors/test_apple_notes.py@@ -28,6 +28,7 @@ def _create_fake_notes_db(db_path: Path) -> None:CREATE TABLE ZICCLOUDSYNCINGOBJECT (Z_PK INTEGER PRIMARY KEY,ZTITLE TEXT,+ ZTITLE1 TEXT,ZMODIFICATIONDATE REAL,ZIDENTIFIER TEXT,ZNOTE INTEGER@@ -44,7 +45,7 @@ def _create_fake_notes_db(db_path: Path) -> None:compressed1 = gzip.compress(html1.encode())conn.execute("INSERT INTO ZICCLOUDSYNCINGOBJECT VALUES "- "(1, 'Shopping List', 694310400.0, 'note-001', 1)"+ "(1, NULL, 'Shopping List', 694310400.0, 'note-001', 1)")conn.execute("INSERT INTO ZICNOTEDATA VALUES (1, ?, 1)", (compressed1,))@@ -53,7 +54,7 @@ def _create_fake_notes_db(db_path: Path) -> None:compressed2 = gzip.compress(html2.encode())conn.execute("INSERT INTO ZICCLOUDSYNCINGOBJECT VALUES "- "(2, 'Meeting Notes', 694396800.0, 'note-002', 2)"+ "(2, NULL, 'Meeting Notes', 694396800.0, 'note-002', 2)")conn.execute("INSERT INTO ZICNOTEDATA VALUES (2, ?, 2)", (compressed2,))