Skip to content

Personal AI, On Personal Devices

Python

fix: Apple Notes protobuf extraction handles HTML too, fix ZTITLE1 column

Jon Saad-Falcon744da567db13
diff --git a/src/openjarvis/connectors/apple_notes.py b/src/openjarvis/connectors/apple_notes.py
index 1755a786..c197f83c 100644
--- a/src/openjarvis/connectors/apple_notes.py
+++ b/src/openjarvis/connectors/apple_notes.py
@@ -16,8 +16,9 @@ epoch of 2001-01-01 00:00:00 UTC. Conversion formula::
Content extraction
------------------
-The ``ZDATA`` column in ``ZICNOTEDATA`` contains gzip-compressed HTML. Plain
-text is obtained by decompressing the bytes and stripping HTML tags.
+The ``ZDATA`` column in ``ZICNOTEDATA`` contains gzip-compressed protobuf
+(``com.apple.notes.ICNote``). Plain text is obtained by decompressing the
+bytes, decoding to UTF-8, and stripping protobuf control bytes.
"""
from __future__ import annotations
@@ -70,24 +71,38 @@ def _apple_ts_to_datetime(apple_seconds: float) -> datetime:
def _extract_text_from_zdata(zdata: bytes) -> str:
- """Decompress gzip bytes and strip HTML tags to return plain text.
+ """Decompress gzip bytes and extract plain text from the protobuf payload.
Parameters
----------
zdata:
- Raw bytes from the ``ZDATA`` column (gzip-compressed HTML).
+ Raw bytes from the ``ZDATA`` column — gzip-compressed protobuf
+ (``com.apple.notes.ICNote``).
Returns
-------
str
- Plain text with all HTML tags removed. Returns an empty string if
- decompression fails.
+ Plain text with protobuf control bytes stripped. Returns an empty
+ string if decompression fails.
"""
try:
- html = gzip.decompress(zdata).decode("utf-8", errors="replace")
+ raw = gzip.decompress(zdata)
except Exception: # noqa: BLE001
return ""
- return re.sub(r"<[^>]+>", "", html)
+
+ text = raw.decode("utf-8", errors="replace")
+ # Strip HTML tags first (handles test fixtures and any HTML-formatted notes)
+ text = re.sub(r"<[^>]+>", "", text)
+ # Strip non-printable control bytes and U+FFFD replacement chars that
+ # come from the protobuf wire format.
+ cleaned = re.sub(r"[\x00-\x09\x0b\x0c\x0e-\x1f\x7f-\x9f\ufffd]+", " ", text)
+ # Collapse whitespace runs
+ cleaned = re.sub(r" {2,}", " ", cleaned)
+ cleaned = re.sub(r"\n{3,}", "\n\n", cleaned)
+ cleaned = cleaned.strip()
+ # Strip short leading protobuf varint artifacts (e.g. "3 3 " or "b b ")
+ cleaned = re.sub(r"^(?:[a-z0-9] ){1,4}", "", cleaned)
+ return cleaned.strip()
# ---------------------------------------------------------------------------
@@ -160,7 +175,9 @@ class AppleNotesConnector(BaseConnector):
try:
rows = conn.execute(
- "SELECT n.ZIDENTIFIER, n.ZTITLE, n.ZMODIFICATIONDATE, d.ZDATA "
+ "SELECT n.ZIDENTIFIER, "
+ " COALESCE(n.ZTITLE1, n.ZTITLE, '') AS title, "
+ " n.ZMODIFICATIONDATE, d.ZDATA "
"FROM ZICCLOUDSYNCINGOBJECT n "
"JOIN ZICNOTEDATA d ON d.ZNOTE = n.Z_PK "
"ORDER BY n.ZMODIFICATIONDATE ASC"
diff --git a/tests/connectors/test_apple_notes.py b/tests/connectors/test_apple_notes.py
index c3ee4ed5..e98a26e2 100644
--- a/tests/connectors/test_apple_notes.py
+++ b/tests/connectors/test_apple_notes.py
@@ -28,6 +28,7 @@ def _create_fake_notes_db(db_path: Path) -> None:
CREATE TABLE ZICCLOUDSYNCINGOBJECT (
Z_PK INTEGER PRIMARY KEY,
ZTITLE TEXT,
+ ZTITLE1 TEXT,
ZMODIFICATIONDATE REAL,
ZIDENTIFIER TEXT,
ZNOTE INTEGER
@@ -44,7 +45,7 @@ def _create_fake_notes_db(db_path: Path) -> None:
compressed1 = gzip.compress(html1.encode())
conn.execute(
"INSERT INTO ZICCLOUDSYNCINGOBJECT VALUES "
- "(1, 'Shopping List', 694310400.0, 'note-001', 1)"
+ "(1, NULL, 'Shopping List', 694310400.0, 'note-001', 1)"
)
conn.execute("INSERT INTO ZICNOTEDATA VALUES (1, ?, 1)", (compressed1,))
@@ -53,7 +54,7 @@ def _create_fake_notes_db(db_path: Path) -> None:
compressed2 = gzip.compress(html2.encode())
conn.execute(
"INSERT INTO ZICCLOUDSYNCINGOBJECT VALUES "
- "(2, 'Meeting Notes', 694396800.0, 'note-002', 2)"
+ "(2, NULL, 'Meeting Notes', 694396800.0, 'note-002', 2)"
)
conn.execute("INSERT INTO ZICNOTEDATA VALUES (2, ?, 2)", (compressed2,))