Skip to content

Native iCloud apps for Windows and Linux, built in Rust. Notes, Reminders, Calendar, Contacts, Drive and Photos, offline and in sync with your iPhone.

Notes and reminders carry the merge records a phone rebuilds them from

Yashwanth Kamireddi4382f1fe1418
diff --git a/crates/quince-icloud/src/ckdocument.rs b/crates/quince-icloud/src/ckdocument.rs
index d507a34..264be7c 100644
--- a/crates/quince-icloud/src/ckdocument.rs
+++ b/crates/quince-icloud/src/ckdocument.rs
@@ -237,6 +237,9 @@ const ATTRIBUTE_RUN: u64 = 5;
/// false. See the comment where they are dropped.
const PROVENANCE: u64 = 3;
+/// The list of writers the records' ids refer to.
+const WRITERS: u64 = 4;
+
/// A protobuf varint field: `(tag << 3) | 0`, then the value.
fn varint_field(tag: u64, value: u64) -> Vec<u8> {
let mut out = Vec::with_capacity(8);
@@ -272,6 +275,9 @@ fn utf16_len(text: &str) -> u64 {
/// order, differing only in the text.
fn document_bytes(text: &str) -> Vec<u8> {
let mut note = length_delimited(TEXT_PATH[2], text.as_bytes());
+ // The records a phone rebuilds the text from. Without them the text is only a cache,
+ // and a phone shows a note with nothing in it. See `topotext`.
+ note.extend(crate::topotext::Topo::fresh(utf16_len(text)).encode());
note.extend(length_delimited(
ATTRIBUTE_RUN,
&varint_field(1, utf16_len(text)),
@@ -508,6 +514,29 @@ fn set_varint_field(message: &[u8], tag: u64, value: u64) -> Option<Vec<u8>> {
fn rebuild_note(note: &[u8], old_text: &str, new_text: &str) -> Option<Vec<u8>> {
let (from, removed, inserted) = utf16_change(old_text, new_text);
+ // The merge records have to say what the text now says, or the phone shows what they
+ // say instead: edits made here arrived as text repeated, or lines that would not go.
+ let records = match crate::topotext::Topo::read(note) {
+ // The usual case: records that agree with the text, and the edit made to them the
+ // way a device would make it.
+ Some(mut topo) if topo.live_length() == utf16_len(old_text) => {
+ topo.edit(from, removed, inserted);
+ topo
+ }
+ // Records that already disagree — a note this app edited before it kept them. No
+ // edit can be placed in them truthfully, so what they hold is marked deleted and the
+ // text written anew. Other devices' characters keep their ids, so they merge.
+ Some(mut topo) => {
+ topo.replace_all(utf16_len(new_text));
+ topo
+ }
+ // Records in a shape this does not understand are not ours to rewrite.
+ None if has_field(note, PROVENANCE) => return None,
+ // None at all: a document written before any of this, or a template stripped of
+ // someone else's history. It gets records of its own.
+ None => crate::topotext::Topo::fresh(utf16_len(new_text)),
+ };
+
// Collect the existing runs, whole.
let mut runs: Vec<Vec<u8>> = Vec::new();
let mut reader = Reader::new(note);
@@ -562,7 +591,12 @@ fn rebuild_note(note: &[u8], old_text: &str, new_text: &str) -> Option<Vec<u8>>
let n = usize::try_from(reader.varint()?).ok()?;
reader.take(n)?;
out.extend_from_slice(&length_delimited(field, new_text.as_bytes()));
+ // The records follow the text, where Apple puts them.
+ out.extend(records.encode());
wrote_text = true;
+ } else if field == PROVENANCE || field == WRITERS {
+ // Written afresh above.
+ reader.skip(wire)?;
} else if field == ATTRIBUTE_RUN && wire == WIRE_LEN {
let n = usize::try_from(reader.varint()?).ok()?;
reader.take(n)?;
@@ -668,21 +702,52 @@ pub fn repair_runs_in_base64(original: &str) -> Option<String> {
Some(B64.encode(gzip(&repair_runs(&raw)?)))
}
-/// Give a document the merge records Apple's notes carry, for tests that need one to copy.
-#[doc(hidden)]
-#[must_use]
-pub fn add_merge_records_for_test(original: &str) -> String {
+/// A document as a phone would have written it: its records replaced by ones from a writer
+/// that is not this app.
+#[cfg(test)]
+pub(crate) fn add_merge_records_for_test(original: &str) -> String {
let raw = B64.decode(original.trim()).unwrap_or_default();
let plain = inflate(&raw).unwrap_or_default();
let note = field_at(&plain, &TEXT_PATH[..2]).unwrap_or_default();
+ let text = std::str::from_utf8(field_at(note, &TEXT_PATH[2..]).unwrap_or_default())
+ .unwrap_or_default();
- let mut with = note.to_vec();
- with.extend(length_delimited(3, &varint_field(5, 1)));
- with.extend(length_delimited(4, &varint_field(1, 1)));
+ let mut with = length_delimited(TEXT_PATH[2], text.as_bytes());
+ with.extend(crate::topotext::Topo::fresh_by([0x50; 16], utf16_len(text)).encode());
+ let mut reader = Reader::new(note);
+ while !reader.done() {
+ let start = reader.at;
+ let Some(key) = reader.varint() else { break };
+ if reader.skip(key & 7).is_none() {
+ break;
+ }
+ if !matches!(key >> 3, 2..=4) {
+ with.extend_from_slice(&note[start..reader.at]);
+ }
+ }
let rebuilt = rewrite(&plain, &TEXT_PATH[..2], &with).unwrap_or(plain);
B64.encode(gzip(&rebuilt))
}
+/// How much text a document's records say it holds, when they can be read.
+#[doc(hidden)]
+#[must_use]
+pub fn debug_records(original: &str) -> Option<u64> {
+ let raw = B64.decode(original.trim()).ok()?;
+ let plain = inflate(&raw)?;
+ Some(crate::topotext::Topo::read(field_at(&plain, &TEXT_PATH[..2])?)?.live_length())
+}
+
+/// How many writers a document's records name.
+#[cfg(test)]
+pub(crate) fn debug_writers(original: &str) -> usize {
+ let raw = B64.decode(original.trim()).unwrap_or_default();
+ let plain = inflate(&raw).unwrap_or_default();
+ field_at(&plain, &TEXT_PATH[..2])
+ .and_then(crate::topotext::Topo::read)
+ .map_or(0, |t| t.writer_clocks().len())
+}
+
/// What a document says about the note it belongs to: whether it names a writer, and whether
/// it still carries a record of which stretch of text came from where.
#[doc(hidden)]
@@ -793,7 +858,13 @@ pub fn is_rewritable(original: &[u8]) -> bool {
let Some(current) = field_at(&plain, TEXT_PATH) else {
return false;
};
- rewrite(&plain, TEXT_PATH, current).is_some_and(|rebuilt| rebuilt == plain)
+ if !rewrite(&plain, TEXT_PATH, current).is_some_and(|rebuilt| rebuilt == plain) {
+ return false;
+ }
+ // Merge records have to be ones an edit can be made to, the same test a save applies.
+ field_at(&plain, &TEXT_PATH[..2]).is_some_and(|note| {
+ !has_field(note, PROVENANCE) || crate::topotext::Topo::read(note).is_some()
+ })
}
/// [`replace_text`] over the base64 a CloudKit field carries, gzipped on the way back.
@@ -1246,6 +1317,32 @@ mod tests {
mod rewrite_tests {
use super::*;
+ /// The writer id a phone put on the fixture's text.
+ const PHONE: [u8; 16] = [0xAB; 16];
+
+ /// The note, with its text, runs and merge records taken out: everything an edit has no
+ /// business changing.
+ fn untouched_by_an_edit(document: &[u8]) -> Vec<u8> {
+ let plain = inflate(document).unwrap();
+ let note = field_at(&plain, &TEXT_PATH[..2]).unwrap();
+ let mut rest = Vec::new();
+ let mut reader = Reader::new(note);
+ while !reader.done() {
+ let start = reader.at;
+ let key = reader.varint().unwrap();
+ reader.skip(key & 7).unwrap();
+ if !matches!(key >> 3, 2..=5) {
+ rest.extend_from_slice(&note[start..reader.at]);
+ }
+ }
+ rewrite(&plain, &TEXT_PATH[..2], &rest).unwrap()
+ }
+
+ fn records(document: &[u8]) -> crate::topotext::Topo {
+ let plain = inflate(document).unwrap();
+ crate::topotext::Topo::read(field_at(&plain, &TEXT_PATH[..2]).unwrap()).unwrap()
+ }
+
fn varint(tag: u64, value: u64) -> Vec<u8> {
let mut out = Vec::new();
write_varint(tag << 3, &mut out);
@@ -1258,18 +1355,8 @@ mod rewrite_tests {
fn document(text: &str) -> Vec<u8> {
let mut note = Vec::new();
note.extend(length_delimited(2, text.as_bytes()));
- // Two of Apple's merge records, which carry no length describing the text.
- for index in 1..=2u64 {
- let mut run = Vec::new();
- run.extend(length_delimited(
- 1,
- &[varint(1, 1), varint(2, 144_927)].concat(),
- ));
- run.extend(varint(2, 34_947));
- run.extend(varint(5, index));
- note.extend(length_delimited(3, &run));
- }
- note.extend(length_delimited(4, &length_delimited(1, &[0xAB; 28])));
+ // Merge records exactly as a phone writes them, by a writer that is not this app.
+ note.extend(crate::topotext::Topo::fresh_by(PHONE, utf16_len(text)).encode());
// The attribute run that makes the text renderable, carrying styling alongside its
// length exactly as Apple's own runs do.
note.extend(length_delimited(
@@ -1300,11 +1387,12 @@ mod rewrite_tests {
text_from_bytes(&after).as_deref(),
Some("a longer body than the one it replaced")
);
- // Everything that is not the text is still there, byte for byte.
+ // Everything that is not the text or its records is still there, byte for byte.
+ assert_eq!(untouched_by_an_edit(&after), untouched_by_an_edit(&before));
+ // And the records describe the new text, not the old.
assert_eq!(
- replace_text(&after, "the original body").unwrap(),
- before,
- "putting the old text back must give the original document"
+ records(&after).live_length(),
+ utf16_len("a longer body than the one it replaced")
);
}
@@ -1464,58 +1552,48 @@ mod rewrite_tests {
assert_eq!(utf16_change("", "new"), (0, 0, 3));
}
- /// The whole point of rewriting in place rather than re-encoding. An edit of the same
- /// length must leave the document byte-identical apart from the text itself, which is
- /// the strongest statement available that nothing else was touched.
+ /// The whole point of rewriting in place rather than re-encoding: an edit touches the
+ /// text and the records that describe it, and nothing else.
#[test]
- fn a_same_length_edit_changes_only_the_text() {
+ fn a_same_length_edit_changes_only_the_text_and_its_records() {
let before = document("first");
let after = replace_text(&before, "FIRST").unwrap();
- assert_eq!(before.len(), after.len());
-
- let differing: Vec<usize> = before
- .iter()
- .zip(&after)
- .enumerate()
- .filter_map(|(i, (a, b))| (a != b).then_some(i))
- .collect();
- assert_eq!(differing.len(), 5, "only the five text bytes moved");
+ assert_eq!(untouched_by_an_edit(&after), untouched_by_an_edit(&before));
+ assert_eq!(
+ field_at(&inflate(&after).unwrap(), TEXT_PATH),
+ Some(&b"FIRST"[..])
+ );
// And the run still covers it, unchanged, because the length did not change.
assert_eq!(debug_attribute_lengths(&after), vec![5]);
}
- /// And when the length does change, the merge records Apple uses to reconcile edits
- /// across devices must come through byte for byte. Those are what a phone consults to
- /// decide whether our version is newer than its own.
+
+ /// What a phone merges on. The characters it wrote and this edit did not touch keep their
+ /// ids, and its entry in the list of writers comes through as it was; what was typed here
+ /// is this app's own. Rewriting the records wholesale would make the phone's copy and
+ /// this one disagree about every character — which is how text came back doubled.
#[test]
- fn the_version_vectors_survive_a_save() {
- let before = document("first");
- let after = replace_text(&before, "second").unwrap();
-
- // Field 3 of the note is Apple's merge state. Pull it out of both and compare.
- let merge_records = |bytes: &[u8]| -> Vec<Vec<u8>> {
- let plain = inflate(bytes).unwrap();
- let note = field_at(&plain, &TEXT_PATH[..2]).unwrap();
- let mut found = Vec::new();
- let mut reader = Reader::new(note);
- while !reader.done() {
- let key = reader.varint().unwrap();
- let (field, wire) = (key >> 3, key & 7);
- if field == 3 && wire == WIRE_LEN {
- let n = usize::try_from(reader.varint().unwrap()).unwrap();
- found.push(reader.take(n).unwrap().to_vec());
- } else {
- reader.skip(wire).unwrap();
- }
- }
- found
- };
+ fn a_phones_characters_keep_their_ids_through_a_save() {
+ let before = document("first draft");
+ let after = replace_text(&before, "first final draft").unwrap();
+ let (was, now) = (records(&before), records(&after));
+ let (was_ids, now_ids) = (was.live_ids(), now.live_ids());
+ assert_eq!(now.live_length(), 17);
assert_eq!(
- merge_records(&after),
- merge_records(&before),
- "the merge state must survive an edit untouched"
+ now_ids[..6],
+ was_ids[..6],
+ "\"first \" is still the phone's"
);
- assert!(!merge_records(&before).is_empty(), "the fixture has some");
+ assert_eq!(
+ now_ids[12..],
+ was_ids[6..],
+ "\"draft\" is still the phone's"
+ );
+ assert!(
+ now_ids[6..12].iter().all(|&(writer, _)| writer == 2),
+ "the typed words are this app's"
+ );
+ assert_eq!(now.writer_clocks()[0], (PHONE.to_vec(), vec![11, 1]));
}
#[test]
@@ -1665,16 +1743,18 @@ mod version_tests {
mod merge_record_tests {
use super::*;
- /// What this app builds from nothing: text, a run, the versions. It reads back here
- /// perfectly and a phone shows it as an empty note.
+ /// What this app builds from nothing carries records of its own that describe its text.
+ /// Without them it read back here perfectly and a phone showed it as an empty note.
#[test]
- fn a_document_built_from_nothing_has_no_merge_records() {
+ fn a_document_built_from_nothing_has_merge_records() {
let built = text_to_base64("Buy milk");
assert_eq!(text_from_base64(&built).as_deref(), Some("Buy milk"));
- assert!(
- !has_merge_records(&built),
- "which is exactly why a phone showed nothing"
- );
+ assert!(has_merge_records(&built));
+ let raw = B64.decode(&built).unwrap();
+ let plain = inflate(&raw).unwrap();
+ let topo = crate::topotext::Topo::read(field_at(&plain, &TEXT_PATH[..2]).unwrap())
+ .expect("records in Apple's shape");
+ assert_eq!(topo.live_length(), 8);
}
/// The shape Apple writes: records saying which writer produced which stretch of text.
@@ -1727,8 +1807,7 @@ mod merge_record_tests {
#[test]
fn swapping_the_text_keeps_what_makes_it_readable() {
let mut note = length_delimited(TEXT_PATH[2], b"Old words");
- note.extend(length_delimited(3, &varint_field(5, 1)));
- note.extend(length_delimited(4, &varint_field(1, 1)));
+ note.extend(crate::topotext::Topo::fresh_by([7; 16], 9).encode());
note.extend(length_delimited(ATTRIBUTE_RUN, &varint_field(1, 9)));
let mut document = varint_field(1, 0);
document.extend(varint_field(2, 0));
diff --git a/crates/quince-icloud/src/lib.rs b/crates/quince-icloud/src/lib.rs
index 260074b..76ce7fd 100644
--- a/crates/quince-icloud/src/lib.rs
+++ b/crates/quince-icloud/src/lib.rs
@@ -11,6 +11,7 @@ pub mod drive;
pub mod notes;
pub mod reminders;
pub mod sync;
+pub mod topotext;
#[derive(Debug, thiserror::Error)]
pub enum Error {
diff --git a/crates/quince-icloud/src/notes.rs b/crates/quince-icloud/src/notes.rs
index d9c3ef9..8145d6d 100644
--- a/crates/quince-icloud/src/notes.rs
+++ b/crates/quince-icloud/src/notes.rs
@@ -763,35 +763,9 @@ mod write_tests {
}
}
- /// A document shaped like the real ones: text surrounded by Apple's merge state.
+ /// A document shaped like the real ones: text with a phone's merge records around it.
fn document(text: &str) -> String {
- use base64::{engine::general_purpose::STANDARD as B64, Engine};
- use std::io::Write;
- fn framed(tag: u8, payload: &[u8]) -> Vec<u8> {
- let mut out = vec![(tag << 3) | 2];
- let mut len = payload.len();
- loop {
- let mut byte = (len & 0x7F) as u8;
- len >>= 7;
- if len != 0 {
- byte |= 0x80;
- }
- out.push(byte);
- if len == 0 {
- break;
- }
- }
- out.extend_from_slice(payload);
- out
- }
- // Version-vector siblings alongside the text, which a save must not disturb.
- let mut note = framed(2, text.as_bytes());
- note.extend(framed(3, &[0x08, 0x01, 0x10, 0x83, 0x8d, 0x02]));
- let proto = framed(2, &framed(3, &note));
-
- let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
- gz.write_all(&proto).unwrap();
- B64.encode(gz.finish().unwrap())
+ crate::ckdocument::add_merge_records_for_test(&crate::ckdocument::text_to_base64(text))
}
async fn server(existing: &str) -> MockServer {
@@ -851,13 +825,10 @@ mod write_tests {
crate::ckdocument::text_from_base64(written).as_deref(),
Some("Curtains\nmeasure the bay window\norder the rail")
);
- // The version vector alongside the text has to have come through the save.
- let raw = crate::ckdocument::debug_inflate(written).unwrap();
- assert!(
- crate::ckdocument::debug_tree(&raw, 6)
- .iter()
- .any(|line| line.contains("3: message")),
- "Apple's merge state must survive an edit"
+ // The records a phone merges on came through the save, describing the new text.
+ assert_eq!(
+ crate::ckdocument::debug_records(written),
+ Some(utf16("Curtains\nmeasure the bay window\norder the rail"))
);
}
@@ -1061,51 +1032,62 @@ mod folder_shape_tests {
}
}
+#[cfg(test)]
+fn utf16(text: &str) -> u64 {
+ text.encode_utf16().count() as u64
+}
+
/// Shaping a new note like one Apple wrote.
#[cfg(test)]
mod template_tests {
use super::*;
- /// A document with the merge records a phone needs, to stand in for one Apple wrote.
+ /// A document with a phone's merge records, to stand in for one Apple wrote.
fn apples_shape(text: &str) -> String {
- let built = crate::ckdocument::text_to_base64(text);
- // Built documents have no merge records, so this checks the fixture is honest about
- // what it is rather than quietly testing nothing.
- assert!(!crate::ckdocument::has_merge_records(&built));
- crate::ckdocument::add_merge_records_for_test(&built)
+ crate::ckdocument::add_merge_records_for_test(&crate::ckdocument::text_to_base64(text))
}
- /// A new note keeps the writer from the note it was shaped by, and none of its history.
+ /// A new note's records are its own: none of the history of the note it was shaped by,
+ /// and the text it holds written by this app.
///
/// The history is lengths measured against text that is not in this note. Carried over, a
/// fifteen-character note came out on a phone as the same word four times with gaps in
- /// between — worse than the blank it was meant to fix.
+ /// between. Carrying none at all left it blank.
#[test]
- fn a_new_note_takes_the_writer_but_not_the_other_notes_history() {
+ fn a_new_note_has_records_of_its_own() {
let template = apples_shape("Something Apple wrote");
- assert_eq!(
- crate::ckdocument::debug_note_fields(&template),
- (true, true),
- "the fixture has both, or this proves nothing"
- );
-
let made = document_like_apples(Some(&template), "Something written here");
assert_eq!(
crate::ckdocument::text_from_base64(&made).as_deref(),
Some("Something written here")
);
- let (writer, history) = crate::ckdocument::debug_note_fields(&made);
- assert!(writer, "the writer travels");
- assert!(!history, "the other note's history does not");
+ assert_eq!(
+ crate::ckdocument::debug_records(&made),
+ Some(utf16("Something written here")),
+ "records that describe exactly this text"
+ );
+ assert_eq!(
+ crate::ckdocument::debug_writers(&made),
+ 1,
+ "this app alone wrote it; the template's writer did not"
+ );
}
- /// A note's own history is its own, and an edit leaves it alone.
+ /// A note's own history is its own, and an edit adds to it rather than replacing it.
#[test]
fn editing_a_note_keeps_the_notes_own_history() {
let mine = apples_shape("What I wrote before");
let edited =
crate::ckdocument::replace_text_in_base64(&mine, "What I wrote after").unwrap();
- assert_eq!(crate::ckdocument::debug_note_fields(&edited), (true, true));
+ assert_eq!(
+ crate::ckdocument::debug_records(&edited),
+ Some(utf16("What I wrote after"))
+ );
+ assert_eq!(
+ crate::ckdocument::debug_writers(&edited),
+ 2,
+ "the phone, then this app"
+ );
}
/// A new account has no note to learn from. Building from nothing is the only option, and
diff --git a/crates/quince-icloud/src/topotext.rs b/crates/quince-icloud/src/topotext.rs
new file mode 100644
index 0000000..e16f169
--- /dev/null
+++ b/crates/quince-icloud/src/topotext.rs
@@ -0,0 +1,607 @@
+//! The merge records inside an Apple Notes or Reminders document.
+//!
+//! The text a document carries is a cache. What Apple's devices actually merge is a list of
+//! substrings, each naming who wrote a stretch of characters and whether it has since been
+//! deleted. A phone reading a document rebuilds the note from those records, so text with no
+//! records behind it shows as an empty note, and text whose records describe something else
+//! shows as whatever the records say — a word repeated, a line that will not go away.
+//!
+//! Measured on real documents written by a phone, every one of them the same:
+//!
+//! - The substrings sit in the note at field 3, in reading order, each pointing at the next
+//! by index (field 5). The first is a start marker with id (0, 0); the last is an end marker
+//! with id (0, 0xFFFFFFFF). Neither holds text.
+//! - A substring is `{1: id, 2: length, 3: timestamp, 4: deleted, 5: next}`, where an id and
+//! a timestamp are `{1: writer, 2: clock}`. Its characters are numbered clock, clock+1, …
+//! - Lengths are UTF-16 units, and the lengths of the ones not deleted add up to the text.
+//! - The writers are listed at field 4 as `{1: {1: uuid, 2: {1: next character},
+//! 2: {1: next timestamp}}, …}`, and an id's writer is its 1-based place in that list.
+//!
+//! An edit here is made the way a device makes one: what was removed is marked deleted, never
+//! dropped, and what was typed is a new substring under this app's own writer. Characters
+//! other devices wrote keep their ids, so their copies of the note line up with this one.
+
+use std::sync::OnceLock;
+
+/// The id of the end marker's characters.
+const END: u64 = 0xFFFF_FFFF;
+
+/// This app's writer id: one per installation, so two computers never number characters
+/// from the same writer. Until one is set, a fresh id per run is still correct — only less
+/// tidy, since each run that edits a note adds itself to that note's list of writers.
+static WRITER: OnceLock<[u8; 16]> = OnceLock::new();
+
+/// Adopt this installation's writer id. Only the first call counts.
+pub fn set_writer(id: [u8; 16]) {
+ let _ = WRITER.set(id);
+}
+
+fn writer() -> [u8; 16] {
+ *WRITER.get_or_init(|| *uuid::Uuid::new_v4().as_bytes())
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+struct Substring {
+ writer: u64,
+ clock: u64,
+ length: u64,
+ stamp_writer: u64,
+ stamp_clock: u64,
+ deleted: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+struct Writer {
+ uuid: Vec<u8>,
+ /// Next character clock, then next timestamp clock. Kept as a list because that is what
+ /// the field is, and a writer entry with some other count must survive untouched.
+ clocks: Vec<u64>,
+}
+
+/// The records of one note, in the order they are read.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) struct Topo {
+ substrings: Vec<Substring>,
+ writers: Vec<Writer>,
+}
+
+// ── Reading ────────────────────────────────────────────────────────────────────────────
+
+/// One field: its number, its wire type, the payload of a length-delimited one, and the
+/// value of a varint.
+type Field<'a> = (u64, u64, &'a [u8], u64);
+
+fn fields(bytes: &[u8]) -> Option<Vec<Field<'_>>> {
+ let mut out = Vec::new();
+ let mut at = 0usize;
+ while at < bytes.len() {
+ let (key, next) = varint(bytes, at)?;
+ at = next;
+ let (field, wire) = (key >> 3, key & 7);
+ match wire {
+ 0 => {
+ let (v, next) = varint(bytes, at)?;
+ at = next;
+ out.push((field, wire, &bytes[0..0], v));
+ }
+ 2 => {
+ let (n, next) = varint(bytes, at)?;
+ let end = next.checked_add(usize::try_from(n).ok()?)?;
+ out.push((field, wire, bytes.get(next..end)?, 0));
+ at = end;
+ }
+ 1 => at = at.checked_add(8).filter(|&e| e <= bytes.len())?,
+ 5 => at = at.checked_add(4).filter(|&e| e <= bytes.len())?,
+ _ => return None,
+ }
+ }
+ Some(out)
+}
+
+fn varint(bytes: &[u8], mut at: usize) -> Option<(u64, usize)> {
+ let mut value = 0u64;
+ for shift in 0..10 {
+ let byte = *bytes.get(at)?;
+ at += 1;
+ value |= u64::from(byte & 0x7F) << (shift * 7);
+ if byte & 0x80 == 0 {
+ return Some((value, at));
+ }
+ }
+ None
+}
+
+fn id(bytes: &[u8]) -> Option<(u64, u64)> {
+ let mut id = (0, 0);
+ for (field, wire, _, v) in fields(bytes)? {
+ match (field, wire) {
+ (1, 0) => id.0 = v,
+ (2, 0) => id.1 = v,
+ _ => {}
+ }
+ }
+ Some(id)
+}
+
+fn substring(bytes: &[u8]) -> Option<(Substring, Vec<u64>)> {
+ let mut s = Substring {
+ writer: 0,
+ clock: 0,
+ length: 0,
+ stamp_writer: 0,
+ stamp_clock: 0,
+ deleted: false,
+ };
+ let mut next = Vec::new();
+ for (field, wire, payload, v) in fields(bytes)? {
+ match (field, wire) {
+ (1, 2) => (s.writer, s.clock) = id(payload)?,
+ (2, 0) => s.length = v,
+ (3, 2) => (s.stamp_writer, s.stamp_clock) = id(payload)?,
+ (4, 0) => s.deleted = v != 0,
+ (5, 0) => next.push(v),
+ // A field this does not know means a document this does not understand.
+ _ => return None,
+ }
+ }
+ Some((s, next))
+}
+
+fn writers(bytes: &[u8]) -> Option<Vec<Writer>> {
+ let mut out = Vec::new();
+ for (field, wire, payload, _) in fields(bytes)? {
+ if (field, wire) != (1, 2) {
+ return None;
+ }
+ let mut w = Writer {
+ uuid: Vec::new(),
+ clocks: Vec::new(),
+ };
+ for (field, wire, payload, _) in fields(payload)? {
+ match (field, wire) {
+ (1, 2) => w.uuid = payload.to_vec(),
+ (2, 2) => {
+ let mut clock = None;
+ for (field, wire, _, v) in fields(payload)? {
+ match (field, wire) {
+ (1, 0) => clock = Some(v),
+ _ => return None,
+ }
+ }
+ w.clocks.push(clock.unwrap_or(0));
+ }
+ _ => return None,
+ }
+ }
+ out.push(w);
+ }
+ Some(out)
+}
+
+impl Topo {
+ /// The records of a note message, or `None` when it has none or they are not shaped the
+ /// way Apple writes them. A graph that is not a single chain in reading order is refused
+ /// rather than guessed at.
+ pub(crate) fn read(note: &[u8]) -> Option<Self> {
+ let mut substrings = Vec::new();
+ let mut links = Vec::new();
+ let mut writer_list = None;
+ for (field, wire, payload, _) in fields(note)? {
+ match (field, wire) {
+ (3, 2) => {
+ let (s, next) = substring(payload)?;
+ substrings.push(s);
+ links.push(next);
+ }
+ (4, 2) => writer_list = Some(writers(payload)?),
+ _ => {}
+ }
+ }
+ if substrings.len() < 2 {
+ return None;
+ }
+ let last = links.len() - 1;
+ for (i, next) in links.iter().enumerate() {
+ let expected: Vec<u64> = if i == last {
+ vec![]
+ } else {
+ vec![i as u64 + 1]
+ };
+ if *next != expected {
+ return None;
+ }
+ }
+ let first = &substrings[0];
+ let end = &substrings[last];
+ if (first.writer, first.clock, first.length) != (0, 0, 0)
+ || (end.writer, end.clock, end.length) != (0, END, 0)
+ {
+ return None;
+ }
+ Some(Self {
+ substrings,
+ writers: writer_list.unwrap_or_default(),
+ })
+ }
+
+ /// Records for a note that holds `length` units of text, all written by this app.
+ pub(crate) fn fresh(length: u64) -> Self {
+ let mut topo = Self {
+ substrings: vec![
+ Substring {
+ writer: 0,
+ clock: 0,
+ length: 0,
+ stamp_writer: 0,
+ stamp_clock: 0,
+ deleted: false,
+ },
+ Substring {
+ writer: 0,
+ clock: END,
+ length: 0,
+ stamp_writer: 0,
+ stamp_clock: END,
+ deleted: false,
+ },
+ ],
+ writers: Vec::new(),
+ };
+ topo.insert(0, length);
+ topo
+ }
+
+ /// How much text the records say is there.
+ pub(crate) fn live_length(&self) -> u64 {
+ self.substrings
+ .iter()
+ .filter(|s| !s.deleted)
+ .map(|s| s.length)
+ .sum()
+ }
+
+ // ── Editing ────────────────────────────────────────────────────────────────────────
+
+ /// Make `at` fall on a boundary between substrings, splitting the one it lands inside.
+ fn split_at(&mut self, at: u64) {
+ let mut offset = 0;
+ for i in 0..self.substrings.len() {
+ let s = &self.substrings[i];
+ if s.deleted {
+ continue;
+ }
+ if offset < at && at < offset + s.length {
+ let keep = at - offset;
+ let mut tail = s.clone();
+ tail.clock += keep;
+ tail.length -= keep;
+ self.substrings[i].length = keep;
+ self.substrings.insert(i + 1, tail);
+ return;
+ }
+ offset += s.length;
+ }
+ }
+
+ /// This app's place in the list of writers, adding it if it is not there yet.
+ fn own_writer(&mut self) -> usize {
+ let me = writer();
+ if let Some(i) = self.writers.iter().position(|w| w.uuid == me) {
+ while self.writers[i].clocks.len() < 2 {
+ self.writers[i].clocks.push(0);
+ }
+ return i;
+ }
+ self.writers.push(Writer {
+ uuid: me.to_vec(),
+ clocks: vec![0, 0],
+ });
+ self.writers.len() - 1
+ }
+
+ /// New text of `length` units at `at`, as a substring of this app's own.
+ fn insert(&mut self, at: u64, length: u64) {
+ if length == 0 {
+ return;
+ }
+ self.split_at(at);
+ let me = self.own_writer();
+ let clock = self.writers[me].clocks[0];
+ let stamp = self.writers[me].clocks[1];
+
+ // Right after the last live character before `at`; the start marker when `at` is 0.
+ let mut offset = 0;
+ let mut place = 1;
+ for (i, s) in self.substrings.iter().enumerate().skip(1) {
+ if offset == at {
+ place = i;
+ break;
+ }
+ if !s.deleted {
+ offset += s.length;
+ }
+ place = i + 1;
+ }
+ let place = place.min(self.substrings.len() - 1);
+
+ self.substrings.insert(
+ place,
+ Substring {
+ writer: me as u64 + 1,
+ clock,
+ length,
+ stamp_writer: me as u64 + 1,
+ stamp_clock: stamp,
+ deleted: false,
+ },
+ );
+ self.writers[me].clocks[0] = clock + length;
+ self.writers[me].clocks[1] = stamp + 1;
+ }
+
+ /// Mark `length` units from `at` deleted. The characters keep their ids, which is what
+ /// lets another device's copy agree that they are gone.
+ fn delete(&mut self, at: u64, length: u64) {
+ if length == 0 {
+ return;
+ }
+ self.split_at(at);
+ self.split_at(at + length);
+ let mut offset = 0;
+ for s in &mut self.substrings {
+ if s.deleted {
+ continue;
+ }
+ if offset >= at && offset + s.length <= at + length && s.length > 0 {
+ s.deleted = true;
+ }
+ offset += s.length;
+ }
+ }
+
+ /// Apply one edit: `removed` units from `from` replaced by `inserted` new ones.
+ pub(crate) fn edit(&mut self, from: u64, removed: u64, inserted: u64) {
+ self.delete(from, removed);
+ self.insert(from, inserted);
+ }
+
+ /// Everything the records say is there replaced by `length` new units. For a document
+ /// whose records already disagree with its text, where no edit can be placed truthfully.
+ pub(crate) fn replace_all(&mut self, length: u64) {
+ let live = self.live_length();
+ self.edit(0, live, length);
+ }
+
+ // ── Writing ────────────────────────────────────────────────────────────────────────
+
+ /// The note fields these records occupy: every substring at 3, then the writers at 4.
+ pub(crate) fn encode(&self) -> Vec<u8> {
+ let mut out = Vec::new();
+ let last = self.substrings.len() - 1;
+ for (i, s) in self.substrings.iter().enumerate() {
+ let mut m = message(1, &pair(s.writer, s.clock));
+ m.extend(varint_field(2, s.length));
+ m.extend(message(3, &pair(s.stamp_writer, s.stamp_clock)));
+ if s.deleted {
+ m.extend(varint_field(4, 1));
+ }
+ if i != last {
+ m.extend(varint_field(5, i as u64 + 1));
+ }
+ out.extend(message(3, &m));
+ }
+ let mut list = Vec::new();
+ for w in &self.writers {
+ let mut entry = message(1, &w.uuid);
+ for clock in &w.clocks {
+ entry.extend(message(2, &varint_field(1, *clock)));
+ }
+ list.extend(message(1, &entry));
+ }
+ out.extend(message(4, &list));
+ out
+ }
+}
+
+fn write_varint(mut value: u64, out: &mut Vec<u8>) {
+ loop {
+ let mut byte = (value & 0x7F) as u8;
+ value >>= 7;
+ if value != 0 {
+ byte |= 0x80;
+ }
+ out.push(byte);
+ if value == 0 {
+ return;
+ }
+ }
+}
+
+fn varint_field(tag: u64, value: u64) -> Vec<u8> {
+ let mut out = Vec::new();
+ write_varint(tag << 3, &mut out);
+ write_varint(value, &mut out);
+ out
+}
+
+fn message(tag: u64, payload: &[u8]) -> Vec<u8> {
+ let mut out = Vec::new();
+ write_varint((tag << 3) | 2, &mut out);
+ write_varint(payload.len() as u64, &mut out);
+ out.extend_from_slice(payload);
+ out
+}
+
+/// An id or timestamp: both halves written even when zero, as Apple writes them.
+fn pair(writer: u64, clock: u64) -> Vec<u8> {
+ let mut out = varint_field(1, writer);
+ out.extend(varint_field(2, clock));
+ out
+}
+
+#[cfg(test)]
+impl Topo {
+ /// Records as another device would have written them for `length` units.
+ pub(crate) fn fresh_by(uuid: [u8; 16], length: u64) -> Self {
+ let mut t = Self::fresh(0);
+ t.writers.push(Writer {
+ uuid: uuid.to_vec(),
+ clocks: vec![length, 1],
+ });
+ t.substrings.insert(
+ 1,
+ Substring {
+ writer: 1,
+ clock: 0,
+ length,
+ stamp_writer: 1,
+ stamp_clock: 0,
+ deleted: false,
+ },
+ );
+ t
+ }
+
+ /// Every writer, with its clocks.
+ pub(crate) fn writer_clocks(&self) -> Vec<(Vec<u8>, Vec<u64>)> {
+ self.writers
+ .iter()
+ .map(|w| (w.uuid.clone(), w.clocks.clone()))
+ .collect()
+ }
+
+ /// Which writer wrote each live unit, in reading order — what a phone rebuilds from.
+ pub(crate) fn live_ids(&self) -> Vec<(u64, u64)> {
+ self.substrings
+ .iter()
+ .filter(|s| !s.deleted)
+ .flat_map(|s| (0..s.length).map(move |k| (s.writer, s.clock + k)))
+ .collect()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn roundtrip(t: &Topo) -> Topo {
+ let mut note = message(2, b"text");
+ note.extend(t.encode());
+ Topo::read(&note).expect("our own records read back")
+ }
+
+ /// The exact bytes a phone wrote for a reminder titled "Charge the laptop 🔋", except for
+ /// the writer id, which is the phone's own.
+ #[test]
+ fn a_fresh_document_is_shaped_like_the_phones() {
+ set_writer([0x67; 16]);
+ let t = Topo::fresh(20);
+ let s = &t.substrings;
+ assert_eq!(s.len(), 3);
+ assert_eq!((s[0].writer, s[0].clock, s[0].length), (0, 0, 0));
+ assert_eq!((s[1].writer, s[1].clock, s[1].length), (1, 0, 20));
+ assert_eq!((s[1].stamp_writer, s[1].stamp_clock), (1, 0));
+ assert_eq!((s[2].writer, s[2].clock, s[2].stamp_clock), (0, END, END));
+ assert_eq!(t.writers[0].clocks, vec![20, 1]);
+ assert_eq!(roundtrip(&t), t);
+ }
+
+ #[test]
+ fn typing_in_the_middle_keeps_everyone_elses_characters() {
+ let mut t = Topo::fresh(10);
+ let before = t.live_ids();
+ t.edit(4, 0, 3);
+ let after = t.live_ids();
+ assert_eq!(t.live_length(), 13);
+ assert_eq!(&after[..4], &before[..4]);
+ assert_eq!(&after[7..], &before[4..]);
+ assert_eq!(roundtrip(&t), t);
+ }
+
+ #[test]
+ fn deleting_marks_characters_rather_than_dropping_them() {
+ let mut t = Topo::fresh(10);
+ let all: u64 = t.substrings.iter().map(|s| s.length).sum();
+ t.edit(2, 5, 0);
+ assert_eq!(t.live_length(), 5);
+ // Nothing vanished: the deleted stretch is still there, marked.
+ assert_eq!(t.substrings.iter().map(|s| s.length).sum::<u64>(), all);
+ assert_eq!(roundtrip(&t), t);
+ }
+
+ #[test]
+ fn replacing_a_word_is_a_delete_and_an_insert() {
+ let mut t = Topo::fresh(11); // "hello world"
+ t.edit(6, 5, 5); // "hello there"
+ assert_eq!(t.live_length(), 11);
+ let ids = t.live_ids();
+ // The kept prefix is the original characters; the new word is new ones.
+ assert_eq!(ids[..6], Topo::fresh(11).live_ids()[..6]);
+ assert!(ids[6..].iter().all(|&(_, c)| c >= 11));
+ }
+
+ #[test]
+ fn many_edits_keep_the_records_consistent() {
+ let mut t = Topo::fresh(0);
+ let mut len = 0u64;
+ // A deterministic scatter of inserts and deletes all over the text.
+ for step in 0..200u64 {
+ let at = (step * 7919) % (len + 1);
+ if step % 3 == 0 && len > 0 {
+ let n = (step % 5 + 1).min(len - at.min(len));
+ t.edit(at.min(len), n, 0);
+ len -= n;
+ } else {
+ t.edit(at, 0, step % 4 + 1);
+ len += step % 4 + 1;
+ }
+ assert_eq!(t.live_length(), len, "step {step}");
+ }
+ let ids = t.live_ids();
+ let unique: std::collections::HashSet<_> = ids.iter().collect();
+ assert_eq!(unique.len(), ids.len(), "no character id is used twice");
+ assert_eq!(roundtrip(&t), t);
+ }
+
+ #[test]
+ fn another_writers_entry_is_left_as_it_was() {
+ let mut t = Topo::fresh(5);
+ t.writers.insert(
+ 0,
+ Writer {
+ uuid: vec![9; 16],
+ clocks: vec![74800, 31],
+ },
+ );
+ // Our own substring now refers to the second writer.
+ t.substrings[1].writer = 2;
+ t.substrings[1].stamp_writer = 2;
+ t.edit(5, 0, 2);
+ assert_eq!(t.writers[0].clocks, vec![74800, 31]);
+ assert_eq!(t.writers.len(), 2);
+ assert_eq!(t.substrings[2].writer, 2);
+ }
+
+ #[test]
+ fn a_graph_that_is_not_one_chain_is_refused() {
+ let t = Topo::fresh(3);
+ let mut bytes = message(2, b"abc");
+ bytes.extend(t.encode());
+ // Point the start marker somewhere other than the next record.
+ let mut forked = message(2, b"abc");
+ let mut s0 = message(1, &pair(0, 0));
+ s0.extend(varint_field(2, 0));
+ s0.extend(message(3, &pair(0, 0)));
+ s0.extend(varint_field(5, 2));
+ forked.extend(message(3, &s0));
+ let rest = Topo {
+ substrings: t.substrings[1..].to_vec(),
+ writers: t.writers.clone(),
+ };
+ forked.extend(rest.encode());
+ assert!(Topo::read(&bytes).is_some());
+ assert!(Topo::read(&forked).is_none());
+ assert!(Topo::read(&[0xFF]).is_none());
+ assert!(Topo::read(&message(2, b"no records")).is_none());
+ }
+}
diff --git a/probe/examples/edit_loop.rs b/probe/examples/edit_loop.rs
index 3f387c8..b38efac 100644
--- a/probe/examples/edit_loop.rs
+++ b/probe/examples/edit_loop.rs
@@ -33,7 +33,7 @@ async fn main() -> Result<()> {
&root,
params.clone(),
Some("DefaultFolder-CloudKit"),
- "edit test",
+ "Quince test",
template.as_deref(),
)
.await?;
@@ -41,14 +41,16 @@ async fn main() -> Result<()> {
// What someone typing actually produces: a growing body, saved after each pause.
let steps = [
- "edit test\n1",
- "edit test\n1\n2",
- "edit test\n1\n2\n3",
- "edit test\n1\n2\n3\n4",
- "edit test\n1\n2\n3\n4\n56",
+ "Quince test\n1",
+ "Quince test\n1\n2",
+ "Quince test\n1\n2\n3",
+ "Quince test\n1\n2\n3\n4",
+ "Quince test\n1\n2\n3\n4\n56",
// And a deletion, which is where an append-only mistake shows up loudest.
- "edit test\n1\n2",
- "short",
+ "Quince test\n1\n2",
+ // A word changed in the middle, then an emoji, which counts as two units.
+ "Quince test\nfirst\n2",
+ "Quince test\nfirst line\nsecond line 🔴",
];
let mut failures = 0;
@@ -73,20 +75,26 @@ async fn main() -> Result<()> {
};
let units: u64 = want.encode_utf16().count() as u64;
let covered: u64 = runs.iter().sum();
- let ok = got == *want && covered == units;
+ // What a phone rebuilds the note from has to say the same.
+ let records = quince_icloud::ckdocument::debug_records(doc);
+ let ok = got == *want && covered == units && records == Some(units);
if !ok {
failures += 1;
}
println!(
- " {} step {n}: {:?} runs cover {covered} of {units}",
+ " {} step {n}: {:?} runs cover {covered} of {units}, records {records:?}",
if ok { "PASS" } else { "FAIL" },
got.chars().take(28).collect::<String>()
);
}
- let note = library_note(&http, &root, &params, &guid).await?;
- quince_icloud::notes::delete_cloudkit(&http, &root, params.clone(), &note).await?;
- println!("\ncleaned up");
+ if std::env::var("KEEP").is_ok() {
+ println!("\nleft on the account to look at on a phone: {guid}");
+ } else {
+ let note = library_note(&http, &root, &params, &guid).await?;
+ quince_icloud::notes::delete_cloudkit(&http, &root, params.clone(), &note).await?;
+ println!("\ncleaned up");
+ }
if failures > 0 {
return Err(anyhow!("{failures} step(s) did not hold"));
}
diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs
index e32378d..f5ffc69 100644
--- a/src-tauri/src/main.rs
+++ b/src-tauri/src/main.rs
@@ -1343,6 +1343,29 @@ async fn contact_photo(
})
}
+/// Where this installation's writer id is kept.
+const WRITER_KEY: &str = "documents.writer";
+
+/// Give note and reminder edits this installation's own writer id.
+///
+/// Apple's devices merge a note by who wrote which characters, so every edit made here is
+/// signed. The same id every run keeps a note's list of writers from growing by one each time
+/// the app starts; a different id on every computer keeps two of them from numbering
+/// characters the same way. If it cannot be stored, a fresh one for this run is still correct.
+fn adopt_writer(library: &mut Library) {
+ let stored = library
+ .state(WRITER_KEY)
+ .ok()
+ .flatten()
+ .and_then(|s| uuid::Uuid::parse_str(&s).ok());
+ let id = stored.unwrap_or_else(|| {
+ let fresh = uuid::Uuid::new_v4();
+ let _ = library.set_state(WRITER_KEY, &fresh.to_string());
+ fresh
+ });
+ quince_icloud::topotext::set_writer(*id.as_bytes());
+}
+
fn main() {
tauri::Builder::default()
.plugin(tauri_plugin_dialog::init())
@@ -1408,7 +1431,9 @@ fn main() {
});
// The local library. Opened once at startup so every command shares one
// connection and the WAL behaves.
- app.manage(Data(Mutex::new(Library::open(&dir)?)));
+ let mut library = Library::open(&dir)?;
+ adopt_writer(&mut library);
+ app.manage(Data(Mutex::new(library)));
spawn_heartbeat(Arc::clone(&client));
spawn_auto_sync(app.handle().clone(), client);
Ok(())