Native iCloud apps for Windows and Linux, built in Rust. Notes, Reminders, Calendar, Contacts, Drive and Photos, offline and in sync with your iPhone.
Rust apple-notesapple-reminderscloudkitcrdtdesktop-appicloudicloud-calendaricloud-driveicloud-notesicloud-photosiphonelinuxlinux-desktoplocal-firstoffline-firstprivacyrusttauritauri-appwindows
Notes and reminders carry the merge records a phone rebuilds them from
Yashwanth Kamireddi4382f1fe1418
diff --git a/crates/quince-icloud/src/ckdocument.rs b/crates/quince-icloud/src/ckdocument.rsindex d507a34..264be7c 100644--- a/crates/quince-icloud/src/ckdocument.rs+++ b/crates/quince-icloud/src/ckdocument.rs@@ -237,6 +237,9 @@ const ATTRIBUTE_RUN: u64 = 5;/// false. See the comment where they are dropped.const PROVENANCE: u64 = 3;+/// The list of writers the records' ids refer to.+const WRITERS: u64 = 4;+/// A protobuf varint field: `(tag << 3) | 0`, then the value.fn varint_field(tag: u64, value: u64) -> Vec<u8> {let mut out = Vec::with_capacity(8);@@ -272,6 +275,9 @@ fn utf16_len(text: &str) -> u64 {/// order, differing only in the text.fn document_bytes(text: &str) -> Vec<u8> {let mut note = length_delimited(TEXT_PATH[2], text.as_bytes());+ // The records a phone rebuilds the text from. Without them the text is only a cache,+ // and a phone shows a note with nothing in it. See `topotext`.+ note.extend(crate::topotext::Topo::fresh(utf16_len(text)).encode());note.extend(length_delimited(ATTRIBUTE_RUN,&varint_field(1, utf16_len(text)),@@ -508,6 +514,29 @@ fn set_varint_field(message: &[u8], tag: u64, value: u64) -> Option<Vec<u8>> {fn rebuild_note(note: &[u8], old_text: &str, new_text: &str) -> Option<Vec<u8>> {let (from, removed, inserted) = utf16_change(old_text, new_text);+ // The merge records have to say what the text now says, or the phone shows what they+ // say instead: edits made here arrived as text repeated, or lines that would not go.+ let records = match crate::topotext::Topo::read(note) {+ // The usual case: records that agree with the text, and the edit made to them the+ // way a device would make it.+ Some(mut topo) if topo.live_length() == utf16_len(old_text) => {+ topo.edit(from, removed, inserted);+ topo+ }+ // Records that already disagree — a note this app edited before it kept them. No+ // edit can be placed in them truthfully, so what they hold is marked deleted and the+ // text written anew. Other devices' characters keep their ids, so they merge.+ Some(mut topo) => {+ topo.replace_all(utf16_len(new_text));+ topo+ }+ // Records in a shape this does not understand are not ours to rewrite.+ None if has_field(note, PROVENANCE) => return None,+ // None at all: a document written before any of this, or a template stripped of+ // someone else's history. It gets records of its own.+ None => crate::topotext::Topo::fresh(utf16_len(new_text)),+ };+// Collect the existing runs, whole.let mut runs: Vec<Vec<u8>> = Vec::new();let mut reader = Reader::new(note);@@ -562,7 +591,12 @@ fn rebuild_note(note: &[u8], old_text: &str, new_text: &str) -> Option<Vec<u8>>let n = usize::try_from(reader.varint()?).ok()?;reader.take(n)?;out.extend_from_slice(&length_delimited(field, new_text.as_bytes()));+ // The records follow the text, where Apple puts them.+ out.extend(records.encode());wrote_text = true;+ } else if field == PROVENANCE || field == WRITERS {+ // Written afresh above.+ reader.skip(wire)?;} else if field == ATTRIBUTE_RUN && wire == WIRE_LEN {let n = usize::try_from(reader.varint()?).ok()?;reader.take(n)?;@@ -668,21 +702,52 @@ pub fn repair_runs_in_base64(original: &str) -> Option<String> {Some(B64.encode(gzip(&repair_runs(&raw)?)))}-/// Give a document the merge records Apple's notes carry, for tests that need one to copy.-#[doc(hidden)]-#[must_use]-pub fn add_merge_records_for_test(original: &str) -> String {+/// A document as a phone would have written it: its records replaced by ones from a writer+/// that is not this app.+#[cfg(test)]+pub(crate) fn add_merge_records_for_test(original: &str) -> String {let raw = B64.decode(original.trim()).unwrap_or_default();let plain = inflate(&raw).unwrap_or_default();let note = field_at(&plain, &TEXT_PATH[..2]).unwrap_or_default();+ let text = std::str::from_utf8(field_at(note, &TEXT_PATH[2..]).unwrap_or_default())+ .unwrap_or_default();- let mut with = note.to_vec();- with.extend(length_delimited(3, &varint_field(5, 1)));- with.extend(length_delimited(4, &varint_field(1, 1)));+ let mut with = length_delimited(TEXT_PATH[2], text.as_bytes());+ with.extend(crate::topotext::Topo::fresh_by([0x50; 16], utf16_len(text)).encode());+ let mut reader = Reader::new(note);+ while !reader.done() {+ let start = reader.at;+ let Some(key) = reader.varint() else { break };+ if reader.skip(key & 7).is_none() {+ break;+ }+ if !matches!(key >> 3, 2..=4) {+ with.extend_from_slice(¬e[start..reader.at]);+ }+ }let rebuilt = rewrite(&plain, &TEXT_PATH[..2], &with).unwrap_or(plain);B64.encode(gzip(&rebuilt))}+/// How much text a document's records say it holds, when they can be read.+#[doc(hidden)]+#[must_use]+pub fn debug_records(original: &str) -> Option<u64> {+ let raw = B64.decode(original.trim()).ok()?;+ let plain = inflate(&raw)?;+ Some(crate::topotext::Topo::read(field_at(&plain, &TEXT_PATH[..2])?)?.live_length())+}++/// How many writers a document's records name.+#[cfg(test)]+pub(crate) fn debug_writers(original: &str) -> usize {+ let raw = B64.decode(original.trim()).unwrap_or_default();+ let plain = inflate(&raw).unwrap_or_default();+ field_at(&plain, &TEXT_PATH[..2])+ .and_then(crate::topotext::Topo::read)+ .map_or(0, |t| t.writer_clocks().len())+}+/// What a document says about the note it belongs to: whether it names a writer, and whether/// it still carries a record of which stretch of text came from where.#[doc(hidden)]@@ -793,7 +858,13 @@ pub fn is_rewritable(original: &[u8]) -> bool {let Some(current) = field_at(&plain, TEXT_PATH) else {return false;};- rewrite(&plain, TEXT_PATH, current).is_some_and(|rebuilt| rebuilt == plain)+ if !rewrite(&plain, TEXT_PATH, current).is_some_and(|rebuilt| rebuilt == plain) {+ return false;+ }+ // Merge records have to be ones an edit can be made to, the same test a save applies.+ field_at(&plain, &TEXT_PATH[..2]).is_some_and(|note| {+ !has_field(note, PROVENANCE) || crate::topotext::Topo::read(note).is_some()+ })}/// [`replace_text`] over the base64 a CloudKit field carries, gzipped on the way back.@@ -1246,6 +1317,32 @@ mod tests {mod rewrite_tests {use super::*;+ /// The writer id a phone put on the fixture's text.+ const PHONE: [u8; 16] = [0xAB; 16];++ /// The note, with its text, runs and merge records taken out: everything an edit has no+ /// business changing.+ fn untouched_by_an_edit(document: &[u8]) -> Vec<u8> {+ let plain = inflate(document).unwrap();+ let note = field_at(&plain, &TEXT_PATH[..2]).unwrap();+ let mut rest = Vec::new();+ let mut reader = Reader::new(note);+ while !reader.done() {+ let start = reader.at;+ let key = reader.varint().unwrap();+ reader.skip(key & 7).unwrap();+ if !matches!(key >> 3, 2..=5) {+ rest.extend_from_slice(¬e[start..reader.at]);+ }+ }+ rewrite(&plain, &TEXT_PATH[..2], &rest).unwrap()+ }++ fn records(document: &[u8]) -> crate::topotext::Topo {+ let plain = inflate(document).unwrap();+ crate::topotext::Topo::read(field_at(&plain, &TEXT_PATH[..2]).unwrap()).unwrap()+ }+fn varint(tag: u64, value: u64) -> Vec<u8> {let mut out = Vec::new();write_varint(tag << 3, &mut out);@@ -1258,18 +1355,8 @@ mod rewrite_tests {fn document(text: &str) -> Vec<u8> {let mut note = Vec::new();note.extend(length_delimited(2, text.as_bytes()));- // Two of Apple's merge records, which carry no length describing the text.- for index in 1..=2u64 {- let mut run = Vec::new();- run.extend(length_delimited(- 1,- &[varint(1, 1), varint(2, 144_927)].concat(),- ));- run.extend(varint(2, 34_947));- run.extend(varint(5, index));- note.extend(length_delimited(3, &run));- }- note.extend(length_delimited(4, &length_delimited(1, &[0xAB; 28])));+ // Merge records exactly as a phone writes them, by a writer that is not this app.+ note.extend(crate::topotext::Topo::fresh_by(PHONE, utf16_len(text)).encode());// The attribute run that makes the text renderable, carrying styling alongside its// length exactly as Apple's own runs do.note.extend(length_delimited(@@ -1300,11 +1387,12 @@ mod rewrite_tests {text_from_bytes(&after).as_deref(),Some("a longer body than the one it replaced"));- // Everything that is not the text is still there, byte for byte.+ // Everything that is not the text or its records is still there, byte for byte.+ assert_eq!(untouched_by_an_edit(&after), untouched_by_an_edit(&before));+ // And the records describe the new text, not the old.assert_eq!(- replace_text(&after, "the original body").unwrap(),- before,- "putting the old text back must give the original document"+ records(&after).live_length(),+ utf16_len("a longer body than the one it replaced"));}@@ -1464,58 +1552,48 @@ mod rewrite_tests {assert_eq!(utf16_change("", "new"), (0, 0, 3));}- /// The whole point of rewriting in place rather than re-encoding. An edit of the same- /// length must leave the document byte-identical apart from the text itself, which is- /// the strongest statement available that nothing else was touched.+ /// The whole point of rewriting in place rather than re-encoding: an edit touches the+ /// text and the records that describe it, and nothing else.#[test]- fn a_same_length_edit_changes_only_the_text() {+ fn a_same_length_edit_changes_only_the_text_and_its_records() {let before = document("first");let after = replace_text(&before, "FIRST").unwrap();- assert_eq!(before.len(), after.len());-- let differing: Vec<usize> = before- .iter()- .zip(&after)- .enumerate()- .filter_map(|(i, (a, b))| (a != b).then_some(i))- .collect();- assert_eq!(differing.len(), 5, "only the five text bytes moved");+ assert_eq!(untouched_by_an_edit(&after), untouched_by_an_edit(&before));+ assert_eq!(+ field_at(&inflate(&after).unwrap(), TEXT_PATH),+ Some(&b"FIRST"[..])+ );// And the run still covers it, unchanged, because the length did not change.assert_eq!(debug_attribute_lengths(&after), vec![5]);}- /// And when the length does change, the merge records Apple uses to reconcile edits- /// across devices must come through byte for byte. Those are what a phone consults to- /// decide whether our version is newer than its own.++ /// What a phone merges on. The characters it wrote and this edit did not touch keep their+ /// ids, and its entry in the list of writers comes through as it was; what was typed here+ /// is this app's own. Rewriting the records wholesale would make the phone's copy and+ /// this one disagree about every character — which is how text came back doubled.#[test]- fn the_version_vectors_survive_a_save() {- let before = document("first");- let after = replace_text(&before, "second").unwrap();-- // Field 3 of the note is Apple's merge state. Pull it out of both and compare.- let merge_records = |bytes: &[u8]| -> Vec<Vec<u8>> {- let plain = inflate(bytes).unwrap();- let note = field_at(&plain, &TEXT_PATH[..2]).unwrap();- let mut found = Vec::new();- let mut reader = Reader::new(note);- while !reader.done() {- let key = reader.varint().unwrap();- let (field, wire) = (key >> 3, key & 7);- if field == 3 && wire == WIRE_LEN {- let n = usize::try_from(reader.varint().unwrap()).unwrap();- found.push(reader.take(n).unwrap().to_vec());- } else {- reader.skip(wire).unwrap();- }- }- found- };+ fn a_phones_characters_keep_their_ids_through_a_save() {+ let before = document("first draft");+ let after = replace_text(&before, "first final draft").unwrap();+ let (was, now) = (records(&before), records(&after));+ let (was_ids, now_ids) = (was.live_ids(), now.live_ids());+ assert_eq!(now.live_length(), 17);assert_eq!(- merge_records(&after),- merge_records(&before),- "the merge state must survive an edit untouched"+ now_ids[..6],+ was_ids[..6],+ "\"first \" is still the phone's");- assert!(!merge_records(&before).is_empty(), "the fixture has some");+ assert_eq!(+ now_ids[12..],+ was_ids[6..],+ "\"draft\" is still the phone's"+ );+ assert!(+ now_ids[6..12].iter().all(|&(writer, _)| writer == 2),+ "the typed words are this app's"+ );+ assert_eq!(now.writer_clocks()[0], (PHONE.to_vec(), vec![11, 1]));}#[test]@@ -1665,16 +1743,18 @@ mod version_tests {mod merge_record_tests {use super::*;- /// What this app builds from nothing: text, a run, the versions. It reads back here- /// perfectly and a phone shows it as an empty note.+ /// What this app builds from nothing carries records of its own that describe its text.+ /// Without them it read back here perfectly and a phone showed it as an empty note.#[test]- fn a_document_built_from_nothing_has_no_merge_records() {+ fn a_document_built_from_nothing_has_merge_records() {let built = text_to_base64("Buy milk");assert_eq!(text_from_base64(&built).as_deref(), Some("Buy milk"));- assert!(- !has_merge_records(&built),- "which is exactly why a phone showed nothing"- );+ assert!(has_merge_records(&built));+ let raw = B64.decode(&built).unwrap();+ let plain = inflate(&raw).unwrap();+ let topo = crate::topotext::Topo::read(field_at(&plain, &TEXT_PATH[..2]).unwrap())+ .expect("records in Apple's shape");+ assert_eq!(topo.live_length(), 8);}/// The shape Apple writes: records saying which writer produced which stretch of text.@@ -1727,8 +1807,7 @@ mod merge_record_tests {#[test]fn swapping_the_text_keeps_what_makes_it_readable() {let mut note = length_delimited(TEXT_PATH[2], b"Old words");- note.extend(length_delimited(3, &varint_field(5, 1)));- note.extend(length_delimited(4, &varint_field(1, 1)));+ note.extend(crate::topotext::Topo::fresh_by([7; 16], 9).encode());note.extend(length_delimited(ATTRIBUTE_RUN, &varint_field(1, 9)));let mut document = varint_field(1, 0);document.extend(varint_field(2, 0));diff --git a/crates/quince-icloud/src/lib.rs b/crates/quince-icloud/src/lib.rsindex 260074b..76ce7fd 100644--- a/crates/quince-icloud/src/lib.rs+++ b/crates/quince-icloud/src/lib.rs@@ -11,6 +11,7 @@ pub mod drive;pub mod notes;pub mod reminders;pub mod sync;+pub mod topotext;#[derive(Debug, thiserror::Error)]pub enum Error {diff --git a/crates/quince-icloud/src/notes.rs b/crates/quince-icloud/src/notes.rsindex d9c3ef9..8145d6d 100644--- a/crates/quince-icloud/src/notes.rs+++ b/crates/quince-icloud/src/notes.rs@@ -763,35 +763,9 @@ mod write_tests {}}- /// A document shaped like the real ones: text surrounded by Apple's merge state.+ /// A document shaped like the real ones: text with a phone's merge records around it.fn document(text: &str) -> String {- use base64::{engine::general_purpose::STANDARD as B64, Engine};- use std::io::Write;- fn framed(tag: u8, payload: &[u8]) -> Vec<u8> {- let mut out = vec![(tag << 3) | 2];- let mut len = payload.len();- loop {- let mut byte = (len & 0x7F) as u8;- len >>= 7;- if len != 0 {- byte |= 0x80;- }- out.push(byte);- if len == 0 {- break;- }- }- out.extend_from_slice(payload);- out- }- // Version-vector siblings alongside the text, which a save must not disturb.- let mut note = framed(2, text.as_bytes());- note.extend(framed(3, &[0x08, 0x01, 0x10, 0x83, 0x8d, 0x02]));- let proto = framed(2, &framed(3, ¬e));-- let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());- gz.write_all(&proto).unwrap();- B64.encode(gz.finish().unwrap())+ crate::ckdocument::add_merge_records_for_test(&crate::ckdocument::text_to_base64(text))}async fn server(existing: &str) -> MockServer {@@ -851,13 +825,10 @@ mod write_tests {crate::ckdocument::text_from_base64(written).as_deref(),Some("Curtains\nmeasure the bay window\norder the rail"));- // The version vector alongside the text has to have come through the save.- let raw = crate::ckdocument::debug_inflate(written).unwrap();- assert!(- crate::ckdocument::debug_tree(&raw, 6)- .iter()- .any(|line| line.contains("3: message")),- "Apple's merge state must survive an edit"+ // The records a phone merges on came through the save, describing the new text.+ assert_eq!(+ crate::ckdocument::debug_records(written),+ Some(utf16("Curtains\nmeasure the bay window\norder the rail")));}@@ -1061,51 +1032,62 @@ mod folder_shape_tests {}}+#[cfg(test)]+fn utf16(text: &str) -> u64 {+ text.encode_utf16().count() as u64+}+/// Shaping a new note like one Apple wrote.#[cfg(test)]mod template_tests {use super::*;- /// A document with the merge records a phone needs, to stand in for one Apple wrote.+ /// A document with a phone's merge records, to stand in for one Apple wrote.fn apples_shape(text: &str) -> String {- let built = crate::ckdocument::text_to_base64(text);- // Built documents have no merge records, so this checks the fixture is honest about- // what it is rather than quietly testing nothing.- assert!(!crate::ckdocument::has_merge_records(&built));- crate::ckdocument::add_merge_records_for_test(&built)+ crate::ckdocument::add_merge_records_for_test(&crate::ckdocument::text_to_base64(text))}- /// A new note keeps the writer from the note it was shaped by, and none of its history.+ /// A new note's records are its own: none of the history of the note it was shaped by,+ /// and the text it holds written by this app.////// The history is lengths measured against text that is not in this note. Carried over, a/// fifteen-character note came out on a phone as the same word four times with gaps in- /// between — worse than the blank it was meant to fix.+ /// between. Carrying none at all left it blank.#[test]- fn a_new_note_takes_the_writer_but_not_the_other_notes_history() {+ fn a_new_note_has_records_of_its_own() {let template = apples_shape("Something Apple wrote");- assert_eq!(- crate::ckdocument::debug_note_fields(&template),- (true, true),- "the fixture has both, or this proves nothing"- );-let made = document_like_apples(Some(&template), "Something written here");assert_eq!(crate::ckdocument::text_from_base64(&made).as_deref(),Some("Something written here"));- let (writer, history) = crate::ckdocument::debug_note_fields(&made);- assert!(writer, "the writer travels");- assert!(!history, "the other note's history does not");+ assert_eq!(+ crate::ckdocument::debug_records(&made),+ Some(utf16("Something written here")),+ "records that describe exactly this text"+ );+ assert_eq!(+ crate::ckdocument::debug_writers(&made),+ 1,+ "this app alone wrote it; the template's writer did not"+ );}- /// A note's own history is its own, and an edit leaves it alone.+ /// A note's own history is its own, and an edit adds to it rather than replacing it.#[test]fn editing_a_note_keeps_the_notes_own_history() {let mine = apples_shape("What I wrote before");let edited =crate::ckdocument::replace_text_in_base64(&mine, "What I wrote after").unwrap();- assert_eq!(crate::ckdocument::debug_note_fields(&edited), (true, true));+ assert_eq!(+ crate::ckdocument::debug_records(&edited),+ Some(utf16("What I wrote after"))+ );+ assert_eq!(+ crate::ckdocument::debug_writers(&edited),+ 2,+ "the phone, then this app"+ );}/// A new account has no note to learn from. Building from nothing is the only option, anddiff --git a/crates/quince-icloud/src/topotext.rs b/crates/quince-icloud/src/topotext.rsnew file mode 100644index 0000000..e16f169--- /dev/null+++ b/crates/quince-icloud/src/topotext.rs@@ -0,0 +1,607 @@+//! The merge records inside an Apple Notes or Reminders document.+//!+//! The text a document carries is a cache. What Apple's devices actually merge is a list of+//! substrings, each naming who wrote a stretch of characters and whether it has since been+//! deleted. A phone reading a document rebuilds the note from those records, so text with no+//! records behind it shows as an empty note, and text whose records describe something else+//! shows as whatever the records say — a word repeated, a line that will not go away.+//!+//! Measured on real documents written by a phone, every one of them the same:+//!+//! - The substrings sit in the note at field 3, in reading order, each pointing at the next+//! by index (field 5). The first is a start marker with id (0, 0); the last is an end marker+//! with id (0, 0xFFFFFFFF). Neither holds text.+//! - A substring is `{1: id, 2: length, 3: timestamp, 4: deleted, 5: next}`, where an id and+//! a timestamp are `{1: writer, 2: clock}`. Its characters are numbered clock, clock+1, …+//! - Lengths are UTF-16 units, and the lengths of the ones not deleted add up to the text.+//! - The writers are listed at field 4 as `{1: {1: uuid, 2: {1: next character},+//! 2: {1: next timestamp}}, …}`, and an id's writer is its 1-based place in that list.+//!+//! An edit here is made the way a device makes one: what was removed is marked deleted, never+//! dropped, and what was typed is a new substring under this app's own writer. Characters+//! other devices wrote keep their ids, so their copies of the note line up with this one.++use std::sync::OnceLock;++/// The id of the end marker's characters.+const END: u64 = 0xFFFF_FFFF;++/// This app's writer id: one per installation, so two computers never number characters+/// from the same writer. Until one is set, a fresh id per run is still correct — only less+/// tidy, since each run that edits a note adds itself to that note's list of writers.+static WRITER: OnceLock<[u8; 16]> = OnceLock::new();++/// Adopt this installation's writer id. Only the first call counts.+pub fn set_writer(id: [u8; 16]) {+ let _ = WRITER.set(id);+}++fn writer() -> [u8; 16] {+ *WRITER.get_or_init(|| *uuid::Uuid::new_v4().as_bytes())+}++#[derive(Debug, Clone, PartialEq, Eq)]+struct Substring {+ writer: u64,+ clock: u64,+ length: u64,+ stamp_writer: u64,+ stamp_clock: u64,+ deleted: bool,+}++#[derive(Debug, Clone, PartialEq, Eq)]+struct Writer {+ uuid: Vec<u8>,+ /// Next character clock, then next timestamp clock. Kept as a list because that is what+ /// the field is, and a writer entry with some other count must survive untouched.+ clocks: Vec<u64>,+}++/// The records of one note, in the order they are read.+#[derive(Debug, Clone, PartialEq, Eq)]+pub(crate) struct Topo {+ substrings: Vec<Substring>,+ writers: Vec<Writer>,+}++// ── Reading ────────────────────────────────────────────────────────────────────────────++/// One field: its number, its wire type, the payload of a length-delimited one, and the+/// value of a varint.+type Field<'a> = (u64, u64, &'a [u8], u64);++fn fields(bytes: &[u8]) -> Option<Vec<Field<'_>>> {+ let mut out = Vec::new();+ let mut at = 0usize;+ while at < bytes.len() {+ let (key, next) = varint(bytes, at)?;+ at = next;+ let (field, wire) = (key >> 3, key & 7);+ match wire {+ 0 => {+ let (v, next) = varint(bytes, at)?;+ at = next;+ out.push((field, wire, &bytes[0..0], v));+ }+ 2 => {+ let (n, next) = varint(bytes, at)?;+ let end = next.checked_add(usize::try_from(n).ok()?)?;+ out.push((field, wire, bytes.get(next..end)?, 0));+ at = end;+ }+ 1 => at = at.checked_add(8).filter(|&e| e <= bytes.len())?,+ 5 => at = at.checked_add(4).filter(|&e| e <= bytes.len())?,+ _ => return None,+ }+ }+ Some(out)+}++fn varint(bytes: &[u8], mut at: usize) -> Option<(u64, usize)> {+ let mut value = 0u64;+ for shift in 0..10 {+ let byte = *bytes.get(at)?;+ at += 1;+ value |= u64::from(byte & 0x7F) << (shift * 7);+ if byte & 0x80 == 0 {+ return Some((value, at));+ }+ }+ None+}++fn id(bytes: &[u8]) -> Option<(u64, u64)> {+ let mut id = (0, 0);+ for (field, wire, _, v) in fields(bytes)? {+ match (field, wire) {+ (1, 0) => id.0 = v,+ (2, 0) => id.1 = v,+ _ => {}+ }+ }+ Some(id)+}++fn substring(bytes: &[u8]) -> Option<(Substring, Vec<u64>)> {+ let mut s = Substring {+ writer: 0,+ clock: 0,+ length: 0,+ stamp_writer: 0,+ stamp_clock: 0,+ deleted: false,+ };+ let mut next = Vec::new();+ for (field, wire, payload, v) in fields(bytes)? {+ match (field, wire) {+ (1, 2) => (s.writer, s.clock) = id(payload)?,+ (2, 0) => s.length = v,+ (3, 2) => (s.stamp_writer, s.stamp_clock) = id(payload)?,+ (4, 0) => s.deleted = v != 0,+ (5, 0) => next.push(v),+ // A field this does not know means a document this does not understand.+ _ => return None,+ }+ }+ Some((s, next))+}++fn writers(bytes: &[u8]) -> Option<Vec<Writer>> {+ let mut out = Vec::new();+ for (field, wire, payload, _) in fields(bytes)? {+ if (field, wire) != (1, 2) {+ return None;+ }+ let mut w = Writer {+ uuid: Vec::new(),+ clocks: Vec::new(),+ };+ for (field, wire, payload, _) in fields(payload)? {+ match (field, wire) {+ (1, 2) => w.uuid = payload.to_vec(),+ (2, 2) => {+ let mut clock = None;+ for (field, wire, _, v) in fields(payload)? {+ match (field, wire) {+ (1, 0) => clock = Some(v),+ _ => return None,+ }+ }+ w.clocks.push(clock.unwrap_or(0));+ }+ _ => return None,+ }+ }+ out.push(w);+ }+ Some(out)+}++impl Topo {+ /// The records of a note message, or `None` when it has none or they are not shaped the+ /// way Apple writes them. A graph that is not a single chain in reading order is refused+ /// rather than guessed at.+ pub(crate) fn read(note: &[u8]) -> Option<Self> {+ let mut substrings = Vec::new();+ let mut links = Vec::new();+ let mut writer_list = None;+ for (field, wire, payload, _) in fields(note)? {+ match (field, wire) {+ (3, 2) => {+ let (s, next) = substring(payload)?;+ substrings.push(s);+ links.push(next);+ }+ (4, 2) => writer_list = Some(writers(payload)?),+ _ => {}+ }+ }+ if substrings.len() < 2 {+ return None;+ }+ let last = links.len() - 1;+ for (i, next) in links.iter().enumerate() {+ let expected: Vec<u64> = if i == last {+ vec![]+ } else {+ vec![i as u64 + 1]+ };+ if *next != expected {+ return None;+ }+ }+ let first = &substrings[0];+ let end = &substrings[last];+ if (first.writer, first.clock, first.length) != (0, 0, 0)+ || (end.writer, end.clock, end.length) != (0, END, 0)+ {+ return None;+ }+ Some(Self {+ substrings,+ writers: writer_list.unwrap_or_default(),+ })+ }++ /// Records for a note that holds `length` units of text, all written by this app.+ pub(crate) fn fresh(length: u64) -> Self {+ let mut topo = Self {+ substrings: vec![+ Substring {+ writer: 0,+ clock: 0,+ length: 0,+ stamp_writer: 0,+ stamp_clock: 0,+ deleted: false,+ },+ Substring {+ writer: 0,+ clock: END,+ length: 0,+ stamp_writer: 0,+ stamp_clock: END,+ deleted: false,+ },+ ],+ writers: Vec::new(),+ };+ topo.insert(0, length);+ topo+ }++ /// How much text the records say is there.+ pub(crate) fn live_length(&self) -> u64 {+ self.substrings+ .iter()+ .filter(|s| !s.deleted)+ .map(|s| s.length)+ .sum()+ }++ // ── Editing ────────────────────────────────────────────────────────────────────────++ /// Make `at` fall on a boundary between substrings, splitting the one it lands inside.+ fn split_at(&mut self, at: u64) {+ let mut offset = 0;+ for i in 0..self.substrings.len() {+ let s = &self.substrings[i];+ if s.deleted {+ continue;+ }+ if offset < at && at < offset + s.length {+ let keep = at - offset;+ let mut tail = s.clone();+ tail.clock += keep;+ tail.length -= keep;+ self.substrings[i].length = keep;+ self.substrings.insert(i + 1, tail);+ return;+ }+ offset += s.length;+ }+ }++ /// This app's place in the list of writers, adding it if it is not there yet.+ fn own_writer(&mut self) -> usize {+ let me = writer();+ if let Some(i) = self.writers.iter().position(|w| w.uuid == me) {+ while self.writers[i].clocks.len() < 2 {+ self.writers[i].clocks.push(0);+ }+ return i;+ }+ self.writers.push(Writer {+ uuid: me.to_vec(),+ clocks: vec![0, 0],+ });+ self.writers.len() - 1+ }++ /// New text of `length` units at `at`, as a substring of this app's own.+ fn insert(&mut self, at: u64, length: u64) {+ if length == 0 {+ return;+ }+ self.split_at(at);+ let me = self.own_writer();+ let clock = self.writers[me].clocks[0];+ let stamp = self.writers[me].clocks[1];++ // Right after the last live character before `at`; the start marker when `at` is 0.+ let mut offset = 0;+ let mut place = 1;+ for (i, s) in self.substrings.iter().enumerate().skip(1) {+ if offset == at {+ place = i;+ break;+ }+ if !s.deleted {+ offset += s.length;+ }+ place = i + 1;+ }+ let place = place.min(self.substrings.len() - 1);++ self.substrings.insert(+ place,+ Substring {+ writer: me as u64 + 1,+ clock,+ length,+ stamp_writer: me as u64 + 1,+ stamp_clock: stamp,+ deleted: false,+ },+ );+ self.writers[me].clocks[0] = clock + length;+ self.writers[me].clocks[1] = stamp + 1;+ }++ /// Mark `length` units from `at` deleted. The characters keep their ids, which is what+ /// lets another device's copy agree that they are gone.+ fn delete(&mut self, at: u64, length: u64) {+ if length == 0 {+ return;+ }+ self.split_at(at);+ self.split_at(at + length);+ let mut offset = 0;+ for s in &mut self.substrings {+ if s.deleted {+ continue;+ }+ if offset >= at && offset + s.length <= at + length && s.length > 0 {+ s.deleted = true;+ }+ offset += s.length;+ }+ }++ /// Apply one edit: `removed` units from `from` replaced by `inserted` new ones.+ pub(crate) fn edit(&mut self, from: u64, removed: u64, inserted: u64) {+ self.delete(from, removed);+ self.insert(from, inserted);+ }++ /// Everything the records say is there replaced by `length` new units. For a document+ /// whose records already disagree with its text, where no edit can be placed truthfully.+ pub(crate) fn replace_all(&mut self, length: u64) {+ let live = self.live_length();+ self.edit(0, live, length);+ }++ // ── Writing ────────────────────────────────────────────────────────────────────────++ /// The note fields these records occupy: every substring at 3, then the writers at 4.+ pub(crate) fn encode(&self) -> Vec<u8> {+ let mut out = Vec::new();+ let last = self.substrings.len() - 1;+ for (i, s) in self.substrings.iter().enumerate() {+ let mut m = message(1, &pair(s.writer, s.clock));+ m.extend(varint_field(2, s.length));+ m.extend(message(3, &pair(s.stamp_writer, s.stamp_clock)));+ if s.deleted {+ m.extend(varint_field(4, 1));+ }+ if i != last {+ m.extend(varint_field(5, i as u64 + 1));+ }+ out.extend(message(3, &m));+ }+ let mut list = Vec::new();+ for w in &self.writers {+ let mut entry = message(1, &w.uuid);+ for clock in &w.clocks {+ entry.extend(message(2, &varint_field(1, *clock)));+ }+ list.extend(message(1, &entry));+ }+ out.extend(message(4, &list));+ out+ }+}++fn write_varint(mut value: u64, out: &mut Vec<u8>) {+ loop {+ let mut byte = (value & 0x7F) as u8;+ value >>= 7;+ if value != 0 {+ byte |= 0x80;+ }+ out.push(byte);+ if value == 0 {+ return;+ }+ }+}++fn varint_field(tag: u64, value: u64) -> Vec<u8> {+ let mut out = Vec::new();+ write_varint(tag << 3, &mut out);+ write_varint(value, &mut out);+ out+}++fn message(tag: u64, payload: &[u8]) -> Vec<u8> {+ let mut out = Vec::new();+ write_varint((tag << 3) | 2, &mut out);+ write_varint(payload.len() as u64, &mut out);+ out.extend_from_slice(payload);+ out+}++/// An id or timestamp: both halves written even when zero, as Apple writes them.+fn pair(writer: u64, clock: u64) -> Vec<u8> {+ let mut out = varint_field(1, writer);+ out.extend(varint_field(2, clock));+ out+}++#[cfg(test)]+impl Topo {+ /// Records as another device would have written them for `length` units.+ pub(crate) fn fresh_by(uuid: [u8; 16], length: u64) -> Self {+ let mut t = Self::fresh(0);+ t.writers.push(Writer {+ uuid: uuid.to_vec(),+ clocks: vec![length, 1],+ });+ t.substrings.insert(+ 1,+ Substring {+ writer: 1,+ clock: 0,+ length,+ stamp_writer: 1,+ stamp_clock: 0,+ deleted: false,+ },+ );+ t+ }++ /// Every writer, with its clocks.+ pub(crate) fn writer_clocks(&self) -> Vec<(Vec<u8>, Vec<u64>)> {+ self.writers+ .iter()+ .map(|w| (w.uuid.clone(), w.clocks.clone()))+ .collect()+ }++ /// Which writer wrote each live unit, in reading order — what a phone rebuilds from.+ pub(crate) fn live_ids(&self) -> Vec<(u64, u64)> {+ self.substrings+ .iter()+ .filter(|s| !s.deleted)+ .flat_map(|s| (0..s.length).map(move |k| (s.writer, s.clock + k)))+ .collect()+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn roundtrip(t: &Topo) -> Topo {+ let mut note = message(2, b"text");+ note.extend(t.encode());+ Topo::read(¬e).expect("our own records read back")+ }++ /// The exact bytes a phone wrote for a reminder titled "Charge the laptop 🔋", except for+ /// the writer id, which is the phone's own.+ #[test]+ fn a_fresh_document_is_shaped_like_the_phones() {+ set_writer([0x67; 16]);+ let t = Topo::fresh(20);+ let s = &t.substrings;+ assert_eq!(s.len(), 3);+ assert_eq!((s[0].writer, s[0].clock, s[0].length), (0, 0, 0));+ assert_eq!((s[1].writer, s[1].clock, s[1].length), (1, 0, 20));+ assert_eq!((s[1].stamp_writer, s[1].stamp_clock), (1, 0));+ assert_eq!((s[2].writer, s[2].clock, s[2].stamp_clock), (0, END, END));+ assert_eq!(t.writers[0].clocks, vec![20, 1]);+ assert_eq!(roundtrip(&t), t);+ }++ #[test]+ fn typing_in_the_middle_keeps_everyone_elses_characters() {+ let mut t = Topo::fresh(10);+ let before = t.live_ids();+ t.edit(4, 0, 3);+ let after = t.live_ids();+ assert_eq!(t.live_length(), 13);+ assert_eq!(&after[..4], &before[..4]);+ assert_eq!(&after[7..], &before[4..]);+ assert_eq!(roundtrip(&t), t);+ }++ #[test]+ fn deleting_marks_characters_rather_than_dropping_them() {+ let mut t = Topo::fresh(10);+ let all: u64 = t.substrings.iter().map(|s| s.length).sum();+ t.edit(2, 5, 0);+ assert_eq!(t.live_length(), 5);+ // Nothing vanished: the deleted stretch is still there, marked.+ assert_eq!(t.substrings.iter().map(|s| s.length).sum::<u64>(), all);+ assert_eq!(roundtrip(&t), t);+ }++ #[test]+ fn replacing_a_word_is_a_delete_and_an_insert() {+ let mut t = Topo::fresh(11); // "hello world"+ t.edit(6, 5, 5); // "hello there"+ assert_eq!(t.live_length(), 11);+ let ids = t.live_ids();+ // The kept prefix is the original characters; the new word is new ones.+ assert_eq!(ids[..6], Topo::fresh(11).live_ids()[..6]);+ assert!(ids[6..].iter().all(|&(_, c)| c >= 11));+ }++ #[test]+ fn many_edits_keep_the_records_consistent() {+ let mut t = Topo::fresh(0);+ let mut len = 0u64;+ // A deterministic scatter of inserts and deletes all over the text.+ for step in 0..200u64 {+ let at = (step * 7919) % (len + 1);+ if step % 3 == 0 && len > 0 {+ let n = (step % 5 + 1).min(len - at.min(len));+ t.edit(at.min(len), n, 0);+ len -= n;+ } else {+ t.edit(at, 0, step % 4 + 1);+ len += step % 4 + 1;+ }+ assert_eq!(t.live_length(), len, "step {step}");+ }+ let ids = t.live_ids();+ let unique: std::collections::HashSet<_> = ids.iter().collect();+ assert_eq!(unique.len(), ids.len(), "no character id is used twice");+ assert_eq!(roundtrip(&t), t);+ }++ #[test]+ fn another_writers_entry_is_left_as_it_was() {+ let mut t = Topo::fresh(5);+ t.writers.insert(+ 0,+ Writer {+ uuid: vec![9; 16],+ clocks: vec![74800, 31],+ },+ );+ // Our own substring now refers to the second writer.+ t.substrings[1].writer = 2;+ t.substrings[1].stamp_writer = 2;+ t.edit(5, 0, 2);+ assert_eq!(t.writers[0].clocks, vec![74800, 31]);+ assert_eq!(t.writers.len(), 2);+ assert_eq!(t.substrings[2].writer, 2);+ }++ #[test]+ fn a_graph_that_is_not_one_chain_is_refused() {+ let t = Topo::fresh(3);+ let mut bytes = message(2, b"abc");+ bytes.extend(t.encode());+ // Point the start marker somewhere other than the next record.+ let mut forked = message(2, b"abc");+ let mut s0 = message(1, &pair(0, 0));+ s0.extend(varint_field(2, 0));+ s0.extend(message(3, &pair(0, 0)));+ s0.extend(varint_field(5, 2));+ forked.extend(message(3, &s0));+ let rest = Topo {+ substrings: t.substrings[1..].to_vec(),+ writers: t.writers.clone(),+ };+ forked.extend(rest.encode());+ assert!(Topo::read(&bytes).is_some());+ assert!(Topo::read(&forked).is_none());+ assert!(Topo::read(&[0xFF]).is_none());+ assert!(Topo::read(&message(2, b"no records")).is_none());+ }+}diff --git a/probe/examples/edit_loop.rs b/probe/examples/edit_loop.rsindex 3f387c8..b38efac 100644--- a/probe/examples/edit_loop.rs+++ b/probe/examples/edit_loop.rs@@ -33,7 +33,7 @@ async fn main() -> Result<()> {&root,params.clone(),Some("DefaultFolder-CloudKit"),- "edit test",+ "Quince test",template.as_deref(),).await?;@@ -41,14 +41,16 @@ async fn main() -> Result<()> {// What someone typing actually produces: a growing body, saved after each pause.let steps = [- "edit test\n1",- "edit test\n1\n2",- "edit test\n1\n2\n3",- "edit test\n1\n2\n3\n4",- "edit test\n1\n2\n3\n4\n56",+ "Quince test\n1",+ "Quince test\n1\n2",+ "Quince test\n1\n2\n3",+ "Quince test\n1\n2\n3\n4",+ "Quince test\n1\n2\n3\n4\n56",// And a deletion, which is where an append-only mistake shows up loudest.- "edit test\n1\n2",- "short",+ "Quince test\n1\n2",+ // A word changed in the middle, then an emoji, which counts as two units.+ "Quince test\nfirst\n2",+ "Quince test\nfirst line\nsecond line 🔴",];let mut failures = 0;@@ -73,20 +75,26 @@ async fn main() -> Result<()> {};let units: u64 = want.encode_utf16().count() as u64;let covered: u64 = runs.iter().sum();- let ok = got == *want && covered == units;+ // What a phone rebuilds the note from has to say the same.+ let records = quince_icloud::ckdocument::debug_records(doc);+ let ok = got == *want && covered == units && records == Some(units);if !ok {failures += 1;}println!(- " {} step {n}: {:?} runs cover {covered} of {units}",+ " {} step {n}: {:?} runs cover {covered} of {units}, records {records:?}",if ok { "PASS" } else { "FAIL" },got.chars().take(28).collect::<String>());}- let note = library_note(&http, &root, ¶ms, &guid).await?;- quince_icloud::notes::delete_cloudkit(&http, &root, params.clone(), ¬e).await?;- println!("\ncleaned up");+ if std::env::var("KEEP").is_ok() {+ println!("\nleft on the account to look at on a phone: {guid}");+ } else {+ let note = library_note(&http, &root, ¶ms, &guid).await?;+ quince_icloud::notes::delete_cloudkit(&http, &root, params.clone(), ¬e).await?;+ println!("\ncleaned up");+ }if failures > 0 {return Err(anyhow!("{failures} step(s) did not hold"));}diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rsindex e32378d..f5ffc69 100644--- a/src-tauri/src/main.rs+++ b/src-tauri/src/main.rs@@ -1343,6 +1343,29 @@ async fn contact_photo(})}+/// Where this installation's writer id is kept.+const WRITER_KEY: &str = "documents.writer";++/// Give note and reminder edits this installation's own writer id.+///+/// Apple's devices merge a note by who wrote which characters, so every edit made here is+/// signed. The same id every run keeps a note's list of writers from growing by one each time+/// the app starts; a different id on every computer keeps two of them from numbering+/// characters the same way. If it cannot be stored, a fresh one for this run is still correct.+fn adopt_writer(library: &mut Library) {+ let stored = library+ .state(WRITER_KEY)+ .ok()+ .flatten()+ .and_then(|s| uuid::Uuid::parse_str(&s).ok());+ let id = stored.unwrap_or_else(|| {+ let fresh = uuid::Uuid::new_v4();+ let _ = library.set_state(WRITER_KEY, &fresh.to_string());+ fresh+ });+ quince_icloud::topotext::set_writer(*id.as_bytes());+}+fn main() {tauri::Builder::default().plugin(tauri_plugin_dialog::init())@@ -1408,7 +1431,9 @@ fn main() {});// The local library. Opened once at startup so every command shares one// connection and the WAL behaves.- app.manage(Data(Mutex::new(Library::open(&dir)?)));+ let mut library = Library::open(&dir)?;+ adopt_writer(&mut library);+ app.manage(Data(Mutex::new(library)));spawn_heartbeat(Arc::clone(&client));spawn_auto_sync(app.handle().clone(), client);Ok(())