Native iCloud apps for Windows and Linux, built in Rust. Notes, Reminders, Calendar, Contacts, Drive and Photos, offline and in sync with your iPhone.
Rust apple-notesapple-reminderscloudkitcrdtdesktop-appicloudicloud-calendaricloud-driveicloud-notesicloud-photosiphonelinuxlinux-desktoplocal-firstoffline-firstprivacyrusttauritauri-appwindows
Lists and groups appear in the phone's order, read from the account record
Yashwanth Kamireddi9a082577de2d
diff --git a/crates/quince-icloud/src/lib.rs b/crates/quince-icloud/src/lib.rsindex 76ce7fd..2ae6df9 100644--- a/crates/quince-icloud/src/lib.rs+++ b/crates/quince-icloud/src/lib.rs@@ -8,6 +8,7 @@ pub mod ckdocument;pub mod cloudkit;pub mod contacts;pub mod drive;+pub mod mergeable;pub mod notes;pub mod reminders;pub mod sync;diff --git a/crates/quince-icloud/src/mergeable.rs b/crates/quince-icloud/src/mergeable.rsnew file mode 100644index 0000000..81c8711--- /dev/null+++ b/crates/quince-icloud/src/mergeable.rs@@ -0,0 +1,291 @@+//! Apple's "mergeable data": the CRDT object graph behind Notes tables and Reminders' list+//! order.+//!+//! Measured on the `ListIDsMergeableOrdering` field of a real Reminders `Account` record. The+//! top-level message holds:+//!+//! - `3`: the objects, each one message. Object 0 is the root.+//! - `4`: key names (`self`, `UUIDIndex`), `5`: type names (`com.apple.CRDT.NSString`,+//! `com.apple.CRDT.NSUUID`, …), `6`: 16-byte UUIDs. Objects refer to all three by index.+//!+//! An object that is a plain value is a map at field 13: `{1: type index, 3: {1: key index,+//! 2: value}}`, where the value carries a string at 4 or an index at 2 — for an `NSUUID`, the+//! index of its UUID.+//!+//! The root is an ordered set at field 16: `{1: ordering, 2: elements}`. The ordering is+//! `{1: array, 2: contents}`, where the array is a text CRDT whose every character is an+//! attachment — `{2: {1: position, 2: uuid}}` in reading order, one per character still+//! there — and the contents are a dictionary `{1: {1: {6: key object}, 2: {6: value+//! object}, 3: timestamp}}` from each attachment's UUID to what sits at that position.+//!+//! So reading the order is: attachments in order, each UUID looked up in the contents, each+//! answer resolved to its string.++use std::collections::HashMap;++type Field<'a> = (u64, u64, &'a [u8], u64);++fn varint(bytes: &[u8], mut at: usize) -> Option<(u64, usize)> {+ let mut value = 0u64;+ for shift in 0..10 {+ let byte = *bytes.get(at)?;+ at += 1;+ value |= u64::from(byte & 0x7F) << (shift * 7);+ if byte & 0x80 == 0 {+ return Some((value, at));+ }+ }+ None+}++fn fields(bytes: &[u8]) -> Option<Vec<Field<'_>>> {+ let mut out = Vec::new();+ let mut at = 0usize;+ while at < bytes.len() {+ let (key, next) = varint(bytes, at)?;+ at = next;+ let (field, wire) = (key >> 3, key & 7);+ match wire {+ 0 => {+ let (v, next) = varint(bytes, at)?;+ at = next;+ out.push((field, wire, &bytes[0..0], v));+ }+ 2 => {+ let (n, next) = varint(bytes, at)?;+ let end = next.checked_add(usize::try_from(n).ok()?)?;+ out.push((field, wire, bytes.get(next..end)?, 0));+ at = end;+ }+ 1 => at = at.checked_add(8).filter(|&e| e <= bytes.len())?,+ 5 => at = at.checked_add(4).filter(|&e| e <= bytes.len())?,+ _ => return None,+ }+ }+ Some(out)+}++fn message<'a>(fields: &[Field<'a>], tag: u64) -> Option<&'a [u8]> {+ fields.iter().find(|f| f.0 == tag && f.1 == 2).map(|f| f.2)+}++fn messages<'a>(fields: &[Field<'a>], tag: u64) -> Vec<&'a [u8]> {+ fields+ .iter()+ .filter(|f| f.0 == tag && f.1 == 2)+ .map(|f| f.2)+ .collect()+}++fn number(fields: &[Field<'_>], tag: u64) -> Option<u64> {+ fields.iter().find(|f| f.0 == tag && f.1 == 0).map(|f| f.3)+}++/// A plain value an object can hold.+#[derive(Debug, Clone, PartialEq, Eq)]+enum Value {+ Text(String),+ Uuid(Vec<u8>),+}++struct Graph<'a> {+ objects: Vec<&'a [u8]>,+ types: Vec<String>,+ uuids: Vec<&'a [u8]>,+}++impl<'a> Graph<'a> {+ fn read(bytes: &'a [u8]) -> Option<Self> {+ let top = fields(bytes)?;+ Some(Self {+ objects: messages(&top, 3),+ types: messages(&top, 5)+ .into_iter()+ .map(|t| String::from_utf8_lossy(t).into_owned())+ .collect(),+ uuids: messages(&top, 6),+ })+ }++ /// The value held by the object at `index`, when it is a string or a UUID.+ fn value(&self, index: u64) -> Option<Value> {+ let object = fields(self.objects.get(usize::try_from(index).ok()?)?)?;+ let map = fields(message(&object, 13)?)?;+ let kind = self.types.get(usize::try_from(number(&map, 1)?).ok()?)?;+ let entry = fields(message(&map, 3)?)?;+ let held = fields(message(&entry, 2)?)?;+ if kind.ends_with("NSString") {+ let text = message(&held, 4)?;+ return Some(Value::Text(String::from_utf8(text.to_vec()).ok()?));+ }+ if kind.ends_with("NSUUID") {+ let at = usize::try_from(number(&held, 2)?).ok()?;+ return Some(Value::Uuid(self.uuids.get(at)?.to_vec()));+ }+ None+ }++ fn object_ref(reference: &[u8]) -> Option<u64> {+ number(&fields(reference)?, 6)+ }+}++/// The strings an ordered set holds, in order. `None` when the bytes are not an ordered set+/// shaped the way Apple writes one; a caller then keeps whatever order it had.+#[must_use]+pub fn ordered_strings(bytes: &[u8]) -> Option<Vec<String>> {+ let graph = Graph::read(bytes)?;+ let root = fields(graph.objects.first()?)?;+ let set = fields(message(&root, 16)?)?;+ let ordering = fields(message(&set, 1)?)?;+ let array = fields(message(&ordering, 1)?)?;++ // What each position's UUID stands for. Written in the order the edits happened, so a+ // later entry for the same UUID is the newer one.+ let mut contents: HashMap<Vec<u8>, String> = HashMap::new();+ if let Some(dictionary) = message(&ordering, 2) {+ for element in messages(&fields(dictionary)?, 1) {+ let element = fields(element)?;+ let key = graph.value(Graph::object_ref(message(&element, 1)?)?);+ let value = graph.value(Graph::object_ref(message(&element, 2)?)?);+ if let (Some(Value::Uuid(key)), Some(Value::Text(value))) = (key, value) {+ contents.insert(key, value);+ }+ }+ }++ let mut positions: Vec<(u64, Vec<u8>)> = Vec::new();+ for attachment in messages(&array, 2) {+ let attachment = fields(attachment)?;+ let uuid = message(&attachment, 2)?;+ positions.push((number(&attachment, 1).unwrap_or(0), uuid.to_vec()));+ }+ positions.sort_by_key(|(position, _)| *position);++ let mut out = Vec::with_capacity(positions.len());+ let mut seen = std::collections::HashSet::new();+ for (_, uuid) in positions {+ // A position whose contents are unknown is skipped, not guessed at; one that names+ // an entry already placed keeps its first place.+ if let Some(id) = contents.get(&uuid) {+ if seen.insert(id.clone()) {+ out.push(id.clone());+ }+ }+ }+ Some(out)+}++#[cfg(test)]+pub(crate) mod build {+ //! An encoder for the same shape, so tests can make one without real account bytes.++ pub(crate) fn varint(mut value: u64, out: &mut Vec<u8>) {+ loop {+ let mut byte = (value & 0x7F) as u8;+ value >>= 7;+ if value != 0 {+ byte |= 0x80;+ }+ out.push(byte);+ if value == 0 {+ return;+ }+ }+ }+ pub(crate) fn num(tag: u64, value: u64) -> Vec<u8> {+ let mut out = Vec::new();+ varint(tag << 3, &mut out);+ varint(value, &mut out);+ out+ }+ pub(crate) fn msg(tag: u64, payload: &[u8]) -> Vec<u8> {+ let mut out = Vec::new();+ varint((tag << 3) | 2, &mut out);+ varint(payload.len() as u64, &mut out);+ out.extend_from_slice(payload);+ out+ }++ /// An ordered set holding `ids` in this order, with positions listed in `shuffle` order+ /// on the wire and a stale contents entry ahead of each real one.+ pub(crate) fn ordered_set(ids: &[&str], wire_order: &[usize]) -> Vec<u8> {+ const STRING: u64 = 0;+ const UUID: u64 = 1;+ let uuid = |i: usize| vec![u8::try_from(i).unwrap(); 16];+ let mut objects: Vec<Vec<u8>> = vec![Vec::new()]; // the root, filled in last+ let mut uuids: Vec<Vec<u8>> = Vec::new();+ let value_object = |kind: u64, value: Vec<u8>| {+ let mut entry = num(1, 0);+ entry.extend(msg(2, &value));+ let mut map = num(1, kind);+ map.extend(msg(3, &entry));+ msg(13, &map)+ };+ let mut dictionary = Vec::new();+ let mut attachments = Vec::new();+ for (i, id) in ids.iter().enumerate() {+ uuids.push(uuid(i));+ objects.push(value_object(UUID, num(2, i as u64)));+ let key = objects.len() - 1;+ // An old answer for the same position, which the later one replaces.+ objects.push(value_object(STRING, msg(4, b"STALE")));+ let stale = objects.len() - 1;+ objects.push(value_object(STRING, msg(4, id.as_bytes())));+ let value = objects.len() - 1;+ for v in [stale, value] {+ let mut element = msg(1, &num(6, key as u64));+ element.extend(msg(2, &num(6, v as u64)));+ element.extend(msg(3, &msg(1, &[num(1, 1), num(2, 0)].concat())));+ dictionary.extend(msg(1, &element));+ }+ }+ for &i in wire_order {+ let mut a = num(1, i as u64);+ a.extend(msg(2, &uuid(i)));+ attachments.extend(msg(2, &a));+ }+ let mut array = msg(1, b"");+ array.extend(attachments);+ let mut ordering = msg(1, &array);+ ordering.extend(msg(2, &dictionary));+ let set = msg(1, &ordering);+ objects[0] = msg(16, &set);++ let mut top = Vec::new();+ for o in &objects {+ top.extend(msg(3, o));+ }+ for k in ["self", "UUIDIndex"] {+ top.extend(msg(4, k.as_bytes()));+ }+ for t in ["com.apple.CRDT.NSString", "com.apple.CRDT.NSUUID"] {+ top.extend(msg(5, t.as_bytes()));+ }+ for u in &uuids {+ top.extend(msg(6, u));+ }+ top+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn the_order_is_read_by_position_and_the_latest_answer_wins() {+ let bytes = build::ordered_set(&["DAILY", "STUDY", "TODO", "LIFE"], &[2, 0, 3, 1]);+ assert_eq!(+ ordered_strings(&bytes).unwrap(),+ vec!["DAILY", "STUDY", "TODO", "LIFE"]+ );+ }++ #[test]+ fn anything_else_is_refused_rather_than_guessed() {+ assert!(ordered_strings(&[]).is_none());+ assert!(ordered_strings(&[0xFF, 0xFF]).is_none());+ assert!(ordered_strings(&build::msg(3, &build::num(1, 1))).is_none());+ }+}diff --git a/crates/quince-icloud/src/reminders.rs b/crates/quince-icloud/src/reminders.rsindex dbfbe6b..4e45be4 100644--- a/crates/quince-icloud/src/reminders.rs+++ b/crates/quince-icloud/src/reminders.rs@@ -602,38 +602,62 @@ fn list_order_from(record: &Value) -> Vec<String> {/// So the question asked of it is whether it still describes the account: most of what it names/// should still be there. A list it has not heard of is just a new one and sorts after; a list/// of names that are mostly gone is a leftover and is ignored.-fn apply_list_order(lists: &mut [ReminderList], order: &[String]) {+fn apply_list_order(+ lists: &mut [ReminderList],+ groups: &mut [quince_core::reminders::ReminderGroup],+ order: &[String],+) {if order.is_empty() {return;}+ let uuid_of = |guid: &str| guid.rsplit('/').next().unwrap_or(guid).to_string();let position = |guid: &str| {- let uuid = guid.rsplit('/').next().unwrap_or(guid);- order.iter().position(|id| id.eq_ignore_ascii_case(uuid))+ let uuid = uuid_of(guid);+ order.iter().position(|id| id.eq_ignore_ascii_case(&uuid))};+ let known: Vec<String> = lists+ .iter()+ .map(|l| uuid_of(&l.guid))+ .chain(groups.iter().map(|g| uuid_of(&g.guid)))+ .collect();let still_here = order.iter()- .filter(|id| {- lists.iter().any(|l| {- l.guid- .rsplit('/')- .next()- .unwrap_or(&l.guid)- .eq_ignore_ascii_case(id)- })- })+ .filter(|id| known.iter().any(|k| k.eq_ignore_ascii_case(id))).count();if still_here * 2 < order.len() {return;}- for list in lists.iter_mut() {- // A list the arrangement has not heard of sits after the ones it has, keeping the- // order it arrived in rather than tying with every other stranger.- list.sort_order = position(&list.guid)+ // A list or group the arrangement has not heard of sits after the ones it has, keeping+ // the order it arrived in rather than tying with every other stranger.+ let place = |guid: &str, arrived: i64| {+ position(guid).and_then(|i| i64::try_from(i).ok())- .unwrap_or_else(|| i64::from(u32::MAX) + list.sort_order);+ .unwrap_or_else(|| i64::from(u32::MAX) + arrived)+ };+ for list in lists.iter_mut() {+ list.sort_order = place(&list.guid, list.sort_order);+ }+ for group in groups.iter_mut() {+ group.sort_order = place(&group.guid, group.sort_order);}}+/// The order lists and groups appear in on the phone.+///+/// It is on the account record, as `ListIDsMergeableOrdering`: one ordered set holding every+/// list and group, a group's lists ordered among the rest by the same sequence. The+/// `ManualSortHint` read before this is a leftover — on a real account seven of its eight+/// ids named lists that no longer exist.+fn account_list_order(record: &Value) -> Vec<String> {+ use base64::{engine::general_purpose::STANDARD as B64, Engine};+ record+ .pointer("/fields/ListIDsMergeableOrdering/value")+ .and_then(Value::as_str)+ .and_then(|encoded| B64.decode(encoded.trim()).ok())+ .and_then(|bytes| crate::mergeable::ordered_strings(&bytes))+ .unwrap_or_default()+}+/// Read reminders from CloudKit into the local library.////// Pass the previous `since` token to fetch only what changed. That distinction matters for@@ -771,6 +795,7 @@ pub fn apply_cloudkit(let mut gone_lists: Vec<String> = Vec::new();let mut list_order: Vec<String> = Vec::new();let mut groups: Vec<quince_core::reminders::ReminderGroup> = Vec::new();+ let mut account_order: Vec<String> = Vec::new();// A RecurrenceRule is its own record pointing back at the reminder it governs.let mut rules: std::collections::HashMap<String, (i64, i64)> = std::collections::HashMap::new();@@ -852,7 +877,13 @@ pub fn apply_cloudkit(}}}- // Carries the order the lists appear in on the user's other devices.+ "Account" => {+ let order = account_list_order(record);+ if !order.is_empty() {+ account_order = order;+ }+ }+ // An older record of the order, used only where the account holds none."ManualSortHint" => {let order = list_order_from(record);if !order.is_empty() {@@ -895,7 +926,26 @@ pub fn apply_cloudkit(}// Match the order the phone shows before storing.- apply_list_order(&mut lists, &list_order);+ let order = if account_order.is_empty() {+ &list_order+ } else {+ &account_order+ };+ apply_list_order(&mut lists, &mut groups, order);+ // An incremental pass that carried no order says nothing about where anything sits, and+ // its walk positions count only what changed. A list already stored keeps its place.+ if incremental && order.is_empty() {+ let stored: std::collections::HashMap<String, i64> = library+ .reminder_lists()?+ .into_iter()+ .map(|l| (l.guid, l.sort_order))+ .collect();+ for list in &mut lists {+ if let Some(&at) = stored.get(&list.guid) {+ list.sort_order = at;+ }+ }+ }if incremental {library.merge_reminder_lists(&lists)?;@@ -2368,7 +2418,11 @@ mod order_tests {#[test]fn lists_take_the_order_the_phone_shows() {let mut lists = vec![list("List/CCC"), list("List/AAA"), list("List/BBB")];- apply_list_order(&mut lists, &["AAA".into(), "BBB".into(), "CCC".into()]);+ apply_list_order(+ &mut lists,+ &mut [],+ &["AAA".into(), "BBB".into(), "CCC".into()],+ );lists.sort_by_key(|l| l.sort_order);let order: Vec<&str> = lists.iter().map(|l| l.guid.as_str()).collect();assert_eq!(order, vec!["List/AAA", "List/BBB", "List/CCC"]);@@ -2378,7 +2432,7 @@ mod order_tests {#[test]fn unmentioned_lists_sort_after_the_rest() {let mut lists = vec![list("List/NEW"), list("List/AAA")];- apply_list_order(&mut lists, &["AAA".into()]);+ apply_list_order(&mut lists, &mut [], &["AAA".into()]);assert!(lists.iter()@@ -2396,7 +2450,7 @@ mod order_tests {#[test]fn matching_ignores_case() {let mut lists = vec![list("List/abc")];- apply_list_order(&mut lists, &["ABC".into()]);+ apply_list_order(&mut lists, &mut [], &["ABC".into()]);assert_eq!(lists[0].sort_order, 0);}@@ -2405,7 +2459,7 @@ mod order_tests {fn an_absent_or_broken_hint_changes_nothing() {let mut lists = vec![list("List/AAA")];lists[0].sort_order = 7;- apply_list_order(&mut lists, &[]);+ apply_list_order(&mut lists, &mut [], &[]);assert_eq!(lists[0].sort_order, 7);let broken = json!({@@ -3306,6 +3360,7 @@ mod list_shape_tests {let before: Vec<i64> = lists.iter().map(|l| l.sort_order).collect();apply_list_order(&mut lists,+ &mut [],&["GONE1".into(), "GONE2".into(), "GONE3".into(), "A".into()],);let after: Vec<i64> = lists.iter().map(|l| l.sort_order).collect();@@ -3328,7 +3383,7 @@ mod list_shape_tests {..Default::default()}).collect();- apply_list_order(&mut lists, &["B".into(), "A".into()]);+ apply_list_order(&mut lists, &mut [], &["B".into(), "A".into()]);lists.sort_by_key(|l| l.sort_order);let order: Vec<&str> = lists.iter().map(|l| l.guid.as_str()).collect();assert_eq!(order, vec!["List/B", "List/A", "List/NEW"]);@@ -3401,9 +3456,9 @@ mod other_accounts_tests {enabled: true,..Default::default()}];- apply_list_order(&mut lists, &[]);+ apply_list_order(&mut lists, &mut [], &[]);assert_eq!(lists[0].sort_order, 0);- apply_list_order(&mut lists, &["SOMETHING-ELSE".into()]);+ apply_list_order(&mut lists, &mut [], &["SOMETHING-ELSE".into()]);assert_eq!(lists[0].sort_order, 0,"an arrangement about other lists is not ours"@@ -3653,3 +3708,123 @@ mod membership_tests {assert!(memberships_from(br#"{"other":[]}"#).is_empty());}}++/// Where lists and groups sit, as the phone has them.+#[cfg(test)]+mod account_order_tests {+ use super::*;+ use serde_json::json;++ fn list(name: &str, title: &str, group: Option<&str>, is_group: bool) -> Value {+ let mut fields = json!({+ "Name": { "type": "STRING", "value": title },+ "IsGroup": { "type": "NUMBER_INT64", "value": i64::from(is_group) },+ });+ if let Some(parent) = group {+ fields["ParentList"] =+ json!({ "type": "REFERENCE", "value": { "recordName": parent } });+ }+ json!({ "recordName": name, "recordType": "List", "fields": fields })+ }++ fn account(order: &[&str]) -> Value {+ use base64::{engine::general_purpose::STANDARD as B64, Engine};+ let wire: Vec<usize> = (0..order.len()).rev().collect();+ let bytes = crate::mergeable::build::ordered_set(order, &wire);+ json!({+ "recordName": "Account/x",+ "recordType": "Account",+ "fields": { "ListIDsMergeableOrdering": { "type": "BYTES", "value": B64.encode(bytes) } }+ })+ }++ fn pass(library: &mut quince_core::Library, records: Vec<Value>, incremental: bool) {+ apply_cloudkit(+ library,+ RemindersFetch {+ memberships: std::collections::HashMap::new(),+ records,+ token: Some("t".into()),+ incremental,+ },+ )+ .unwrap();+ }++ fn walked() -> Vec<Value> {+ // Walked in an order that is not the phone's.+ vec![+ list("List/GOALS", "Life Goals", Some("List/LIFE"), false),+ list("List/LIFE", "Life", None, true),+ list("List/DAILY", "Daily", Some("List/LIFE"), false),+ list("List/UNI", "University", None, true),+ list("List/TODO", "To-Do", None, false),+ ]+ }++ /// The account record's order is the phone's, for groups as much as lists. The stale+ /// hint beside it named lists that no longer exist, so the sidebar followed the walk.+ #[test]+ fn lists_and_groups_follow_the_account_record() {+ let mut library = quince_core::Library::in_memory().unwrap();+ let mut records = walked();+ records.push(account(&["DAILY", "TODO", "UNI", "LIFE", "GOALS"]));+ pass(&mut library, records, false);++ let order = |lists: Vec<ReminderList>| -> Vec<String> {+ let mut lists = lists;+ lists.sort_by_key(|l| l.sort_order);+ lists.into_iter().map(|l| l.title).collect()+ };+ assert_eq!(+ order(library.reminder_lists().unwrap()),+ ["Daily", "To-Do", "Life Goals"]+ );+ let groups: Vec<String> = library+ .reminder_groups()+ .unwrap()+ .into_iter()+ .map(|g| g.title)+ .collect();+ assert_eq!(groups, ["University", "Life"]);+ // And groups and lists share one sequence: University is before Life Goals's group.+ let uni = library.reminder_groups().unwrap()[0].sort_order;+ let todo = library+ .reminder_lists()+ .unwrap()+ .iter()+ .find(|l| l.title == "To-Do")+ .unwrap()+ .sort_order;+ assert!(todo < uni);+ }++ /// A later pass with no order in it leaves every list where it was.+ #[test]+ fn a_pass_without_an_order_keeps_the_places_it_has() {+ let mut library = quince_core::Library::in_memory().unwrap();+ let mut records = walked();+ records.push(account(&["DAILY", "TODO", "UNI", "LIFE", "GOALS"]));+ pass(&mut library, records, false);+ let before: Vec<(String, i64)> = library+ .reminder_lists()+ .unwrap()+ .into_iter()+ .map(|l| (l.title, l.sort_order))+ .collect();++ // Only Life Goals changed; it arrives first in this walk.+ pass(+ &mut library,+ vec![list("List/GOALS", "Life Goals", Some("List/LIFE"), false)],+ true,+ );+ let after: Vec<(String, i64)> = library+ .reminder_lists()+ .unwrap()+ .into_iter()+ .map(|l| (l.title, l.sort_order))+ .collect();+ assert_eq!(before, after);+ }+}