Native iCloud apps for Windows and Linux, built in Rust. Notes, Reminders, Calendar, Contacts, Drive and Photos, offline and in sync with your iPhone.
Rust apple-notesapple-reminderscloudkitcrdtdesktop-appicloudicloud-calendaricloud-driveicloud-notesicloud-photosiphonelinuxlinux-desktoplocal-firstoffline-firstprivacyrusttauritauri-appwindows
A full pass is the floor under tombstones, and it has to be a whole one
Yashwanth Kamireddiea0519c7721d
diff --git a/crates/quince-core/src/health.rs b/crates/quince-core/src/health.rsindex a01eeee..720bb7c 100644--- a/crates/quince-core/src/health.rs+++ b/crates/quince-core/src/health.rs@@ -9,6 +9,12 @@ use crate::{Library, Result};use rusqlite::params;use serde::{Deserialize, Serialize};+fn now() -> i64 {+ std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .map_or(0, |d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX))+}+/// One service's standing after the last refresh.#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]#[serde(rename_all = "camelCase")]@@ -44,6 +50,37 @@ impl Library {Ok(())}+ /// Whether this service is due a full pass rather than a delta.+ ///+ /// A delta cannot remove a record whose tombstone it never saw, so a full pass runs+ /// occasionally to reconcile against the whole truth. Never having done one counts as+ /// due, which is what heals a library that predates this bookkeeping.+ pub fn needs_full_sync(&self, service: &str, every_seconds: i64) -> Result<bool> {+ let last: Option<i64> = self+ .conn+ .query_row(+ "SELECT last_full_at FROM sync_state WHERE service = ?1",+ params![service],+ |row| row.get(0),+ )+ .ok()+ .flatten();+ Ok(match last {+ None => true,+ Some(at) => now().saturating_sub(at) >= every_seconds,+ })+ }++ /// Record that a service reconciled against everything Apple has.+ pub fn mark_full_sync(&self, service: &str) -> Result<()> {+ self.conn.execute(+ "INSERT INTO sync_state (service, last_full_at) VALUES (?1, ?2)+ ON CONFLICT(service) DO UPDATE SET last_full_at = excluded.last_full_at",+ params![service, now()],+ )?;+ Ok(())+ }+/// Every service the library has ever tried to refresh.pub fn sync_health(&self) -> Result<Vec<ServiceHealth>> {let mut stmt = self.conn.prepare(@@ -155,6 +192,53 @@ mod tests {);}+ /// A library that has never done a full pass is due one — which is what heals a library+ /// that predates this bookkeeping and is holding records whose tombstones it missed.+ #[test]+ fn never_having_reconciled_counts_as_due() {+ let lib = library();+ assert!(lib.needs_full_sync("reminders", 86_400).unwrap());+ lib.mark_synced("reminders", Some("token")).unwrap();+ assert!(+ lib.needs_full_sync("reminders", 86_400).unwrap(),+ "a delta is not a full pass"+ );+ }++ #[test]+ fn a_recent_full_pass_is_not_repeated() {+ let lib = library();+ lib.mark_full_sync("reminders").unwrap();+ assert!(!lib.needs_full_sync("reminders", 86_400).unwrap());+ }++ #[test]+ fn a_stale_full_pass_comes_due_again() {+ let lib = library();+ lib.mark_full_sync("reminders").unwrap();+ lib.conn+ .execute(+ "UPDATE sync_state SET last_full_at = last_full_at - 90000 WHERE service = 'reminders'",+ [],+ )+ .unwrap();+ assert!(lib.needs_full_sync("reminders", 86_400).unwrap());+ }++ /// Recording a full pass must not disturb the freshness stamp or the resume token.+ #[test]+ fn recording_a_full_pass_leaves_everything_else_alone() {+ let lib = library();+ lib.mark_synced("reminders", Some("token")).unwrap();+ let before = lib.last_synced("reminders").unwrap();+ lib.mark_full_sync("reminders").unwrap();+ assert_eq!(lib.last_synced("reminders").unwrap(), before);+ assert_eq!(+ lib.sync_token("reminders").unwrap().as_deref(),+ Some("token")+ );+ }+#[test]fn a_service_that_never_synced_makes_the_whole_claim_unknown() {let lib = library();diff --git a/crates/quince-core/src/notes.rs b/crates/quince-core/src/notes.rsindex 3dd055f..79a0f3f 100644--- a/crates/quince-core/src/notes.rs+++ b/crates/quince-core/src/notes.rs@@ -103,6 +103,36 @@ impl Library {Ok(())}+ /// Keep only these notes, dropping everything else.+ ///+ /// The same floor a full reminder sync has: a tombstone arrives once, so a sync that+ /// consumed one before it was understood would leave a deleted note on screen forever —+ /// and saving one of those writes it back to iCloud.+ pub fn retain_notes(&mut self, keep: &[String]) -> Result<usize> {+ let tx = self.conn.transaction()?;+ let removed = {+ let mut known: std::collections::HashSet<&str> =+ keep.iter().map(String::as_str).collect();+ let mut stmt = tx.prepare("SELECT guid FROM notes")?;+ let all: Vec<String> = stmt+ .query_map([], |row| row.get::<_, String>(0))?+ .collect::<rusqlite::Result<Vec<_>>>()?;+ drop(stmt);++ let mut delete = tx.prepare("DELETE FROM notes WHERE guid = ?1")?;+ let mut count = 0;+ for guid in &all {+ if !known.remove(guid.as_str()) {+ delete.execute(params![guid])?;+ count += 1;+ }+ }+ count+ };+ tx.commit()?;+ Ok(removed)+ }+pub fn forget_notes(&mut self, note_guids: &[String]) -> Result<()> {let tx = self.conn.transaction()?;{diff --git a/crates/quince-core/src/reminders.rs b/crates/quince-core/src/reminders.rsindex fe92eff..63909d6 100644--- a/crates/quince-core/src/reminders.rs+++ b/crates/quince-core/src/reminders.rs@@ -310,6 +310,39 @@ impl Library {Ok(())}+ /// Keep only these reminders, dropping everything else.+ ///+ /// A full sync is the whole truth: Apple reported every reminder it has, so anything+ /// still here that is not in that set no longer exists. Tombstones cover the usual+ /// case, but a tombstone is only ever delivered once — if a sync consumed one before+ /// the code knew what to do with it, that record would otherwise stay on screen+ /// forever. This is the floor under that.+ pub fn retain_reminders(&mut self, keep: &[String]) -> Result<usize> {+ let tx = self.conn.transaction()?;+ let removed = {+ let mut known: std::collections::HashSet<&str> =+ keep.iter().map(String::as_str).collect();+ // A reminder with no list would fail its foreign key anyway; leave those alone.+ let mut stmt = tx.prepare("SELECT guid FROM reminders")?;+ let all: Vec<String> = stmt+ .query_map([], |row| row.get::<_, String>(0))?+ .collect::<rusqlite::Result<Vec<_>>>()?;+ drop(stmt);++ let mut delete = tx.prepare("DELETE FROM reminders WHERE guid = ?1")?;+ let mut count = 0;+ for guid in &all {+ if !known.remove(guid.as_str()) {+ delete.execute(params![guid])?;+ count += 1;+ }+ }+ count+ };+ tx.commit()?;+ Ok(removed)+ }+pub fn put_reminders(&mut self, reminders: &[Reminder]) -> Result<()> {let tx = self.conn.transaction()?;{diff --git a/crates/quince-core/src/schema.rs b/crates/quince-core/src/schema.rsindex b03c5b3..a59c34f 100644--- a/crates/quince-core/src/schema.rs+++ b/crates/quince-core/src/schema.rs@@ -188,4 +188,13 @@ pub const MIGRATIONS: &[&str] = &[r#"ALTER TABLE notes ADD COLUMN editable INTEGER NOT NULL DEFAULT 1;"#,+ // 10 — when a service last did a full pass, rather than a delta.+ //+ // A delta is cheap and is what almost every refresh should be, but it can only ever+ // apply what Apple chose to send. A tombstone is delivered once; a sync that consumed+ // one before the code knew what to do with it left that record stranded forever. A full+ // pass now and then is the floor under that, and this is what schedules it.+ r#"+ ALTER TABLE sync_state ADD COLUMN last_full_at INTEGER;+ "#,];diff --git a/crates/quince-icloud/src/cloudkit.rs b/crates/quince-icloud/src/cloudkit.rsindex 22c634f..c0d5d2f 100644--- a/crates/quince-icloud/src/cloudkit.rs+++ b/crates/quince-icloud/src/cloudkit.rs@@ -126,19 +126,17 @@ impl<'a> ZoneClient<'a> {////// Returns the records and the final sync token, which a later call can resume from so a/// refresh fetches only what changed.- pub async fn all_records(- &self,- zone: &str,- since: Option<&str>,- ) -> Result<(Vec<Value>, Option<String>)> {+ pub async fn all_records(&self, zone: &str, since: Option<&str>) -> Result<ZoneWalk> {let mut records = Vec::new();let mut token = since.map(ToString::to_string);+ let mut complete = false;for _ in 0..MAX_PAGES {let page = self.changes(zone, token.as_deref()).await?;records.extend(page.records);token = page.sync_token;if !page.more_coming {+ complete = true;break;}// A page that says "more coming" but hands back no cursor would loop forever.@@ -146,10 +144,26 @@ impl<'a> ZoneClient<'a> {break;}}- Ok((records, token))+ Ok(ZoneWalk {+ records,+ token,+ complete,+ })}}+/// What one walk of a zone returned.+///+/// `complete` is the part that matters. A walk stops at a page cap, and one that stopped+/// early has seen only part of the zone — treating that as the whole truth and pruning+/// everything it did not mention would delete records that are perfectly fine. Nothing+/// authoritative may be done with a walk that did not finish.+pub struct ZoneWalk {+ pub records: Vec<Value>,+ pub token: Option<String>,+ pub complete: bool,+}+/// Field helpers. CloudKit wraps every value as `{"type": …, "value": …}`.pub mod field {use serde_json::Value;@@ -241,7 +255,7 @@ mod tests {let http = reqwest::Client::new();let client = ZoneClient::new(&http, &server.uri(), "com.apple.reminders", vec![]);- let (records, token) = client.all_records("Reminders", None).await.unwrap();+ let ZoneWalk { records, token, .. } = client.all_records("Reminders", None).await.unwrap();assert_eq!(records.len(), 3, "every page must be collected");assert_eq!(@@ -265,7 +279,7 @@ mod tests {let http = reqwest::Client::new();let client = ZoneClient::new(&http, &server.uri(), "com.apple.reminders", vec![]);- let (records, _) = client.all_records("Reminders", None).await.unwrap();+ let ZoneWalk { records, .. } = client.all_records("Reminders", None).await.unwrap();assert_eq!(records.len(), 1);}@@ -600,6 +614,79 @@ mod modify_tests {}}+#[cfg(test)]+mod walk_tests {+ use super::*;+ use wiremock::matchers::{method, path_regex};+ use wiremock::{Mock, MockServer, ResponseTemplate};++ /// A walk that runs out of pages has seen only part of the zone. Saying so is what+ /// stops a caller treating it as the whole truth and deleting everything it missed.+ #[tokio::test]+ async fn a_walk_that_never_ends_is_reported_incomplete() {+ let server = MockServer::start().await;+ // Every page claims there is more, forever.+ Mock::given(method("POST"))+ .and(path_regex(r".*/changes/zone$"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "zones": [{+ "records": [{ "recordName": "Reminder/1", "recordType": "Reminder" }],+ "moreComing": true,+ "syncToken": "next"+ }]+ })))+ .mount(&server)+ .await;++ let http = reqwest::Client::new();+ let client = ZoneClient::new(&http, &server.uri(), "com.apple.reminders", vec![]);+ let walk = client.all_records("Reminders", None).await.unwrap();+ assert!(!walk.complete, "hitting the page cap is not completion");+ }++ #[tokio::test]+ async fn a_walk_that_finishes_says_so() {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path_regex(r".*/changes/zone$"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "zones": [{+ "records": [{ "recordName": "Reminder/1", "recordType": "Reminder" }],+ "moreComing": false,+ "syncToken": "done"+ }]+ })))+ .mount(&server)+ .await;++ let http = reqwest::Client::new();+ let client = ZoneClient::new(&http, &server.uri(), "com.apple.reminders", vec![]);+ let walk = client.all_records("Reminders", None).await.unwrap();+ assert!(walk.complete);+ assert_eq!(walk.records.len(), 1);+ assert_eq!(walk.token.as_deref(), Some("done"));+ }++ /// "More coming" with no cursor to follow would loop forever; stopping is right, and+ /// calling it complete would not be.+ #[tokio::test]+ async fn more_coming_with_no_cursor_stops_and_admits_it() {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path_regex(r".*/changes/zone$"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "zones": [{ "records": [], "moreComing": true }]+ })))+ .mount(&server)+ .await;++ let http = reqwest::Client::new();+ let client = ZoneClient::new(&http, &server.uri(), "com.apple.reminders", vec![]);+ let walk = client.all_records("Reminders", None).await.unwrap();+ assert!(!walk.complete);+ }+}+#[cfg(test)]mod conflict_tests {use super::*;diff --git a/crates/quince-icloud/src/notes.rs b/crates/quince-icloud/src/notes.rsindex b19b96e..acfe148 100644--- a/crates/quince-icloud/src/notes.rs+++ b/crates/quince-icloud/src/notes.rs@@ -95,6 +95,7 @@ pub async fn sync_cloudkit(pub struct NotesFetch {records: Vec<Value>,token: Option<String>,+ incremental: bool,}/// Ask Apple what changed. Touches no local state.@@ -105,8 +106,14 @@ pub async fn fetch_cloudkit(since: Option<&str>,) -> Result<NotesFetch> {let client = ZoneClient::new(http, ckdatabasews_root, CONTAINER, params);- let (records, token) = client.all_records(ZONE, since).await?;- Ok(NotesFetch { records, token })+ let walk = client.all_records(ZONE, since).await?;+ Ok(NotesFetch {+ records: walk.records,+ token: walk.token,+ // A walk cut short by the page cap is not the whole truth, whatever it was asked+ // for — pruning against a partial view would delete records that are perfectly fine.+ incremental: since.is_some() || !walk.complete,+ })}/// Store a fetched pass. Touches no network.@@ -114,7 +121,11 @@ pub fn apply_cloudkit(library: &mut quince_core::Library,fetched: NotesFetch,) -> Result<(usize, usize, Option<String>)> {- let NotesFetch { records, token } = fetched;+ let NotesFetch {+ records,+ token,+ incremental,+ } = fetched;let mut folders: Vec<NoteFolder> = Vec::new();let mut notes: Vec<Note> = Vec::new();@@ -203,6 +214,12 @@ pub fn apply_cloudkit(if !gone.is_empty() {library.forget_notes(&gone)?;}+ // A full pass is the whole truth; an incremental one only carries changes, where+ // absence means unchanged rather than deleted.+ if !incremental {+ let keep: Vec<String> = notes.iter().map(|n| n.guid.clone()).collect();+ library.retain_notes(&keep)?;+ }library.mark_synced("notes", token.as_deref())?;Ok((folders.len(), notes.len(), token))}diff --git a/crates/quince-icloud/src/reminders.rs b/crates/quince-icloud/src/reminders.rsindex f1eacaf..f28cf82 100644--- a/crates/quince-icloud/src/reminders.rs+++ b/crates/quince-icloud/src/reminders.rs@@ -554,11 +554,13 @@ pub async fn fetch_cloudkit(since: Option<&str>,) -> Result<RemindersFetch> {let client = ZoneClient::new(http, ckdatabasews_root, CONTAINER, params);- let (records, token) = client.all_records(ZONE, since).await?;+ let walk = client.all_records(ZONE, since).await?;Ok(RemindersFetch {- records,- token,- incremental: since.is_some(),+ records: walk.records,+ token: walk.token,+ // A walk that stopped at its page cap saw only part of the zone, so it cannot be+ // treated as the whole truth however it was asked for.+ incremental: since.is_some() || !walk.complete,})}@@ -685,6 +687,13 @@ pub fn apply_cloudkit(if !gone_reminders.is_empty() || !gone_lists.is_empty() {library.forget_reminders(&gone_reminders, &gone_lists)?;}+ // A full pass saw everything Apple has, so anything else here is gone. An incremental+ // pass saw only changes, where absence means unchanged — pruning there would empty the+ // library on the first refresh.+ if !incremental {+ let keep: Vec<String> = reminders.iter().map(|r| r.guid.clone()).collect();+ library.retain_reminders(&keep)?;+ }library.mark_synced("reminders", token.as_deref())?;Ok((lists.len(), reminders.len(), token))}@@ -2321,6 +2330,44 @@ mod tombstone_tests {);}+ /// The floor under tombstones: a tombstone is delivered once, so a sync that consumed+ /// one before this code knew what to do with it would leave that record on screen+ /// forever. A full pass saw everything Apple has, so anything else is gone.+ #[test]+ fn a_full_pass_removes_what_apple_no_longer_reports() {+ let mut library = library_with("Reminder/ghost");+ apply_cloudkit(+ &mut library,+ RemindersFetch {+ records: vec![json!({+ "recordName": "List/a",+ "recordType": "List",+ "fields": { "Name": { "type": "STRING", "value": "Daily" } }+ })],+ token: Some("t".into()),+ incremental: false,+ },+ )+ .unwrap();+ assert!(+ library.reminders_in("List/a", true).unwrap().is_empty(),+ "a full pass is the whole truth"+ );+ }++ /// And the opposite must never happen: an incremental pass carries only changes, so+ /// pruning there would empty the library on the first refresh.+ #[test]+ fn an_incremental_pass_never_prunes() {+ let mut library = library_with("Reminder/1");+ apply(&mut library, vec![]);+ assert_eq!(+ library.reminders_in("List/a", true).unwrap().len(),+ 1,+ "absence in a delta means unchanged, not deleted"+ );+ }+/// A name that matches nothing must not disturb what is there.#[test]fn a_tombstone_for_something_we_never_had_is_harmless() {diff --git a/probe/src/main.rs b/probe/src/main.rsindex 04191c3..b87b69d 100644--- a/probe/src/main.rs+++ b/probe/src/main.rs@@ -624,7 +624,7 @@ async fn main() -> Result<()> {" {zone_name:<12} error: {}",e.to_string().chars().take(70).collect::<String>()),- Ok((records, _)) => {+ Ok(quince_icloud::cloudkit::ZoneWalk { records, .. }) => {let mut census: std::collections::BTreeMap<String, usize> =std::collections::BTreeMap::new();for r in &records {@@ -1001,7 +1001,9 @@ async fn main() -> Result<()> {"com.apple.notes",client.service_params(),);- if let Ok((records, _)) = zc.all_records("Notes", None).await {+ if let Ok(quince_icloud::cloudkit::ZoneWalk { records, .. }) =+ zc.all_records("Notes", None).await+ {let (mut safe, mut refused) = (0, 0);for rec in records.iter()@@ -1276,7 +1278,9 @@ async fn main() -> Result<()> {"com.apple.notes",client.service_params(),);- if let Ok((records, _)) = zc.all_records("Notes", None).await {+ if let Ok(quince_icloud::cloudkit::ZoneWalk { records, .. }) =+ zc.all_records("Notes", None).await+ {for rec in records.iter().filter(|r| {@@ -1433,7 +1437,9 @@ async fn main() -> Result<()> {"com.apple.reminders",client.service_params(),);- if let Ok((records, _)) = zc.all_records("Reminders", None).await {+ if let Ok(quince_icloud::cloudkit::ZoneWalk { records, .. }) =+ zc.all_records("Reminders", None).await+ {for rec in records.iter().filter(|r| r.get("recordType").and_then(|t| t.as_str()) == Some("RecurrenceRule"))diff --git a/src-tauri/src/refresh.rs b/src-tauri/src/refresh.rsindex d1f7957..f0be33d 100644--- a/src-tauri/src/refresh.rs+++ b/src-tauri/src/refresh.rs@@ -56,6 +56,15 @@ pub struct Refresh {pub took_ms: u64,}+/// How often a service reconciles against everything Apple has, rather than the changes+/// since last time.+///+/// A delta is what almost every refresh should be — seconds rather than a minute. But a+/// delta can only apply what Apple chose to send, and a tombstone is sent once, so a record+/// whose removal was missed would stay forever. Once a day is often enough to be a floor+/// and rare enough that nobody notices the cost.+const FULL_SYNC_EVERY: i64 = 24 * 60 * 60;+/// The services one pass covers, in the order a person would notice them going stale.const COVERED: [(&str, &str); 4] = [("calendar", "Calendar"),@@ -153,6 +162,7 @@ pub async fn all(client: &Arc<Mutex<Client>>, data: &Mutex<Library>) -> RefreshNone => unavailable("reminders", "Reminders"),Some(root) => {let since = token(data, "reminders").await;+ let full = since.is_none();let fetched = quince_icloud::reminders::fetch_cloudkit(&ctx.http,root,@@ -160,14 +170,23 @@ pub async fn all(client: &Arc<Mutex<Client>>, data: &Mutex<Library>) -> Refreshsince.as_deref(),).await;- store("reminders", "Reminders", data, fetched, |library, f| {- let (lists, count, _) = quince_icloud::reminders::apply_cloudkit(library, f)?;- Ok(format!(- "{count} reminder{} in {lists} list{}",- plural(count),- plural(lists)- ))- })+ store(+ "reminders",+ "Reminders",+ data,+ fetched,+ move |library, f| {+ let (lists, count, _) = quince_icloud::reminders::apply_cloudkit(library, f)?;+ if full {+ library.mark_full_sync("reminders")?;+ }+ Ok(format!(+ "{count} reminder{} in {lists} list{}",+ plural(count),+ plural(lists)+ ))+ },+ ).await}});@@ -176,6 +195,7 @@ pub async fn all(client: &Arc<Mutex<Client>>, data: &Mutex<Library>) -> RefreshNone => unavailable("notes", "Notes"),Some(root) => {let since = token(data, "notes").await;+ let full = since.is_none();let fetched = quince_icloud::notes::fetch_cloudkit(&ctx.http,root,@@ -183,8 +203,11 @@ pub async fn all(client: &Arc<Mutex<Client>>, data: &Mutex<Library>) -> Refreshsince.as_deref(),).await;- store("notes", "Notes", data, fetched, |library, f| {+ store("notes", "Notes", data, fetched, move |library, f| {let (folders, notes, _) = quince_icloud::notes::apply_cloudkit(library, f)?;+ if full {+ library.mark_full_sync("notes")?;+ }Ok(format!("{notes} note{} in {folders} folder{}",plural(notes),@@ -275,8 +298,16 @@ fn unavailable(service: &'static str, label: &'static str) -> ServiceReport {}}+/// Where the last sync got to — or `None` when this service is due a full pass, which is+/// what makes the next fetch walk everything.async fn token(data: &Mutex<Library>, service: &str) -> Option<String> {let library = data.lock().await;+ if library+ .needs_full_sync(service, FULL_SYNC_EVERY)+ .unwrap_or(false)+ {+ return None;+ }library.sync_token(service).ok().flatten()}