Skip to content

Native iCloud apps for Windows and Linux, built in Rust. Notes, Reminders, Calendar, Contacts, Drive and Photos, offline and in sync with your iPhone.

Search and sorting ignore case and accents everywhere

Yashwanth Kamireddifec060a61d78
diff --git a/Cargo.lock b/Cargo.lock
index 2846d4a..20d2007 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3225,6 +3225,7 @@ dependencies = [
name = "quince-core"
version = "0.1.0"
dependencies = [
+ "icu_normalizer",
"jiff",
"rusqlite",
"serde",
diff --git a/crates/quince-core/Cargo.toml b/crates/quince-core/Cargo.toml
index 7203092..cb51c4a 100644
--- a/crates/quince-core/Cargo.toml
+++ b/crates/quince-core/Cargo.toml
@@ -10,6 +10,7 @@ rusqlite = { version = "0.32", features = ["bundled"] }
jiff = "0.2.37"
serde.workspace = true
thiserror = "2"
+icu_normalizer = { version = "2.3", default-features = false, features = ["compiled_data"] }
[dev-dependencies]
serde_json.workspace = true
diff --git a/crates/quince-core/src/contacts.rs b/crates/quince-core/src/contacts.rs
index 8bf8f95..3d11158 100644
--- a/crates/quince-core/src/contacts.rs
+++ b/crates/quince-core/src/contacts.rs
@@ -90,7 +90,7 @@ impl Contact {
} else {
key
};
- key.to_lowercase()
+ crate::fold::fold(&key)
}
}
@@ -202,26 +202,19 @@ impl Library {
/// The address book in display order, optionally filtered.
///
/// The filter matches names and any stored method, so searching a phone number finds the
- /// person it belongs to.
+ /// person it belongs to. Every word must match, in any order, without regard to case or
+ /// accents, and a number matches however it was typed: "98765 43210" finds
+ /// "+91 98765-43210". That is done here rather than in SQL, which folds ASCII alone and
+ /// knows nothing about phone formatting.
pub fn contacts(&self, search: Option<&str>) -> Result<Vec<Contact>> {
- let needle = search
- .map(str::trim)
- .filter(|s| !s.is_empty())
- .map(|s| format!("%{}%", s.to_lowercase()));
+ let words = crate::fold::words(search.unwrap_or(""));
let mut stmt = self.conn.prepare(
"SELECT c.guid, c.first_name, c.middle_name, c.last_name, c.company, c.is_company,
c.birthday, c.notes, c.photo_url, c.photo_signature, c.photo_crop, c.etag
- FROM contacts c
- WHERE ?1 IS NULL
- OR lower(coalesce(c.first_name, '') || ' ' || coalesce(c.middle_name, '')
- || ' ' || coalesce(c.last_name, '')
- || ' ' || coalesce(c.company, '')) LIKE ?1
- OR EXISTS (SELECT 1 FROM contact_methods m
- WHERE m.contact_guid = c.guid AND lower(m.value) LIKE ?1)
- ORDER BY c.sort_name",
+ FROM contacts c",
)?;
- let rows = stmt.query_map(params![needle], |row| {
+ let rows = stmt.query_map([], |row| {
Ok(Contact {
guid: row.get(0)?,
first_name: row.get(1)?,
@@ -240,21 +233,85 @@ impl Library {
})?;
let mut contacts = rows.collect::<rusqlite::Result<Vec<_>>>()?;
- let mut methods = self.conn.prepare(
- "SELECT kind, label, value FROM contact_methods
- WHERE contact_guid = ?1 ORDER BY position",
+ // One pass over every method rather than a query per person: an address book of a
+ // few thousand would otherwise cost a few thousand queries on every keystroke.
+ let mut methods: std::collections::HashMap<String, Vec<ContactMethod>> =
+ std::collections::HashMap::new();
+ let mut stmt = self.conn.prepare(
+ "SELECT contact_guid, kind, label, value FROM contact_methods
+ ORDER BY contact_guid, position",
)?;
+ let rows = stmt.query_map([], |row| {
+ Ok((
+ row.get::<_, String>(0)?,
+ ContactMethod {
+ kind: row.get(1)?,
+ label: row.get(2)?,
+ value: row.get(3)?,
+ },
+ ))
+ })?;
+ for row in rows {
+ let (guid, method) = row?;
+ methods.entry(guid).or_default().push(method);
+ }
for contact in &mut contacts {
- let rows = methods.query_map(params![contact.guid], |row| {
- Ok(ContactMethod {
- kind: row.get(0)?,
- label: row.get(1)?,
- value: row.get(2)?,
- })
- })?;
- contact.methods = rows.collect::<rusqlite::Result<Vec<_>>>()?;
+ contact.methods = methods.remove(&contact.guid).unwrap_or_default();
}
- Ok(contacts)
+
+ if !words.is_empty() {
+ contacts.retain(|c| c.matches(&words));
+ }
+ let mut keyed: Vec<(String, Contact)> =
+ contacts.into_iter().map(|c| (c.sort_key(), c)).collect();
+ keyed.sort_by(|(a, _), (b, _)| crate::fold::natural(a, b));
+ Ok(keyed.into_iter().map(|(_, c)| c).collect())
+ }
+}
+
+/// Only the digits of a number, so its formatting stops mattering.
+fn digits(text: &str) -> String {
+ text.chars().filter(char::is_ascii_digit).collect()
+}
+
+/// True for a query word that reads as (part of) a phone number rather than a name.
+fn looks_like_number(word: &str) -> bool {
+ word.chars().any(|c| c.is_ascii_digit())
+ && word
+ .chars()
+ .all(|c| c.is_ascii_digit() || matches!(c, '+' | '-' | '(' | ')' | '.' | '/'))
+}
+
+impl Contact {
+ fn matches(&self, words: &[String]) -> bool {
+ let mut text = [
+ self.first_name.as_deref(),
+ self.middle_name.as_deref(),
+ self.last_name.as_deref(),
+ self.company.as_deref(),
+ ]
+ .into_iter()
+ .flatten()
+ .collect::<Vec<_>>()
+ .join(" ");
+ for m in &self.methods {
+ text.push('\n');
+ text.push_str(&m.value);
+ }
+ let text = crate::fold::fold(&text);
+ let numbers: Vec<String> = self
+ .methods
+ .iter()
+ .filter(|m| m.kind == "phone")
+ .map(|m| digits(&m.value))
+ .collect();
+ words.iter().all(|w| {
+ text.contains(w.as_str())
+ || (looks_like_number(w) && {
+ let d = digits(w);
+ numbers.iter().any(|n| n.contains(&d))
+ })
+ })
}
}
@@ -284,6 +341,60 @@ mod tests {
}
}
+ #[test]
+ fn search_ignores_case_accents_and_word_order() {
+ let mut lib = Library::in_memory().unwrap();
+ lib.put_contacts(&[
+ contact("1", Some("Émile"), Some("Zola")),
+ contact("2", Some("Zoë"), Some("Ahuja")),
+ ])
+ .unwrap();
+ let found = |q: &str| -> Vec<String> {
+ lib.contacts(Some(q))
+ .unwrap()
+ .iter()
+ .map(|c| c.guid.clone())
+ .collect()
+ };
+ assert_eq!(found("emile"), ["1"]);
+ assert_eq!(found("ÉMILE"), ["1"]);
+ assert_eq!(found("zola emile"), ["1"]);
+ assert_eq!(found("zoe"), ["2"]);
+ // A wildcard character is a character, not a pattern.
+ assert!(found("%").is_empty());
+ assert!(found("_").is_empty());
+ }
+
+ #[test]
+ fn a_number_matches_however_it_was_typed() {
+ let mut lib = Library::in_memory().unwrap();
+ let mut c = contact("1", Some("Meera"), None);
+ c.methods[0].value = "+91 98765-43210".into();
+ lib.put_contacts(&[c]).unwrap();
+ for typed in ["9876543210", "98765 43210", "(987) 65", "+91-98765"] {
+ assert_eq!(lib.contacts(Some(typed)).unwrap().len(), 1, "{typed}");
+ }
+ assert!(lib.contacts(Some("12345")).unwrap().is_empty());
+ }
+
+ #[test]
+ fn accented_names_sort_with_their_letter() {
+ let mut lib = Library::in_memory().unwrap();
+ lib.put_contacts(&[
+ contact("1", Some("Zoya"), Some("Zutshi")),
+ contact("2", Some("Émile"), Some("Écrivain")),
+ contact("3", Some("Aarav"), Some("Ahuja")),
+ ])
+ .unwrap();
+ let order: Vec<String> = lib
+ .contacts(None)
+ .unwrap()
+ .iter()
+ .map(|c| c.guid.clone())
+ .collect();
+ assert_eq!(order, ["3", "2", "1"]);
+ }
+
#[test]
fn contacts_sort_by_last_name_the_way_apple_does() {
let mut lib = Library::in_memory().unwrap();
diff --git a/crates/quince-core/src/drive.rs b/crates/quince-core/src/drive.rs
index 69eb8c2..8959ae9 100644
--- a/crates/quince-core/src/drive.rs
+++ b/crates/quince-core/src/drive.rs
@@ -111,13 +111,15 @@ impl Library {
}
/// What is inside a folder, folders first and then by name — how a file browser reads.
+ ///
+ /// Named the way Finder orders them: numbers as numbers and accents ignored, so "Scan 10"
+ /// follows "Scan 9" and "Été" sits among the E's. SQL's `lower` can do neither.
pub fn drive_children(&self, parent_id: &str) -> Result<Vec<DriveItem>> {
let mut stmt = self.conn.prepare(
"SELECT drivewsid, parent_id, name, extension, kind, size, child_count,
created_at, modified_at, etag
FROM drive_items
- WHERE parent_id = ?1
- ORDER BY kind = 'FILE', lower(name)",
+ WHERE parent_id = ?1",
)?;
let rows = stmt.query_map(params![parent_id], |row| {
Ok(DriveItem {
@@ -133,7 +135,13 @@ impl Library {
etag: row.get(9)?,
})
})?;
- Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
+ let mut items = rows.collect::<rusqlite::Result<Vec<_>>>()?;
+ items.sort_by(|a, b| {
+ (!a.is_container())
+ .cmp(&!b.is_container())
+ .then_with(|| crate::fold::natural(&a.name, &b.name))
+ });
+ Ok(items)
}
/// One item by its identifier, for building a breadcrumb trail.
@@ -207,6 +215,29 @@ mod tests {
assert_eq!(names, vec!["apples", "Zebra", "Archive", "notes"]);
}
+ /// Finder's order: "Scan 10" after "Scan 9", and "Été" among the E's rather than after Z.
+ #[test]
+ fn names_sort_the_way_finder_sorts_them() {
+ let mut lib = Library::in_memory().unwrap();
+ lib.put_drive_children(
+ ROOT,
+ &[
+ item("a", "Scan 10", "FILE"),
+ item("b", "Zoo", "FILE"),
+ item("c", "Scan 9", "FILE"),
+ item("d", "Été", "FILE"),
+ ],
+ )
+ .unwrap();
+ let names: Vec<String> = lib
+ .drive_children(ROOT)
+ .unwrap()
+ .iter()
+ .map(|i| i.name.clone())
+ .collect();
+ assert_eq!(names, vec!["Été", "Scan 9", "Scan 10", "Zoo"]);
+ }
+
#[test]
fn an_extension_shows_on_files_but_not_folders() {
assert_eq!(item("a", "report", "FILE").display_name(), "report.pdf");
diff --git a/crates/quince-core/src/fold.rs b/crates/quince-core/src/fold.rs
new file mode 100644
index 0000000..0033d39
--- /dev/null
+++ b/crates/quince-core/src/fold.rs
@@ -0,0 +1,114 @@
+//! Comparing text the way a person reads it rather than the way it is encoded.
+//!
+//! SQLite folds case for ASCII alone, and byte order puts "Émile" after "Zoya". Apple's apps
+//! search and sort without regard to case or accents, so every search and every name-sorted
+//! view goes through here instead.
+
+use std::cmp::Ordering;
+
+/// Accents that sit on a letter without changing which letter it is.
+///
+/// Named ranges rather than every combining mark: the vowel signs of Devanagari, Tamil and
+/// the rest are combining marks too, and dropping them turns one word into another.
+fn is_accent(c: char) -> bool {
+ matches!(c,
+ '\u{0300}'..='\u{036F}'
+ | '\u{1AB0}'..='\u{1AFF}'
+ | '\u{1DC0}'..='\u{1DFF}'
+ | '\u{20D0}'..='\u{20FF}'
+ | '\u{FE20}'..='\u{FE2F}')
+}
+
+/// `text` with case and accents gone, so "Émile", "EMILE" and "emile" are one word.
+#[must_use]
+pub fn fold(text: &str) -> String {
+ icu_normalizer::DecomposingNormalizerBorrowed::new_nfd()
+ .normalize(text)
+ .chars()
+ .filter(|&c| !is_accent(c))
+ .flat_map(char::to_lowercase)
+ .collect()
+}
+
+/// The words of a query, folded; empty when there is nothing to look for.
+#[must_use]
+pub fn words(query: &str) -> Vec<String> {
+ query.split_whitespace().map(fold).collect()
+}
+
+/// True when every word appears somewhere in `haystack`, which must already be folded.
+#[must_use]
+pub fn holds_all(haystack: &str, words: &[String]) -> bool {
+ words.iter().all(|w| haystack.contains(w.as_str()))
+}
+
+/// Names in the order Finder lists them: case and accents ignored, and runs of digits
+/// compared as numbers, so "Scan 2" comes before "Scan 10".
+#[must_use]
+pub fn natural(a: &str, b: &str) -> Ordering {
+ let (a, b) = (fold(a), fold(b));
+ let (mut x, mut y) = (a.chars().peekable(), b.chars().peekable());
+ loop {
+ match (x.peek().copied(), y.peek().copied()) {
+ (None, None) => return Ordering::Equal,
+ (None, Some(_)) => return Ordering::Less,
+ (Some(_), None) => return Ordering::Greater,
+ (Some(p), Some(q)) if p.is_ascii_digit() && q.is_ascii_digit() => {
+ let m: String = std::iter::from_fn(|| x.next_if(char::is_ascii_digit)).collect();
+ let n: String = std::iter::from_fn(|| y.next_if(char::is_ascii_digit)).collect();
+ let (m, n) = (m.trim_start_matches('0'), n.trim_start_matches('0'));
+ // More digits is a bigger number; equal lengths compare digit by digit. No
+ // parsing, so a forty-digit run in a file name cannot overflow anything.
+ let order = m.len().cmp(&n.len()).then_with(|| m.cmp(n));
+ if order != Ordering::Equal {
+ return order;
+ }
+ }
+ (Some(p), Some(q)) => {
+ if p != q {
+ return p.cmp(&q);
+ }
+ x.next();
+ y.next();
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn case_and_accents_fold_away() {
+ assert_eq!(fold("Émile"), "emile");
+ assert_eq!(fold("ZOË"), "zoe");
+ assert_eq!(fold("Straße"), "straße");
+ // Composed and decomposed spellings of the same name are the same name.
+ assert_eq!(fold("Jose\u{301}"), fold("José"));
+ }
+
+ #[test]
+ fn vowel_signs_are_not_accents() {
+ // "कि" and "क" are different syllables; folding one into the other would be wrong.
+ assert_ne!(fold("किताब"), fold("कताब"));
+ assert_eq!(fold("किताब"), "किताब");
+ }
+
+ #[test]
+ fn numbers_in_names_sort_as_numbers() {
+ let mut names = vec!["Scan 10", "scan 2", "Scan 1", "Éclair", "apple", "Zebra"];
+ names.sort_by(|a, b| natural(a, b));
+ assert_eq!(
+ names,
+ ["apple", "Éclair", "Scan 1", "scan 2", "Scan 10", "Zebra"]
+ );
+ }
+
+ #[test]
+ fn long_digit_runs_do_not_overflow() {
+ let big = format!("IMG {}", "9".repeat(40));
+ assert_eq!(natural("IMG 1", &big), Ordering::Less);
+ assert_eq!(natural("IMG 007", "IMG 7"), Ordering::Equal);
+ }
+}
diff --git a/crates/quince-core/src/lib.rs b/crates/quince-core/src/lib.rs
index ba3b101..dc8a0d0 100644
--- a/crates/quince-core/src/lib.rs
+++ b/crates/quince-core/src/lib.rs
@@ -7,6 +7,7 @@
pub mod calendar;
pub mod contacts;
pub mod drive;
+pub mod fold;
pub mod health;
pub mod notes;
pub mod reminders;
diff --git a/crates/quince-core/src/notes.rs b/crates/quince-core/src/notes.rs
index e918000..de1d70b 100644
--- a/crates/quince-core/src/notes.rs
+++ b/crates/quince-core/src/notes.rs
@@ -262,12 +262,13 @@ impl Library {
/// Notes whose title or text holds every word of `query`, in the list's own order.
///
/// Matched here rather than in SQL, because SQLite folds case for ASCII alone: a search
- /// for "Café" would miss "café", and so would anything written in a script with case of
- /// its own. Every word has to appear, in any order, which is how Apple's search behaves.
+ /// for "café" would miss "CAFÉ", and so would anything written in a script with case of
+ /// its own. Accents are ignored as they are on the phone, so "cafe" finds it too. Every
+ /// word has to appear, in any order, which is how Apple's search behaves.
/// Recently Deleted is left out, as it is on the phone. Bodies are read to match against
/// and never returned — the results cross into the window like any other list.
pub fn search_notes(&self, query: &str) -> Result<Vec<Note>> {
- let words: Vec<String> = query.split_whitespace().map(str::to_lowercase).collect();
+ let words = crate::fold::words(query);
if words.is_empty() {
return Ok(Vec::new());
}
@@ -301,8 +302,8 @@ impl Library {
let mut found = Vec::new();
for row in rows {
let (note, body) = row?;
- let haystack = format!("{}\n{}", note.title, body).to_lowercase();
- if words.iter().all(|w| haystack.contains(w.as_str())) {
+ let haystack = format!("{}\n{}", note.title, body);
+ if crate::fold::holds_all(&crate::fold::fold(&haystack), &words) {
found.push(note);
}
}
@@ -382,6 +383,12 @@ mod search_tests {
assert_eq!(found(&library(), "తెలుగు"), vec!["C"]);
}
+ /// Accents are ignored, as on the phone: "cafe" finds "Café".
+ #[test]
+ fn accents_are_ignored() {
+ assert_eq!(found(&library(), "cafe"), vec!["B"]);
+ }
+
/// Every word must be there, in any order.
#[test]
fn every_word_has_to_appear() {
diff --git a/crates/quince-core/src/reminders.rs b/crates/quince-core/src/reminders.rs
index f5f168a..c45a71f 100644
--- a/crates/quince-core/src/reminders.rs
+++ b/crates/quince-core/src/reminders.rs
@@ -630,7 +630,7 @@ impl Library {
/// ASCII alone. Bounded like every other view, so a search that matches everything on a
/// ten-year account does not carry all of it into the window.
pub fn search_reminders(&self, query: &str) -> Result<Vec<Reminder>> {
- let words: Vec<String> = query.split_whitespace().map(str::to_lowercase).collect();
+ let words = crate::fold::words(query);
if words.is_empty() {
return Ok(Vec::new());
}
@@ -665,9 +665,8 @@ impl Library {
let mut found = Vec::new();
for row in rows {
let r = row?;
- let haystack =
- format!("{}\n{}", r.title, r.notes.as_deref().unwrap_or("")).to_lowercase();
- if words.iter().all(|w| haystack.contains(w.as_str())) {
+ let haystack = format!("{}\n{}", r.title, r.notes.as_deref().unwrap_or(""));
+ if crate::fold::holds_all(&crate::fold::fold(&haystack), &words) {
found.push(r);
if found.len() == MOST_AT_ONCE {
break;
@@ -1670,6 +1669,7 @@ mod search_tests {
#[test]
fn case_folds_beyond_ascii_and_every_word_counts() {
assert_eq!(found("CAFÉ"), vec!["D"]);
+ assert_eq!(found("cafe"), vec!["D"]);
assert_eq!(found("run evening"), vec!["A"]);
assert!(found("evening plants").is_empty());
assert!(found(" ").is_empty());