Zulip server and web application. Open-source team chat that helps teams stay productive and focused.
typeahead: Add diacritic-prefix bucket to triage_raw.
Aadyot1fc5159f5346b
diff --git a/web/src/typeahead.ts b/web/src/typeahead.tsindex 06ac256f06d..0241749671d 100644--- a/web/src/typeahead.ts+++ b/web/src/typeahead.ts@@ -259,32 +259,47 @@ export function triage_raw<T>(get_items: (x: T) => string[],): {exact_matches: T[];+ // Only populated when the query itself contains diacritics; holds items+ // whose lowercased form starts with the lowercased query (diacritics intact).+ begins_with_case_insensitive_diacritic_matches: T[];begins_with_case_sensitive_matches: T[];begins_with_case_insensitive_matches: T[];word_boundary_matches: T[];no_matches: T[];} {/*- We split objs into five groups:+ We split objs into six groups:- entire string exact match- - match prefix exactly with `query`- - match prefix case-insensitively- - match word boundary prefix case-insensitively+ - case-insensitive prefix match preserving diacritics in the query+ - prefix match with `query` exactly (case-sensitive)+ - prefix match case-insensitively+ - word-boundary prefix match case-insensitively- otherand return an object of these.++ A query with diacritics also matches after stripping diacritics from+ both sides, so "gaë" matches "Gael" but ranks below "Gaël". A query+ without diacritics gets no such fallback, so "a" doesn't prefix-match+ "Ądam".*/const exact_matches = [];+ const begins_with_case_insensitive_diacritic_matches = [];const begins_with_case_sensitive_matches = [];const begins_with_case_insensitive_matches = [];const word_boundary_matches = [];const no_matches = [];const lower_query = query ? query.toLowerCase() : "";+ const diacritic_stripped_query = remove_diacritics(lower_query);+ const query_has_diacritics = lower_query !== diacritic_stripped_query;- const word_boundary_match_regex = new RegExp(+ const lower_word_boundary_regex = new RegExp(`[${word_boundary_chars}]${_.escapeRegExp(lower_query)}`,);+ const diacritic_stripped_word_boundary_regex = new RegExp(+ `[${word_boundary_chars}]${_.escapeRegExp(diacritic_stripped_query)}`,+ );for (const obj of objs) {const items = get_items(obj);@@ -293,11 +308,28 @@ export function triage_raw<T>(if (lower_items.includes(lower_query)) {exact_matches.push(obj);+ } else if (+ query_has_diacritics &&+ lower_items.some((item) => item.startsWith(lower_query))+ ) {+ begins_with_case_insensitive_diacritic_matches.push(obj);} else if (items.some((item) => item.startsWith(query))) {begins_with_case_sensitive_matches.push(obj);} else if (lower_items.some((item) => item.startsWith(lower_query))) {begins_with_case_insensitive_matches.push(obj);- } else if (lower_items.some((item) => word_boundary_match_regex.test(item))) {+ } else if (+ query_has_diacritics &&+ lower_items.some((item) => remove_diacritics(item).startsWith(diacritic_stripped_query))+ ) {+ begins_with_case_insensitive_matches.push(obj);+ } else if (lower_items.some((item) => lower_word_boundary_regex.test(item))) {+ word_boundary_matches.push(obj);+ } else if (+ query_has_diacritics &&+ lower_items.some((item) =>+ diacritic_stripped_word_boundary_regex.test(remove_diacritics(item)),+ )+ ) {word_boundary_matches.push(obj);} else {no_matches.push(obj);@@ -306,6 +338,7 @@ export function triage_raw<T>(return {exact_matches,+ begins_with_case_insensitive_diacritic_matches,begins_with_case_sensitive_matches,begins_with_case_insensitive_matches,word_boundary_matches,@@ -321,6 +354,7 @@ export function triage<T>(): {matches: T[]; rest: T[]} {const {exact_matches,+ begins_with_case_insensitive_diacritic_matches,begins_with_case_sensitive_matches,begins_with_case_insensitive_matches,word_boundary_matches,@@ -335,6 +369,7 @@ export function triage<T>(return {matches: [...exact_matches.toSorted(sorting_comparator),+ ...begins_with_case_insensitive_diacritic_matches.toSorted(sorting_comparator),...beginning_matches_sorted,...word_boundary_matches.toSorted(sorting_comparator),],@@ -345,6 +380,7 @@ export function triage<T>(return {matches: [...exact_matches,+ ...begins_with_case_insensitive_diacritic_matches,...begins_with_case_sensitive_matches,...begins_with_case_insensitive_matches,...word_boundary_matches,diff --git a/web/tests/typeahead.test.cjs b/web/tests/typeahead.test.cjsindex a3938d21854..b5200869c12 100644--- a/web/tests/typeahead.test.cjs+++ b/web/tests/typeahead.test.cjs@@ -208,6 +208,84 @@ run_test("triage: prioritise word boundary matches to arbitrary substring matche);});+run_test("triage: prioritize exact diacritic matches over diacritic-stripped matches", () => {+ const exact_a = {name: "a"};+ const exact_diacritic = {name: "ą"};+ const aaron = {name: "aaron"};+ const aa = {name: "Aa"};+ const adam_diacritic = {name: "Ądam"};+ const john_a = {name: "John a"};+ const zoe = {name: "Zoe"};++ const users = [exact_a, exact_diacritic, aaron, aa, adam_diacritic, john_a, zoe];++ assert.deepEqual(+ typeahead.triage("a", users, (r) => r.name),+ {+ matches: [exact_a, aaron, aa, john_a],+ rest: [exact_diacritic, adam_diacritic, zoe],+ },+ );++ assert.deepEqual(+ typeahead.triage("A", users, (r) => r.name),+ {+ matches: [exact_a, aa, aaron, john_a],+ rest: [exact_diacritic, adam_diacritic, zoe],+ },+ );++ assert.deepEqual(+ typeahead.triage("ą", users, (r) => r.name),+ {+ matches: [exact_diacritic, adam_diacritic, exact_a, aaron, aa, john_a],+ rest: [zoe],+ },+ );++ assert.deepEqual(+ typeahead.triage("Ą", users, (r) => r.name),+ {+ matches: [exact_diacritic, adam_diacritic, exact_a, aaron, aa, john_a],+ rest: [zoe],+ },+ );+});++run_test("triage: ASCII query matches diacritic item via word boundary", () => {+ const luke_adam = {name: "Łuke adam"};+ const zoe = {name: "Zoe"};+ const users = [luke_adam, zoe];++ assert.deepEqual(+ typeahead.triage("a", users, (r) => r.name),+ {+ matches: [luke_adam],+ rest: [zoe],+ },+ );+});++run_test("triage_raw: diacritic-prefix and diacritic-stripped fallback", () => {+ const adam_diacritic = {names: ["Ądam"]};+ const joe_ascii = {names: ["Joe Maria"]};+ const alex_joe = {names: ["Alex joe"]};+ const zoe = {names: ["Zoe"]};+ const items = [adam_diacritic, joe_ascii, alex_joe, zoe];++ const diacritic_result = typeahead.triage_raw("ą", items, (item) => item.names);+ assert.deepEqual(diacritic_result.begins_with_case_insensitive_diacritic_matches, [+ adam_diacritic,+ ]);+ // The query has diacritics, so the diacritic-stripped fallback still lets an+ // ASCII candidate ("Alex joe") land in the case-insensitive bucket.+ assert.deepEqual(diacritic_result.begins_with_case_insensitive_matches, [alex_joe]);++ const fallback_result = typeahead.triage_raw("joé", items, (item) => item.names);+ assert.deepEqual(fallback_result.begins_with_case_insensitive_matches, [joe_ascii]);+ assert.deepEqual(fallback_result.word_boundary_matches, [alex_joe]);+});+function sort_emojis(emojis, query) {return typeahead.sort_emojis(emojis, query).map((emoji) => emoji.emoji_name);}diff --git a/web/tests/typeahead_helper.test.cjs b/web/tests/typeahead_helper.test.cjsindex 1e2918227d5..94e92bc1ba5 100644--- a/web/tests/typeahead_helper.test.cjs+++ b/web/tests/typeahead_helper.test.cjs@@ -1904,6 +1904,25 @@ test("render_person with matching custom profile field but email hidden", ({assert.ok(rendered);});+test("query_matches_person_name keeps diacritics when the query has them", () => {+ // When should_remove_diacritics is false (used when the query itself has+ // diacritics), the name keeps its diacritics, so a diacritic query matches+ // a name with the same diacritics but not its ASCII counterpart.+ const adam_diacritic = make_user({+ email: "[email protected]",+ full_name: "Ądam",+ user_id: 101,+ });+ const adam_ascii = make_user({+ email: "[email protected]",+ full_name: "adam",+ user_id: 102,+ });++ assert.equal(th.query_matches_person_name("ądam", user_item(adam_diacritic), false), true);+ assert.equal(th.query_matches_person_name("ądam", user_item(adam_ascii), false), false);+});+test("query_matches_person matches custom profile fields when they are enabled for user matching ", ({override,}) => {