Skip to content

armgit / zulip

Public Imported from GitHub
0

Zulip server and web application. Open-source team chat that helps teams stay productive and focused.

typeahead: Add diacritic-prefix bucket to triage_raw.

Aadyot1fc5159f5346b
diff --git a/web/src/typeahead.ts b/web/src/typeahead.ts
index 06ac256f06d..0241749671d 100644
--- a/web/src/typeahead.ts
+++ b/web/src/typeahead.ts
@@ -259,32 +259,47 @@ export function triage_raw<T>(
get_items: (x: T) => string[],
): {
exact_matches: T[];
+ // Only populated when the query itself contains diacritics; holds items
+ // whose lowercased form starts with the lowercased query (diacritics intact).
+ begins_with_case_insensitive_diacritic_matches: T[];
begins_with_case_sensitive_matches: T[];
begins_with_case_insensitive_matches: T[];
word_boundary_matches: T[];
no_matches: T[];
} {
/*
- We split objs into five groups:
+ We split objs into six groups:
- entire string exact match
- - match prefix exactly with `query`
- - match prefix case-insensitively
- - match word boundary prefix case-insensitively
+ - case-insensitive prefix match preserving diacritics in the query
+ - prefix match with `query` exactly (case-sensitive)
+ - prefix match case-insensitively
+ - word-boundary prefix match case-insensitively
- other
and return an object of these.
+
+ A query with diacritics also matches after stripping diacritics from
+ both sides, so "gaë" matches "Gael" but ranks below "Gaël". A query
+ without diacritics gets no such fallback, so "a" doesn't prefix-match
+ "Ądam".
*/
const exact_matches = [];
+ const begins_with_case_insensitive_diacritic_matches = [];
const begins_with_case_sensitive_matches = [];
const begins_with_case_insensitive_matches = [];
const word_boundary_matches = [];
const no_matches = [];
const lower_query = query ? query.toLowerCase() : "";
+ const diacritic_stripped_query = remove_diacritics(lower_query);
+ const query_has_diacritics = lower_query !== diacritic_stripped_query;
- const word_boundary_match_regex = new RegExp(
+ const lower_word_boundary_regex = new RegExp(
`[${word_boundary_chars}]${_.escapeRegExp(lower_query)}`,
);
+ const diacritic_stripped_word_boundary_regex = new RegExp(
+ `[${word_boundary_chars}]${_.escapeRegExp(diacritic_stripped_query)}`,
+ );
for (const obj of objs) {
const items = get_items(obj);
@@ -293,11 +308,28 @@ export function triage_raw<T>(
if (lower_items.includes(lower_query)) {
exact_matches.push(obj);
+ } else if (
+ query_has_diacritics &&
+ lower_items.some((item) => item.startsWith(lower_query))
+ ) {
+ begins_with_case_insensitive_diacritic_matches.push(obj);
} else if (items.some((item) => item.startsWith(query))) {
begins_with_case_sensitive_matches.push(obj);
} else if (lower_items.some((item) => item.startsWith(lower_query))) {
begins_with_case_insensitive_matches.push(obj);
- } else if (lower_items.some((item) => word_boundary_match_regex.test(item))) {
+ } else if (
+ query_has_diacritics &&
+ lower_items.some((item) => remove_diacritics(item).startsWith(diacritic_stripped_query))
+ ) {
+ begins_with_case_insensitive_matches.push(obj);
+ } else if (lower_items.some((item) => lower_word_boundary_regex.test(item))) {
+ word_boundary_matches.push(obj);
+ } else if (
+ query_has_diacritics &&
+ lower_items.some((item) =>
+ diacritic_stripped_word_boundary_regex.test(remove_diacritics(item)),
+ )
+ ) {
word_boundary_matches.push(obj);
} else {
no_matches.push(obj);
@@ -306,6 +338,7 @@ export function triage_raw<T>(
return {
exact_matches,
+ begins_with_case_insensitive_diacritic_matches,
begins_with_case_sensitive_matches,
begins_with_case_insensitive_matches,
word_boundary_matches,
@@ -321,6 +354,7 @@ export function triage<T>(
): {matches: T[]; rest: T[]} {
const {
exact_matches,
+ begins_with_case_insensitive_diacritic_matches,
begins_with_case_sensitive_matches,
begins_with_case_insensitive_matches,
word_boundary_matches,
@@ -335,6 +369,7 @@ export function triage<T>(
return {
matches: [
...exact_matches.toSorted(sorting_comparator),
+ ...begins_with_case_insensitive_diacritic_matches.toSorted(sorting_comparator),
...beginning_matches_sorted,
...word_boundary_matches.toSorted(sorting_comparator),
],
@@ -345,6 +380,7 @@ export function triage<T>(
return {
matches: [
...exact_matches,
+ ...begins_with_case_insensitive_diacritic_matches,
...begins_with_case_sensitive_matches,
...begins_with_case_insensitive_matches,
...word_boundary_matches,
diff --git a/web/tests/typeahead.test.cjs b/web/tests/typeahead.test.cjs
index a3938d21854..b5200869c12 100644
--- a/web/tests/typeahead.test.cjs
+++ b/web/tests/typeahead.test.cjs
@@ -208,6 +208,84 @@ run_test("triage: prioritise word boundary matches to arbitrary substring matche
);
});
+run_test("triage: prioritize exact diacritic matches over diacritic-stripped matches", () => {
+ const exact_a = {name: "a"};
+ const exact_diacritic = {name: "ą"};
+ const aaron = {name: "aaron"};
+ const aa = {name: "Aa"};
+ const adam_diacritic = {name: "Ądam"};
+ const john_a = {name: "John a"};
+ const zoe = {name: "Zoe"};
+
+ const users = [exact_a, exact_diacritic, aaron, aa, adam_diacritic, john_a, zoe];
+
+ assert.deepEqual(
+ typeahead.triage("a", users, (r) => r.name),
+ {
+ matches: [exact_a, aaron, aa, john_a],
+ rest: [exact_diacritic, adam_diacritic, zoe],
+ },
+ );
+
+ assert.deepEqual(
+ typeahead.triage("A", users, (r) => r.name),
+ {
+ matches: [exact_a, aa, aaron, john_a],
+ rest: [exact_diacritic, adam_diacritic, zoe],
+ },
+ );
+
+ assert.deepEqual(
+ typeahead.triage("ą", users, (r) => r.name),
+ {
+ matches: [exact_diacritic, adam_diacritic, exact_a, aaron, aa, john_a],
+ rest: [zoe],
+ },
+ );
+
+ assert.deepEqual(
+ typeahead.triage("Ą", users, (r) => r.name),
+ {
+ matches: [exact_diacritic, adam_diacritic, exact_a, aaron, aa, john_a],
+ rest: [zoe],
+ },
+ );
+});
+
+run_test("triage: ASCII query matches diacritic item via word boundary", () => {
+ const luke_adam = {name: "Łuke adam"};
+ const zoe = {name: "Zoe"};
+ const users = [luke_adam, zoe];
+
+ assert.deepEqual(
+ typeahead.triage("a", users, (r) => r.name),
+ {
+ matches: [luke_adam],
+ rest: [zoe],
+ },
+ );
+});
+
+run_test("triage_raw: diacritic-prefix and diacritic-stripped fallback", () => {
+ const adam_diacritic = {names: ["Ądam"]};
+ const joe_ascii = {names: ["Joe Maria"]};
+ const alex_joe = {names: ["Alex joe"]};
+ const zoe = {names: ["Zoe"]};
+ const items = [adam_diacritic, joe_ascii, alex_joe, zoe];
+
+ const diacritic_result = typeahead.triage_raw("ą", items, (item) => item.names);
+ assert.deepEqual(diacritic_result.begins_with_case_insensitive_diacritic_matches, [
+ adam_diacritic,
+ ]);
+ // The query has diacritics, so the diacritic-stripped fallback still lets an
+ // ASCII candidate ("Alex joe") land in the case-insensitive bucket.
+ assert.deepEqual(diacritic_result.begins_with_case_insensitive_matches, [alex_joe]);
+
+ const fallback_result = typeahead.triage_raw("joé", items, (item) => item.names);
+ assert.deepEqual(fallback_result.begins_with_case_insensitive_matches, [joe_ascii]);
+ assert.deepEqual(fallback_result.word_boundary_matches, [alex_joe]);
+});
+
function sort_emojis(emojis, query) {
return typeahead.sort_emojis(emojis, query).map((emoji) => emoji.emoji_name);
}
diff --git a/web/tests/typeahead_helper.test.cjs b/web/tests/typeahead_helper.test.cjs
index 1e2918227d5..94e92bc1ba5 100644
--- a/web/tests/typeahead_helper.test.cjs
+++ b/web/tests/typeahead_helper.test.cjs
@@ -1904,6 +1904,25 @@ test("render_person with matching custom profile field but email hidden", ({
assert.ok(rendered);
});
+test("query_matches_person_name keeps diacritics when the query has them", () => {
+ // When should_remove_diacritics is false (used when the query itself has
+ // diacritics), the name keeps its diacritics, so a diacritic query matches
+ // a name with the same diacritics but not its ASCII counterpart.
+ const adam_diacritic = make_user({
+ email: "[email protected]",
+ full_name: "Ądam",
+ user_id: 101,
+ });
+ const adam_ascii = make_user({
+ email: "[email protected]",
+ full_name: "adam",
+ user_id: 102,
+ });
+
+ assert.equal(th.query_matches_person_name("ądam", user_item(adam_diacritic), false), true);
+ assert.equal(th.query_matches_person_name("ądam", user_item(adam_ascii), false), false);
+});
+
test("query_matches_person matches custom profile fields when they are enabled for user matching ", ({
override,
}) => {