Zulip server and web application. Open-source team chat that helps teams stay productive and focused.
people: Fetch users in batches to bound the request URL length.
Aman Agrawald01fc67b90a2
diff --git a/web/src/people.ts b/web/src/people.tsindex db2e904c5ec..16a823cc33e 100644--- a/web/src/people.ts+++ b/web/src/people.ts@@ -1874,14 +1874,18 @@ async function start_fetch_for_requested_users(): Promise<void> {fetch_users_storage.promise_for_pending = undefined;- let fetched_users;- for (let num_attempts = 1; ; num_attempts += 1) {+ const user_ids_remaining = get_valid_user_ids_to_fetch(user_ids_pending_fetch);+ let num_consecutive_failures = 0;+ while (user_ids_remaining.size > 0) {try {- fetched_users = await fetch_users(user_ids_pending_fetch);- break;+ // Each batch lands in our data structures as it arrives, so+ // that its users are usable, and are not requested again, while+ // the later batches are still in flight.+ add_fetched_users(await fetch_next_user_batch(user_ids_remaining));+ num_consecutive_failures = 0;} catch (error) {- // Retry on error.- const retry_delay_secs = get_retry_backoff_seconds(undefined, num_attempts);+ num_consecutive_failures += 1;+ const retry_delay_secs = get_retry_backoff_seconds(undefined, num_consecutive_failures);// Since users are in `valid_user_ids`, we expect// the fetch to eventually succeed, so we log a warning@@ -1896,17 +1900,6 @@ async function start_fetch_for_requested_users(): Promise<void> {}}- for (const user of fetched_users) {- if (user.is_active) {- add_active_user(user);- } else {- if (!user.is_deleted) {- non_active_user_dict.set(user.user_id, user);- }- _add_user(user);- }- }-// Resolve promises waiting on this fetch after updating the data locally.fetch_users_storage.promise_for_in_transit.get(user_ids_pending_fetch)!.resolve();// Clean up in transit promise for this fetch.@@ -2033,24 +2026,19 @@ export function get_users_that_match_role_ids(return users;}-export async function fetch_users(user_ids: Set<number>): Promise<UsersFetchResponse["members"]> {- // Requested users outside the set of known valid user IDs likely- // reflect some sort of Zulip bug, so fetch and log them.- const invalid_user_ids = user_ids.difference(valid_user_ids);- if (invalid_user_ids.size > 0) {- blueslip.error("Ignored invalid user_ids: " + [...invalid_user_ids].join(", "));- }+// nginx and the development proxy both reject a request line over 8KB,+// which 500 IDs stay inside even at the int4 maximum user ID. Batches go+// out one at a time, since concurrent ones would burst against the user's+// rate limit.+const MAX_USER_IDS_PER_FETCH = 500;- const user_ids_to_fetch = valid_user_ids.intersection(user_ids);- if (user_ids_to_fetch.size === 0) {- return [];- }+async function fetch_user_batch(user_ids: number[]): Promise<UsersFetchResponse["members"]> {return new Promise((resolve, reject) => {fetch_users_from_server({// POST /register obtains custom profile field data if and only if// the current user is not a spectator. Mimic this behavior.include_custom_profile_fields: !page_params.is_spectator,- user_ids: JSON.stringify([...user_ids_to_fetch]),+ user_ids: JSON.stringify(user_ids),success(users) {resolve(users);},@@ -2070,6 +2058,41 @@ export async function fetch_users(user_ids: Set<number>): Promise<UsersFetchResp});}+function get_valid_user_ids_to_fetch(user_ids: Set<number>): Set<number> {+ // Requested users outside the set of known valid user IDs likely+ // reflect some sort of Zulip bug, so fetch and log them.+ const invalid_user_ids = user_ids.difference(valid_user_ids);+ if (invalid_user_ids.size > 0) {+ blueslip.error("Ignored invalid user_ids: " + [...invalid_user_ids].join(", "));+ }++ return valid_user_ids.intersection(user_ids);+}++async function fetch_next_user_batch(+ user_ids_remaining: Set<number>,+): Promise<UsersFetchResponse["members"]> {+ const batch = [...user_ids_remaining].slice(0, MAX_USER_IDS_PER_FETCH);+ const fetched_users = await fetch_user_batch(batch);+ for (const user_id of batch) {+ user_ids_remaining.delete(user_id);+ }+ return fetched_users;+}++function add_fetched_users(fetched_users: UsersFetchResponse["members"]): void {+ for (const user of fetched_users) {+ if (user.is_active) {+ add_active_user(user);+ } else {+ if (!user.is_deleted) {+ non_active_user_dict.set(user.user_id, user);+ }+ _add_user(user);+ }+ }+}+export async function initialize(my_user_id: number,people_params: StateData["people"],diff --git a/web/tests/people.test.cjs b/web/tests/people.test.cjsindex a1c0ef4e565..e79b38c09f3 100644--- a/web/tests/people.test.cjs+++ b/web/tests/people.test.cjs@@ -1674,7 +1674,7 @@ run_test("reset MockDate", () => {clock.reset();});-run_test("fetch_users retry", async ({override, override_rewire}) => {+run_test("user fetch retries a failed request", async ({override, override_rewire}) => {initialize();people.add_valid_user_id(1);let retry_count = 1;@@ -1724,7 +1724,7 @@ run_test("fetch_users retry", async ({override, override_rewire}) => {await people.fetch_users_from_ids_internal([1]);});-run_test("fetch_users", async ({override}) => {+run_test("user fetch during initialize", async ({override}) => {initialize();people.init();@@ -1835,29 +1835,163 @@ run_test("fetch_users", async ({override}) => {assert.equal(people.get_by_email("[email protected]").user_id, 16);assert.equal(people.get_by_email("[email protected]").user_id, 15);- override(channel, "get", ({url, _data, _success, error}) => {- assert.equal(url, "/json/users");- // Return error response.- error({responseJSON: {msg: "test error"}});- });+ blueslip.expect("error", "Ignored invalid user_ids: 1, 2");+ assert.deepEqual(await people.get_or_fetch_users_from_ids([1, 2]), []);+ assert.equal(user_GET_request_calls, 1);+});++function make_fetched_user(user_id) {+ return {+ email: `user${user_id}@example.com`,+ user_id,+ full_name: `User ${user_id}`,+ delivery_email: "",+ date_joined: "",+ is_active: true,+ is_owner: false,+ is_admin: false,+ is_guest: false,+ role: Role.MEMBER,+ avatar_url: "",+ avatar_version: 1,+ is_bot: false,+ is_imported_stub: false,+ };+}- await assert.rejects(- async () => {- await people.fetch_users(new Set([15, 16]));+function make_params_for_users_missing_from_register(my_user_id, missing_user_ids) {+ return {+ people_params: {+ realm_users: [+ {+ email: "[email protected]",+ user_id: my_user_id,+ full_name: "Me Myself",+ },+ ],+ realm_non_active_users: [],+ cross_realm_bots: [],},- (err) => {- // Check that the error is an instance of Error and has the correct message- assert.ok(err instanceof Error);- assert.equal(err.message, "test error");- return true;+ user_group_params: {+ realm_user_groups: [+ make_user_group({+ is_system_group: true,+ members: [my_user_id, ...missing_user_ids],+ }),+ ],},+ };+}++run_test("user fetch batches a large request", async ({override}) => {+ initialize();+ people.init();++ const my_user_id = 42;+ const user_ids_to_fetch = Array.from({length: 1200}, (_unused, index) => 1000 + index);+ const {people_params, user_group_params} = make_params_for_users_missing_from_register(+ my_user_id,+ user_ids_to_fetch,);- blueslip.expect("error", "Ignored invalid user_ids: 1, 2");- await people.fetch_users(new Set([1, 2]));+ const requested_batches = [];+ override(channel, "get", ({url, data, success}) => {+ assert.equal(url, "/json/users");+ const batch = JSON.parse(data.user_ids);+ // Users from the earlier batches are already known by the time we+ // request this one, rather than only once every batch has arrived.+ assert.ok(requested_batches.flat().every((user_id) => people.is_known_user_id(user_id)));+ requested_batches.push(batch);+ success({+ members: batch.map((user_id) => make_fetched_user(user_id)),+ result: "success",+ msg: "",+ });+ });++ await people.initialize(my_user_id, people_params, user_group_params);++ assert.deepEqual(+ requested_batches.map((batch) => batch.length),+ [500, 500, 200],+ );+ assert.deepEqual(+ requested_batches.flat().toSorted((a, b) => a - b),+ user_ids_to_fetch,+ );++ for (const user_id of user_ids_to_fetch) {+ assert.equal(people.get_by_user_id(user_id).full_name, `User ${user_id}`);+ }+});++run_test("user fetch resumes after a failed batch", async ({override, override_rewire}) => {+ // Inaccessible users make a batch succeed while returning no members,+ // which is what requires tracking progress by the IDs we sent.+ initialize();+ people.init();+ override(settings_data, "user_can_access_all_other_users", () => false);++ const my_user_id = 42;+ const inaccessible_user_ids = Array.from({length: 1200}, (_unused, index) => 1000 + index);+ const {people_params, user_group_params} = make_params_for_users_missing_from_register(+ my_user_id,+ inaccessible_user_ids,+ );++ const response_with_no_accessible_users = {members: [], result: "success", msg: ""};+ const requested_batches = [];+ // Requests 2 and 4 are the first attempts at the second and third+ // batches, so a success falls between the two failures.+ const failing_request_numbers = new Set([2, 4]);+ override(channel, "get", ({url, data, success, error}) => {+ assert.equal(url, "/json/users");+ requested_batches.push(JSON.parse(data.user_ids));++ if (failing_request_numbers.has(requested_batches.length)) {+ error({responseJSON: {msg: "test error"}});+ return;+ }++ success(response_with_no_accessible_users);+ });++ const backoff_attempt_counts = [];+ override_rewire(retry_backoff, "get_retry_backoff_seconds", (_xhr, attempts) => {+ backoff_attempt_counts.push(attempts);+ return 0;+ });+ blueslip.expect(+ "warn",+ "Fetch for users failed, retrying after 0 seconds. Error: test error",+ 2,+ );++ await people.initialize(my_user_id, people_params, user_group_params);++ // Only the failed batches are requested twice; the rest are not refetched.+ assert.deepEqual(+ requested_batches.map((batch) => batch.length),+ [500, 500, 500, 200, 200],+ );+ assert.deepEqual(requested_batches[1], requested_batches[2]);+ assert.deepEqual(requested_batches[3], requested_batches[4]);++ const request_count_by_user_id = new Map();+ for (const user_id of requested_batches.flat()) {+ request_count_by_user_id.set(user_id, (request_count_by_user_id.get(user_id) ?? 0) + 1);+ }+ assert.ok(requested_batches[0].every((user_id) => request_count_by_user_id.get(user_id) === 1));++ // The counter restarts at 1 because a batch succeeded between the failures.+ assert.deepEqual(backoff_attempt_counts, [1, 1]);++ for (const user_id of inaccessible_user_ids) {+ assert.ok(people.get_by_user_id(user_id).is_inaccessible_user);+ }});-run_test("fetch_users corner case", async ({override, override_rewire}) => {+run_test("user fetch corner case", async ({override, override_rewire}) => {// We test here for the following sequence of fetches:// 1st request for [1, 2].// 2nd request for [1, 2, 3] while the first one is in-flight.