Skip to content

armgit / zulip

Public Imported from GitHub
0

Zulip server and web application. Open-source team chat that helps teams stay productive and focused.

people: Fetch users in batches to bound the request URL length.

Aman Agrawald01fc67b90a2
diff --git a/web/src/people.ts b/web/src/people.ts
index db2e904c5ec..16a823cc33e 100644
--- a/web/src/people.ts
+++ b/web/src/people.ts
@@ -1874,14 +1874,18 @@ async function start_fetch_for_requested_users(): Promise<void> {
fetch_users_storage.promise_for_pending = undefined;
- let fetched_users;
- for (let num_attempts = 1; ; num_attempts += 1) {
+ const user_ids_remaining = get_valid_user_ids_to_fetch(user_ids_pending_fetch);
+ let num_consecutive_failures = 0;
+ while (user_ids_remaining.size > 0) {
try {
- fetched_users = await fetch_users(user_ids_pending_fetch);
- break;
+ // Each batch lands in our data structures as it arrives, so
+ // that its users are usable, and are not requested again, while
+ // the later batches are still in flight.
+ add_fetched_users(await fetch_next_user_batch(user_ids_remaining));
+ num_consecutive_failures = 0;
} catch (error) {
- // Retry on error.
- const retry_delay_secs = get_retry_backoff_seconds(undefined, num_attempts);
+ num_consecutive_failures += 1;
+ const retry_delay_secs = get_retry_backoff_seconds(undefined, num_consecutive_failures);
// Since users are in `valid_user_ids`, we expect
// the fetch to eventually succeed, so we log a warning
@@ -1896,17 +1900,6 @@ async function start_fetch_for_requested_users(): Promise<void> {
}
}
- for (const user of fetched_users) {
- if (user.is_active) {
- add_active_user(user);
- } else {
- if (!user.is_deleted) {
- non_active_user_dict.set(user.user_id, user);
- }
- _add_user(user);
- }
- }
-
// Resolve promises waiting on this fetch after updating the data locally.
fetch_users_storage.promise_for_in_transit.get(user_ids_pending_fetch)!.resolve();
// Clean up in transit promise for this fetch.
@@ -2033,24 +2026,19 @@ export function get_users_that_match_role_ids(
return users;
}
-export async function fetch_users(user_ids: Set<number>): Promise<UsersFetchResponse["members"]> {
- // Requested users outside the set of known valid user IDs likely
- // reflect some sort of Zulip bug, so fetch and log them.
- const invalid_user_ids = user_ids.difference(valid_user_ids);
- if (invalid_user_ids.size > 0) {
- blueslip.error("Ignored invalid user_ids: " + [...invalid_user_ids].join(", "));
- }
+// nginx and the development proxy both reject a request line over 8KB,
+// which 500 IDs stay inside even at the int4 maximum user ID. Batches go
+// out one at a time, since concurrent ones would burst against the user's
+// rate limit.
+const MAX_USER_IDS_PER_FETCH = 500;
- const user_ids_to_fetch = valid_user_ids.intersection(user_ids);
- if (user_ids_to_fetch.size === 0) {
- return [];
- }
+async function fetch_user_batch(user_ids: number[]): Promise<UsersFetchResponse["members"]> {
return new Promise((resolve, reject) => {
fetch_users_from_server({
// POST /register obtains custom profile field data if and only if
// the current user is not a spectator. Mimic this behavior.
include_custom_profile_fields: !page_params.is_spectator,
- user_ids: JSON.stringify([...user_ids_to_fetch]),
+ user_ids: JSON.stringify(user_ids),
success(users) {
resolve(users);
},
@@ -2070,6 +2058,41 @@ export async function fetch_users(user_ids: Set<number>): Promise<UsersFetchResp
});
}
+function get_valid_user_ids_to_fetch(user_ids: Set<number>): Set<number> {
+ // Requested users outside the set of known valid user IDs likely
+ // reflect some sort of Zulip bug, so fetch and log them.
+ const invalid_user_ids = user_ids.difference(valid_user_ids);
+ if (invalid_user_ids.size > 0) {
+ blueslip.error("Ignored invalid user_ids: " + [...invalid_user_ids].join(", "));
+ }
+
+ return valid_user_ids.intersection(user_ids);
+}
+
+async function fetch_next_user_batch(
+ user_ids_remaining: Set<number>,
+): Promise<UsersFetchResponse["members"]> {
+ const batch = [...user_ids_remaining].slice(0, MAX_USER_IDS_PER_FETCH);
+ const fetched_users = await fetch_user_batch(batch);
+ for (const user_id of batch) {
+ user_ids_remaining.delete(user_id);
+ }
+ return fetched_users;
+}
+
+function add_fetched_users(fetched_users: UsersFetchResponse["members"]): void {
+ for (const user of fetched_users) {
+ if (user.is_active) {
+ add_active_user(user);
+ } else {
+ if (!user.is_deleted) {
+ non_active_user_dict.set(user.user_id, user);
+ }
+ _add_user(user);
+ }
+ }
+}
+
export async function initialize(
my_user_id: number,
people_params: StateData["people"],
diff --git a/web/tests/people.test.cjs b/web/tests/people.test.cjs
index a1c0ef4e565..e79b38c09f3 100644
--- a/web/tests/people.test.cjs
+++ b/web/tests/people.test.cjs
@@ -1674,7 +1674,7 @@ run_test("reset MockDate", () => {
clock.reset();
});
-run_test("fetch_users retry", async ({override, override_rewire}) => {
+run_test("user fetch retries a failed request", async ({override, override_rewire}) => {
initialize();
people.add_valid_user_id(1);
let retry_count = 1;
@@ -1724,7 +1724,7 @@ run_test("fetch_users retry", async ({override, override_rewire}) => {
await people.fetch_users_from_ids_internal([1]);
});
-run_test("fetch_users", async ({override}) => {
+run_test("user fetch during initialize", async ({override}) => {
initialize();
people.init();
@@ -1835,29 +1835,163 @@ run_test("fetch_users", async ({override}) => {
assert.equal(people.get_by_email("[email protected]").user_id, 16);
assert.equal(people.get_by_email("[email protected]").user_id, 15);
- override(channel, "get", ({url, _data, _success, error}) => {
- assert.equal(url, "/json/users");
- // Return error response.
- error({responseJSON: {msg: "test error"}});
- });
+ blueslip.expect("error", "Ignored invalid user_ids: 1, 2");
+ assert.deepEqual(await people.get_or_fetch_users_from_ids([1, 2]), []);
+ assert.equal(user_GET_request_calls, 1);
+});
+
+function make_fetched_user(user_id) {
+ return {
+ email: `user${user_id}@example.com`,
+ user_id,
+ full_name: `User ${user_id}`,
+ delivery_email: "",
+ date_joined: "",
+ is_active: true,
+ is_owner: false,
+ is_admin: false,
+ is_guest: false,
+ role: Role.MEMBER,
+ avatar_url: "",
+ avatar_version: 1,
+ is_bot: false,
+ is_imported_stub: false,
+ };
+}
- await assert.rejects(
- async () => {
- await people.fetch_users(new Set([15, 16]));
+function make_params_for_users_missing_from_register(my_user_id, missing_user_ids) {
+ return {
+ people_params: {
+ realm_users: [
+ {
+ email: "[email protected]",
+ user_id: my_user_id,
+ full_name: "Me Myself",
+ },
+ ],
+ realm_non_active_users: [],
+ cross_realm_bots: [],
},
- (err) => {
- // Check that the error is an instance of Error and has the correct message
- assert.ok(err instanceof Error);
- assert.equal(err.message, "test error");
- return true;
+ user_group_params: {
+ realm_user_groups: [
+ make_user_group({
+ is_system_group: true,
+ members: [my_user_id, ...missing_user_ids],
+ }),
+ ],
},
+ };
+}
+
+run_test("user fetch batches a large request", async ({override}) => {
+ initialize();
+ people.init();
+
+ const my_user_id = 42;
+ const user_ids_to_fetch = Array.from({length: 1200}, (_unused, index) => 1000 + index);
+ const {people_params, user_group_params} = make_params_for_users_missing_from_register(
+ my_user_id,
+ user_ids_to_fetch,
);
- blueslip.expect("error", "Ignored invalid user_ids: 1, 2");
- await people.fetch_users(new Set([1, 2]));
+ const requested_batches = [];
+ override(channel, "get", ({url, data, success}) => {
+ assert.equal(url, "/json/users");
+ const batch = JSON.parse(data.user_ids);
+ // Users from the earlier batches are already known by the time we
+ // request this one, rather than only once every batch has arrived.
+ assert.ok(requested_batches.flat().every((user_id) => people.is_known_user_id(user_id)));
+ requested_batches.push(batch);
+ success({
+ members: batch.map((user_id) => make_fetched_user(user_id)),
+ result: "success",
+ msg: "",
+ });
+ });
+
+ await people.initialize(my_user_id, people_params, user_group_params);
+
+ assert.deepEqual(
+ requested_batches.map((batch) => batch.length),
+ [500, 500, 200],
+ );
+ assert.deepEqual(
+ requested_batches.flat().toSorted((a, b) => a - b),
+ user_ids_to_fetch,
+ );
+
+ for (const user_id of user_ids_to_fetch) {
+ assert.equal(people.get_by_user_id(user_id).full_name, `User ${user_id}`);
+ }
+});
+
+run_test("user fetch resumes after a failed batch", async ({override, override_rewire}) => {
+ // Inaccessible users make a batch succeed while returning no members,
+ // which is what requires tracking progress by the IDs we sent.
+ initialize();
+ people.init();
+ override(settings_data, "user_can_access_all_other_users", () => false);
+
+ const my_user_id = 42;
+ const inaccessible_user_ids = Array.from({length: 1200}, (_unused, index) => 1000 + index);
+ const {people_params, user_group_params} = make_params_for_users_missing_from_register(
+ my_user_id,
+ inaccessible_user_ids,
+ );
+
+ const response_with_no_accessible_users = {members: [], result: "success", msg: ""};
+ const requested_batches = [];
+ // Requests 2 and 4 are the first attempts at the second and third
+ // batches, so a success falls between the two failures.
+ const failing_request_numbers = new Set([2, 4]);
+ override(channel, "get", ({url, data, success, error}) => {
+ assert.equal(url, "/json/users");
+ requested_batches.push(JSON.parse(data.user_ids));
+
+ if (failing_request_numbers.has(requested_batches.length)) {
+ error({responseJSON: {msg: "test error"}});
+ return;
+ }
+
+ success(response_with_no_accessible_users);
+ });
+
+ const backoff_attempt_counts = [];
+ override_rewire(retry_backoff, "get_retry_backoff_seconds", (_xhr, attempts) => {
+ backoff_attempt_counts.push(attempts);
+ return 0;
+ });
+ blueslip.expect(
+ "warn",
+ "Fetch for users failed, retrying after 0 seconds. Error: test error",
+ 2,
+ );
+
+ await people.initialize(my_user_id, people_params, user_group_params);
+
+ // Only the failed batches are requested twice; the rest are not refetched.
+ assert.deepEqual(
+ requested_batches.map((batch) => batch.length),
+ [500, 500, 500, 200, 200],
+ );
+ assert.deepEqual(requested_batches[1], requested_batches[2]);
+ assert.deepEqual(requested_batches[3], requested_batches[4]);
+
+ const request_count_by_user_id = new Map();
+ for (const user_id of requested_batches.flat()) {
+ request_count_by_user_id.set(user_id, (request_count_by_user_id.get(user_id) ?? 0) + 1);
+ }
+ assert.ok(requested_batches[0].every((user_id) => request_count_by_user_id.get(user_id) === 1));
+
+ // The counter restarts at 1 because a batch succeeded between the failures.
+ assert.deepEqual(backoff_attempt_counts, [1, 1]);
+
+ for (const user_id of inaccessible_user_ids) {
+ assert.ok(people.get_by_user_id(user_id).is_inaccessible_user);
+ }
});
-run_test("fetch_users corner case", async ({override, override_rewire}) => {
+run_test("user fetch corner case", async ({override, override_rewire}) => {
// We test here for the following sequence of fetches:
// 1st request for [1, 2].
// 2nd request for [1, 2, 3] while the first one is in-flight.