Zulip server and web application. Open-source team chat that helps teams stay productive and focused.
reactions: Retry transient reaction request failures with backoff.
Evy Kassirerff416e0f3c3e
diff --git a/web/src/reactions.ts b/web/src/reactions.tsindex a411f060996..39a08868025 100644--- a/web/src/reactions.ts+++ b/web/src/reactions.ts@@ -16,12 +16,18 @@ import * as message_store from "./message_store.ts";import type {Message, MessageCleanReaction, RawMessage} from "./message_store.ts";import {page_params} from "./page_params.ts";import * as people from "./people.ts";+import {get_retry_backoff_seconds} from "./retry_backoff.ts";import * as spectators from "./spectators.ts";import {current_user} from "./state_data.ts";import {user_settings} from "./user_settings.ts";const waiting_for_server_request_ids = new Set<string>();+// How many times to retry a reaction request that fails with a+// transient server (5xx) or network error before giving up and+// reporting the failure.+const MAX_SEND_REACTION_RETRIES = 5;+export type ReactionEvent = {message_id: number;user_id: number;@@ -102,31 +108,91 @@ function update_ui_and_send_reaction_ajax(remove_reaction(reaction);}+ send_reaction_request(operation, reaction, rendering_details, reaction_request_id, 0);+}++function send_reaction_request(+ operation: "add" | "remove",+ reaction: ReactionEvent,+ rendering_details: EmojiRenderingDetails,+ reaction_request_id: string,+ attempt: number,+): void {+ waiting_for_server_request_ids.add(reaction_request_id);const args = {- url: "/json/messages/" + message.id + "/reactions",+ url: "/json/messages/" + reaction.message_id + "/reactions",data: rendering_details,success() {waiting_for_server_request_ids.delete(reaction_request_id);},error(xhr: JQuery.jqXHR) {- waiting_for_server_request_ids.delete(reaction_request_id);- if (xhr.readyState !== 0) {- const parsed = z.object({code: z.string()}).safeParse(xhr.responseJSON);- if (- parsed.success &&- (parsed.data.code === "REACTION_ALREADY_EXISTS" ||- parsed.data.code === "REACTION_DOES_NOT_EXIST")- ) {- // Don't send error report for simple precondition failures caused by race- // conditions; the user already got what they wanted+ // Network drop (status 0) and gateway/server errors deserve some+ // retries with exponential backoff.+ const is_transient_error = xhr.status === 0 || xhr.status >= 500;+ let parsed;+ if (xhr.readyState === 0) {+ // client cancelled the request+ waiting_for_server_request_ids.delete(reaction_request_id);+ } else if (+ (parsed = z.object({code: z.string()}).safeParse(xhr.responseJSON)).success &&+ (parsed.data.code === "REACTION_ALREADY_EXISTS" ||+ parsed.data.code === "REACTION_DOES_NOT_EXIST")+ ) {+ // Don't send error report for simple precondition failures caused by race+ // conditions; the user already got what they wanted+ waiting_for_server_request_ids.delete(reaction_request_id);+ } else if (+ (parsed = z+ .object({code: z.literal("RATE_LIMIT_HIT"), ["retry-after"]: z.number()})+ .safeParse(xhr.responseJSON)).success+ ) {+ // If we hit the rate limit, just retry after the server-specified+ // delay. This is self-resolving, so we retry indefinitely and+ // don't count it against the transient-error retry budget.+ const milliseconds_to_wait = 1000 * parsed.data["retry-after"];+ setTimeout(() => {+ send_reaction_request(+ operation,+ reaction,+ rendering_details,+ reaction_request_id,+ 0,+ );+ }, milliseconds_to_wait);+ } else if (is_transient_error && attempt < MAX_SEND_REACTION_RETRIES) {+ // Use the slower backoff, since a restarting server can take+ // a while to come back.+ const delay_secs = get_retry_backoff_seconds(xhr, attempt, false, true);+ setTimeout(() => {+ send_reaction_request(+ operation,+ reaction,+ rendering_details,+ reaction_request_id,+ attempt + 1,+ );+ }, delay_secs * 1000);+ } else {+ waiting_for_server_request_ids.delete(reaction_request_id);+ if (is_transient_error) {+ // The transient failure persisted past our retry budget,+ // so the server is likely having a real problem.+ blueslip.error("Error sending reaction after retries", {+ status: xhr.status,+ body: xhr.responseText,+ });} else {- blueslip.error(channel.xhr_error_message("Error sending reaction", xhr));+ // Any other error -- e.g. a 4xx bad request -- is a+ // client-side problem that retrying won't fix.+ blueslip.error("Error sending reaction", {+ status: xhr.status,+ body: xhr.responseText,+ });}}},};- waiting_for_server_request_ids.add(reaction_request_id);if (operation === "add") {void channel.post(args);} else if (operation === "remove") {diff --git a/web/tests/reactions.test.cjs b/web/tests/reactions.test.cjsindex a560ea0a53c..efb13b711cd 100644--- a/web/tests/reactions.test.cjs+++ b/web/tests/reactions.test.cjs@@ -61,6 +61,7 @@ const emoji = zrequire("emoji");const emoji_codes = zrequire("../../static/generated/emoji/emoji_codes.json");const people = zrequire("people");const reactions = zrequire("reactions");+const retry_backoff = zrequire("retry_backoff");const {set_current_user, set_realm} = zrequire("state_data");const {initialize_user_settings} = zrequire("user_settings");@@ -358,10 +359,8 @@ test("sending", ({override, override_rewire}) => {args.success();// similarly, we only exercise the failure codepath- // Since this path calls blueslip.warn, we need to handle it.- blueslip.expect("error", "XHR error message.");- channel.xhr_error_message = () => "XHR error message.";- args.error({readyState: 4, responseJSON: {msg: "Some error message"}});+ blueslip.expect("error", "Error sending reaction");+ args.error({readyState: 4, status: 400, responseJSON: {msg: "Some error message"}});}emoji_name = "alien"; // not set yet{@@ -421,6 +420,89 @@ test("sending", ({override, override_rewire}) => {});});+test("sending precondition failures", ({override_rewire}) => {+ const message = sample_message_with_clean_reactions();+ override_rewire(reactions, "add_reaction", noop);+ const stub = make_stub();+ channel.post = stub.f;++ reactions.toggle_emoji_reaction(message, "octopus");+ assert.equal(stub.num_calls, 1);++ // A precondition failure isn't reported (no blueslip.expect here)+ // because the user already got what they wanted.+ stub.get_args("args").args.error({+ readyState: 4,+ status: 400,+ responseJSON: {code: "REACTION_ALREADY_EXISTS", msg: "Reaction already exists."},+ });++ // The request is no longer in flight, so the user can send another one.+ reactions.toggle_emoji_reaction(message, "octopus");+ assert.equal(stub.num_calls, 2);++ // A cancelled request isn't reported either.+ stub.get_args("args").args.error({readyState: 0, status: 0});+ reactions.toggle_emoji_reaction(message, "octopus");+ assert.equal(stub.num_calls, 3);+ stub.get_args("args").args.success();+});++test("sending failure retries", ({override_rewire}) => {+ const message = sample_message_with_clean_reactions();+ override_rewire(reactions, "add_reaction", noop);+ override_rewire(retry_backoff, "get_retry_backoff_seconds", () => 10);+ const stub = make_stub();+ channel.post = stub.f;++ let timeout_callback;+ let timeout_delay_ms;+ set_global("setTimeout", (f, delay) => {+ timeout_callback = f;+ timeout_delay_ms = delay;+ });++ reactions.toggle_emoji_reaction(message, "airplane");+ assert.equal(stub.num_calls, 1);++ // A rate-limited request is retried after the server-specified delay,+ // without reporting an error.+ stub.get_args("args").args.error({+ readyState: 4,+ status: 429,+ responseJSON: {code: "RATE_LIMIT_HIT", "retry-after": 60},+ });+ assert.equal(timeout_delay_ms, 60 * 1000);++ // Clicking again while a retry is pending doesn't send a duplicate+ // request.+ reactions.toggle_emoji_reaction(message, "airplane");+ assert.equal(stub.num_calls, 1);++ timeout_callback();+ assert.equal(stub.num_calls, 2);++ // Transient server errors are retried with backoff, and reported only+ // once we give up.+ for (let attempt = 0; attempt < 5; attempt += 1) {+ stub.get_args("args").args.error({+ readyState: 4,+ status: 502,+ responseText: "Bad gateway",+ });+ assert.equal(timeout_delay_ms, 10 * 1000);+ timeout_callback();+ assert.equal(stub.num_calls, 3 + attempt);+ }+ blueslip.expect("error", "Error sending reaction after retries");+ stub.get_args("args").args.error({readyState: 4, status: 502, responseText: "Bad gateway"});++ // The request is no longer in flight, so the user can try again.+ reactions.toggle_emoji_reaction(message, "airplane");+ assert.equal(stub.num_calls, 8);+ stub.get_args("args").args.success();+});+test("prevent_simultaneous_requests_updating_reaction", ({override_rewire}) => {const message = sample_message_with_clean_reactions();override_rewire(reactions, "add_reaction", noop);