Skip to content

armgit / zulip

Public Imported from GitHub
0

Zulip server and web application. Open-source team chat that helps teams stay productive and focused.

reactions: Retry transient reaction request failures with backoff.

Evy Kassirerff416e0f3c3e
diff --git a/web/src/reactions.ts b/web/src/reactions.ts
index a411f060996..39a08868025 100644
--- a/web/src/reactions.ts
+++ b/web/src/reactions.ts
@@ -16,12 +16,18 @@ import * as message_store from "./message_store.ts";
import type {Message, MessageCleanReaction, RawMessage} from "./message_store.ts";
import {page_params} from "./page_params.ts";
import * as people from "./people.ts";
+import {get_retry_backoff_seconds} from "./retry_backoff.ts";
import * as spectators from "./spectators.ts";
import {current_user} from "./state_data.ts";
import {user_settings} from "./user_settings.ts";
const waiting_for_server_request_ids = new Set<string>();
+// How many times to retry a reaction request that fails with a
+// transient server (5xx) or network error before giving up and
+// reporting the failure.
+const MAX_SEND_REACTION_RETRIES = 5;
+
export type ReactionEvent = {
message_id: number;
user_id: number;
@@ -102,31 +108,91 @@ function update_ui_and_send_reaction_ajax(
remove_reaction(reaction);
}
+ send_reaction_request(operation, reaction, rendering_details, reaction_request_id, 0);
+}
+
+function send_reaction_request(
+ operation: "add" | "remove",
+ reaction: ReactionEvent,
+ rendering_details: EmojiRenderingDetails,
+ reaction_request_id: string,
+ attempt: number,
+): void {
+ waiting_for_server_request_ids.add(reaction_request_id);
const args = {
- url: "/json/messages/" + message.id + "/reactions",
+ url: "/json/messages/" + reaction.message_id + "/reactions",
data: rendering_details,
success() {
waiting_for_server_request_ids.delete(reaction_request_id);
},
error(xhr: JQuery.jqXHR) {
- waiting_for_server_request_ids.delete(reaction_request_id);
- if (xhr.readyState !== 0) {
- const parsed = z.object({code: z.string()}).safeParse(xhr.responseJSON);
- if (
- parsed.success &&
- (parsed.data.code === "REACTION_ALREADY_EXISTS" ||
- parsed.data.code === "REACTION_DOES_NOT_EXIST")
- ) {
- // Don't send error report for simple precondition failures caused by race
- // conditions; the user already got what they wanted
+ // Network drop (status 0) and gateway/server errors deserve some
+ // retries with exponential backoff.
+ const is_transient_error = xhr.status === 0 || xhr.status >= 500;
+ let parsed;
+ if (xhr.readyState === 0) {
+ // client cancelled the request
+ waiting_for_server_request_ids.delete(reaction_request_id);
+ } else if (
+ (parsed = z.object({code: z.string()}).safeParse(xhr.responseJSON)).success &&
+ (parsed.data.code === "REACTION_ALREADY_EXISTS" ||
+ parsed.data.code === "REACTION_DOES_NOT_EXIST")
+ ) {
+ // Don't send error report for simple precondition failures caused by race
+ // conditions; the user already got what they wanted
+ waiting_for_server_request_ids.delete(reaction_request_id);
+ } else if (
+ (parsed = z
+ .object({code: z.literal("RATE_LIMIT_HIT"), ["retry-after"]: z.number()})
+ .safeParse(xhr.responseJSON)).success
+ ) {
+ // If we hit the rate limit, just retry after the server-specified
+ // delay. This is self-resolving, so we retry indefinitely and
+ // don't count it against the transient-error retry budget.
+ const milliseconds_to_wait = 1000 * parsed.data["retry-after"];
+ setTimeout(() => {
+ send_reaction_request(
+ operation,
+ reaction,
+ rendering_details,
+ reaction_request_id,
+ 0,
+ );
+ }, milliseconds_to_wait);
+ } else if (is_transient_error && attempt < MAX_SEND_REACTION_RETRIES) {
+ // Use the slower backoff, since a restarting server can take
+ // a while to come back.
+ const delay_secs = get_retry_backoff_seconds(xhr, attempt, false, true);
+ setTimeout(() => {
+ send_reaction_request(
+ operation,
+ reaction,
+ rendering_details,
+ reaction_request_id,
+ attempt + 1,
+ );
+ }, delay_secs * 1000);
+ } else {
+ waiting_for_server_request_ids.delete(reaction_request_id);
+ if (is_transient_error) {
+ // The transient failure persisted past our retry budget,
+ // so the server is likely having a real problem.
+ blueslip.error("Error sending reaction after retries", {
+ status: xhr.status,
+ body: xhr.responseText,
+ });
} else {
- blueslip.error(channel.xhr_error_message("Error sending reaction", xhr));
+ // Any other error -- e.g. a 4xx bad request -- is a
+ // client-side problem that retrying won't fix.
+ blueslip.error("Error sending reaction", {
+ status: xhr.status,
+ body: xhr.responseText,
+ });
}
}
},
};
- waiting_for_server_request_ids.add(reaction_request_id);
if (operation === "add") {
void channel.post(args);
} else if (operation === "remove") {
diff --git a/web/tests/reactions.test.cjs b/web/tests/reactions.test.cjs
index a560ea0a53c..efb13b711cd 100644
--- a/web/tests/reactions.test.cjs
+++ b/web/tests/reactions.test.cjs
@@ -61,6 +61,7 @@ const emoji = zrequire("emoji");
const emoji_codes = zrequire("../../static/generated/emoji/emoji_codes.json");
const people = zrequire("people");
const reactions = zrequire("reactions");
+const retry_backoff = zrequire("retry_backoff");
const {set_current_user, set_realm} = zrequire("state_data");
const {initialize_user_settings} = zrequire("user_settings");
@@ -358,10 +359,8 @@ test("sending", ({override, override_rewire}) => {
args.success();
// similarly, we only exercise the failure codepath
- // Since this path calls blueslip.warn, we need to handle it.
- blueslip.expect("error", "XHR error message.");
- channel.xhr_error_message = () => "XHR error message.";
- args.error({readyState: 4, responseJSON: {msg: "Some error message"}});
+ blueslip.expect("error", "Error sending reaction");
+ args.error({readyState: 4, status: 400, responseJSON: {msg: "Some error message"}});
}
emoji_name = "alien"; // not set yet
{
@@ -421,6 +420,89 @@ test("sending", ({override, override_rewire}) => {
});
});
+test("sending precondition failures", ({override_rewire}) => {
+ const message = sample_message_with_clean_reactions();
+ override_rewire(reactions, "add_reaction", noop);
+ const stub = make_stub();
+ channel.post = stub.f;
+
+ reactions.toggle_emoji_reaction(message, "octopus");
+ assert.equal(stub.num_calls, 1);
+
+ // A precondition failure isn't reported (no blueslip.expect here)
+ // because the user already got what they wanted.
+ stub.get_args("args").args.error({
+ readyState: 4,
+ status: 400,
+ responseJSON: {code: "REACTION_ALREADY_EXISTS", msg: "Reaction already exists."},
+ });
+
+ // The request is no longer in flight, so the user can send another one.
+ reactions.toggle_emoji_reaction(message, "octopus");
+ assert.equal(stub.num_calls, 2);
+
+ // A cancelled request isn't reported either.
+ stub.get_args("args").args.error({readyState: 0, status: 0});
+ reactions.toggle_emoji_reaction(message, "octopus");
+ assert.equal(stub.num_calls, 3);
+ stub.get_args("args").args.success();
+});
+
+test("sending failure retries", ({override_rewire}) => {
+ const message = sample_message_with_clean_reactions();
+ override_rewire(reactions, "add_reaction", noop);
+ override_rewire(retry_backoff, "get_retry_backoff_seconds", () => 10);
+ const stub = make_stub();
+ channel.post = stub.f;
+
+ let timeout_callback;
+ let timeout_delay_ms;
+ set_global("setTimeout", (f, delay) => {
+ timeout_callback = f;
+ timeout_delay_ms = delay;
+ });
+
+ reactions.toggle_emoji_reaction(message, "airplane");
+ assert.equal(stub.num_calls, 1);
+
+ // A rate-limited request is retried after the server-specified delay,
+ // without reporting an error.
+ stub.get_args("args").args.error({
+ readyState: 4,
+ status: 429,
+ responseJSON: {code: "RATE_LIMIT_HIT", "retry-after": 60},
+ });
+ assert.equal(timeout_delay_ms, 60 * 1000);
+
+ // Clicking again while a retry is pending doesn't send a duplicate
+ // request.
+ reactions.toggle_emoji_reaction(message, "airplane");
+ assert.equal(stub.num_calls, 1);
+
+ timeout_callback();
+ assert.equal(stub.num_calls, 2);
+
+ // Transient server errors are retried with backoff, and reported only
+ // once we give up.
+ for (let attempt = 0; attempt < 5; attempt += 1) {
+ stub.get_args("args").args.error({
+ readyState: 4,
+ status: 502,
+ responseText: "Bad gateway",
+ });
+ assert.equal(timeout_delay_ms, 10 * 1000);
+ timeout_callback();
+ assert.equal(stub.num_calls, 3 + attempt);
+ }
+ blueslip.expect("error", "Error sending reaction after retries");
+ stub.get_args("args").args.error({readyState: 4, status: 502, responseText: "Bad gateway"});
+
+ // The request is no longer in flight, so the user can try again.
+ reactions.toggle_emoji_reaction(message, "airplane");
+ assert.equal(stub.num_calls, 8);
+ stub.get_args("args").args.success();
+});
+
test("prevent_simultaneous_requests_updating_reaction", ({override_rewire}) => {
const message = sample_message_with_clean_reactions();
override_rewire(reactions, "add_reaction", noop);