Skip to content

A platform for community discussion. Free, open, simple.

DEV: Fail fast when the qunit browser stalls (#41919)

Sérgio Saquetimdbea1f184f03
diff --git a/bin/qunit b/bin/qunit
index 8c1bba68c76..55424de8822 100755
--- a/bin/qunit
+++ b/bin/qunit
@@ -11,6 +11,9 @@ require "securerandom"
require_relative "../lib/chrome_installed_checker"
class QunitRunner
+ DEFAULT_BROWSER_START_TIMEOUT = 45
+ DEFAULT_BROWSER_INACTIVITY_TIMEOUT = 30
+
def initialize(args)
@file_paths = []
@verbose = ENV["ACTIONS_STEP_DEBUG"] == "true"
@@ -29,6 +32,11 @@ class QunitRunner
@plugin_targets = []
@headless = true
+ @browser_watchdog = boolean_env("QUNIT_BROWSER_WATCHDOG", default: true)
+ @browser_start_timeout =
+ positive_integer_env("QUNIT_BROWSER_START_TIMEOUT", DEFAULT_BROWSER_START_TIMEOUT)
+ @browser_inactivity_timeout =
+ positive_integer_env("QUNIT_BROWSER_INACTIVITY_TIMEOUT", DEFAULT_BROWSER_INACTIVITY_TIMEOUT)
parse_options(args)
end
@@ -114,6 +122,27 @@ class QunitRunner
@headless = h
end
+ opts.on(
+ "--[no-]browser-watchdog",
+ "Fail fast if the browser stalls on startup or between tests (env: QUNIT_BROWSER_WATCHDOG)",
+ ) { |enabled| @browser_watchdog = enabled }
+
+ opts.on(
+ "--browser-start-timeout SECONDS",
+ Integer,
+ "Browser connection timeout (default: 45; env: QUNIT_BROWSER_START_TIMEOUT)",
+ ) do |seconds|
+ @browser_start_timeout = positive_integer(seconds, "--browser-start-timeout")
+ end
+
+ opts.on(
+ "--browser-inactivity-timeout SECONDS",
+ Integer,
+ "Browser inactivity timeout outside active tests (default: 30; env: QUNIT_BROWSER_INACTIVITY_TIMEOUT)",
+ ) do |seconds|
+ @browser_inactivity_timeout = positive_integer(seconds, "--browser-inactivity-timeout")
+ end
+
opts.on("-v", "--verbose", "Verbose output") { @verbose = true }
opts.on("-h", "--help", "Show this help message") do
@@ -246,6 +275,9 @@ class QunitRunner
env["TESTEM_DEFAULT_BROWSER"] = ENV["TESTEM_DEFAULT_BROWSER"] || detect_browser
env["PLUGIN_TARGETS"] = @plugin_targets.join(",") if @plugin_targets.any?
env["QUNIT_HEADLESS"] = "0" if !@headless
+ env["QUNIT_BROWSER_WATCHDOG"] = @browser_watchdog ? "1" : "0"
+ env["QUNIT_BROWSER_START_TIMEOUT"] = @browser_start_timeout.to_s
+ env["QUNIT_BROWSER_INACTIVITY_TIMEOUT"] = @browser_inactivity_timeout.to_s
parallel = !@plugin_targets.any? && present_string(@parallel)
reuse_build = ENV["QUNIT_REUSE_BUILD"] == "1" || !@standalone_mode
@@ -613,6 +645,28 @@ class QunitRunner
str.empty? ? nil : str
end
+ def boolean_env(name, default:)
+ value = ENV[name]
+ return default if value.nil?
+ return true if %w[1 true].include?(value.downcase)
+ return false if %w[0 false].include?(value.downcase)
+
+ abort "Error: #{name} must be one of: 1, 0, true, false"
+ end
+
+ def positive_integer_env(name, default)
+ value = ENV[name]
+ return default if value.nil?
+
+ positive_integer(Integer(value, exception: false), name)
+ end
+
+ def positive_integer(value, name)
+ abort "Error: #{name} must be greater than 0" if !value || value <= 0
+
+ value
+ end
+
def resolve_plugin_namespace(directory_name)
@plugin_namespace_cache ||= {}
return @plugin_namespace_cache[directory_name] if @plugin_namespace_cache.key?(directory_name)
diff --git a/frontend/discourse/patch-testem-browser-watchdog.js b/frontend/discourse/patch-testem-browser-watchdog.js
new file mode 100644
index 00000000000..d12f7152dc6
--- /dev/null
+++ b/frontend/discourse/patch-testem-browser-watchdog.js
@@ -0,0 +1,88 @@
+const BrowserTestRunner = require("testem/lib/runners/browser_test_runner");
+
+const patchedRunnerClasses = new WeakSet();
+
+// Patches testem's browser runner so a browser that connects but then stops making
+// test progress (for example a stalled or wedged headless Chrome) fails the run
+// instead of hanging indefinitely. The failure is fast and reports a clear reason, so
+// when the run was launched in an automated way (for example by an agent or in CI) the
+// caller can react and retry the whole run rather than blocking on a wedged browser.
+function installBrowserWatchdog(
+ RunnerClass,
+ {
+ inactivityTimeout,
+ setTimeout: setTimer = setTimeout,
+ clearTimeout: clearTimer = clearTimeout,
+ }
+) {
+ if (patchedRunnerClasses.has(RunnerClass)) {
+ return;
+ }
+ patchedRunnerClasses.add(RunnerClass);
+
+ function clearWatchdog(runner) {
+ if (runner.browserWatchdogTimer) {
+ clearTimer(runner.browserWatchdogTimer);
+ runner.browserWatchdogTimer = undefined;
+ }
+ }
+
+ function scheduleWatchdog(runner) {
+ clearWatchdog(runner);
+ runner.browserWatchdogTimer = setTimer(() => {
+ runner.browserWatchdogTimer = undefined;
+ if (runner.finished) {
+ return;
+ }
+
+ const message =
+ `Browser made no test progress for ${inactivityTimeout} seconds outside ` +
+ `an active test; failing fast so the run can be retried.`;
+
+ // Surface the reason on stderr as well as through the test result, so it is
+ // visible in the console output even when the reporter summary is truncated.
+ // eslint-disable-next-line no-console
+ console.error(`\n[browser-watchdog] ${message}`);
+
+ runner.reportResults(new Error(message), 0);
+ }, inactivityTimeout * 1000);
+ }
+
+ const originalTryAttach = RunnerClass.prototype.tryAttach;
+ RunnerClass.prototype.tryAttach = function (browser, id, socket) {
+ const result = originalTryAttach.call(this, browser, id, socket);
+ if (result === false) {
+ return result;
+ }
+
+ scheduleWatchdog(this);
+ socket.on("tests-start", () => clearWatchdog(this));
+ socket.on("test-result", () => scheduleWatchdog(this));
+ socket.on("all-test-results", () => clearWatchdog(this));
+ socket.on("after-tests-complete", () => clearWatchdog(this));
+ socket.on("disconnect", () => clearWatchdog(this));
+
+ return result;
+ };
+
+ const originalFinish = RunnerClass.prototype.finish;
+ RunnerClass.prototype.finish = function (...args) {
+ clearWatchdog(this);
+ return originalFinish.apply(this, args);
+ };
+}
+
+function patchTestemBrowserWatchdog() {
+ if (process.env.QUNIT_BROWSER_WATCHDOG !== "1") {
+ return;
+ }
+
+ installBrowserWatchdog(BrowserTestRunner, {
+ inactivityTimeout: parseInt(
+ process.env.QUNIT_BROWSER_INACTIVITY_TIMEOUT,
+ 10
+ ),
+ });
+}
+
+module.exports = patchTestemBrowserWatchdog;
diff --git a/frontend/discourse/testem.js b/frontend/discourse/testem.js
index 05e09947af9..34f88ae27d1 100644
--- a/frontend/discourse/testem.js
+++ b/frontend/discourse/testem.js
@@ -4,6 +4,7 @@ const displayUtils = require("testem/lib/utils/displayutils");
const colors = require("@colors/colors/safe");
require("./patch-testem-output")();
+require("./patch-testem-browser-watchdog")();
const SANDBOX_DISABLE_VALUES = ["1", "true"];
const sandboxDisabled =
@@ -269,7 +270,10 @@ module.exports = {
socket_server_options: {
maxHttpBufferSize: 1e8, // 100MB
},
- browser_start_timeout: 120,
+ browser_start_timeout:
+ process.env.QUNIT_BROWSER_WATCHDOG === "1"
+ ? parseInt(process.env.QUNIT_BROWSER_START_TIMEOUT, 10)
+ : 120,
browser_disconnect_timeout: 30,
browser_args: {
Chromium: [
diff --git a/spec/tasks/qunit_cli_spec.rb b/spec/tasks/qunit_cli_spec.rb
index a2a4cb1efc0..c086da8917e 100644
--- a/spec/tasks/qunit_cli_spec.rb
+++ b/spec/tasks/qunit_cli_spec.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
describe "bin/qunit" do
- def run(*args)
- out, err, status = Open3.capture3("bin/qunit", "--dry-run", *args, chdir: Rails.root.to_s)
+ def run(*args, env: {})
+ out, err, status = Open3.capture3(env, "bin/qunit", "--dry-run", *args, chdir: Rails.root.to_s)
parsed_args, parsed_env =
if parsed_result = out.match(/Executing: (?<args>\[.+?\])\nwith env: (?<env>\{.+?\})/m)
@@ -25,6 +25,14 @@ describe "bin/qunit" do
let(:chat_test_file) { Dir.glob("#{Rails.root.join("plugins/chat/test/**/*-test.js")}").first }
+ let(:default_watchdog_env) do
+ {
+ "QUNIT_BROWSER_WATCHDOG" => "1",
+ "QUNIT_BROWSER_START_TIMEOUT" => "45",
+ "QUNIT_BROWSER_INACTIVITY_TIMEOUT" => "30",
+ }
+ end
+
it "runs all core tests by default" do
result = run
expect(result.status).to eq(0)
@@ -44,9 +52,11 @@ describe "bin/qunit" do
],
)
expect(result.env).to match(
- "UNICORN_PORT" => a_truthy_value,
- "TESTEM_DEFAULT_BROWSER" => a_truthy_value,
- "LOAD_PLUGINS" => "0",
+ default_watchdog_env.merge(
+ "UNICORN_PORT" => a_truthy_value,
+ "TESTEM_DEFAULT_BROWSER" => a_truthy_value,
+ "LOAD_PLUGINS" => "0",
+ ),
)
end
@@ -71,9 +81,11 @@ describe "bin/qunit" do
],
)
expect(result.env).to match(
- "UNICORN_PORT" => a_truthy_value,
- "TESTEM_DEFAULT_BROWSER" => a_truthy_value,
- "LOAD_PLUGINS" => "0",
+ default_watchdog_env.merge(
+ "UNICORN_PORT" => a_truthy_value,
+ "TESTEM_DEFAULT_BROWSER" => a_truthy_value,
+ "LOAD_PLUGINS" => "0",
+ ),
)
end
@@ -96,10 +108,12 @@ describe "bin/qunit" do
],
)
expect(result.env).to match(
- "UNICORN_PORT" => a_truthy_value,
- "TESTEM_DEFAULT_BROWSER" => a_truthy_value,
- "LOAD_PLUGINS" => "1",
- "PLUGIN_TARGETS" => a_string_matching(/,/),
+ default_watchdog_env.merge(
+ "UNICORN_PORT" => a_truthy_value,
+ "TESTEM_DEFAULT_BROWSER" => a_truthy_value,
+ "LOAD_PLUGINS" => "1",
+ "PLUGIN_TARGETS" => a_string_matching(/,/),
+ ),
)
end
@@ -122,10 +136,12 @@ describe "bin/qunit" do
],
)
expect(result.env).to match(
- "UNICORN_PORT" => a_truthy_value,
- "TESTEM_DEFAULT_BROWSER" => a_truthy_value,
- "LOAD_PLUGINS" => "1",
- "PLUGIN_TARGETS" => "chat,discourse-local-dates",
+ default_watchdog_env.merge(
+ "UNICORN_PORT" => a_truthy_value,
+ "TESTEM_DEFAULT_BROWSER" => a_truthy_value,
+ "LOAD_PLUGINS" => "1",
+ "PLUGIN_TARGETS" => "chat,discourse-local-dates",
+ ),
)
end
@@ -153,9 +169,11 @@ describe "bin/qunit" do
],
)
expect(result.env).to match(
- "UNICORN_PORT" => a_truthy_value,
- "TESTEM_DEFAULT_BROWSER" => a_truthy_value,
- "LOAD_PLUGINS" => "1",
+ default_watchdog_env.merge(
+ "UNICORN_PORT" => a_truthy_value,
+ "TESTEM_DEFAULT_BROWSER" => a_truthy_value,
+ "LOAD_PLUGINS" => "1",
+ ),
)
end
@@ -172,4 +190,60 @@ describe "bin/qunit" do
expect(result.status).to eq(0)
expect(result.launched_server).to eq(true)
end
+
+ it "enables the browser watchdog with default settings" do
+ result = run
+
+ expect(result.env).to include(
+ "QUNIT_BROWSER_WATCHDOG" => "1",
+ "QUNIT_BROWSER_START_TIMEOUT" => "45",
+ "QUNIT_BROWSER_INACTIVITY_TIMEOUT" => "30",
+ )
+ end
+
+ it "allows browser watchdog settings to be configured through the environment" do
+ result =
+ run(
+ env: {
+ "QUNIT_BROWSER_WATCHDOG" => "0",
+ "QUNIT_BROWSER_START_TIMEOUT" => "60",
+ "QUNIT_BROWSER_INACTIVITY_TIMEOUT" => "40",
+ },
+ )
+
+ expect(result.env).to include(
+ "QUNIT_BROWSER_WATCHDOG" => "0",
+ "QUNIT_BROWSER_START_TIMEOUT" => "60",
+ "QUNIT_BROWSER_INACTIVITY_TIMEOUT" => "40",
+ )
+ end
+
+ it "gives browser watchdog CLI settings precedence over the environment" do
+ result =
+ run(
+ "--browser-watchdog",
+ "--browser-start-timeout",
+ "70",
+ "--browser-inactivity-timeout",
+ "50",
+ env: {
+ "QUNIT_BROWSER_WATCHDOG" => "0",
+ "QUNIT_BROWSER_START_TIMEOUT" => "60",
+ "QUNIT_BROWSER_INACTIVITY_TIMEOUT" => "40",
+ },
+ )
+
+ expect(result.env).to include(
+ "QUNIT_BROWSER_WATCHDOG" => "1",
+ "QUNIT_BROWSER_START_TIMEOUT" => "70",
+ "QUNIT_BROWSER_INACTIVITY_TIMEOUT" => "50",
+ )
+ end
+
+ it "rejects non-positive browser watchdog settings" do
+ result = run("--browser-inactivity-timeout", "0")
+
+ expect(result.status).to eq(1)
+ expect(result.err).to include("--browser-inactivity-timeout must be greater than 0")
+ end
end