Skip to content

Jitsi Meet - Secure, Simple and Scalable Video Conferences that you use as a standalone app or embed in your web application.

fix(tests): detect teardown crashes via suite completion, not log text

damencho10cdbce46186
diff --git a/tests/wdio.conf.ts b/tests/wdio.conf.ts
index c757229836..0aefbe8aed 100644
--- a/tests/wdio.conf.ts
+++ b/tests/wdio.conf.ts
@@ -209,32 +209,64 @@ const TEST_RESULTS_DIR = 'test-results';
const keepAlive: Array<any> = [];
+// Titles of the tests that failed in this worker. Recorded by afterTest and handed to the launcher
+// by the `after` hook, so onWorkerEnd can tell a clean run from a failed one even when the
+// reporters never got the chance to flush their results.
+const failedTestTitles: Array<string> = [];
+
// Tracks browser-session lifecycle events written by each worker's
// beforeSession/afterSession hooks. The launcher's onComplete reads it to
// compute the peak number of concurrent browser sessions during the run.
const CONCURRENCY_LOG_PATH = path.join(TEST_RESULTS_DIR, 'concurrency.jsonl');
/**
- * Detects the "tests completed but session teardown crashed" signature in a worker's wdio log.
- * Selenium Grid (especially with Firefox) sometimes takes longer than `connectionRetryTimeout`
- * to acknowledge session deletion; the runner exits non-zero even though the test ran fine.
+ * Path of the sentinel a worker writes once its whole spec file has finished running.
*
- * Two variants of the same benign pattern are recognised, both gated on the suite having run to
- * completion (the `afterSuite` hook finished):
- * 1. An explicit DELETE-method WebDriverError timeout in the log tail.
- * 2. The worker was killed mid-teardown: the framework `after` hook finished and a session
- * `deleteSession()`/`[DELETE] .../session/` was issued, then the log simply truncates with no
- * error line ever written. This is what happens when the grid drops the worker during DELETE.
+ * Written by the framework `after` hook (which @wdio/mocha-framework runs after every test and
+ * hook, and before the runner deletes the sessions) and read by the launcher's `onWorkerEnd`: a
+ * worker that exits non-zero *with* this file present got through its whole spec file, so whatever
+ * killed it happened while tearing the session(s) down.
*
- * Both require `afterSuite` to have finished, so every test body and hook executed; the only thing
- * left was tearing the session(s) down. A genuine test failure that managed to write results is
- * already handled earlier by the `xmlHasNamedTests` early-return in `onWorkerEnd`, so it never
- * reaches here.
+ * This has to be a file rather than something parsed out of the worker log. The worker's debug
+ * output goes to stdout, which the launcher forwards to the console; WDIO_LOG_PATH only ever
+ * receives launcher-level lines, so the hook traces a log-scraping check would need are absent.
+ *
+ * @param cid - The worker (capability) id.
+ */
+function completionSentinelPath(cid: string): string {
+ return path.join(TEST_RESULTS_DIR, `completed-${cid}.json`);
+}
+
+/**
+ * Reads a worker's completion sentinel, if it wrote one.
+ *
+ * @param cid - The worker (capability) id.
+ * @returns The failed test titles and mocha failure count the worker recorded, or undefined when
+ * it never reached its `after` hook.
+ */
+function readWorkerCompletion(cid: string): { failed: Array<string>; failures: number; } | undefined {
+ try {
+ const parsed = JSON.parse(fs.readFileSync(completionSentinelPath(cid), 'utf8'));
+
+ return {
+ failed: Array.isArray(parsed.failed) ? parsed.failed : [],
+ failures: typeof parsed.failures === 'number' ? parsed.failures : 0
+ };
+ } catch {
+ // No sentinel, or it is unreadable — the worker did not finish its spec file.
+ return undefined;
+ }
+}
+
+/**
+ * Extracts the last error line from a worker's wdio log, so a synthesised report can say what
+ * actually went wrong instead of only that the worker exited non-zero.
*
* @param specFile - The worker's first spec file (path or URL); used to locate its wdio log.
* @param cid - The worker (capability) id.
+ * @returns The error line, trimmed and truncated, or undefined if the log has none.
*/
-function isPostTestSessionDeleteTimeout(specFile: string | undefined, cid: string): boolean {
+function readWorkerError(specFile: string | undefined, cid: string): string | undefined {
// @wdio/local-runner writes the worker log to `${specBaseName}-${cid}.log`, where specBaseName
// strips only the final extension (so `displayName.spec.ts` -> `displayName.spec`), falling back
// to `wdio-${cid}.log` when no spec is associated. Mirror that exactly so the log is found.
@@ -250,7 +282,7 @@ function isPostTestSessionDeleteTimeout(specFile: string | undefined, cid: strin
const logPath = candidates.find(p => fs.existsSync(p));
if (!logPath) {
- return false;
+ return undefined;
}
try {
@@ -263,21 +295,14 @@ function isPostTestSessionDeleteTimeout(specFile: string | undefined, cid: strin
fs.readSync(fd, buf, 0, tailSize, Math.max(0, stats.size - tailSize));
fs.closeSync(fd);
- const tail = buf.toString('utf8');
- const afterSuiteFinished = tail.includes('Finished to run "afterSuite" hook');
-
- // Variant 1: the grid was slow to ack the DELETE and the runner logged an explicit timeout.
- const deleteTimeout = /WebDriverError:[^\n]*timeout[^\n]*method "DELETE"/i.test(tail);
-
- // Variant 2: the worker was killed during teardown. The framework `after` hook finished and
- // a session DELETE was issued, but the log truncates before any error is written.
- const afterHookFinished = tail.includes('Finished to run "after" hook');
- const deleteIssued = tail.includes('COMMAND deleteSession()')
- || /\[DELETE\][^\n]*\/session\//.test(tail);
-
- return afterSuiteFinished && (deleteTimeout || (afterHookFinished && deleteIssued));
+ return buf.toString('utf8')
+ .split('\n')
+ .reverse()
+ .find(line => line.includes('ERROR'))
+ ?.trim()
+ .slice(0, 500);
} catch {
- return false;
+ return undefined;
}
}
@@ -365,6 +390,16 @@ export const config: WebdriverIO.MultiremoteConfig = {
} catch {
// best effort — if we can't reset, onComplete will still parse what it sees
}
+
+ // Drop completion sentinels left behind by an earlier run in the same workspace; a stale
+ // one would make onWorkerEnd read a genuine crash as a teardown-only crash.
+ try {
+ fs.readdirSync(TEST_RESULTS_DIR)
+ .filter(name => (/^completed-.*\.json$/).test(name))
+ .forEach(name => fs.unlinkSync(path.join(TEST_RESULTS_DIR, name)));
+ } catch {
+ // best effort — the directory may not exist yet
+ }
},
/**
@@ -466,14 +501,43 @@ export const config: WebdriverIO.MultiremoteConfig = {
}
},
- after() {
+ /**
+ * Gets executed after all tests in the spec file are done, before the sessions are deleted.
+ *
+ * @param {number} result - Mocha's number of failing tests for this spec file.
+ * @returns {void}
+ */
+ after(result) {
const { ctx }: any = global;
ctx?.webhooksProxy?.disconnect();
keepAlive.forEach(clearInterval);
+
+ // Tell the launcher this worker got through every test and hook. If it still exits
+ // non-zero, the failure was in session teardown, which onWorkerEnd reports separately.
+ const cid = process.env.WDIO_WORKER_ID;
+
+ if (cid) {
+ try {
+ fs.writeFileSync(completionSentinelPath(cid), JSON.stringify({
+ failed: failedTestTitles,
+ failures: typeof result === 'number' ? result : 1
+ }));
+ } catch {
+ // best effort — without it, a teardown crash is reported as a runner crash
+ }
+ }
},
async beforeSession(c, capabilities_, spec, cid) {
+ // Clear any completion sentinel left by an earlier attempt at this spec — worker ids can
+ // be reused when a spec file is retried — so onWorkerEnd never reads a stale completion.
+ try {
+ fs.rmSync(completionSentinelPath(cid), { force: true });
+ } catch {
+ // best effort — onPrepare already cleared sentinels from previous runs
+ }
+
// Record this worker's browser sessions opening, so the launcher's
// onComplete can compute peak concurrent sessions across the run.
try {
@@ -606,6 +670,10 @@ export const config: WebdriverIO.MultiremoteConfig = {
if (error) {
+ // Record the failure for the `after` hook, so the launcher still knows this spec had
+ // failing tests if the worker dies before the reporters flush.
+ failedTestTitles.push(test.title);
+
// Skip the remaining tests in the same describe block (but not other describe blocks in the file).
ctx.failedSuite = test.parent;
@@ -692,11 +760,11 @@ export const config: WebdriverIO.MultiremoteConfig = {
/**
* Gets executed after a worker process has exited.
* Handles three crash scenarios:
- * 1. Session teardown crash AFTER tests completed (common with Firefox on Selenium Grid):
- * afterSuite ran, then the session DELETE either timed out or the worker was killed
- * mid-teardown, so the runner exited non-zero. Tests actually passed. We skip the misleading
- * "crashed" synthesis; allure already has accurate per-test results, and we summarise the
- * teardown anomaly in a passing JUnit entry. See isPostTestSessionDeleteTimeout.
+ * 1. Session teardown crash AFTER the spec file completed (common with Firefox on Selenium
+ * Grid): the worker wrote its completion sentinel, then the session DELETE either timed out
+ * or the worker was killed mid-teardown, so the runner exited non-zero. The tests themselves
+ * ran, so we report their outcome (from the sentinel) rather than a misleading "crashed",
+ * and record the teardown anomaly in the entry. See completionSentinelPath.
* 2. Session DELETE timeout BEFORE tests completed: JUnit reporter never flushes,
* leaving a zero-byte XML. Synthesise a failure entry.
* 3. Session INIT timeout: JUnit reporter writes a non-empty XML but with empty name/classname,
@@ -733,31 +801,46 @@ export const config: WebdriverIO.MultiremoteConfig = {
return;
}
- // Detect the "tests passed but session teardown crashed" pattern from the worker's wdio
- // log. Pass the raw spec path so the log file name is derived the same way
- // @wdio/local-runner derives it (only the final extension is stripped).
- if (isPostTestSessionDeleteTimeout(workerSpecs?.[0], cid)) {
- const teardownName = 'Session teardown crashed after tests completed';
- const teardownMessage
- = `Session teardown crashed after tests completed (worker exit ${exitCode}). `
- + 'The suite ran to completion; this entry just records the teardown anomaly.';
+ // A worker that wrote its completion sentinel ran every test and hook in the spec file, so
+ // it died in session teardown. Report what the tests actually did. Pass the raw spec path
+ // so the log file name is derived the same way @wdio/local-runner derives it (only the
+ // final extension is stripped).
+ const completion = readWorkerCompletion(cid);
+
+ if (completion) {
+ const failureCount = Math.max(completion.failures, completion.failed.length);
+ const clean = failureCount === 0;
+ const workerError = readWorkerError(workerSpecs?.[0], cid);
+ const teardownDetail = `Session teardown crashed (worker exit ${exitCode}), so the `
+ + `reporters never wrote their results.${workerError ? ` Worker error: ${workerError}` : ''}`;
+ const teardownName = clean
+ ? 'Session teardown crashed after tests completed'
+ : 'Tests failed, then session teardown crashed';
+ const teardownMessage = clean
+ ? `The spec file ran to completion with no failures. ${teardownDetail}`
+ : `The spec file ran to completion with ${failureCount} failing test(s)`
+ + `${completion.failed.length ? ` (${completion.failed.join(', ')})` : ''}. `
+ + teardownDetail;
const b = junitReportBuilder.newBuilder();
-
- b.testSuite().name(specName).testCase()
+ const testCase = b.testSuite().name(specName).testCase()
.name(teardownName)
.className(specName)
.standardOutput(teardownMessage);
+
+ if (!clean) {
+ testCase.failure(teardownMessage);
+ }
b.writeTo(xmlPath);
// When the worker is killed mid-teardown the reporters never flush, so allure has no
// per-test results for this spec and it would otherwise vanish from the report. Write a
- // passing entry so the spec stays visible with the teardown anomaly recorded. If allure
- // already captured the per-test results (e.g. a slow-DELETE timeout after a clean
- // flush), this is just an extra informational passing entry alongside them.
+ // single entry so the spec stays visible, carrying the outcome the worker recorded. If
+ // allure already captured the per-test results (e.g. a slow-DELETE timeout after a
+ // clean flush), this is just an extra informational entry alongside them.
const allureResult = {
uuid: randomUUID(),
name: teardownName,
- status: 'passed',
+ status: clean ? 'passed' : 'failed',
statusDetails: { message: teardownMessage },
stage: 'finished',
steps: [],
@@ -773,8 +856,8 @@ export const config: WebdriverIO.MultiremoteConfig = {
= path.join(TEST_RESULTS_DIR, 'allure-results', `${allureResult.uuid}-result.json`);
fs.writeFileSync(allurePath, JSON.stringify(allureResult));
- console.log(`[onWorkerEnd] Worker ${cid} (${specName}): session teardown crashed after `
- + 'afterSuite — synthesising passing JUnit + allure entry; suite ran to completion.');
+ console.log(`[onWorkerEnd] Worker ${cid} (${specName}): ${teardownName} — synthesising `
+ + `${clean ? 'passing' : 'failing'} JUnit + allure entry. ${teardownMessage}`);
return;
}