Jitsi Meet - Secure, Simple and Scalable Video Conferences that you use as a standalone app or embed in your web application.
TypeScript debiandeep-videojitsijitsi-meetscalable-video-conferencessfuvideovideo-communicationvideo-conferencingwebrtc
fix(tests): detect teardown crashes via suite completion, not log text
damencho10cdbce46186
diff --git a/tests/wdio.conf.ts b/tests/wdio.conf.tsindex c757229836..0aefbe8aed 100644--- a/tests/wdio.conf.ts+++ b/tests/wdio.conf.ts@@ -209,32 +209,64 @@ const TEST_RESULTS_DIR = 'test-results';const keepAlive: Array<any> = [];+// Titles of the tests that failed in this worker. Recorded by afterTest and handed to the launcher+// by the `after` hook, so onWorkerEnd can tell a clean run from a failed one even when the+// reporters never got the chance to flush their results.+const failedTestTitles: Array<string> = [];+// Tracks browser-session lifecycle events written by each worker's// beforeSession/afterSession hooks. The launcher's onComplete reads it to// compute the peak number of concurrent browser sessions during the run.const CONCURRENCY_LOG_PATH = path.join(TEST_RESULTS_DIR, 'concurrency.jsonl');/**- * Detects the "tests completed but session teardown crashed" signature in a worker's wdio log.- * Selenium Grid (especially with Firefox) sometimes takes longer than `connectionRetryTimeout`- * to acknowledge session deletion; the runner exits non-zero even though the test ran fine.+ * Path of the sentinel a worker writes once its whole spec file has finished running.*- * Two variants of the same benign pattern are recognised, both gated on the suite having run to- * completion (the `afterSuite` hook finished):- * 1. An explicit DELETE-method WebDriverError timeout in the log tail.- * 2. The worker was killed mid-teardown: the framework `after` hook finished and a session- * `deleteSession()`/`[DELETE] .../session/` was issued, then the log simply truncates with no- * error line ever written. This is what happens when the grid drops the worker during DELETE.+ * Written by the framework `after` hook (which @wdio/mocha-framework runs after every test and+ * hook, and before the runner deletes the sessions) and read by the launcher's `onWorkerEnd`: a+ * worker that exits non-zero *with* this file present got through its whole spec file, so whatever+ * killed it happened while tearing the session(s) down.*- * Both require `afterSuite` to have finished, so every test body and hook executed; the only thing- * left was tearing the session(s) down. A genuine test failure that managed to write results is- * already handled earlier by the `xmlHasNamedTests` early-return in `onWorkerEnd`, so it never- * reaches here.+ * This has to be a file rather than something parsed out of the worker log. The worker's debug+ * output goes to stdout, which the launcher forwards to the console; WDIO_LOG_PATH only ever+ * receives launcher-level lines, so the hook traces a log-scraping check would need are absent.+ *+ * @param cid - The worker (capability) id.+ */+function completionSentinelPath(cid: string): string {+ return path.join(TEST_RESULTS_DIR, `completed-${cid}.json`);+}++/**+ * Reads a worker's completion sentinel, if it wrote one.+ *+ * @param cid - The worker (capability) id.+ * @returns The failed test titles and mocha failure count the worker recorded, or undefined when+ * it never reached its `after` hook.+ */+function readWorkerCompletion(cid: string): { failed: Array<string>; failures: number; } | undefined {+ try {+ const parsed = JSON.parse(fs.readFileSync(completionSentinelPath(cid), 'utf8'));++ return {+ failed: Array.isArray(parsed.failed) ? parsed.failed : [],+ failures: typeof parsed.failures === 'number' ? parsed.failures : 0+ };+ } catch {+ // No sentinel, or it is unreadable — the worker did not finish its spec file.+ return undefined;+ }+}++/**+ * Extracts the last error line from a worker's wdio log, so a synthesised report can say what+ * actually went wrong instead of only that the worker exited non-zero.** @param specFile - The worker's first spec file (path or URL); used to locate its wdio log.* @param cid - The worker (capability) id.+ * @returns The error line, trimmed and truncated, or undefined if the log has none.*/-function isPostTestSessionDeleteTimeout(specFile: string | undefined, cid: string): boolean {+function readWorkerError(specFile: string | undefined, cid: string): string | undefined {// @wdio/local-runner writes the worker log to `${specBaseName}-${cid}.log`, where specBaseName// strips only the final extension (so `displayName.spec.ts` -> `displayName.spec`), falling back// to `wdio-${cid}.log` when no spec is associated. Mirror that exactly so the log is found.@@ -250,7 +282,7 @@ function isPostTestSessionDeleteTimeout(specFile: string | undefined, cid: strinconst logPath = candidates.find(p => fs.existsSync(p));if (!logPath) {- return false;+ return undefined;}try {@@ -263,21 +295,14 @@ function isPostTestSessionDeleteTimeout(specFile: string | undefined, cid: strinfs.readSync(fd, buf, 0, tailSize, Math.max(0, stats.size - tailSize));fs.closeSync(fd);- const tail = buf.toString('utf8');- const afterSuiteFinished = tail.includes('Finished to run "afterSuite" hook');-- // Variant 1: the grid was slow to ack the DELETE and the runner logged an explicit timeout.- const deleteTimeout = /WebDriverError:[^\n]*timeout[^\n]*method "DELETE"/i.test(tail);-- // Variant 2: the worker was killed during teardown. The framework `after` hook finished and- // a session DELETE was issued, but the log truncates before any error is written.- const afterHookFinished = tail.includes('Finished to run "after" hook');- const deleteIssued = tail.includes('COMMAND deleteSession()')- || /\[DELETE\][^\n]*\/session\//.test(tail);-- return afterSuiteFinished && (deleteTimeout || (afterHookFinished && deleteIssued));+ return buf.toString('utf8')+ .split('\n')+ .reverse()+ .find(line => line.includes('ERROR'))+ ?.trim()+ .slice(0, 500);} catch {- return false;+ return undefined;}}@@ -365,6 +390,16 @@ export const config: WebdriverIO.MultiremoteConfig = {} catch {// best effort — if we can't reset, onComplete will still parse what it sees}++ // Drop completion sentinels left behind by an earlier run in the same workspace; a stale+ // one would make onWorkerEnd read a genuine crash as a teardown-only crash.+ try {+ fs.readdirSync(TEST_RESULTS_DIR)+ .filter(name => (/^completed-.*\.json$/).test(name))+ .forEach(name => fs.unlinkSync(path.join(TEST_RESULTS_DIR, name)));+ } catch {+ // best effort — the directory may not exist yet+ }},/**@@ -466,14 +501,43 @@ export const config: WebdriverIO.MultiremoteConfig = {}},- after() {+ /**+ * Gets executed after all tests in the spec file are done, before the sessions are deleted.+ *+ * @param {number} result - Mocha's number of failing tests for this spec file.+ * @returns {void}+ */+ after(result) {const { ctx }: any = global;ctx?.webhooksProxy?.disconnect();keepAlive.forEach(clearInterval);++ // Tell the launcher this worker got through every test and hook. If it still exits+ // non-zero, the failure was in session teardown, which onWorkerEnd reports separately.+ const cid = process.env.WDIO_WORKER_ID;++ if (cid) {+ try {+ fs.writeFileSync(completionSentinelPath(cid), JSON.stringify({+ failed: failedTestTitles,+ failures: typeof result === 'number' ? result : 1+ }));+ } catch {+ // best effort — without it, a teardown crash is reported as a runner crash+ }+ }},async beforeSession(c, capabilities_, spec, cid) {+ // Clear any completion sentinel left by an earlier attempt at this spec — worker ids can+ // be reused when a spec file is retried — so onWorkerEnd never reads a stale completion.+ try {+ fs.rmSync(completionSentinelPath(cid), { force: true });+ } catch {+ // best effort — onPrepare already cleared sentinels from previous runs+ }+// Record this worker's browser sessions opening, so the launcher's// onComplete can compute peak concurrent sessions across the run.try {@@ -606,6 +670,10 @@ export const config: WebdriverIO.MultiremoteConfig = {if (error) {+ // Record the failure for the `after` hook, so the launcher still knows this spec had+ // failing tests if the worker dies before the reporters flush.+ failedTestTitles.push(test.title);+// Skip the remaining tests in the same describe block (but not other describe blocks in the file).ctx.failedSuite = test.parent;@@ -692,11 +760,11 @@ export const config: WebdriverIO.MultiremoteConfig = {/*** Gets executed after a worker process has exited.* Handles three crash scenarios:- * 1. Session teardown crash AFTER tests completed (common with Firefox on Selenium Grid):- * afterSuite ran, then the session DELETE either timed out or the worker was killed- * mid-teardown, so the runner exited non-zero. Tests actually passed. We skip the misleading- * "crashed" synthesis; allure already has accurate per-test results, and we summarise the- * teardown anomaly in a passing JUnit entry. See isPostTestSessionDeleteTimeout.+ * 1. Session teardown crash AFTER the spec file completed (common with Firefox on Selenium+ * Grid): the worker wrote its completion sentinel, then the session DELETE either timed out+ * or the worker was killed mid-teardown, so the runner exited non-zero. The tests themselves+ * ran, so we report their outcome (from the sentinel) rather than a misleading "crashed",+ * and record the teardown anomaly in the entry. See completionSentinelPath.* 2. Session DELETE timeout BEFORE tests completed: JUnit reporter never flushes,* leaving a zero-byte XML. Synthesise a failure entry.* 3. Session INIT timeout: JUnit reporter writes a non-empty XML but with empty name/classname,@@ -733,31 +801,46 @@ export const config: WebdriverIO.MultiremoteConfig = {return;}- // Detect the "tests passed but session teardown crashed" pattern from the worker's wdio- // log. Pass the raw spec path so the log file name is derived the same way- // @wdio/local-runner derives it (only the final extension is stripped).- if (isPostTestSessionDeleteTimeout(workerSpecs?.[0], cid)) {- const teardownName = 'Session teardown crashed after tests completed';- const teardownMessage- = `Session teardown crashed after tests completed (worker exit ${exitCode}). `- + 'The suite ran to completion; this entry just records the teardown anomaly.';+ // A worker that wrote its completion sentinel ran every test and hook in the spec file, so+ // it died in session teardown. Report what the tests actually did. Pass the raw spec path+ // so the log file name is derived the same way @wdio/local-runner derives it (only the+ // final extension is stripped).+ const completion = readWorkerCompletion(cid);++ if (completion) {+ const failureCount = Math.max(completion.failures, completion.failed.length);+ const clean = failureCount === 0;+ const workerError = readWorkerError(workerSpecs?.[0], cid);+ const teardownDetail = `Session teardown crashed (worker exit ${exitCode}), so the `+ + `reporters never wrote their results.${workerError ? ` Worker error: ${workerError}` : ''}`;+ const teardownName = clean+ ? 'Session teardown crashed after tests completed'+ : 'Tests failed, then session teardown crashed';+ const teardownMessage = clean+ ? `The spec file ran to completion with no failures. ${teardownDetail}`+ : `The spec file ran to completion with ${failureCount} failing test(s)`+ + `${completion.failed.length ? ` (${completion.failed.join(', ')})` : ''}. `+ + teardownDetail;const b = junitReportBuilder.newBuilder();-- b.testSuite().name(specName).testCase()+ const testCase = b.testSuite().name(specName).testCase().name(teardownName).className(specName).standardOutput(teardownMessage);++ if (!clean) {+ testCase.failure(teardownMessage);+ }b.writeTo(xmlPath);// When the worker is killed mid-teardown the reporters never flush, so allure has no// per-test results for this spec and it would otherwise vanish from the report. Write a- // passing entry so the spec stays visible with the teardown anomaly recorded. If allure- // already captured the per-test results (e.g. a slow-DELETE timeout after a clean- // flush), this is just an extra informational passing entry alongside them.+ // single entry so the spec stays visible, carrying the outcome the worker recorded. If+ // allure already captured the per-test results (e.g. a slow-DELETE timeout after a+ // clean flush), this is just an extra informational entry alongside them.const allureResult = {uuid: randomUUID(),name: teardownName,- status: 'passed',+ status: clean ? 'passed' : 'failed',statusDetails: { message: teardownMessage },stage: 'finished',steps: [],@@ -773,8 +856,8 @@ export const config: WebdriverIO.MultiremoteConfig = {= path.join(TEST_RESULTS_DIR, 'allure-results', `${allureResult.uuid}-result.json`);fs.writeFileSync(allurePath, JSON.stringify(allureResult));- console.log(`[onWorkerEnd] Worker ${cid} (${specName}): session teardown crashed after `- + 'afterSuite — synthesising passing JUnit + allure entry; suite ran to completion.');+ console.log(`[onWorkerEnd] Worker ${cid} (${specName}): ${teardownName} — synthesising `+ + `${clean ? 'passing' : 'failing'} JUnit + allure entry. ${teardownMessage}`);return;}