Skip to content

armgit / mocha

Public Imported from GitHub
0

☕️ Classic, reliable, trusted test framework for Node.js and the browser

17.8 KB
Raw Blame
"use strict";
const Mocha = require("../../../lib/mocha.cjs");
const {
  runMochaAsync,
  invokeMochaAsync,
  getSummary,
  resolveFixturePath,
} = require("../helpers.cjs");
const psList = require("ps-list").default;

const REPORTER_FIXTURE_PATH = resolveFixturePath(
  "options/parallel/test-a.fixture.js",
);

/**
 * Get child processes of a given PID using ps-list
 * @param {number} pid - Parent process ID
 * @param {Object} options - Options object
 * @param {boolean} options.root - Include the parent PID in results
 * @returns {Promise<number[]>} Array of child process PIDs
 */
async function getChildPids(pid) {
  return (
    (await psList())
      // ppid = parent process ID
      .filter((proc) => proc.ppid === pid)
      .map((proc) => proc.pid)
  );
}

/**
 * Check if a process exists using ps-list
 * @param {number} pid - Process ID to check
 * @param {Object} options - Options object
 * @param {boolean} options.root - Include the PID itself in results
 * @returns {Promise<boolean>} Array with the PID if exists, null otherwise
 */
async function checkProcessExists(pid) {
  return (await psList()).some((proc) => proc.pid === pid);
}

/**
 * Run a test fixture with the same reporter in both parallel and serial modes,
 * returning both outputs for comparison
 * @param {string} reporter - Reporter name
 * @returns {{actual: import('../helpers.cjs').Summary, expected: import('../helpers.cjs').Summary}}
 */
async function compareReporters(reporter) {
  const [actual, expected] = await Promise.all([
    runMochaAsync(REPORTER_FIXTURE_PATH, [
      "--reporter",
      reporter,
      "--no-parallel",
    ]),
    runMochaAsync(REPORTER_FIXTURE_PATH, [
      "--reporter",
      reporter,
      "--parallel",
    ]),
  ]);

  // the test duration is non-deterministic, so we just fudge it
  actual.output = expected.output = expected.output.replace(/\d+m?s/g, "100ms");

  return { actual, expected };
}

/**
 * Many (but not all) reporters can use this assertion to compare output of serial vs. parallel
 * @param {string} reporter - Reporter name
 */
async function assertReporterOutputEquality(reporter) {
  const { actual, expected } = await compareReporters(reporter);
  return expect(actual, "to satisfy", {
    passing: expected.passing,
    failing: expected.failing,
    pending: expected.pending,
    code: expected.code,
    output: expected.output,
  });
}

/**
 * Polls a process for its list of children PIDs. Returns the first non-empty list found
 * @param {number} pid - Process PID
 * @returns {number[]} Child PIDs
 */
async function waitForChildPids(pid) {
  let childPids = [];
  while (!childPids.length) {
    childPids = await getChildPids(pid);
    await new Promise((resolve) => setTimeout(resolve, 100));
  }
  return childPids;
}

describe("--parallel", function () {
  this.timeout(20000);

  describe("when a test has a syntax error", function () {
    describe("when there is only a single test file", function () {
      it("should fail gracefully", async function () {
        return expect(
          runMochaAsync("options/parallel/syntax-err.fixture.js", [
            "--parallel",
          ]),
          "when fulfilled",
          "to have failed with output",
          /SyntaxError/,
        );
      });
    });

    describe("when there are multiple test files", function () {
      it("should fail gracefully", async function () {
        return expect(
          invokeMochaAsync(
            [
              resolveFixturePath("options/parallel/syntax-err.fixture.js"),
              "--parallel",
            ],
            "pipe",
          )[1],
          "when fulfilled",
          "to have failed",
        );
      });
    });
  });

  describe("when used with CJS tests", function () {
    it("should have the same result as with --no-parallel", async function () {
      const expected = await runMochaAsync(
        "options/parallel/test-*.fixture.js",
        ["--no-parallel"],
      );
      return expect(
        runMochaAsync("options/parallel/test-*.fixture.js", ["--parallel"]),
        "to be fulfilled with value satisfying",
        {
          passing: expected.passing,
          failing: expected.failing,
          pending: expected.pending,
          code: expected.code,
        },
      );
    });
  });

  describe("when used with ESM tests", function () {
    it("should have the same result as with --no-parallel", async function () {
      const expected = getSummary(
        await invokeMochaAsync([
          "--no-parallel",
          resolveFixturePath("esm/*.fixture.js"),
        ])[1],
      );

      const actual = getSummary(
        await invokeMochaAsync([
          "--parallel",
          resolveFixturePath("esm/*.fixture.js"),
        ])[1],
      );

      return expect(actual, "to satisfy", {
        pending: expected.pending,
        passing: expected.passing,
        failing: expected.failing,
      });
    });
  });

  describe("when used with --retries", function () {
    it("should retry tests appropriately", async function () {
      return expect(
        runMochaAsync("options/parallel/retries-*.fixture.js", ["--parallel"]),
        "when fulfilled",
        "to satisfy",
        expect
          .it("to have failed")
          .and("to have passed test count", 1)
          .and("to have pending test count", 0)
          .and("to have failed test count", 1)
          .and("to contain output", /count: 3/),
      );
    });
  });

  describe("when used with --allow-uncaught", function () {
    it("should bubble up an exception", async function () {
      return expect(
        invokeMochaAsync(
          [
            resolveFixturePath("options/parallel/uncaught.fixture.js"),
            "--parallel",
            "--allow-uncaught",
          ],
          "pipe",
        )[1],
        "when fulfilled",
        "to satisfy",
        expect
          .it("to contain output", /Error: existential isolation/i)
          .and("to have exit code", 1),
      );
    });
  });

  describe("when used with --file", function () {
    it("should error out", async function () {
      return expect(
        invokeMochaAsync(
          [
            "--file",
            resolveFixturePath("options/parallel/test-a.fixture.js"),
            "--parallel",
          ],
          "pipe",
        )[1],
        "when fulfilled",
        "to contain output",
        /mutually exclusive with --file/,
      );
    });
  });

  describe("when used with --sort", function () {
    it("should error out", async function () {
      return expect(
        invokeMochaAsync(
          [
            "--sort",
            resolveFixturePath("options/parallel/test-*.fixture.js"),
            "--parallel",
          ],
          "pipe",
        )[1],
        "when fulfilled",
        "to contain output",
        /mutually exclusive with --sort/,
      );
    });
  });

  describe("when used with exclusive tests", function () {
    it("should error out", async function () {
      return expect(
        invokeMochaAsync(
          [
            resolveFixturePath("options/parallel/exclusive-test-*.fixture.js"),
            "--parallel",
          ],
          "pipe",
        )[1],
        "when fulfilled",
        "to contain output",
        /`\.only` is not supported in parallel mode/,
      );
    });
  });

  describe("when used with --bail", function () {
    it("should skip some tests", async function () {
      const result = await runMochaAsync("options/parallel/test-*.fixture.js", [
        "--parallel",
        "--bail",
      ]);
      // we don't know _exactly_ how many tests will be skipped here
      // due to the --bail, but the number of tests completed should be
      // less than the total, which is 5.
      return expect(
        result.passing + result.pending + result.failing,
        "to be less than",
        5,
      );
    });

    it("should fail", async function () {
      return expect(
        runMochaAsync("options/parallel/test-*.fixture.js", [
          "--parallel",
          "--bail",
        ]),
        "when fulfilled",
        "to have failed",
      );
    });
  });

  describe('when encountering a "bail" in context', function () {
    it("should skip some tests", async function () {
      const result = await runMochaAsync("options/parallel/bail.fixture.js", [
        "--parallel",
      ]);
      return expect(
        result.passing + result.pending + result.failing,
        "to be less than",
        2,
      );
    });

    it("should fail", async function () {
      return expect(
        runMochaAsync("options/parallel/bail.fixture.js", [
          "--parallel",
          "--bail",
        ]),
        "when fulfilled",
        "to have failed",
      );
    });
  });

  describe('when used with "grep"', function () {
    it("should be equivalent to running in serial", async function () {
      const expected = await runMochaAsync(
        "options/parallel/test-*.fixture.js",
        ["--no-parallel", '--grep="suite d"'],
      );
      return expect(
        runMochaAsync("options/parallel/test-*.fixture.js", [
          "--parallel",
          '--grep="suite d"',
        ]),
        "to be fulfilled with value satisfying",
        {
          passing: expected.passing,
          failing: expected.failing,
          pending: expected.pending,
          code: expected.code,
        },
      );
    });
  });

  describe("reporter equivalence", function () {
    // each reporter name is duplicated; one is in all lower-case
    // 'base' is abstract, 'html' is browser-only, others are incompatible
    const DENY = ["progress", "base", "html", "markdown", "json-stream"];
    Object.keys(Mocha.reporters)
      .filter((name) => /^[a-z]/.test(name) && DENY.indexOf(name) === -1)
      .forEach((reporter) => {
        describe(`when multiple test files run with --reporter=${reporter}`, function () {
          it("should have the same result as when run with --no-parallel", async function () {
            // note that the output may not be in the same order, as running file
            // order is non-deterministic in parallel mode
            const expected = await runMochaAsync(
              "options/parallel/test-*.fixture.js",
              ["--reporter", reporter, "--no-parallel"],
            );
            return expect(
              runMochaAsync("options/parallel/test-*.fixture.js", [
                "--reporter",
                reporter,
                "--parallel",
              ]),
              "to be fulfilled with value satisfying",
              {
                passing: expected.passing,
                failing: expected.failing,
                pending: expected.pending,
                code: expected.code,
              },
            );
          });
        });
      });

    describe("when a single test file is run with --reporter=dot", function () {
      it("should have the same output as when run with --no-parallel", async function () {
        return assertReporterOutputEquality.call(this, "dot");
      });
    });

    describe("when a single test file is run with --reporter=doc", function () {
      it("should have the same output as when run with --no-parallel", async function () {
        return assertReporterOutputEquality.call(this, "doc");
      });
    });

    describe("when a single test file is run with --reporter=tap", function () {
      it("should have the same output as when run with --no-parallel", async function () {
        return assertReporterOutputEquality.call(this, "tap");
      });
    });

    describe("when a single test file is run with --reporter=list", function () {
      it("should have the same output as when run with --no-parallel", async function () {
        return assertReporterOutputEquality.call(this, "list");
      });
    });

    describe("when a single test file is run with --reporter=min", function () {
      it("should have the same output as when run with --no-parallel", async function () {
        return assertReporterOutputEquality.call(this, "min");
      });
    });

    describe("when a single test file is run with --reporter=spec", function () {
      it("should have the same output as when run with --no-parallel", async function () {
        return assertReporterOutputEquality.call(this, "spec");
      });
    });

    describe("when a single test file is run with --reporter=nyan", function () {
      it("should have the same output as when run with --no-parallel", async function () {
        return assertReporterOutputEquality.call(this, "nyan");
      });
    });

    describe("when a single test file is run with --reporter=landing", function () {
      it("should have the same output as when run with --no-parallel", async function () {
        return assertReporterOutputEquality.call(this, "landing");
      });
    });

    describe("when a single test file is run with --reporter=progress", function () {
      it("should fail due to incompatibility", async function () {
        return expect(
          invokeMochaAsync(
            [
              resolveFixturePath("options/parallel/test-a.fixture.js"),
              "--reporter=progress",
              "--parallel",
            ],
            "pipe",
          )[1],
          "when fulfilled",
          "to satisfy",
          expect
            .it("to have failed")
            .and("to contain output", /mutually exclusive/),
        );
      });
    });

    describe("when a single test file is run with --reporter=markdown", function () {
      it("should fail due to incompatibility", async function () {
        return expect(
          invokeMochaAsync(
            [
              resolveFixturePath("options/parallel/test-a.fixture.js"),
              "--reporter=markdown",
              "--parallel",
            ],
            "pipe",
          )[1],
          "when fulfilled",
          "to satisfy",
          expect
            .it("to have failed")
            .and("to contain output", /mutually exclusive/),
        );
      });
    });

    describe("when a single test file is run with --reporter=json-stream", function () {
      it("should fail due to incompatibility", async function () {
        return expect(
          invokeMochaAsync(
            [
              resolveFixturePath("options/parallel/test-a.fixture.js"),
              "--reporter=json-stream",
              "--parallel",
            ],
            "pipe",
          )[1],
          "when fulfilled",
          "to satisfy",
          expect
            .it("to have failed")
            .and("to contain output", /mutually exclusive/),
        );
      });
    });

    describe("when a single test file is run with --reporter=json", function () {
      it("should have the same output as when run with --no-parallel", async function () {
        // this one has some timings/durations that we can safely ignore
        const { expected, actual } = await compareReporters("json");
        expected.output = JSON.parse(expected.output);
        actual.output = JSON.parse(actual.output);
        expect(actual, "to satisfy", {
          passing: expected.passing,
          failing: expected.failing,
          pending: expected.pending,
          code: expected.code,
        });
        expect(actual.output, "to satisfy", {
          stats: {
            suites: expected.output.stats.suites,
            tests: expected.output.stats.tests,
            passes: expected.output.stats.passes,
            pending: expected.output.stats.pending,
            failures: expected.output.stats.failures,
          },
        });
        expect(actual.output.tests, "to satisfy", expected.output.tests);
      });
    });

    describe("when a single test file is run with --reporter=xunit", function () {
      it("should have the same output as when run with --no-parallel", async function () {
        // durations need replacing
        const { expected, actual } = await compareReporters("xunit");
        expected.output = expected.output
          .replace(/time=".+?"/g, 'time="0.5"')
          .replace(/timestamp=".+?"/g, 'timestamp="some-timestamp');
        actual.output = actual.output
          .replace(/time=".+?"/g, 'time="0.5"')
          .replace(/timestamp=".+?"/g, 'timestamp="some-timestamp');
        return expect(actual, "to satisfy", {
          passing: expected.passing,
          failing: expected.failing,
          pending: expected.pending,
          code: expected.code,
          output: expected.output,
        });
      });
    });
  });

  // - `waitForChildPids` polls for created children
  // - `await promise` waits for `invokeMochaAsync` to complete, which should
  //   kill all children
  // - `checkProcessExists` validates that each child was killed
  // - if any weren't killed, the test fails
  describe("pool shutdown", function () {
    describe("during normal operation", function () {
      it("should not leave orphaned processes around", async function () {
        const [{ pid }, promise] = invokeMochaAsync([
          resolveFixturePath("options/parallel/test-*.fixture.js"),
          "--parallel",
        ]);
        const childPids = await waitForChildPids(pid);
        await promise;
        return expect(
          Promise.all(
            [pid, ...childPids].map(
              async (childPid) => await checkProcessExists(childPid),
            ),
          ),
          "when fulfilled",
          "to have items satisfying",
          false,
        );
      });
    });

    describe("during operation with --bail", function () {
      it("should not leave orphaned processes around", async function () {
        const [{ pid }, promise] = invokeMochaAsync([
          resolveFixturePath("options/parallel/test-*.fixture.js"),
          "--bail",
          "--parallel",
        ]);
        const childPids = await waitForChildPids(pid);
        await promise;
        return expect(
          Promise.all(
            [pid, ...childPids].map(
              async (childPid) => await checkProcessExists(childPid),
            ),
          ),
          "when fulfilled",
          "to have items satisfying",
          false,
        );
      });
    });
  });
});