The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Scan for import-time side effects in parallel (#83493)
Romeo Van Snicka4c8cbdf02e9
diff --git a/frontend/lint/scripts/side-effect-files.js b/frontend/lint/scripts/side-effect-files.jsindex d596069ea09..f69ba32a013 100644--- a/frontend/lint/scripts/side-effect-files.js+++ b/frontend/lint/scripts/side-effect-files.js@@ -2,7 +2,14 @@// Without a flag it exits 1 when the registry has drifted, `--update` adds new files as unclassified and drops clean ones, and `--verbose` prints every finding.const fs = require("fs");+const os = require("os");const path = require("path");+const {+ Worker,+ isMainThread,+ parentPort,+ workerData,+} = require("worker_threads");const { Linter } = require("eslint");const micromatch = require("micromatch");@@ -69,10 +76,10 @@ function listSourceFiles() {.sort();}-function scanEffectFiles() {+function scanFiles(files) {const linter = new Linter();const findings = new Map();- for (const file of listSourceFiles()) {+ for (const file of files) {// The registry records what a file itself does at import, so its imports of other listed files don't count.const messages = linter.verify(readSource(file), LINTER_CONFIG, {@@ -90,6 +97,46 @@ function scanEffectFiles() {return findings;}+// Parsing every source file with the typescript-eslint parser is the whole cost+// of this scan, and each file is independent, so the list is split across+// workers. The workers load eslint through node's own require, which also skips+// the transform a test runner would otherwise apply to it.+const SCAN_WORKER_COUNT = Math.min(+ 8,+ Math.max(1, os.availableParallelism() - 1),+);+const SCAN_TASK = "scanEffectFiles";++async function scanEffectFiles() {+ const files = listSourceFiles();+ if (SCAN_WORKER_COUNT === 1) {+ return scanFiles(files);+ }++ const chunks = Array.from({ length: SCAN_WORKER_COUNT }, () => []);+ files.forEach((file, index) => {+ chunks[index % SCAN_WORKER_COUNT].push(file);+ });++ const scanned = await Promise.all(+ chunks.map(+ (chunk) =>+ new Promise((resolve, reject) => {+ const worker = new Worker(__filename, {+ workerData: { task: SCAN_TASK, files: chunk },+ });+ worker.on("message", resolve);+ worker.on("error", reject);+ }),+ ),+ );++ // listSourceFiles is sorted and callers rely on that order for their reports.+ return new Map(+ scanned.flat().sort(([left], [right]) => (left < right ? -1 : 1)),+ );+}+// Listed packages that no source file imports.function unimportedPackages(registry, files) {const unseen = new Map(@@ -150,9 +197,9 @@ function updateRegistry(registryPath, { missing, stale }) {}/* eslint-disable no-console */-function main(argv) {+async function main(argv) {const started = Date.now();- const findings = scanEffectFiles();+ const findings = await scanEffectFiles();const effectFiles = [...findings.keys()];const registry = loadRegistry(DEFAULT_REGISTRY_PATH);const diff = diffRegistry(registry, effectFiles);@@ -207,6 +254,12 @@ module.exports = {unimportedPackages,};-if (require.main === module) {- process.exit(main(process.argv.slice(2)));+if (!isMainThread && workerData?.task === SCAN_TASK) {+ parentPort.postMessage([...scanFiles(workerData.files)]);+}++// isMainThread matters: a worker's entry module is its own require.main, so+// without it every worker would run main and spawn workers of its own.+if (isMainThread && require.main === module) {+ main(process.argv.slice(2)).then((code) => process.exit(code));}diff --git a/frontend/lint/tests/side-effect-files.unit.spec.js b/frontend/lint/tests/side-effect-files.unit.spec.jsindex 21adb5a7b05..adbef664751 100644--- a/frontend/lint/tests/side-effect-files.unit.spec.js+++ b/frontend/lint/tests/side-effect-files.unit.spec.js@@ -23,7 +23,12 @@ const REPO_ROOT = path.resolve(__dirname, "../../..");// anywhere fails CI until it is classified.describe("side-effect-files.json", () => {const registry = loadRegistry(DEFAULT_REGISTRY_PATH);- const effectFiles = [...scanEffectFiles().keys()];+ // The scan runs in worker threads, so the files it reports arrive async.+ let effectFiles;++ beforeAll(async () => {+ effectFiles = [...(await scanEffectFiles()).keys()];+ });it("lists every file the rule reports, with no stale global or entry entries", () => {const { missing, stale } = diffRegistry(registry, effectFiles);