The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
// Generates frontend/lint/side-effect-files.json by linting the whole tree with `metabase/no-module-side-effects`.
// Without a flag it exits 1 when the registry has drifted, `--update` adds new files as unclassified and drops clean ones, and `--verbose` prints every finding.
const fs = require("fs");
const os = require("os");
const path = require("path");
const {
Worker,
isMainThread,
parentPort,
workerData,
} = require("worker_threads");
const { Linter } = require("eslint");
const micromatch = require("micromatch");
const tseslint = require("typescript-eslint");
const rule = require("../eslint-plugin-metabase/rules/no-module-side-effects");
const {
NO_MODULE_SIDE_EFFECTS_IGNORES,
NO_MODULE_SIDE_EFFECTS_OPTIONS,
} = require("../no-module-side-effects-options");
const {
DEFAULT_REGISTRY_PATH,
classify,
loadRegistry,
validateRegistry,
} = require("../side-effect-registry");
const REPO_ROOT = path.resolve(__dirname, "../../..");
const SOURCE_ROOTS = ["frontend/src", "enterprise/frontend/src"];
const SOURCE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".jsx"]);
// Generated by the SDK package build and gitignored, so it is not source.
const GENERATED_DIRECTORY = "cli-snippets-tmp";
const RULE_NAME = "metabase/no-module-side-effects";
const LINTER_CONFIG = [
{
files: ["**/*.{ts,tsx,js,jsx}"],
languageOptions: {
parser: tseslint.parser,
parserOptions: { ecmaFeatures: { jsx: true } },
sourceType: "module",
},
plugins: { metabase: { rules: { "no-module-side-effects": rule } } },
rules: { [RULE_NAME]: ["error", NO_MODULE_SIDE_EFFECTS_OPTIONS] },
},
];
function readSource(file) {
return fs.readFileSync(path.join(REPO_ROOT, file), "utf8");
}
function listSourceFiles() {
return SOURCE_ROOTS.flatMap((root) =>
fs
.readdirSync(path.join(REPO_ROOT, root), {
recursive: true,
withFileTypes: true,
})
.filter(
(entry) =>
entry.isFile() &&
SOURCE_EXTENSIONS.has(path.extname(entry.name)) &&
!entry.parentPath.includes(GENERATED_DIRECTORY),
)
.map((entry) =>
path
.relative(REPO_ROOT, path.join(entry.parentPath, entry.name))
.replaceAll("\\", "/"),
),
)
.filter((file) => !micromatch.isMatch(file, NO_MODULE_SIDE_EFFECTS_IGNORES))
.sort();
}
function scanFiles(files) {
const linter = new Linter();
const findings = new Map();
for (const file of files) {
// The registry records what a file itself does at import, so its imports of other listed files don't count.
const messages = linter
.verify(readSource(file), LINTER_CONFIG, {
filename: path.join(REPO_ROOT, file),
})
.filter(
(message) =>
message.ruleId === RULE_NAME &&
message.messageId !== "importsGlobalEffect",
);
if (messages.length > 0) {
findings.set(file, messages);
}
}
return findings;
}
// Parsing every source file with the typescript-eslint parser is the whole cost
// of this scan, and each file is independent, so the list is split across
// workers. The workers load eslint through node's own require, which also skips
// the transform a test runner would otherwise apply to it.
const SCAN_WORKER_COUNT = Math.min(
8,
Math.max(1, os.availableParallelism() - 1),
);
const SCAN_TASK = "scanEffectFiles";
async function scanEffectFiles() {
const files = listSourceFiles();
if (SCAN_WORKER_COUNT === 1) {
return scanFiles(files);
}
const chunks = Array.from({ length: SCAN_WORKER_COUNT }, () => []);
files.forEach((file, index) => {
chunks[index % SCAN_WORKER_COUNT].push(file);
});
const scanned = await Promise.all(
chunks.map(
(chunk) =>
new Promise((resolve, reject) => {
const worker = new Worker(__filename, {
workerData: { task: SCAN_TASK, files: chunk },
});
worker.on("message", resolve);
worker.on("error", reject);
}),
),
);
// listSourceFiles is sorted and callers rely on that order for their reports.
return new Map(
scanned.flat().sort(([left], [right]) => (left < right ? -1 : 1)),
);
}
// Listed packages that no source file imports.
function unimportedPackages(registry, files) {
const unseen = new Map(
Object.keys(registry.packages).map((specifier) => [
specifier,
new RegExp(`["'\`]${escapeRegExp(specifier)}(/[^"'\`]*)?["'\`]`),
]),
);
for (const file of files) {
if (unseen.size === 0) {
break;
}
const source = readSource(file);
for (const [specifier, pattern] of unseen) {
if (pattern.test(source)) {
unseen.delete(specifier);
}
}
}
return [...unseen.keys()];
}
function escapeRegExp(text) {
return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
function diffRegistry(registry, effectFiles) {
const effectSet = new Set(effectFiles);
const missing = effectFiles.filter(
(file) => classify(registry, file) == null,
);
const stale = [
...Object.keys(registry.files).filter((key) => !key.includes("*")),
...registry.unclassified,
].filter((file) => !effectSet.has(file));
return { missing, stale };
}
// A stale "global" or "entry" entry keeps rejecting imports of a clean file, whereas a stale "self" entry affects nothing.
function enforcedStale(registry, stale) {
return stale.filter((file) => classify(registry, file) !== "self");
}
function updateRegistry(registryPath, { missing, stale }) {
const raw = JSON.parse(fs.readFileSync(registryPath, "utf8"));
const staleSet = new Set(stale);
const files = Object.fromEntries(
Object.entries(raw.files || {}).filter(([key]) => !staleSet.has(key)),
);
const unclassified = [
...(raw.unclassified || []).filter((file) => !staleSet.has(file)),
...missing,
].sort();
fs.writeFileSync(
registryPath,
`${JSON.stringify({ ...raw, files, unclassified }, null, 2)}\n`,
);
}
/* eslint-disable no-console */
async function main(argv) {
const started = Date.now();
const findings = await scanEffectFiles();
const effectFiles = [...findings.keys()];
const registry = loadRegistry(DEFAULT_REGISTRY_PATH);
const diff = diffRegistry(registry, effectFiles);
const problems = [
...validateRegistry(registry, effectFiles),
...unimportedPackages(registry, listSourceFiles()).map(
(specifier) => `packages: nothing imports "${specifier}"`,
),
];
const seconds = ((Date.now() - started) / 1000).toFixed(1);
console.log(
`${effectFiles.length} files with import-time effects (${seconds}s)`,
);
if (argv.includes("--verbose")) {
for (const [file, messages] of findings) {
console.log(` ${file}`);
for (const message of messages) {
console.log(` ${message.line}: ${message.message}`);
}
}
}
if (argv.includes("--update")) {
updateRegistry(DEFAULT_REGISTRY_PATH, diff);
console.log(
`added ${diff.missing.length} as unclassified, removed ${diff.stale.length}`,
);
return 0;
}
for (const problem of problems) {
console.log(`invalid: ${problem}`);
}
for (const file of diff.missing) {
console.log(`missing: ${file}`);
}
for (const file of diff.stale) {
console.log(`clean: ${file}`);
}
const drifted = problems.length + diff.missing.length + diff.stale.length;
if (drifted > 0) {
console.log(
"\nregistry out of date. Run: bun frontend/lint/scripts/side-effect-files.js --update",
);
}
return drifted > 0 ? 1 : 0;
}
module.exports = {
listSourceFiles,
scanEffectFiles,
diffRegistry,
enforcedStale,
unimportedPackages,
};
if (!isMainThread && workerData?.task === SCAN_TASK) {
parentPort.postMessage([...scanFiles(workerData.files)]);
}
// isMainThread matters: a worker's entry module is its own require.main, so
// without it every worker would run main and spawn workers of its own.
if (isMainThread && require.main === module) {
main(process.argv.slice(2)).then((code) => process.exit(code));
}