🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(web): Limit check command cpu usage (#19162)
Ben Bachem3fb279c71af3
diff --git a/scripts/check.mjs b/scripts/check.mjsindex 8e7d0e8deb..6073c0e78a 100644--- a/scripts/check.mjs+++ b/scripts/check.mjs@@ -11,6 +11,7 @@ import {writeFileSync,} from "node:fs";import { dirname, relative, resolve } from "node:path";+import { availableParallelism } from "node:os";import { fileURLToPath } from "node:url";import { stripVTControlCharacters } from "node:util";@@ -89,6 +90,7 @@ export function parseOptions(args) {base: "origin/main",all: false,dryRun: false,+ cpus: Math.max(1, Math.floor(availableParallelism() / 2)),};for (let index = 0; index < args.length; index++) {const arg = args[index];@@ -108,6 +110,14 @@ export function parseOptions(args) {}const key = arg === "--only" || arg === "-o" ? "only" : "skip";options[key] = [...new Set([...(options[key] ?? []), ...names])];+ } else if (arg === "--cpus") {+ const value = args[++index];+ if (+ !/^[1-9]\d*$/.test(value ?? "") ||+ !Number.isSafeInteger(Number(value))+ )+ throw new Error("--cpus requires a positive integer");+ options.cpus = Number(value);} else if (arg === "--base") {const base = args[++index];if (!base || base.startsWith("-"))@@ -189,6 +199,7 @@ export function createPlan(taskGraph,exists,changes,+ options.cpus ?? parseOptions([]).cpus,);return [...planRepositoryChecks(@@ -293,6 +304,7 @@ function planPackageChecks(taskGraph,exists,changes,+ cpus,) {const selected = selectPackageChecks(codeFiles,@@ -310,6 +322,7 @@ function planPackageChecks(const prepare = (taskId) => {const task = taskGraph.tasks.get(taskId);if (!task) throw new Error(`Missing Turbo task graph for ${taskId}`);+ const directPrerequisites = [];for (const dependency of task.dependencies) {// Related suites need build/generation prerequisites, not upstream suites.if (@@ -319,7 +332,22 @@ function planPackageChecks()continue;targets.add(dependency);+ directPrerequisites.push(dependency);}+ const prerequisites = new Set();+ const visit = (taskId) => {+ if (prerequisites.has(taskId)) return;+ prerequisites.add(taskId);+ const prerequisite = taskGraph.tasks.get(taskId);+ if (!prerequisite)+ throw new Error(`Missing Turbo task graph for ${taskId}`);+ for (const dependency of prerequisite.dependencies) visit(dependency);+ };+ for (const dependency of directPrerequisites) visit(dependency);+ // Turbo traverses packages without a matching script but does not execute them.+ return [...prerequisites].filter(+ (taskId) => taskGraph.tasks.get(taskId).command !== "<NONEXISTENT>",+ );};// File-only TypeScript checks miss downstream errors. Keep project-wide// incremental checks and let Turbo run prerequisite builds/code generation;@@ -349,6 +377,32 @@ function planPackageChecks(pkg.scripts[task.name].includes(task.runner),);for (const task of checks) {+ if (+ task.runner === "eslint" &&+ pkg.scripts[task.name].includes("eslint")+ ) {+ // Turbo cannot append ESLint-only flags to a mixed task batch.+ // Keep dependency preparation in Turbo; ESLint retains its own cache.+ const prerequisites = prepare(`${pkg.name}#${task.name}`);+ const details = related+ ? planNarrowedCheck(task, pkg, ownFiles, base, exists)+ : { label: `${task.label} · ${pkg.name}`, args: [] };+ packageChecks.push({+ ...details,+ args: [+ "--filter",+ pkg.name,+ "run",+ task.name,+ ...details.args,+ "--concurrency",+ "off",+ ],+ requiresTurbo: true,+ prerequisites,+ });+ continue;+ }if (!related) {full.get(task.name).push(pkg.name);continue;@@ -399,6 +453,7 @@ function planPackageChecks("--output-logs=errors-only","--ui=stream","--log-prefix=none",+ `--concurrency=${Math.max(1, cpus - 1)}`,...targets,"--continue=dependencies-successful",],@@ -739,7 +794,7 @@ async function runCheck(check, outputDirectory, checkIndex) {result = await new Promise((resolve) => {const child = spawn("pnpm", args, {cwd: root,- env: createOutput().childEnv,+ env: createCheckEnv(),stdio: ["ignore", output, output],});let error;@@ -777,31 +832,63 @@ async function runCheck(check, outputDirectory, checkIndex) {}// Do not run a suite whose prerequisite generation/build failed.if (result.error || result.signal || result.status !== 0) {- return { failure: { ...result, outputFile }, reports };+ return {+ failure: { ...result, outputFile },+ reports,+ successfulTurboTasks: check.isTurboBatch+ ? collectDiagnostics(outputFile, true).successfulTaskIds+ : [],+ };}}return { failure: null, reports };}-async function* runPlan(plan, outputDirectory) {+export function createCheckEnv(env = process.env) {+ return createOutput({+ ...env,+ VITEST_MAX_WORKERS: "1",+ CARGO_BUILD_JOBS: "1",+ RUST_TEST_THREADS: "1",+ RAYON_NUM_THREADS: "1",+ GOMAXPROCS: "1",+ UV_THREADPOOL_SIZE: "1",+ }).childEnv;+}++export async function* runPlan(plan, outputDirectory, cpus, run = runCheck) {const pending = new Map();+ const weights = new Map();const hasTurbo = plan.some((check) => check.isTurboBatch);let nextIndex = 0;let foreground = null;let turboFailed = false;+ let successfulTurboTasks = new Set();try {while (nextIndex < plan.length || pending.size > 0) {- if (foreground === null && nextIndex < plan.length) {+ const weight = plan[nextIndex]?.isTurboBatch ? Math.max(1, cpus - 1) : 1;+ const used = [...weights.values()].reduce((sum, value) => sum + value, 0);+ if (+ foreground === null &&+ nextIndex < plan.length &&+ used + weight <= cpus+ ) {const checkIndex = nextIndex++;const check = plan[checkIndex];- if (check.requiresTurbo && turboFailed) {+ if (+ check.requiresTurbo &&+ turboFailed &&+ !check.prerequisites?.every((taskId) =>+ successfulTurboTasks.has(taskId),+ )+ ) {yield { type: "skipped", check, checkIndex };continue;}const promise = (async () => {const startedAt = performance.now();try {- const { failure, reports } = await runCheck(+ const { failure, reports, successfulTurboTasks } = await run(check,outputDirectory,checkIndex,@@ -812,6 +899,7 @@ async function* runPlan(plan, outputDirectory) {checkIndex,result: failure,reports,+ successfulTurboTasks,durationMs: performance.now() - startedAt,};} catch (error) {@@ -819,6 +907,7 @@ async function* runPlan(plan, outputDirectory) {}})();pending.set(checkIndex, promise);+ weights.set(checkIndex, weight);if (!check.requiresTurbo && !(check.isKnip && hasTurbo)) {foreground = checkIndex;}@@ -827,8 +916,12 @@ async function* runPlan(plan, outputDirectory) {}const outcome = await Promise.race(pending.values());pending.delete(outcome.checkIndex);+ weights.delete(outcome.checkIndex);if (outcome.checkIndex === foreground) foreground = null;- if (outcome.check.isTurboBatch && outcome.result) turboFailed = true;+ if (outcome.check.isTurboBatch && outcome.result) {+ turboFailed = true;+ successfulTurboTasks = new Set(outcome.successfulTurboTasks);+ }yield outcome;}} finally {@@ -848,11 +941,18 @@ function collectDiagnostics(outputFile, isTurboBatch) {.match(/^\s*Summary:\s+(.+\.json)\s*$/m)?.[1]?.trim();let failedTasks = [];+ let successfulTaskIds = [];if (summaryPath) {try {const summary = JSON.parse(readFileSync(resolve(root, summaryPath), "utf8"),);+ successfulTaskIds = summary.tasks+ .filter(+ (task) =>+ task.execution?.exitCode === 0 || task.cache?.status === "HIT",+ )+ .map((task) => task.taskId);failedTasks = summary.tasks.filter((task) =>Number.isInteger(task.execution?.exitCode) &&@@ -878,6 +978,7 @@ function collectDiagnostics(outputFile, isTurboBatch) {return {captured,tasks,+ successfulTaskIds,complete: tasks.length > 0 && tasks.every((task) => task.diagnostics),};}@@ -885,6 +986,9 @@ function collectDiagnostics(outputFile, isTurboBatch) {async function main() {const options = parseOptions(process.argv.slice(2));if (options.help) {+ console.log(+ "--cpus <n>: concurrent task budget (default: half available CPUs, minimum 1).\nNested worker pools are limited to one worker during check. This is not a hard CPU quota.\n",+ );console.log("Usage: pnpm check [--branch] [--base <ref>] [--all] [--only <checks> | --skip <checks>] [--dry-run]\n\nDefault: all applicable checks for staged, unstaged, and untracked changes.\n--branch: changes since merge-base with origin/main, plus local changes.\n--base: override branch target (implies --branch).\n--all: check every package within the selected checks.\n--only, -o: run only the comma-separated checks.\n--skip, -s: exclude the comma-separated checks. Cannot combine with --only.\nChecks: format, typecheck, lint, knip, tests. Build/generation prerequisites still run.\n--dry-run: print checks without running them.",);@@ -900,6 +1004,7 @@ async function main() {: git(["rev-parse", "HEAD"]).trim();const files = collectChanges(git, base);const output = createOutput();+ console.log(`Task concurrency budget: ${options.cpus}`);printHeader(files, options, output);const { packages, taskGraph } = loadWorkspace(base);const plan = createPlan(files, packages, options, base, taskGraph);@@ -911,7 +1016,7 @@ async function main() {printPlan(plan);return;}- const results = await executePlan(plan, output);+ const results = await executePlan(plan, output, options.cpus);if (results.interrupted) {process.exitCode = 1;return;@@ -958,7 +1063,7 @@ function printPlan(plan) {}}-async function executePlan(plan, { terminal, paint, indent }) {+async function executePlan(plan, { terminal, paint, indent }, cpus) {mkdirSync(resolve(root, ".turbo"), { recursive: true });const outputDirectory = mkdtempSync(resolve(root, ".turbo/check-"));let retainLogs = false;@@ -982,7 +1087,7 @@ async function executePlan(plan, { terminal, paint, indent }) {progressLines = 0;};try {- for await (const outcome of runPlan(plan, outputDirectory)) {+ for await (const outcome of runPlan(plan, outputDirectory, cpus)) {clearProgress();if (outcome.type === "started") {active.set(outcome.checkIndex, outcome.check);diff --git a/scripts/check.test.mjs b/scripts/check.test.mjsnew file mode 100644index 0000000000..597dd2687d--- /dev/null+++ b/scripts/check.test.mjs@@ -0,0 +1,152 @@+import assert from "node:assert/strict";+import test from "node:test";+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { resolve } from "node:path";+import { createPlan, runPlan } from "./check.mjs";++test("failed Turbo run reads successful and cached prerequisites from its summary", async () => {+ const directory = mkdtempSync(resolve(".check-test-"));+ const previousPath = process.env.PATH;+ try {+ const summaryPath = resolve(directory, "summary.json");+ writeFileSync(+ summaryPath,+ JSON.stringify({+ tasks: [+ { taskId: "plugin#build", cache: { status: "HIT" } },+ { taskId: "a#build", execution: { exitCode: 0 } },+ { taskId: "web#typecheck", execution: { exitCode: 1 } },+ { taskId: "b#build", execution: { exitCode: 1 } },+ ],+ }),+ );+ writeFileSync(+ resolve(directory, "pnpm"),+ `#!${process.execPath}\nconsole.log(${JSON.stringify(`Summary: ${summaryPath}`)}); process.exit(1);\n`,+ { mode: 0o700 },+ );+ process.env.PATH = `${directory}:${previousPath ?? ""}`;+ for (const summary of ["valid", "missing", "invalid"]) {+ if (summary === "missing") rmSync(summaryPath);+ if (summary === "invalid") writeFileSync(summaryPath, "not JSON");+ const plan = [+ { isTurboBatch: true, args: [] },+ {+ requiresTurbo: true,+ prerequisites: ["plugin#build", "a#build"],+ steps: [],+ },+ { requiresTurbo: true, prerequisites: ["b#build"], steps: [] },+ { requiresTurbo: true, prerequisites: ["blocked#build"], steps: [] },+ ];+ const skipped = [];+ for await (const outcome of runPlan(plan, directory, 2)) {+ if (outcome.type === "skipped") skipped.push(outcome.checkIndex);+ if (outcome.error) throw outcome.error;+ }+ assert.deepEqual(skipped, summary === "valid" ? [2, 3] : [1, 2, 3]);+ }+ } finally {+ if (previousPath === undefined) delete process.env.PATH;+ else process.env.PATH = previousPath;+ rmSync(directory, { recursive: true, force: true });+ }+});++test("split-out lint records transitive prerequisites, excluding scriptless tasks", () => {+ const tasks = new Map([+ ["web#lint", { dependencies: ["plugin#build", "shared#build"] }],+ ["plugin#build", { command: "tsc", dependencies: [] }],+ ["shared#build", { command: "tsc", dependencies: ["config#build"] }],+ [+ "config#build",+ { command: "<NONEXISTENT>", dependencies: ["base#build"] },+ ],+ ["base#build", { command: "tsc", dependencies: [] }],+ ]);+ const plan = createPlan(+ [],+ [+ {+ name: "web",+ path: "web",+ scripts: { lint: "eslint" },+ dependencies: [],+ },+ ],+ { all: true, only: ["lint"], cpus: 2 },+ "HEAD",+ { tasks, affected: new Set() },+ );+ assert.deepEqual(plan[1].prerequisites, [+ "plugin#build",+ "shared#build",+ "base#build",+ ]);+});++for (const cpus of [1, 2, 4]) {+ test(`unrelated Turbo failures do not skip lint (cpus=${cpus})`, async () => {+ const plan = [+ { label: "Turbo", isTurboBatch: true },+ {+ label: "Lint A",+ requiresTurbo: true,+ prerequisites: ["plugin#build", "a#build"],+ },+ {+ label: "Lint B",+ requiresTurbo: true,+ prerequisites: ["plugin#build", "b#build"],+ },+ {+ label: "Lint C",+ requiresTurbo: true,+ prerequisites: ["plugin#build", "c#build"],+ },+ ];+ const ran = [];+ const skipped = [];+ for await (const outcome of runPlan(plan, ".", cpus, async (check) => {+ ran.push(check.label);+ return {+ failure: check.isTurboBatch ? { status: 1 } : null,+ reports: [],+ // A's prerequisites succeeded; B failed and C was blocked or omitted.+ successfulTurboTasks: ["plugin#build", "a#build"],+ };+ })) {+ if (outcome.type === "skipped") skipped.push(outcome.check.label);+ }+ assert.deepEqual(ran, ["Turbo", "Lint A"]);+ assert.deepEqual(skipped, ["Lint B", "Lint C"]);+ });+}++test("missing Turbo summary conservatively skips dependent lint", async () => {+ const plan = [+ { isTurboBatch: true },+ { requiresTurbo: true, prerequisites: ["plugin#build"] },+ ];+ const outcomes = [];+ for await (const outcome of runPlan(plan, ".", 2, async () => ({+ failure: { status: 1 },+ reports: [],+ })))+ outcomes.push(outcome.type);+ assert.deepEqual(outcomes, ["started", "completed", "skipped"]);+});++test("successful Turbo batch needs no summary to run dependent lint", async () => {+ const plan = [+ { isTurboBatch: true },+ { requiresTurbo: true, prerequisites: ["plugin#build"] },+ ];+ const outcomes = [];+ for await (const outcome of runPlan(plan, ".", 2, async () => ({+ failure: null,+ reports: [],+ })))+ outcomes.push(outcome.type);+ assert.deepEqual(outcomes, ["started", "completed", "started", "completed"]);+});