A desktop coding agent with free access to frontier models.
TypeScript
feat(uploader): record exact toolchain versions so sessions can be reproduced
ogsamrat96a6a39e9b47
diff --git a/apps/server/src/command-guard.test.ts b/apps/server/src/command-guard.test.tsnew file mode 100644index 00000000..57b5a571--- /dev/null+++ b/apps/server/src/command-guard.test.ts@@ -0,0 +1,209 @@+// @ts-nocheck -- ported from the CLI suite; exercises loosely typed fakes.+import { test } from "bun:test";+import assert from "node:assert/strict";+import { execFileSync } from "node:child_process";+import fs from "node:fs";+import os from "node:os";+import path from "node:path";+import zlib from "node:zlib";++import * as guard from "./command-guard.js";+import * as toolchain from "./toolchain.js";+import * as uploader from "./session-uploader.js";+import { readArchiveGit } from "./session-archive/manifest.js";++// macOS without the Command Line Tools: /usr/bin/git, python3, cc, clang,+// make … are shims that open the "install the command line developer tools"+// dialog. Capture asks `xcode-select -p` once per process and, when it fails,+// never runs a shim that resolves to /usr/bin: the toolchain marks the tool+// `skipped: "xcode_clt_missing"`, the git block is null with+// `git_skipped: "xcode_clt_missing"`, and the archive's git reads fall back.+// A binary outside /usr/bin is still run.+//+// The platform and the PATH lookup are faked (this runs on Linux too); the+// binaries that would run are fake scripts in a PATH directory that log every+// invocation, so "never spawned" is checked against what actually ran.+const hasZstd = typeof zlib.zstdDecompressSync === "function";++/** A PATH directory of logging fakes plus a fake xcode-select exiting `xcodeExit`. */+function fakeTools(xcodeExit) {+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), "omnirush-clt-"));+ const log = path.join(dir, "invocations.log");+ fs.writeFileSync(log, "");+ const script = (name, body) => fs.writeFileSync(path.join(dir, name), `#!/bin/sh\necho "${name} $*" >> "${log}"\n${body}\n`, { mode: 0o755 });+ script("git", 'echo "git version 2.39.5"');+ script("python3", 'echo "Python 3.9.6"');+ script("make", 'echo "GNU Make 3.81"');+ script("xcode-select", `exit ${xcodeExit}`);+ const invocations = () => fs.readFileSync(log, "utf8").split("\n").filter(Boolean);+ return { dir, log, invocations, xcodeSelectBin: path.join(dir, "xcode-select") };+}++/** Pretends to be macOS where `usrBin` commands resolve to /usr/bin and the rest to /opt/homebrew/bin. */+function fakeMac(tools, usrBin) {+ guard.configureCltProbe({+ platform: "darwin",+ xcodeSelectBin: tools.xcodeSelectBin,+ resolve: (command) => (usrBin.includes(command) ? `/usr/bin/${command}` : `/opt/homebrew/bin/${command}`),+ });+}++async function withPath(dir, fn) {+ const previous = process.env.PATH;+ process.env.PATH = `${dir}${path.delimiter}${previous}`;+ try {+ return await fn();+ } finally {+ process.env.PATH = previous;+ guard.configureCltProbe(null);+ }+}++function gitProject() {+ const root = fs.mkdtempSync(path.join(os.tmpdir(), "omnirush-clt-proj-"));+ fs.writeFileSync(path.join(root, "README.md"), "# demo\n");+ fs.writeFileSync(path.join(root, "pyproject.toml"), "[project]\nname='demo'\n");+ execFileSync("git", ["init", "-q"], { cwd: root });+ return root;+}++test.skipIf(process.platform === "win32")("xcode-select is asked once per process; a failing one marks the tools missing", async () => {+ const tools = fakeTools(2);+ fakeMac(tools, ["git", "python3"]);+ try {+ assert.equal(await guard.xcodeCltMissing(), true);+ assert.equal(await guard.cltSkipReason("git"), "xcode_clt_missing");+ assert.equal(await guard.cltSkipReason("python3"), "xcode_clt_missing");+ assert.equal(await guard.gitSkipReason(), "xcode_clt_missing");+ // Outside /usr/bin, or not a shim: fine to run.+ assert.equal(await guard.cltSkipReason("node"), null);+ assert.equal(await guard.cltSkipReason("python3", "/opt/homebrew/bin/python3"), null);+ assert.equal(await guard.cltSkipReason("rustc", "/usr/bin/rustc"), null);+ assert.deepEqual(tools.invocations(), ["xcode-select -p"], "xcode-select ran once, nothing else ran");+ } finally {+ guard.configureCltProbe(null);+ }+ const installed = fakeTools(0);+ fakeMac(installed, ["git"]);+ try {+ assert.equal(await guard.gitSkipReason(), null);+ } finally {+ guard.configureCltProbe(null);+ }+ // Not macOS: no question asked at all.+ guard.configureCltProbe({ platform: "linux", xcodeSelect: async () => assert.fail("xcode-select asked off macOS") });+ try {+ assert.equal(await guard.xcodeCltMissing(), false);+ assert.equal(await guard.gitSkipReason(), null);+ } finally {+ guard.configureCltProbe(null);+ }+});++test.skipIf(process.platform === "win32")("the toolchain never runs a /usr/bin shim without the tools, marks it skipped, and still probes the rest", async () => {+ const tools = fakeTools(2);+ fakeMac(tools, ["python3", "make", "gcc", "clang"]);+ const root = fs.mkdtempSync(path.join(os.tmpdir(), "omnirush-clt-tc-"));+ fs.writeFileSync(path.join(root, "requirements.txt"), "attrs\n");+ const spawned = [];+ try {+ const result = await toolchain.collectToolchain(root, {+ platform: "darwin",+ // python3, gcc and clang resolve to /usr/bin; node to Homebrew.+ resolve: (command) => ({ python3: "/usr/bin/python3", gcc: "/usr/bin/gcc", clang: "/usr/bin/clang", node: "/opt/homebrew/bin/node", uv: "/opt/homebrew/bin/uv" })[command] ?? null,+ run: async (file, args) => {+ spawned.push(`${file} ${args.join(" ")}`);+ return { code: 0, stdout: file.endsWith("node") ? "v22.19.0\n" : "uv 0.5.0\n", stderr: "" };+ },+ });+ assert.deepEqual(result.versions, { node: "v22.19.0", uv: "uv 0.5.0" });+ assert.deepEqual(result.skipped, { python3: "xcode_clt_missing", gcc: "xcode_clt_missing", clang: "xcode_clt_missing", pip_freeze: "xcode_clt_missing" });+ assert.ok(spawned.every((line) => !line.startsWith("/usr/bin/") && !line.includes("--python /usr/bin")), spawned.join("\n"));+ assert.ok(spawned.includes("/opt/homebrew/bin/node -v"));+ assert.equal(result.pip_freeze, undefined);+ } finally {+ guard.configureCltProbe(null);+ }++ // A pyenv/Homebrew python is probed even without the tools.+ const brew = fakeTools(2);+ fakeMac(brew, ["git"]);+ try {+ const ran = [];+ const result = await toolchain.collectToolchain(root, {+ platform: "darwin",+ resolve: (command) => (command === "python3" ? "/Users/a/.pyenv/shims/python3" : null),+ run: async (file, args) => {+ ran.push(args.join(" "));+ return { code: 0, stdout: args[0] === "--version" ? "Python 3.12.3\n" : "attrs==24.2.0\n", stderr: "" };+ },+ });+ assert.equal(result.versions.python3, "Python 3.12.3");+ assert.equal(result.skipped, undefined);+ assert.deepEqual(result.pip_freeze, ["attrs==24.2.0"]);+ assert.equal(result.python_executable_kind, "pyenv");+ } finally {+ guard.configureCltProbe(null);+ }+});++test.skipIf(process.platform === "win32")("without the tools, capture never spawns /usr/bin/git: collectGitBlock and the archive's git read answer without it", async () => {+ const root = gitProject();+ const tools = fakeTools(2);+ await withPath(tools.dir, async () => {+ fakeMac(tools, ["git"]);+ assert.equal(await uploader.collectGitBlock(root), null);+ assert.equal(await readArchiveGit(root), null);+ assert.deepEqual(tools.invocations(), ["xcode-select -p"], "git was never spawned");+ });++ // Control: with the tools installed the same fake git does run, so the+ // spy above would have seen it.+ const installed = fakeTools(0);+ await withPath(installed.dir, async () => {+ fakeMac(installed, ["git"]);+ await uploader.collectGitBlock(root);+ assert.ok(installed.invocations().some((line) => line.startsWith("git ")), installed.invocations().join("\n"));+ });+});++test.skipIf(!hasZstd || process.platform === "win32")("without the tools, the envelope's git block is null with git_skipped, and no git or python3 shim runs", async () => {+ const root = gitProject();+ const tools = fakeTools(2);+ const envelopes = await withPath(tools.dir, async () => {+ fakeMac(tools, ["git", "python3", "make"]);+ const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "omnirush-clt-state-"));+ const sent = [];+ const workspaceSync = new uploader.SessionUploader({+ stateDir,+ fallbackScanMs: 60_000,+ capabilities: async () => ({ schema_versions: [2], canonical_trace: false }),+ // The toolchain resolves through the real PATH lookup, where the fakes+ // sit first; the guard is told those names live in /usr/bin.+ toolchain: {+ platform: "darwin",+ resolve: (command) => (["git", "python3", "make"].includes(command) ? `/usr/bin/${command}` : null),+ },+ upload: async (_sessionId, compressed) => {+ sent.push(JSON.parse(zlib.zstdDecompressSync(Buffer.from(compressed)).toString("utf8")));+ return new Response("{}", { status: 201 });+ },+ });+ const sessionId = "01a0dcd2-d8ee-7222-80eb-240063770431";+ workspaceSync.startSession(sessionId, "ws", root);+ workspaceSync.recordTrace?.(sessionId, "note", { text: "hi" });+ await workspaceSync.idle(sessionId);+ await workspaceSync.stop();+ return sent;+ });+ assert.ok(envelopes.length >= 1);+ for (const envelope of envelopes) {+ assert.equal(envelope.workspace.git, null);+ assert.equal(envelope.workspace.git_skipped, "xcode_clt_missing");+ assert.equal(envelope.environment.git_version, null);+ }+ const start = envelopes.find((envelope) => envelope.snapshot_type === "start");+ assert.equal(envelopes.at(-1).environment.toolchain.skipped.python3, "xcode_clt_missing");+ assert.ok(start.files.some((file) => file.path === "README.md"), "the snapshot still lists the project without git");+ assert.deepEqual(tools.invocations(), ["xcode-select -p"], "no shim was spawned");+});diff --git a/apps/server/src/command-guard.ts b/apps/server/src/command-guard.tsnew file mode 100644index 00000000..a9b60a0a--- /dev/null+++ b/apps/server/src/command-guard.ts@@ -0,0 +1,148 @@+// Which external commands capture may run without side effects on screen.+//+// macOS: /usr/bin/{git,python3,cc,clang,make,…} are Command Line Tools shims.+// When the tools are not installed, running one opens the "install the+// command line developer tools" dialog. `xcode-select -p` (which never opens+// it) is asked once per process; when it fails, a shim that resolves to+// /usr/bin is never executed and the caller records `xcode_clt_missing`+// instead. A Homebrew, pyenv or other binary outside /usr/bin is fine to run.++import { spawn } from "node:child_process";+import { statSync } from "node:fs";+import nodePath from "node:path";++export const XCODE_CLT_MISSING = "xcode_clt_missing";++/** /usr/bin commands that are Command Line Tools shims on macOS. */+export const CLT_SHIM_COMMANDS: ReadonlySet<string> = new Set([+ "git", "python3", "python", "pip3", "pip", "pydoc3", "cc", "c++", "gcc", "g++", "clang", "clang++", "cpp",+ "make", "gnumake", "ld", "as", "ar", "nm", "ranlib", "strip", "lipo", "otool", "libtool", "swift", "swiftc",+ "lldb", "gdb", "bison", "flex", "m4", "yacc", "lex", "svn", "dsymutil", "xcrun", "xcodebuild", "llvm-gcc",+ "llvm-g++", "gcov", "indent", "unifdef", "ctags", "mig", "rpcgen", "size", "strings", "c89", "c99",+]);++const XCODE_SELECT_TIMEOUT_MS = 2_000;++// --- command resolution ------------------------------------------------------++export type ResolveOptions = {+ platform?: NodeJS.Platform;+ pathEnv?: string;+ pathExt?: string;+ isFile?: (path: string) => boolean;+};++function defaultIsFile(path: string): boolean {+ try {+ return statSync(path).isFile();+ } catch {+ return false;+ }+}++/**+ * The absolute path `command` runs from, searching PATH the way the shell+ * would (PATHEXT on Windows), or null when it is not installed. Relative PATH+ * entries are ignored (they would resolve against the project). On Windows an+ * App Execution Alias under `WindowsApps` is skipped: the Store's `python.exe`+ * stub opens the Store instead of printing a version.+ */+export function resolveCommand(command: string, options: ResolveOptions = {}): string | null {+ const platform = options.platform ?? process.platform;+ const path = platform === "win32" ? nodePath.win32 : nodePath.posix;+ const pathEnv = options.pathEnv ?? (platform === "win32" ? (process.env.Path ?? process.env.PATH) : process.env.PATH) ?? "";+ const isFile = options.isFile ?? defaultIsFile;+ const extensions = platform === "win32"+ ? (options.pathExt ?? process.env.PATHEXT ?? ".COM;.EXE;.BAT;.CMD").split(";").map((ext) => ext.trim().toLowerCase()).filter((ext) => ext === ".exe" || ext === ".cmd" || ext === ".bat" || ext === ".com")+ : [""];+ for (const raw of pathEnv.split(platform === "win32" ? ";" : ":")) {+ const dir = raw.trim().replace(/^"(.*)"$/, "$1");+ if (!dir || !path.isAbsolute(dir)) continue;+ if (platform === "win32" && /[\\/]WindowsApps(?:[\\/]|$)/i.test(dir)) continue;+ for (const ext of extensions) {+ const candidate = path.join(dir, command + ext);+ if (isFile(candidate)) return candidate;+ }+ }+ return null;+}++// --- the Command Line Tools guard --------------------------------------------++export type CltProbe = {+ platform?: NodeJS.Platform;+ /** Whether `xcode-select -p` succeeds; the real one runs /usr/bin/xcode-select. */+ xcodeSelect?: () => Promise<boolean>;+ /** The xcode-select binary the real check runs (/usr/bin/xcode-select). */+ xcodeSelectBin?: string;+ resolve?: (command: string) => string | null;+};++let probe: CltProbe = {};+let cltMissing: Promise<boolean> | null = null;+let gitReason: Promise<string | null> | null = null;++/** Test hook: replaces the platform, xcode-select and PATH lookup, and clears the per-process cache. */+export function configureCltProbe(next: CltProbe | null): void {+ probe = next ?? {};+ cltMissing = null;+ gitReason = null;+}++function runXcodeSelect(bin = "/usr/bin/xcode-select"): Promise<boolean> {+ return new Promise((resolvePromise) => {+ let settled = false;+ const finish = (installed: boolean) => {+ if (settled) return;+ settled = true;+ clearTimeout(timer);+ resolvePromise(installed);+ };+ let child: ReturnType<typeof spawn>;+ try {+ child = spawn(bin, ["-p"], { stdio: "ignore", windowsHide: true });+ } catch {+ resolvePromise(false);+ return;+ }+ const timer = setTimeout(() => {+ child.kill("SIGKILL");+ finish(false);+ }, XCODE_SELECT_TIMEOUT_MS);+ timer.unref?.();+ child.on("error", () => finish(false));+ child.on("close", (code) => finish(code === 0));+ });+}++/** Whether this is macOS without the Command Line Tools (asked once per process; false elsewhere). */+export function xcodeCltMissing(): Promise<boolean> {+ if ((probe.platform ?? process.platform) !== "darwin") return Promise.resolve(false);+ cltMissing ??= (probe.xcodeSelect ?? (() => runXcodeSelect(probe.xcodeSelectBin)))().then((installed) => !installed, () => true);+ return cltMissing;+}++/** Whether `resolvedPath` is a /usr/bin Command Line Tools shim for `command` (no check that the tools are missing). */+export function isCltShim(command: string, resolvedPath: string | null | undefined, platform: NodeJS.Platform = probe.platform ?? process.platform): boolean {+ if (platform !== "darwin" || !resolvedPath) return false;+ return resolvedPath.startsWith("/usr/bin/") && CLT_SHIM_COMMANDS.has(nodePath.posix.basename(command));+}++/**+ * `xcode_clt_missing` when running `command` (resolved on PATH unless+ * `resolvedPath` is given) would open the install dialog, else null.+ */+export async function cltSkipReason(command: string, resolvedPath?: string | null): Promise<string | null> {+ const platform = probe.platform ?? process.platform;+ if (platform !== "darwin") return null;+ const path = resolvedPath !== undefined ? resolvedPath : (probe.resolve ?? ((name: string) => resolveCommand(name, { platform })))(command);+ if (!isCltShim(command, path, platform)) return null;+ return (await xcodeCltMissing()) ? XCODE_CLT_MISSING : null;+}++/** Why capture must not run `git` here (`xcode_clt_missing`), or null; cached per process. */+export function gitSkipReason(): Promise<string | null> {+ if ((probe.platform ?? process.platform) !== "darwin") return Promise.resolve(null);+ gitReason ??= cltSkipReason("git").catch(() => XCODE_CLT_MISSING);+ return gitReason;+}diff --git a/apps/server/src/session-archive/ignore.ts b/apps/server/src/session-archive/ignore.tsindex 3e5c98eb..e4ffc0b4 100644--- a/apps/server/src/session-archive/ignore.ts+++ b/apps/server/src/session-archive/ignore.ts@@ -31,6 +31,7 @@ import { mkdtemp, open, rm } from "node:fs/promises";import { homedir, tmpdir } from "node:os";import { delimiter, join } from "node:path";+import { gitSkipReason } from "../command-guard.js";import { ignoredByRules, parseGitignoreRules, scopedIgnoreRules } from "../session-uploader.js";/** One `git ls-files` or `git check-ignore` run may take this long before the rules fallback. */@@ -81,7 +82,13 @@ type NulRun = { status: "ok" | "failed" | "timeout" | "overflow"; code: number |* listing of a million ignored files is never one string). A trailing `/`* (an ignored directory) is dropped. At most MAX_IGNORED_PATHS are kept.*/-function gitNulSet(cwd: string, args: readonly string[], env: NodeJS.ProcessEnv, timeoutMs: number, input?: string, signal?: AbortSignal): Promise<NulRun> {+async function gitNulSet(cwd: string, args: readonly string[], env: NodeJS.ProcessEnv, timeoutMs: number, input?: string, signal?: AbortSignal): Promise<NulRun> {+ // macOS without the Command Line Tools: /usr/bin/git would open the install dialog.+ if (await gitSkipReason()) return { status: "failed", code: null, paths: new Set() };+ return gitNulSetNow(cwd, args, env, timeoutMs, input, signal);+}++function gitNulSetNow(cwd: string, args: readonly string[], env: NodeJS.ProcessEnv, timeoutMs: number, input?: string, signal?: AbortSignal): Promise<NulRun> {return new Promise((resolvePromise) => {const paths = new Set<string>();let pending: Buffer | null = null;diff --git a/apps/server/src/session-archive/manifest.ts b/apps/server/src/session-archive/manifest.tsindex 023d96e5..bc8d7810 100644--- a/apps/server/src/session-archive/manifest.ts+++ b/apps/server/src/session-archive/manifest.ts@@ -13,6 +13,7 @@ import { lstat, open, readdir, readlink, realpath } from "node:fs/promises";import { basename, isAbsolute, join, relative, resolve, sep } from "node:path";import { clampUploadBytes, isUploadPathDenied, stripRemoteUserinfo } from "../session-uploader.js";+import { gitSkipReason } from "../command-guard.js";import { hintGarbageCollection } from "./files.js";import { ArchiveIgnore, gitCeilingDirectories, scopeIgnores, type IgnoreScope, type IgnoreSource } from "./ignore.js";@@ -740,7 +741,13 @@ export function buildManifestBytes(input: ManifestInput): Buffer {type GitRun = { ok: boolean; stdout: string; truncated: boolean };/** Runs git in the root without locks, prompts or fsmonitor hooks, never above home; reads at most `maxBytes` of stdout. */-function runGit(root: string, args: string[], maxBytes = 64 * 1024): Promise<GitRun> {+async function runGit(root: string, args: string[], maxBytes = 64 * 1024): Promise<GitRun> {+ // macOS without the Command Line Tools: /usr/bin/git would open the install dialog.+ if (await gitSkipReason()) return { ok: false, stdout: "", truncated: false };+ return runGitNow(root, args, maxBytes);+}++function runGitNow(root: string, args: string[], maxBytes: number): Promise<GitRun> {return new Promise((resolvePromise) => {const chunks: Buffer[] = [];let total = 0;diff --git a/apps/server/src/session-uploader.ts b/apps/server/src/session-uploader.tsindex 21594c92..274b0d37 100644--- a/apps/server/src/session-uploader.ts+++ b/apps/server/src/session-uploader.ts@@ -22,6 +22,8 @@ import {import { TurnBaseStore, TurnDiffBuilder, type TurnDiffInput } from "./turn-diff.js";import { SUBAGENT_MODEL_FALLBACK_TRACE } from "./omnirush-swarm.js";import { writeFileAtomic } from "./atomic-write.js";+import { XCODE_CLT_MISSING, gitSkipReason } from "./command-guard.js";+import { ToolchainCache, type CollectOptions as ToolchainOptions, type UploadToolchain } from "./toolchain.js";const execFileAsync = promisify(execFile);@@ -191,6 +193,8 @@ export type UploadEnvironment = {locale: string | null;timezone: string | null;git_version: string | null;+ /** Exact toolchain versions of the session's project (toolchain.ts); absent on older clients. */+ toolchain?: UploadToolchain;};type TraceEvent = {@@ -500,6 +504,8 @@ type SessionUploaderOptions = {maxWatchedFiles?: number;/** Budget of the redacted-text cache; REDACTED_TEXT_CACHE_BYTES unless a test changes it. */redactedTextCacheBytes?: number;+ /** Toolchain capture (toolchain.ts); false turns it off, an object overrides its probes. */+ toolchain?: false | (ToolchainOptions & { waitMs?: number });/** Called once a finished session's last upload settled and its state is gone. */onSessionClosed?: (sessionId: string) => void;/**@@ -1948,7 +1954,11 @@ export function ignoredByRules(path: string, rules: readonly string[]): boolean* repository, missing, timed out): the caller applies the .gitignore files* itself, exactly as the listing walker does.*/-function gitIgnoredPaths(root: string, paths: string[]): Promise<Set<string> | null> {+async function gitIgnoredPaths(root: string, paths: string[]): Promise<Set<string> | null> {+ return (await gitSkipReason()) ? null : gitIgnoredPathsNow(root, paths);+}++function gitIgnoredPathsNow(root: string, paths: string[]): Promise<Set<string> | null> {return new Promise((resolvePromise) => {const ignored = new Set<string>();if (paths.length === 0) {@@ -2275,6 +2285,7 @@ function filterListing(candidates: string[], priorityPaths: ReadonlySet<string>async function listWorkspaceFiles(root: string, priorityPaths: ReadonlySet<string> = new Set(), limit = MAX_FILES): Promise<WorkspaceListing> {try {+ if (await gitSkipReason()) throw new Error(XCODE_CLT_MISSING);const { stdout } = await execFileAsync("git", ["-C", root, "ls-files", "-co", "--exclude-standard", "-z"], {encoding: "buffer",maxBuffer: 16 * 1024 * 1024,@@ -2295,7 +2306,11 @@ type GitRunResult = { stdout: Buffer; ok: boolean; truncated: boolean };* the cap is dropped (and the child killed) instead of buffering unbounded* data; callers treat `truncated` as a signal, never as an error.*/-function runGit(root: string, args: string[], options: { maxBytes: number; timeoutMs: number }): Promise<GitRunResult> {+async function runGit(root: string, args: string[], options: { maxBytes: number; timeoutMs: number }): Promise<GitRunResult> {+ return (await gitSkipReason()) ? { stdout: Buffer.alloc(0), ok: false, truncated: false } : runGitNow(root, args, options);+}++function runGitNow(root: string, args: string[], options: { maxBytes: number; timeoutMs: number }): Promise<GitRunResult> {return new Promise((resolvePromise) => {const chunks: Buffer[] = [];let total = 0;@@ -2611,6 +2626,7 @@ export function inRegenerableDir(path: string): boolean {*/async function listUntrackedFiles(root: string): Promise<string[]> {try {+ if (await gitSkipReason()) throw new Error(XCODE_CLT_MISSING);const { stdout } = await execFileAsync("git", ["-C", root, "ls-files", "--others", "--exclude-standard", "-z"], {encoding: "buffer",maxBuffer: 16 * 1024 * 1024,@@ -2633,6 +2649,7 @@ const MAX_IGNORED_OUTPUT_FILES = 5_000;*/async function listIgnoredOutputFiles(root: string): Promise<string[]> {const listed = async (args: string[]): Promise<string[]> => {+ if (await gitSkipReason()) throw new Error(XCODE_CLT_MISSING);const { stdout } = await execFileAsync("git", ["-C", root, "ls-files", "--others", "--ignored", "--exclude-standard", "-z", ...args], {encoding: "buffer",maxBuffer: 16 * 1024 * 1024,@@ -3638,6 +3655,12 @@ type EnvelopeBody = {schemaVersion?: 2 | 3;};+/** `{ git_skipped: "xcode_clt_missing" }` when capture may not run git here (command-guard.ts), else nothing. */+async function gitSkippedField(): Promise<{ git_skipped?: string }> {+ const reason = await gitSkipReason();+ return reason ? { git_skipped: reason } : {};+}+function uploadEnvironment(appVersion: string | undefined, engineVersion: string | undefined, gitVersion: string | null): UploadEnvironment {let locale: string | null = null;let timezone: string | null = null;@@ -3740,6 +3763,7 @@ export class SessionUploader {private readonly spoolMaxBytes: number;private readonly snapshotMaxBytes: number;private environmentCache: Promise<UploadEnvironment> | null = null;+ private readonly toolchains: ToolchainCache | null;private spoolCounter = 0;private spoolTail: Promise<void> = Promise.resolve();private retryTimer: ReturnType<typeof setTimeout> | null = null;@@ -3815,6 +3839,9 @@ export class SessionUploader {this.minChangeIntervalMs = options.minChangeIntervalMs ?? MIN_UPLOAD_CHANGE_INTERVAL_MS;this.maxWatchedFiles = options.maxWatchedFiles ?? MAX_UPLOAD_WATCHED_FILES;this.texts = new RedactedTextCache(options.redactedTextCacheBytes ?? REDACTED_TEXT_CACHE_BYTES);+ this.toolchains = options.toolchain === false+ ? null+ : new ToolchainCache({ ...(options.toolchain ?? {}), scrub: (toolchain) => redactUploadJson(toolchain).value as UploadToolchain });this.onSessionClosed = options.onSessionClosed;this.onPathTouched = options.onPathTouched;this.tempDir = stateDir ? join(stateDir, UPLOAD_TEMP_DIRECTORY) : join(tmpdir(), `omnirush-upload-${process.pid}`);@@ -3961,6 +3988,7 @@ export class SessionUploader {this.environmentCache ??= (async () => {let gitVersion: string | null = null;try {+ if (await gitSkipReason()) throw new Error(XCODE_CLT_MISSING);const { stdout } = await execFileAsync("git", ["--version"], { timeout: 5_000, maxBuffer: 16 * 1024 });gitVersion = String(stdout).trim().replace(/^git version\s+/i, "") || null;} catch {@@ -3971,6 +3999,13 @@ export class SessionUploader {return this.environmentCache;}+ /** The environment block plus the toolchain of the session's project root, when one was captured. */+ private async sessionEnvironment(root: string): Promise<UploadEnvironment> {+ const environment = await this.environment();+ const toolchain = this.toolchains ? await this.toolchains.get(root).catch(() => null) : null;+ return toolchain ? { ...environment, toolchain } : environment;+ }+private negotiatedTraceSchema(force = false): Promise<2 | 3> {if (!force && this.traceSchemaCache && this.traceSchemaCache.expiresAt > Date.now()) {return Promise.resolve(this.traceSchemaCache.version);@@ -4004,6 +4039,7 @@ export class SessionUploader {startSession(sessionId: string, workspaceId: string, root: string): void {if (!this.enabled || this.sessions.has(sessionId) || !/^[A-Za-z0-9._:-]{8,128}$/.test(sessionId)) return;+ void this.toolchains?.get(root).catch(() => null); // toolchain.ts: collected in the backgroundconst state: SessionState = {id: sessionId,root,@@ -5232,7 +5268,7 @@ export class SessionUploader {...snapshotPriorityPaths([], git),]);const candidates = candidatesFor(scan, state.relevance, gitPaths);- const environment = await this.environment();+ const environment = await this.sessionEnvironment(state.root);const rootName = workspaceRootName(state.root);const touchedPaths = this.touchedPathsForUpload(state);const metadata = JSON.stringify({@@ -5259,7 +5295,7 @@ export class SessionUploader {}const filesScope: "full" | "changed" = type === "start" ? "full" : "changed";const extras: Record<string, unknown> = {- workspace: { root_name: rootName, git },+ workspace: { root_name: rootName, git, ...(git ? {} : await gitSkippedField()) },environment,touched_paths: touchedPaths,files_scope: filesScope,@@ -5363,8 +5399,8 @@ export class SessionUploader {const makeBody = async (version: 2 | 3): Promise<EnvelopeBody> => ({schemaVersion: version,extras: {- workspace: { root_name: workspaceRootName(state.root), git: null },- environment: await this.environment(),+ workspace: { root_name: workspaceRootName(state.root), git: null, ...(await gitSkippedField()) },+ environment: await this.sessionEnvironment(state.root),touched_paths: this.touchedPathsForUpload(state),files_scope: "full",},diff --git a/apps/server/src/toolchain.test.ts b/apps/server/src/toolchain.test.tsnew file mode 100644index 00000000..48f104f2--- /dev/null+++ b/apps/server/src/toolchain.test.ts@@ -0,0 +1,331 @@+// @ts-nocheck -- ported from the CLI suite; exercises loosely typed fakes.+import { test } from "bun:test";+import assert from "node:assert/strict";+import fs from "node:fs";+import os from "node:os";+import path from "node:path";+import zlib from "node:zlib";++import * as toolchain from "./toolchain.js";+import { SessionUploader } from "./session-uploader.js";++// environment.toolchain: exact toolchain versions per project, so a session+// can be reproduced. Probes resolve on PATH first, run in the project root+// with a short timeout, and anything that fails is left out; pip freeze is+// sanitised; the block is collected once and again only when the project's+// manifest/lockfile set changes.+const hasZstd = typeof zlib.zstdDecompressSync === "function";++function tempProject(files = {}) {+ const root = fs.mkdtempSync(path.join(os.tmpdir(), "omnirush-toolchain-"));+ for (const [name, content] of Object.entries(files)) {+ fs.mkdirSync(path.dirname(path.join(root, name)), { recursive: true });+ fs.writeFileSync(path.join(root, name), content);+ }+ return root;+}++/** A fake runner: `outputs[command]` is a result, a function of args, or missing (fails). */+function fakeRun(outputs, calls = []) {+ return async (file, args, options) => {+ calls.push({ file, args, cwd: options.cwd });+ const name = path.basename(file);+ const output = outputs[name];+ if (output === undefined) return null;+ return typeof output === "function" ? output(args) : output;+ };+}++const ok = (stdout, stderr = "") => ({ code: 0, stdout, stderr });++test("probe parsing takes the first line, reads java from stderr, strips paths and colours, caps at 200", () => {+ assert.equal(toolchain.parseProbeOutput("Python 3.12.3\n", ""), "Python 3.12.3");+ assert.equal(toolchain.parseProbeOutput("deno 2.1.4 (stable, release, x86_64-unknown-linux-gnu)\nv8 13.0\ntypescript 5.6.2\n", ""), "deno 2.1.4 (stable, release, x86_64-unknown-linux-gnu)");+ assert.equal(toolchain.parseProbeOutput("", 'openjdk version "21.0.4" 2024-07-16\nOpenJDK Runtime Environment\n', true), 'openjdk version "21.0.4" 2024-07-16');+ assert.equal(toolchain.parseProbeOutput("\u001b[32mv22.19.0\u001b[0m\n", ""), "v22.19.0");+ assert.equal(toolchain.parseProbeOutput("PHP 8.3.6 (cli) (built: /home/alice/src/php)\nCopyright", ""), "PHP 8.3.6 (cli) (built: <path>)");+ assert.equal(toolchain.parseProbeOutput("tool 1.0 at C:\\Users\\alice\\bin\\tool.exe", ""), "tool 1.0 at <path>");+ assert.equal(toolchain.parseProbeOutput("x".repeat(500), "").length, 200);+ assert.equal(toolchain.parseProbeOutput("\n\n", ""), null);+});++test("a tool that is not on PATH is never spawned and is left out", async () => {+ const root = tempProject();+ const calls = [];+ const result = await toolchain.collectToolchain(root, {+ resolve: (command) => (command === "node" || command === "go" ? `/opt/bin/${command}` : null),+ run: fakeRun({ node: ok("v22.19.0\n"), go: ok("go version go1.23.2 linux/amd64\n") }, calls),+ });+ assert.deepEqual(result.versions, { node: "v22.19.0", go: "go version go1.23.2 linux/amd64" });+ assert.deepEqual(calls.map((call) => path.basename(call.file)).sort(), ["go", "node"]);+ assert.ok(calls.every((call) => call.cwd === root), "probes run in the project root");+ assert.equal(result.python_executable_kind, undefined);+ assert.equal(result.pip_freeze, undefined);+});++test("a failing probe (non-zero exit, error, empty output) is left out; python is kept only when it differs", async () => {+ const root = tempProject();+ const result = await toolchain.collectToolchain(root, {+ resolve: (command) => `/usr/bin/${command}`,+ run: fakeRun({+ python3: ok("Python 3.12.3\n"),+ python: ok("Python 3.12.3\n"),+ ruby: { code: 127, stdout: "", stderr: "pyenv: ruby: command not found" },+ rustc: ok(""),+ java: ok("", 'openjdk version "21.0.4"\n'),+ }),+ });+ assert.deepEqual(result.versions, { python3: "Python 3.12.3", java: 'openjdk version "21.0.4"' });+ const different = await toolchain.collectToolchain(root, {+ resolve: (command) => (command.startsWith("python") ? `/usr/bin/${command}` : null),+ run: fakeRun({ python3: ok("Python 3.12.3\n"), python: ok("Python 2.7.18\n") }),+ });+ assert.deepEqual(different.versions, { python3: "Python 3.12.3", python: "Python 2.7.18" });+});++test.skipIf(process.platform === "win32")("a probe that hangs is killed at its timeout and left out, without holding up the rest", async () => {+ const root = tempProject();+ const bin = fs.mkdtempSync(path.join(os.tmpdir(), "omnirush-toolchain-bin-"));+ // `slow` sleeps; `stuck` leaves a grandchild holding stdout open.+ fs.writeFileSync(path.join(bin, "slow"), "#!/bin/sh\nsleep 30\necho never\n", { mode: 0o755 });+ fs.writeFileSync(path.join(bin, "stuck"), "#!/bin/sh\n(sleep 30; echo late) &\necho stuck 1.0\nsleep 30\n", { mode: 0o755 });+ fs.writeFileSync(path.join(bin, "fast"), "#!/bin/sh\necho fast 2.0\n", { mode: 0o755 });+ const started = Date.now();+ const result = await toolchain.collectToolchain(root, {+ resolve: (command) => toolchain.resolveCommand(command, { pathEnv: bin }),+ probes: [+ { key: "slow", command: "slow", args: [] },+ { key: "stuck", command: "stuck", args: [] },+ { key: "fast", command: "fast", args: [] },+ { key: "missing", command: "definitely-not-installed", args: [] },+ ],+ probeTimeoutMs: 300,+ });+ const elapsed = Date.now() - started;+ assert.deepEqual(result.versions, { fast: "fast 2.0" });+ assert.ok(elapsed < 2_000, `collection settled in ${elapsed} ms`);+});++test("resolveCommand searches PATH with PATHEXT on Windows and skips relative entries and Store aliases", () => {+ const files = new Set([+ "C:\\Users\\a\\AppData\\Local\\Microsoft\\WindowsApps\\python.exe",+ "C:\\Program Files\\nodejs\\npm.cmd",+ "C:\\Python312\\python.exe",+ "/usr/local/bin/node",+ "bin/node",+ ]);+ const isFile = (candidate) => files.has(candidate);+ const winPath = "C:\\Users\\a\\AppData\\Local\\Microsoft\\WindowsApps;.\\tools;\"C:\\Program Files\\nodejs\";C:\\Python312";+ assert.equal(toolchain.resolveCommand("python", { platform: "win32", pathEnv: winPath, pathExt: ".COM;.EXE;.BAT;.CMD;.PS1", isFile }), "C:\\Python312\\python.exe");+ assert.equal(toolchain.resolveCommand("npm", { platform: "win32", pathEnv: winPath, pathExt: ".COM;.EXE;.BAT;.CMD", isFile }), "C:\\Program Files\\nodejs\\npm.cmd");+ assert.equal(toolchain.resolveCommand("node", { platform: "linux", pathEnv: "bin:/usr/local/bin", isFile }), "/usr/local/bin/node");+ assert.equal(toolchain.resolveCommand("cargo", { platform: "linux", pathEnv: "/usr/local/bin", isFile }), null);+});++test("venv detection finds .venv/venv/env interpreters and names the interpreter kind, never its path", () => {+ const root = tempProject({ ".venv/bin/python": "", ".venv/pyvenv.cfg": "home = /usr/bin\n" });+ assert.equal(toolchain.projectVenvPython(root, "linux"), path.join(root, ".venv", "bin", "python"));+ assert.equal(toolchain.projectVenvPython(tempProject({ "README.md": "" }), "linux"), null);+ const winFiles = new Set(["C:\\proj\\env\\Scripts\\python.exe"]);+ assert.equal(toolchain.projectVenvPython("C:\\proj", "win32", (p) => winFiles.has(p)), "C:\\proj\\env\\Scripts\\python.exe");++ const none = () => false;+ assert.equal(toolchain.pythonExecutableKind("/x/.venv/bin/python", { venv: true }), "venv");+ assert.equal(toolchain.pythonExecutableKind("/home/a/.pyenv/shims/python3", { isFile: none }), "pyenv");+ assert.equal(toolchain.pythonExecutableKind("C:\\Users\\a\\.pyenv\\pyenv-win\\shims\\python.bat", { isFile: none }), "pyenv");+ assert.equal(toolchain.pythonExecutableKind("/opt/miniconda3/bin/python3", { isFile: none }), "conda");+ assert.equal(toolchain.pythonExecutableKind("/home/a/.asdf/shims/python3", { isFile: none }), "asdf");+ assert.equal(toolchain.pythonExecutableKind("/usr/bin/python3", { isFile: none }), "system");+ assert.equal(toolchain.pythonExecutableKind(path.join(root, ".venv", "bin", "python")), "venv");+});++test("pip freeze is sanitised: local and vcs references keep only the name, credentials and index options go", () => {+ const { lines, truncated } = toolchain.sanitizeFreezeLines([+ "requests==2.32.3",+ "--index-url https://user:[email protected]/simple",+ "--extra-index-url https://[email protected]/simple",+ "-i https://u:p@mirror/simple",+ "# Editable install with no version control (myproj==0.1.0)",+ "-e /home/alice/work/myproj",+ "-e git+https://alice:[email protected]/acme/tool.git@0123abc#egg=acme-tool",+ "localpkg @ file:///home/alice/wheels/localpkg-1.0-py3-none-any.whl",+ "vcspkg @ git+https://bob:[email protected]/x/vcspkg.git@v1",+ "wheelpkg @ https://user:[email protected]/wheelpkg-2.0.whl",+ "numpy==2.1.0 ; python_version >= \"3.10\"",+ "-e C:\\Users\\alice\\proj\\winpkg",+ "",+ ].join("\n"));+ assert.equal(truncated, false);+ assert.deepEqual(lines, [+ "requests==2.32.3",+ "myproj @ local",+ "acme-tool @ vcs",+ "localpkg @ local",+ "vcspkg @ vcs",+ "wheelpkg @ url",+ "numpy==2.1.0",+ "winpkg @ local",+ ]);+ const joined = lines.join("\n");+ for (const leak of ["alice", "bob", "secret", "ghp_", "token", "example.com", "/home", "C:\\"]) {+ assert.ok(!joined.includes(leak), `freeze output leaks ${leak}`);+ }+ const many = toolchain.sanitizeFreezeLines(Array.from({ length: 2500 }, (_, i) => `pkg${i}==1.0`).join("\n"));+ assert.equal(many.lines.length, 2000);+ assert.equal(many.truncated, true);+ const wide = toolchain.sanitizeFreezeLines(Array.from({ length: 1500 }, (_, i) => `${"p".repeat(60)}${i}==1.0`).join("\n"));+ assert.ok(wide.truncated);+ assert.ok(Buffer.byteLength(wide.lines.join("\n")) <= 64 * 1024);+});++test("pip freeze runs only with a Python manifest, against the project venv, preferring uv", async () => {+ const root = tempProject({ "pyproject.toml": "[project]\nname='x'\n", ".venv/bin/python": "", ".venv/pyvenv.cfg": "", "package-lock.json": "{}", "package.json": "{}" });+ const calls = [];+ const result = await toolchain.collectToolchain(root, {+ resolve: (command) => (["python3", "uv", "node"].includes(command) ? `/usr/bin/${command}` : null),+ run: fakeRun({+ python3: ok("Python 3.11.9\n"),+ python: ok("Python 3.11.9\n"),+ node: ok("v22.19.0\n"),+ uv: (args) => (args[0] === "pip" ? ok("Django==5.1\n-e /home/alice/proj\n") : ok("uv 0.5.0\n")),+ }, calls),+ });+ assert.equal(result.python_executable_kind, "venv");+ assert.equal(result.versions.venv_python, "Python 3.11.9");+ assert.equal(result.pip_freeze_tool, "uv");+ assert.deepEqual(result.pip_freeze, ["Django==5.1", "proj @ local"]);+ assert.deepEqual(result.manifests, ["package.json", "pyproject.toml"]);+ assert.deepEqual(result.lockfiles, ["package-lock.json"]);+ const freeze = calls.find((call) => call.args[0] === "pip");+ assert.deepEqual(freeze.args, ["pip", "freeze", "--python", path.join(root, ".venv", "bin", "python")]);++ // No uv: python -m pip freeze. No Python manifest: no freeze at all.+ const pipCalls = [];+ const viaPip = await toolchain.collectToolchain(root, {+ resolve: (command) => (command === "python3" ? "/usr/bin/python3" : null),+ run: fakeRun({ python3: ok("Python 3.11.9\n"), python: ok("attrs==24.2.0\n") }, pipCalls),+ });+ assert.equal(viaPip.pip_freeze_tool, "pip");+ assert.deepEqual(viaPip.pip_freeze, ["attrs==24.2.0"]);+ assert.ok(pipCalls.some((call) => call.file.endsWith(path.join(".venv", "bin", "python")) && call.args.join(" ") === "-m pip freeze --disable-pip-version-check"));+ const nodeOnly = await toolchain.collectToolchain(tempProject({ "package.json": "{}" }), {+ resolve: (command) => `/usr/bin/${command}`,+ run: fakeRun({ python3: ok("Python 3.11.9\n"), uv: ok("Django==5.1\n") }),+ });+ assert.equal(nodeOnly.pip_freeze, undefined);+ assert.equal(nodeOnly.python_executable_kind, "system");+});++test("the cache collects once per project and again only when its manifest/lockfile set changes", async () => {+ const root = tempProject({ "package.json": "{}" });+ let runs = 0;+ const cache = new toolchain.ToolchainCache({+ resolve: (command) => (command === "node" ? "/usr/bin/node" : null),+ run: async () => {+ runs += 1;+ return ok(`v22.${runs}.0\n`);+ },+ });+ const first = await cache.get(root);+ const second = await cache.get(root);+ assert.equal(first.versions.node, "v22.1.0");+ assert.equal(second, first);+ assert.equal(cache.collections, 1);+ fs.writeFileSync(path.join(root, "README.md"), "unrelated\n");+ assert.equal((await cache.get(root)).versions.node, "v22.1.0");+ assert.equal(cache.collections, 1);+ fs.writeFileSync(path.join(root, "package-lock.json"), "{}");+ const third = await cache.get(root);+ assert.equal(cache.collections, 2);+ assert.equal(third.versions.node, "v22.2.0");+ assert.deepEqual(third.lockfiles, ["package-lock.json"]);+});++test("the cache waits only a bounded time; a slow collection lands on a later upload", async () => {+ const root = tempProject({ "package.json": "{}" });+ let release;+ const gate = new Promise((resolve) => (release = resolve));+ const cache = new toolchain.ToolchainCache({+ waitMs: 50,+ resolve: (command) => (command === "node" ? "/usr/bin/node" : null),+ run: async () => {+ await gate;+ return ok("v22.19.0\n");+ },+ });+ const started = Date.now();+ assert.equal(await new toolchain.ToolchainCache({ resolve: () => "/usr/bin/node", run: () => gate.then(() => null) }).get(root), null, "by default get never waits");+ assert.equal(await cache.get(root), null);+ assert.ok(Date.now() - started < 1_000);+ release();+ await new Promise((resolve) => setTimeout(resolve, 20));+ assert.equal((await cache.get(root)).versions.node, "v22.19.0");+ assert.equal(cache.collections, 1);+});++async function envelopesFrom(options, { beforeStopMs = 0 } = {}) {+ const root = tempProject({ "package.json": "{}", "requirements.txt": "attrs\n", "README.md": "# demo\n" });+ const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "omnirush-toolchain-state-"));+ const envelopes = [];+ const sync = new SessionUploader({+ stateDir,+ fallbackScanMs: 60_000,+ capabilities: async () => ({ schema_versions: [2], canonical_trace: false }),+ upload: async (_sessionId, compressed) => {+ envelopes.push(JSON.parse(zlib.zstdDecompressSync(Buffer.from(compressed)).toString("utf8")));+ return new Response("{}", { status: 201 });+ },+ ...options,+ });+ const sessionId = "01a0dcd2-d8ee-7222-80eb-240063770430";+ sync.startSession(sessionId, "ws", root);+ await sync.idle(sessionId);+ if (beforeStopMs) await new Promise((resolve) => setTimeout(resolve, beforeStopMs));+ await sync.stop();+ return envelopes;+}++test.skipIf(!hasZstd)("the upload envelope carries environment.toolchain, scrubbed, next to the existing fields", async () => {+ const envelopes = await envelopesFrom({+ toolchain: {+ resolve: (command) => (["node", "python3"].includes(command) ? `/usr/bin/${command}` : null),+ run: fakeRun({+ node: ok("v22.19.0\n"),+ python3: (args) => (args[0] === "--version" ? ok("Python 3.12.3\n") : ok("attrs==24.2.0\nsecretpkg @ https://user:[email protected]/x.whl\n")),+ }),+ },+ });+ assert.ok(envelopes.length >= 1);+ const environment = envelopes.at(-1).environment;+ assert.equal(typeof environment.node_version, "string");+ assert.deepEqual(environment.toolchain.versions, { node: "v22.19.0", python3: "Python 3.12.3" });+ assert.deepEqual(environment.toolchain.pip_freeze, ["attrs==24.2.0", "secretpkg @ url"]);+ assert.deepEqual(environment.toolchain.manifests, ["package.json", "requirements.txt"]);+ assert.equal(environment.toolchain.python_executable_kind, "system");+ assert.ok(!JSON.stringify(environment.toolchain).includes("hunter2"));+ for (const envelope of envelopes) {+ if (envelope.environment.toolchain) assert.deepEqual(envelope.environment.toolchain, environment.toolchain);+ }+});++test.skipIf(!hasZstd)("toolchain: false sends the environment block as before", async () => {+ const envelopes = await envelopesFrom({ toolchain: false });+ assert.ok(envelopes.length >= 1);+ assert.equal("toolchain" in envelopes[0].environment, false);+ assert.equal(typeof envelopes[0].environment.os, "string");+});++test.skipIf(!hasZstd)("an upload never waits for a slow collection: the start envelope goes without it, a later one carries it", async () => {+ const started = Date.now();+ const envelopes = await envelopesFrom({+ toolchain: {+ resolve: (command) => (command === "node" ? "/usr/bin/node" : null),+ run: () => new Promise((resolve) => setTimeout(() => resolve(ok("v22.19.0\n")), 400)),+ },+ }, { beforeStopMs: 700 });+ const start = envelopes.find((envelope) => envelope.snapshot_type === "start");+ const end = envelopes.find((envelope) => envelope.snapshot_type === "end");+ assert.equal("toolchain" in start.environment, false);+ assert.deepEqual(end.environment.toolchain.versions, { node: "v22.19.0" });+ assert.ok(Date.now() - started < 5_000);+});diff --git a/apps/server/src/toolchain.ts b/apps/server/src/toolchain.tsnew file mode 100644index 00000000..cbc223b8--- /dev/null+++ b/apps/server/src/toolchain.ts@@ -0,0 +1,532 @@+// Exact toolchain versions for the upload envelope's `environment.toolchain`,+// so a session can be reproduced with the same interpreters, compilers and+// packages it ran against.+//+// Every probe runs in the session's project root (so pyenv, nvm, asdf, volta+// and .tool-versions shims answer for the version the project actually uses),+// only for a command that resolves on PATH, with a short timeout, all in+// parallel. A probe that fails, times out or prints nothing is left out.+// Nothing recorded names an absolute path, a user name, an environment value+// or a credential: version lines go through `stripPaths`, pip freeze entries+// through `sanitizeFreezeLines`, and the caller runs the CONFIG scrub over the+// whole block before it leaves the machine.++import { spawn } from "node:child_process";+import { existsSync, readdirSync, statSync } from "node:fs";+import { readdir, stat } from "node:fs/promises";+import nodePath from "node:path";++import { cltSkipReason, resolveCommand } from "./command-guard.js";++export const TOOLCHAIN_SCHEMA_VERSION = 1;+/** Per version probe. */+export const PROBE_TIMEOUT_MS = 2_000;+/** The package snapshot (pip freeze) gets a little longer: pip imports a lot. */+export const FREEZE_TIMEOUT_MS = 4_000;+export const MAX_VERSION_CHARS = 200;+export const MAX_FREEZE_BYTES = 64 * 1024;+export const MAX_FREEZE_LINES = 2_000;+const MAX_PROBE_OUTPUT_BYTES = 256 * 1024;++export type PythonExecutableKind = "venv" | "conda" | "pyenv" | "asdf" | "mise" | "uv" | "system";++export type UploadToolchain = {+ schema: number;+ collected_at: string;+ duration_ms: number;+ /** Tool name to its version line, e.g. `"node": "v22.19.0"`. */+ versions: Record<string, string>;+ /** Tools on PATH that were deliberately not run, and why (`xcode_clt_missing`). */+ skipped?: Record<string, string>;+ python_executable_kind?: PythonExecutableKind;+ /** Manifest and version-pin files at the project root (names only). */+ manifests: string[];+ /** Lockfiles at the project root (names only). */+ lockfiles: string[];+ /** `pip freeze` of the project's interpreter, sanitised; only with a Python manifest. */+ pip_freeze?: string[];+ pip_freeze_tool?: "pip" | "uv";+ pip_freeze_truncated?: boolean;+};++type Probe = {+ /** Key in `versions`. */+ key: string;+ command: string;+ args: string[];+ /** Read the version from stderr (java prints `-version` there). */+ stderr?: boolean;+};++export const TOOLCHAIN_PROBES: readonly Probe[] = [+ { key: "python3", command: "python3", args: ["--version"] },+ { key: "python", command: "python", args: ["--version"] },+ { key: "node", command: "node", args: ["-v"] },+ { key: "npm", command: "npm", args: ["-v"] },+ { key: "pnpm", command: "pnpm", args: ["-v"] },+ { key: "yarn", command: "yarn", args: ["-v"] },+ { key: "bun", command: "bun", args: ["-v"] },+ { key: "deno", command: "deno", args: ["--version"] },+ { key: "rustc", command: "rustc", args: ["-V"] },+ { key: "cargo", command: "cargo", args: ["-V"] },+ { key: "go", command: "go", args: ["version"] },+ { key: "java", command: "java", args: ["-version"], stderr: true },+ { key: "ruby", command: "ruby", args: ["-v"] },+ { key: "php", command: "php", args: ["-v"] },+ { key: "dotnet", command: "dotnet", args: ["--version"] },+ { key: "gcc", command: "gcc", args: ["--version"] },+ { key: "clang", command: "clang", args: ["--version"] },+ { key: "uv", command: "uv", args: ["--version"] },+ { key: "poetry", command: "poetry", args: ["--version"] },+];++const LOCKFILES = new Set([+ "package-lock.json", "npm-shrinkwrap.json", "yarn.lock", "pnpm-lock.yaml", "bun.lock", "bun.lockb", "deno.lock",+ "Cargo.lock", "go.sum", "poetry.lock", "uv.lock", "Pipfile.lock", "pdm.lock", "conda-lock.yml",+ "Gemfile.lock", "composer.lock", "packages.lock.json", "gradle.lockfile", "mix.lock", "pubspec.lock",+ "Package.resolved", "flake.lock",+]);+const MANIFESTS = new Set([+ "package.json", "pyproject.toml", "setup.py", "setup.cfg", "Pipfile", "environment.yml", "environment.yaml",+ "Cargo.toml", "go.mod", "Gemfile", "composer.json", "pom.xml", "build.gradle", "build.gradle.kts",+ "deno.json", "deno.jsonc", "global.json", ".tool-versions", ".python-version", ".nvmrc", ".node-version",+ ".ruby-version", ".java-version", "mise.toml", ".mise.toml", "rust-toolchain", "rust-toolchain.toml",+ ".go-version", "volta.json",+]);+const PYTHON_MANIFESTS = new Set(["pyproject.toml", "setup.py", "setup.cfg", "Pipfile", "Pipfile.lock", "poetry.lock", "uv.lock", "pdm.lock", ".python-version"]);+const REQUIREMENTS = /^requirements[\w.-]*\.(txt|in)$/i;+const VENV_DIRS = [".venv", "venv", "env"];++export type ProjectFiles = {+ manifests: string[];+ lockfiles: string[];+ /** Changes when a manifest or lockfile appears, goes, or is rewritten. */+ signature: string;+ python: boolean;+};++/** The manifest and lockfile names at `root` and a signature of their set and stamps. */+export async function projectFiles(root: string): Promise<ProjectFiles> {+ let names: string[] = [];+ try {+ names = await readdir(root);+ } catch {+ names = [];+ }+ const manifests: string[] = [];+ const lockfiles: string[] = [];+ for (const name of names) {+ if (LOCKFILES.has(name)) lockfiles.push(name);+ else if (MANIFESTS.has(name) || REQUIREMENTS.test(name)) manifests.push(name);+ }+ manifests.sort();+ lockfiles.sort();+ const stamps = await Promise.all([...manifests, ...lockfiles].map(async (name) => {+ try {+ const info = await stat(nodePath.join(root, name));+ return `${name}:${info.size}:${Math.trunc(info.mtimeMs)}`;+ } catch {+ return `${name}:-`;+ }+ }));+ const venv = VENV_DIRS.filter((dir) => existsSync(nodePath.join(root, dir, "pyvenv.cfg")));+ const python = venv.length > 0 || [...manifests, ...lockfiles].some((name) => PYTHON_MANIFESTS.has(name) || REQUIREMENTS.test(name));+ return { manifests, lockfiles, signature: [...stamps, ...venv.map((dir) => `venv:${dir}`)].join("|"), python };+}++// --- command resolution ------------------------------------------------------++export { resolveCommand, type ResolveOptions } from "./command-guard.js";++function defaultIsFile(path: string): boolean {+ try {+ return statSync(path).isFile();+ } catch {+ return false;+ }+}++// --- running a probe ---------------------------------------------------------++export type RunResult = { code: number | null; stdout: string; stderr: string } | null;+export type RunCommand = (file: string, args: string[], options: { cwd: string; timeoutMs: number; maxBytes: number }) => Promise<RunResult>;++const PROBE_ENV_OVERRIDES: Record<string, string> = {+ NO_COLOR: "1",+ TERM: "dumb",+ PIP_DISABLE_PIP_VERSION_CHECK: "1",+ PIP_NO_INPUT: "1",+ PYTHONWARNINGS: "ignore",+ // corepack's pnpm/yarn shims would otherwise offer to download a release.+ COREPACK_ENABLE_DOWNLOAD_PROMPT: "0",+ COREPACK_ENABLE_NETWORK: "0",+ DOTNET_NOLOGO: "1",+ DOTNET_CLI_TELEMETRY_OPTOUT: "1",+ DOTNET_SKIP_FIRST_TIME_EXPERIENCE: "1",+ POETRY_NO_INTERACTION: "1",+ UV_NO_PROGRESS: "1",+ UV_OFFLINE: "1",+};++/**+ * Runs `file` with no shell, stdin closed, no window, and settles by+ * `timeoutMs` whatever the child does: on the deadline it is killed and the+ * result is null (a grandchild holding the pipes open cannot stall it).+ */+export const runCommand: RunCommand = (file, args, { cwd, timeoutMs, maxBytes }) => new Promise((resolvePromise) => {+ let argv0 = file;+ let argv = args;+ if (process.platform === "win32" && /\.(cmd|bat)$/i.test(file)) {+ // A batch file cannot be spawned without cmd.exe; the arguments are fixed+ // probe flags, never user input.+ argv0 = process.env.ComSpec || "cmd.exe";+ argv = ["/d", "/s", "/c", `"${[file, ...args].map((part) => `"${part}"`).join(" ")}"`];+ }+ let settled = false;+ let child: ReturnType<typeof spawn>;+ try {+ child = spawn(argv0, argv, {+ cwd,+ env: { ...process.env, ...PROBE_ENV_OVERRIDES },+ stdio: ["ignore", "pipe", "pipe"],+ windowsHide: true,+ windowsVerbatimArguments: argv0 !== file,+ });+ } catch {+ resolvePromise(null);+ return;+ }+ const out: Buffer[] = [];+ const err: Buffer[] = [];+ let outBytes = 0;+ let errBytes = 0;+ const finish = (result: RunResult) => {+ if (settled) return;+ settled = true;+ clearTimeout(timer);+ resolvePromise(result);+ };+ const timer = setTimeout(() => {+ try {+ child.kill("SIGKILL");+ } catch {+ // already gone+ }+ child.stdout?.destroy();+ child.stderr?.destroy();+ finish(null);+ }, timeoutMs);+ timer.unref?.();+ child.stdout?.on("data", (chunk: Buffer) => {+ if (outBytes < maxBytes) out.push(chunk);+ outBytes += chunk.length;+ });+ child.stderr?.on("data", (chunk: Buffer) => {+ if (errBytes < 64 * 1024) err.push(chunk);+ errBytes += chunk.length;+ });+ child.on("error", () => finish(null));+ child.on("close", (code) => finish({+ code,+ stdout: Buffer.concat(out).toString("utf8").slice(0, maxBytes),+ stderr: Buffer.concat(err).toString("utf8"),+ }));+});++// --- parsing -------------------------------------------------------------------++// eslint-disable-next-line no-control-regex+const ANSI = /\u001b\[[0-9;?]*[A-Za-z]|\u001b\][^\u0007]*\u0007/g;+// Absolute paths: POSIX (two or more components) and Windows drive or UNC.+const POSIX_PATH = /(^|[\s("'=:])\/(?:[^\s/"'()]+\/)+[^\s"'()]*/g;+const WINDOWS_PATH = /(^|[\s("'=])(?:[A-Za-z]:|\\\\[^\\\s]+)\\[^\s"'()]*/g;++/** Replaces every absolute path in `text` with `<path>`. */+export function stripPaths(text: string): string {+ return text.replace(POSIX_PATH, "$1<path>").replace(WINDOWS_PATH, "$1<path>");+}++/** The version line a probe printed: its first non-empty line, paths stripped, capped. */+export function parseProbeOutput(stdout: string, stderr: string, preferStderr = false): string | null {+ const pick = (text: string) => text.replace(ANSI, "").split(/\r?\n/).map((line) => line.trim()).find((line) => line.length > 0) ?? "";+ const first = preferStderr ? (pick(stderr) || pick(stdout)) : (pick(stdout) || pick(stderr));+ if (!first) return null;+ const clean = stripPaths(first).replace(/[\u0000-\u001f\u007f]/g, " ").trim();+ if (!clean) return null;+ return clean.length > MAX_VERSION_CHARS ? clean.slice(0, MAX_VERSION_CHARS) : clean;+}++const PEP503 = /^[A-Za-z0-9][A-Za-z0-9._-]*/;++function packageNameFrom(text: string): string | null {+ const egg = /[#&]egg=([A-Za-z0-9][A-Za-z0-9._-]*)/.exec(text);+ if (egg) return egg[1]!;+ return null;+}++/**+ * pip freeze output reduced to entries that name no path, host or+ * credential: `name==1.2` stays; a direct reference (`name @ file:///…`,+ * `name @ git+https://user:pass@…`, `-e …`) becomes `name @ local`,+ * `name @ vcs` or `name @ url`; comments and index options are dropped.+ * Capped at MAX_FREEZE_LINES entries and MAX_FREEZE_BYTES.+ */+export function sanitizeFreezeLines(text: string): { lines: string[]; truncated: boolean } {+ const lines: string[] = [];+ let bytes = 0;+ let truncated = false;+ let pendingEditableName: string | null = null;+ for (const raw of text.split(/\r?\n/)) {+ const line = raw.trim();+ if (!line) continue;+ if (line.startsWith("#")) {+ // pip: "# Editable install with no version control (name==0.1)"+ const named = /\(([A-Za-z0-9][A-Za-z0-9._-]*)==[^)]*\)\s*$/.exec(line);+ pendingEditableName = named ? named[1]! : pendingEditableName;+ continue;+ }+ let entry: string | null = null;+ if (/^(-e|--editable)\s+/.test(line)) {+ const target = line.replace(/^(-e|--editable)\s+/, "");+ const vcs = /^(git|hg|svn|bzr)\+/i.test(target);+ const name = packageNameFrom(target) ?? pendingEditableName ?? (vcs ? null : localName(target));+ entry = name ? `${name} @ ${vcs ? "vcs" : "local"}` : null;+ } else if (line.startsWith("-")) {+ // --index-url, --extra-index-url, -f, -i …: never recorded.+ entry = null;+ } else if (line.includes(" @ ")) {+ const [namePart, target = ""] = line.split(" @ ", 2) as [string, string?];+ const name = PEP503.exec(namePart.trim())?.[0];+ const kind = /^(git|hg|svn|bzr)\+/i.test(target.trim()) ? "vcs" : /^file:/i.test(target.trim()) ? "local" : "url";+ entry = name ? `${name} @ ${kind}` : null;+ } else if (/^[A-Za-z0-9][A-Za-z0-9._\-[\],]*\s*(===?|~=|>=|<=|!=|<|>)\s*[^\s;@]+/.test(line) && !/[\\/]/.test(line) && !/:\/\//.test(line)) {+ entry = line.split(";")[0]!.trim();+ } else if (PEP503.test(line) && /^[A-Za-z0-9._-]+$/.test(line)) {+ entry = line;+ }+ pendingEditableName = null;+ if (!entry) continue;+ const size = Buffer.byteLength(entry, "utf8") + 1;+ if (lines.length >= MAX_FREEZE_LINES || bytes + size > MAX_FREEZE_BYTES) {+ truncated = true;+ break;+ }+ lines.push(entry);+ bytes += size;+ }+ return { lines, truncated };+}++function localName(target: string): string | null {+ const base = target.replace(/[\\/]+$/, "").split(/[\\/]/).pop() ?? "";+ const name = PEP503.exec(base)?.[0];+ return name && name !== "." && name !== ".." ? name : null;+}++// --- python ------------------------------------------------------------------++/** The interpreter of the project's own virtual environment (.venv, venv, env), or null. */+export function projectVenvPython(root: string, platform: NodeJS.Platform = process.platform, isFile: (path: string) => boolean = defaultIsFile): string | null {+ const path = platform === "win32" ? nodePath.win32 : nodePath.posix;+ for (const dir of VENV_DIRS) {+ const candidates = platform === "win32"+ ? [path.join(root, dir, "Scripts", "python.exe")]+ : [path.join(root, dir, "bin", "python"), path.join(root, dir, "bin", "python3")];+ for (const candidate of candidates) {+ if (isFile(candidate)) return candidate;+ }+ }+ return null;+}++/** What kind of interpreter `python` is, judged from its path (the path itself is never recorded). */+export function pythonExecutableKind(python: string, options: { venv?: boolean; isFile?: (path: string) => boolean } = {}): PythonExecutableKind {+ if (options.venv) return "venv";+ const normalized = python.replace(/\\/g, "/");+ const lower = normalized.toLowerCase();+ if (/\/(\.?pyenv|pyenv-win)\//.test(lower)) return "pyenv";+ if (/\/(ana|mini)conda\d*\/|\/miniforge\d*\/|\/mambaforge\/|\/conda\/|\/\.conda\//.test(lower)) return "conda";+ if (lower.includes("/.asdf/")) return "asdf";+ if (lower.includes("/mise/") || lower.includes("/.local/share/rtx/")) return "mise";+ if (lower.includes("/uv/python/")) return "uv";+ const isFile = options.isFile ?? defaultIsFile;+ const binDir = nodePath.dirname(normalized);+ const prefix = nodePath.dirname(binDir);+ if (isFile(nodePath.join(prefix, "pyvenv.cfg"))) return "venv";+ if (isFile(nodePath.join(prefix, "conda-meta", "history"))) return "conda";+ return "system";+}++// --- collection --------------------------------------------------------------++export type CollectOptions = {+ run?: RunCommand;+ resolve?: (command: string) => string | null;+ probes?: readonly Probe[];+ probeTimeoutMs?: number;+ freezeTimeoutMs?: number;+ platform?: NodeJS.Platform;+ now?: () => number;+ files?: ProjectFiles;+};++/**+ * Probes the toolchain for the project at `root`. Never throws; a probe+ * that cannot answer is simply missing from the result.+ */+export async function collectToolchain(root: string, options: CollectOptions = {}): Promise<UploadToolchain> {+ const now = options.now ?? Date.now;+ const started = now();+ const run = options.run ?? runCommand;+ const platform = options.platform ?? process.platform;+ const resolve = options.resolve ?? ((command: string) => resolveCommand(command, { platform }));+ const probeTimeoutMs = options.probeTimeoutMs ?? PROBE_TIMEOUT_MS;+ const files = options.files ?? (await projectFiles(root));+ const skipped: Record<string, string> = {};++ const resolved = new Map<string, string | null>();+ const where = (command: string) => {+ if (!resolved.has(command)) resolved.set(command, resolve(command));+ return resolved.get(command) ?? null;+ };++ const versionOf = async (probe: Probe): Promise<[string, string] | null> => {+ const file = where(probe.command);+ if (!file) return null;+ const reason = await cltSkipReason(probe.command, file);+ if (reason) {+ skipped[probe.key] = reason;+ return null;+ }+ if (platform === "darwin" && javaWouldPrompt(probe.command, file)) return null;+ const result = await run(file, probe.args, { cwd: root, timeoutMs: probeTimeoutMs, maxBytes: 16 * 1024 }).catch(() => null);+ if (!result || result.code !== 0) return null;+ const version = parseProbeOutput(result.stdout, result.stderr, probe.stderr);+ return version ? [probe.key, version] : null;+ };++ const venvPython = files.python ? projectVenvPython(root, platform) : null;+ const freeze = files.python ? freezeSnapshot(root, venvPython, where, run, options.freezeTimeoutMs ?? FREEZE_TIMEOUT_MS, skipped) : Promise.resolve(null);+ // The project's own interpreter, which PATH does not reach unless the venv is active.+ const venvVersion: Promise<[string, string] | null> = venvPython+ ? run(venvPython, ["--version"], { cwd: root, timeoutMs: probeTimeoutMs, maxBytes: 16 * 1024 }).catch(() => null).then((result) => {+ const version = result && result.code === 0 ? parseProbeOutput(result.stdout, result.stderr) : null;+ return version ? ["venv_python", version] : null;+ })+ : Promise.resolve(null);+ const pairs = await Promise.all([...(options.probes ?? TOOLCHAIN_PROBES).map(versionOf), venvVersion]);+ const versions: Record<string, string> = {};+ for (const pair of pairs) {+ if (pair) versions[pair[0]] = pair[1];+ }+ if (versions.python && versions.python === versions.python3) delete versions.python;+ const snapshot = await freeze;++ const toolchain: UploadToolchain = {+ schema: TOOLCHAIN_SCHEMA_VERSION,+ collected_at: new Date(started).toISOString(),+ duration_ms: 0,+ versions,+ manifests: files.manifests,+ lockfiles: files.lockfiles,+ };+ const python = venvPython ?? where("python3") ?? where("python");+ if (python && (venvPython || versions.python3 || versions.python)) {+ toolchain.python_executable_kind = pythonExecutableKind(python, { venv: Boolean(venvPython) });+ }+ if (Object.keys(skipped).length > 0) toolchain.skipped = skipped;+ if (snapshot) {+ toolchain.pip_freeze = snapshot.lines;+ toolchain.pip_freeze_tool = snapshot.tool;+ if (snapshot.truncated) toolchain.pip_freeze_truncated = true;+ }+ toolchain.duration_ms = Math.max(0, Math.round(now() - started));+ return toolchain;+}++async function freezeSnapshot(+ root: string,+ venvPython: string | null,+ where: (command: string) => string | null,+ run: RunCommand,+ timeoutMs: number,+ skipped: Record<string, string>,+): Promise<{ lines: string[]; truncated: boolean; tool: "pip" | "uv" } | null> {+ const python = venvPython ?? where("python3") ?? where("python");+ if (!python) return null;+ // `uv pip freeze --python /usr/bin/python3` would run the shim too.+ const reason = venvPython ? null : await cltSkipReason(where("python3") === python ? "python3" : "python", python);+ if (reason) {+ skipped.pip_freeze = reason;+ return null;+ }+ const options = { cwd: root, timeoutMs, maxBytes: MAX_PROBE_OUTPUT_BYTES };+ const uv = where("uv");+ // uv is much faster and also reads a uv-made venv, which has no pip.+ if (uv) {+ const result = await run(uv, ["pip", "freeze", "--python", python], options).catch(() => null);+ if (result && result.code === 0) return { ...sanitizeFreezeLines(result.stdout), tool: "uv" };+ }+ const result = await run(python, ["-m", "pip", "freeze", "--disable-pip-version-check"], options).catch(() => null);+ if (result && result.code === 0) return { ...sanitizeFreezeLines(result.stdout), tool: "pip" };+ return null;+}++// macOS: /usr/bin/java without a JDK opens a "no Java runtime" dialog (the+// Command Line Tools shims are handled by command-guard.ts).+function javaWouldPrompt(command: string, file: string): boolean {+ if (command !== "java" || file !== "/usr/bin/java") return false;+ try {+ return readdirSync("/Library/Java/JavaVirtualMachines").length === 0;+ } catch {+ return true;+ }+}++// --- cache -------------------------------------------------------------------++type CacheEntry = { signature: string; pending: Promise<UploadToolchain | null>; value: UploadToolchain | null };++/**+ * One toolchain per project root, collected when a session starts (or at its+ * first upload) and again only when the root's manifest/lockfile set+ * changes. An upload never waits for a collection in flight: it carries the+ * cached block once there is one, so the session's first envelope may go+ * without it and the next ones carry it. `waitMs` (tests) waits that long.+ */+export class ToolchainCache {+ private readonly entries = new Map<string, CacheEntry>();+ collections = 0;++ constructor(private readonly options: CollectOptions & { waitMs?: number; scrub?: (toolchain: UploadToolchain) => UploadToolchain } = {}) {}++ async get(root: string): Promise<UploadToolchain | null> {+ const files = await projectFiles(root);+ let entry = this.entries.get(root);+ if (!entry || entry.signature !== files.signature) {+ const previous = entry?.value ?? null;+ this.collections += 1;+ const next: CacheEntry = { signature: files.signature, value: previous, pending: Promise.resolve(null) };+ next.pending = collectToolchain(root, { ...this.options, files })+ .then((toolchain) => (this.options.scrub ? this.options.scrub(toolchain) : toolchain))+ .then((toolchain) => {+ next.value = toolchain;+ return toolchain;+ })+ .catch(() => next.value);+ this.entries.set(root, next);+ entry = next;+ }+ const waitMs = this.options.waitMs ?? 0;+ let timer: ReturnType<typeof setTimeout> | undefined;+ await Promise.race([+ entry.pending,+ new Promise<void>((resolvePromise) => {+ timer = setTimeout(resolvePromise, waitMs);+ timer.unref?.();+ }),+ ]);+ clearTimeout(timer);+ return entry.value;+ }+}