A free, open source, and extensible speech-to-text application that works completely offline.
chore(nix): add bun node_modules normalization scripts (#1256)
Evgeny Khudobaaf6ec6c903e4
diff --git a/.nix/scripts/canonicalize-node-modules.ts b/.nix/scripts/canonicalize-node-modules.tsnew file mode 100644index 00000000..32356967--- /dev/null+++ b/.nix/scripts/canonicalize-node-modules.ts@@ -0,0 +1,97 @@+/**+ * Canonicalize bun's internal node_modules symlinks for reproducible FOD hashes.+ *+ * Isolated-install layout produced by `bun install --linker=isolated`:+ *+ * node_modules/+ * ├── react → .bun/[email protected]/node_modules/react (symlink)+ * ├── minimatch → .bun/[email protected]/node_modules/minimatch (symlink)+ * └── .bun/+ * ├── [email protected]/+ * │ └── node_modules/+ * │ └── react/ ← real package content+ * ├── [email protected]/+ * │ └── node_modules/+ * │ └── minimatch/ ← real package content+ * └── node_modules/ ← target of this script+ * ├── react → ../[email protected]/node_modules/react+ * ├── minimatch → ../[email protected]/node_modules/minimatch+ * └── @babel/+ * └── core → ../../@[email protected]+…/node_modules/@babel/core+ *+ * Real package content lives in .bun/<pkg>@<ver>/node_modules/<pkg>/.+ * The .bun/node_modules/ directory (linkRoot) holds only symlinks — it acts+ * as a fallback upward-resolution path for packages inside .bun/.+ *+ * Bun's creation order for those symlinks is not guaranteed to be stable+ * across hosts or filesystems, which can break fixed-output derivation hashes.+ * This script reads the existing symlinks, removes them, and recreates them+ * in lexicographic order while preserving the exact targets bun picked.+ */++import { lstat, mkdir, readdir, readlink, rm, symlink } from "fs/promises";+import { join } from "path";++type LinkEntry = {+ slug: string;+ target: string;+};++async function isDirectory(path: string) {+ try {+ const info = await lstat(path);+ return info.isDirectory();+ } catch {+ return false;+ }+}++async function collectLinks(dir: string, prefix: string): Promise<LinkEntry[]> {+ const result: LinkEntry[] = [];+ const names = await readdir(dir);+ for (const name of names) {+ const full = join(dir, name);+ const info = await lstat(full);+ if (info.isSymbolicLink()) {+ const target = await readlink(full);+ const slug = prefix ? `${prefix}/${name}` : name;+ result.push({ slug, target });+ } else if (info.isDirectory() && !prefix && name.startsWith("@")) {+ result.push(...(await collectLinks(full, name)));+ }+ }+ return result;+}++export async function canonicalizeNodeModules(): Promise<void> {+ const root = process.cwd();+ const linkRoot = join(root, "node_modules/.bun/node_modules");++ if (!(await isDirectory(linkRoot))) {+ console.log(+ "[canonicalize-node-modules] no .bun/node_modules directory, skipping",+ );+ return;+ }++ const entries = await collectLinks(linkRoot, "");+ entries.sort((a, b) => a.slug.localeCompare(b.slug));++ await rm(linkRoot, { recursive: true, force: true });+ await mkdir(linkRoot, { recursive: true });++ for (const { slug, target } of entries) {+ const parts = slug.split("/");+ const leaf = parts.pop();+ if (!leaf) continue;+ const parent = join(linkRoot, ...parts);+ await mkdir(parent, { recursive: true });+ await symlink(target, join(parent, leaf));+ }++ console.log(`[canonicalize-node-modules] rebuilt ${entries.length} links`);+}++if (import.meta.main) {+ await canonicalizeNodeModules();+}diff --git a/.nix/scripts/heal-peer-dep-bins.ts b/.nix/scripts/heal-peer-dep-bins.tsnew file mode 100644index 00000000..80eb8888--- /dev/null+++ b/.nix/scripts/heal-peer-dep-bins.ts@@ -0,0 +1,229 @@+/**+ * Heal missing `.bin/<peer>` symlinks produced by bun's isolated installer.+ *+ * ## The bug+ *+ * Bun's `--linker=isolated` installer creates `.bin/<name>` symlinks inside+ * each package's private node_modules/.bin/ for every dependency that has a+ * `bin` field in its manifest — regular, optional, AND peer dependencies all+ * go through the same code path (`Installer.zig::linkDependencyBins`), and+ * the decision to link is made purely on whether the source file exists on+ * disk at the moment the linker looks (`bin.zig`:+ *+ * if (!bun.sys.exists(abs_target)) {+ * this.skipped_due_to_missing_bin = true;+ * return;+ * }+ *+ * ). For most dependencies the installer blocks the consuming package on the+ * provider via `isTaskBlocked`, so by the time `linkDependencyBins` runs for+ * package A the provider's file is guaranteed to be in place. But for+ * circular peer dependency pairs — A declares B as a peer, B (transitively)+ * depends on A — that blocking would deadlock, so bun's `Store.isCycle`+ * detector explicitly bypasses it and lets both sides run in parallel.+ *+ * The consequence is a plain timing race between two worker threads. Which+ * side wins depends on anything that shifts the relative scheduling of the+ * two workers — CPU load, thread-pool size, filesystem write latency and+ * caching, the kernel scheduler, NICE / cgroup limits — so the same bun+ * version with the same bun.lock and the same install flags can produce+ * different `.bin/` sets not just between different hosts but in principle+ * between two consecutive runs on the same host. In practice we have+ * observed divergence between a local NixOS sandbox, a GitHub Actions+ * ubuntu-latest runner, and a GitHub Actions macos-latest runner, which is+ * enough to break any single-hash FOD.+ *+ * Concretely, the Handy install hits this with+ * - update-browserslist-db/.bin/browserslist (update-browserslist-db+ * declares browserslist as a peer, browserslist has+ * update-browserslist-db in its regular dependencies → cycle), and+ * - @eslint-community/eslint-utils/.bin/eslint (eslint has eslint-utils+ * as a regular dep, eslint-utils declares eslint as a peer → cycle).+ *+ * There is no bun configuration flag or env var that makes the outcome+ * deterministic, and no upstream issue yet tracks this specific symptom+ * (oven-sh/bun#28147 is the closest family match, different project). See+ * the header of `canonicalize-node-modules.ts` for our sibling normalization+ * pass that rebuilds `.bun/node_modules/` in sorted order.+ *+ * ## The fix this script applies+ *+ * For every package under `node_modules/.bun/` we walk its declared+ * `peerDependencies`, find each resolved peer inside the package's private+ * `node_modules/`, and create any `.bin/<name> → ../<peerName>/<path>`+ * symlinks that bun's installer "intended" to create but may have skipped.+ * Entries that already exist are left alone (the script is idempotent).+ *+ * This is the "fix by adding" approach — we produce the complete `.bin/`+ * set that bun would have produced without the race, rather than stripping+ * the inconsistent subset. Advantages:+ *+ * - Matches bun's intended behavior; if bun ever fixes the race upstream,+ * this script becomes a no-op (every entry it would add already exists)+ * and the FOD hash is unchanged.+ * - Preserves `.bin/` entries that real code might depend on. We don't+ * rely on the (true but brittle) argument that peer-dep `.bin/` entries+ * are dead code in Tauri apps.+ * - Easy to explain in review: we're patching a known upstream race bug+ * with the exact output the upstream code is trying to produce.+ */++import { lstat, mkdir, readdir, readlink, symlink } from "fs/promises";+import { join } from "path";++type Manifest = {+ name?: string;+ bin?: string | Record<string, string>;+ peerDependencies?: Record<string, string>;+};++async function isDirectory(path: string) {+ try {+ const info = await lstat(path);+ return info.isDirectory();+ } catch {+ return false;+ }+}++async function readManifest(path: string): Promise<Manifest | null> {+ const file = Bun.file(path);+ if (!(await file.exists())) return null;+ return (await file.json()) as Manifest;+}++// Parse a .bun entry directory name (e.g. "@[email protected]+a1c3dd1b9adf390b")+// back into an npm package name ("@babel/core"). Returns null for entries that+// do not look like <name>@<version>[+<peer-hash>].+function parsePkgName(bunEntry: string): string | null {+ const at = bunEntry.startsWith("@")+ ? bunEntry.indexOf("@", 1)+ : bunEntry.indexOf("@");+ if (at <= 0) return null;+ return bunEntry.slice(0, at).replace(/\+/g, "/");+}++// Unscoped name used as the default bin name when `bin` is a bare string.+// For "@scope/pkg" returns "pkg"; for "pkg" returns "pkg".+function defaultBinName(pkgName: string): string {+ const slash = pkgName.lastIndexOf("/");+ return slash >= 0 ? pkgName.slice(slash + 1) : pkgName;+}++type BinSpec = { name: string; path: string };++function parseBinField(pkgName: string, binField: Manifest["bin"]): BinSpec[] {+ if (!binField) return [];+ if (typeof binField === "string") {+ return [{ name: defaultBinName(pkgName), path: binField }];+ }+ return Object.entries(binField).map(([name, path]) => ({+ name: defaultBinName(name),+ path,+ }));+}++// Produce the relative symlink target that bun itself uses for a .bin entry+// sitting inside `.bun/<containing>/node_modules/.bin/`, pointing to a file+// under `.bun/<containing>/node_modules/<peerName>/...`.+function binTarget(peerName: string, binPath: string): string {+ const clean = binPath.replace(/^\.\//, "");+ return `../${peerName}/${clean}`;+}++type HealedEntry = {+ containingEntry: string;+ containingPkg: string;+ peerName: string;+ binName: string;+ target: string;+};++export async function healPeerDepBins(): Promise<void> {+ const root = process.cwd();+ const bunRoot = join(root, "node_modules/.bun");++ if (!(await isDirectory(bunRoot))) {+ console.log("[heal-peer-dep-bins] no .bun directory, skipping");+ return;+ }++ const bunEntries = (await readdir(bunRoot)).sort();+ const healed: HealedEntry[] = [];++ for (const entry of bunEntries) {+ const pkgName = parsePkgName(entry);+ if (!pkgName) continue;+ const containingNodeModules = join(bunRoot, entry, "node_modules");+ if (!(await isDirectory(containingNodeModules))) continue;++ const manifest = await readManifest(+ join(containingNodeModules, pkgName, "package.json"),+ );+ if (!manifest) continue;++ const peers = Object.keys(manifest.peerDependencies ?? {});+ if (peers.length === 0) continue;++ const binRoot = join(containingNodeModules, ".bin");++ for (const peerName of peers) {+ // Peer may be optional and unresolved, or may not even be a real package+ // directory in this install layout (e.g. bundled peer). Skip anything we+ // cannot verify as "the peer's package.json is reachable from here".+ const peerManifest = await readManifest(+ join(containingNodeModules, peerName, "package.json"),+ );+ if (!peerManifest) continue;++ const bins = parseBinField(peerName, peerManifest.bin);+ if (bins.length === 0) continue;++ // Ensure the bin directory exists (it may be absent entirely if bun's+ // race skipped every link it would have created for this package).+ await mkdir(binRoot, { recursive: true });++ for (const bin of bins) {+ const linkPath = join(binRoot, bin.name);+ const target = binTarget(peerName, bin.path);++ // Idempotent: skip if anything already occupies this path. We do not+ // overwrite entries bun created; if bun already wrote a .bin/<name>,+ // that is either the correct target (race won) or a close variant we+ // should not second-guess.+ try {+ await lstat(linkPath);+ continue;+ } catch {+ // Does not exist — fall through to create.+ }++ await symlink(target, linkPath);+ healed.push({+ containingEntry: entry,+ containingPkg: pkgName,+ peerName,+ binName: bin.name,+ target,+ });+ }+ }+ }++ if (healed.length > 0) {+ console.log(+ `[heal-peer-dep-bins] healed ${healed.length} missing peer .bin entries:`,+ );+ for (const h of healed) {+ console.log(+ ` ${h.containingEntry}/node_modules/.bin/${h.binName} → ${h.target} (peer ${h.peerName} of ${h.containingPkg})`,+ );+ }+ } else {+ console.log("[heal-peer-dep-bins] nothing to heal");+ }+}++if (import.meta.main) {+ await healPeerDepBins();+}diff --git a/.nix/scripts/normalize-bun-binaries.ts b/.nix/scripts/normalize-bun-binaries.tsnew file mode 100644index 00000000..a5c2daf9--- /dev/null+++ b/.nix/scripts/normalize-bun-binaries.ts@@ -0,0 +1,102 @@+/**+ * Normalize .bin symlinks inside bun's internal module directories.+ *+ * In an isolated install, every package under .bun/ gets its own private+ * node_modules/ with a .bin/ directory holding symlinks to its dependencies'+ * executables:+ *+ * node_modules/.bun/+ * ├── @[email protected]+…/+ * │ └── node_modules/+ * │ ├── vite → ../../[email protected]+…/node_modules/vite (peer symlink)+ * │ └── .bin/ ← target of this script+ * │ └── vite → ../vite/bin/vite.js+ * ├── [email protected]+…/+ * │ └── node_modules/+ * │ └── .bin/+ * │ └── eslint → ../eslint/bin/eslint.js+ * └── [email protected]+…/+ * └── node_modules/+ * └── vite/ ← real package content+ * └── bin/vite.js+ *+ * Real executables live in .bun/<pkg>@<ver>/node_modules/<pkg>/…; every .bin/+ * entry is just a relative symlink reached through the peer symlinks in the+ * same node_modules/.+ *+ * Bun's creation order for those .bin/ symlinks is not guaranteed to be stable+ * across hosts or filesystems, which can break fixed-output derivation hashes.+ * This script reads the .bin/ symlinks bun produced, removes them, and+ * recreates them in lexicographic order while preserving the exact targets+ * bun picked.+ */++import { lstat, mkdir, readdir, readlink, rm, symlink } from "fs/promises";+import { join } from "path";++type BinEntry = {+ name: string;+ target: string;+};++async function isDirectory(path: string) {+ try {+ const info = await lstat(path);+ return info.isDirectory();+ } catch {+ return false;+ }+}++async function collectBinLinks(binRoot: string): Promise<BinEntry[]> {+ const entries: BinEntry[] = [];+ let names: string[];+ try {+ names = await readdir(binRoot);+ } catch {+ return entries;+ }+ for (const name of names) {+ const full = join(binRoot, name);+ const info = await lstat(full);+ if (!info.isSymbolicLink()) continue;+ entries.push({ name, target: await readlink(full) });+ }+ return entries;+}++export async function normalizeBunBinaries(): Promise<void> {+ const root = process.cwd();+ const bunRoot = join(root, "node_modules/.bun");++ if (!(await isDirectory(bunRoot))) {+ console.log("[normalize-bun-binaries] no .bun directory, skipping");+ return;+ }++ const bunEntries = (await readdir(bunRoot)).sort();+ let rewritten = 0;++ for (const entry of bunEntries) {+ const binRoot = join(bunRoot, entry, "node_modules", ".bin");+ if (!(await isDirectory(binRoot))) continue;++ const bins = await collectBinLinks(binRoot);+ if (bins.length === 0) continue;+ bins.sort((a, b) => a.name.localeCompare(b.name));++ await rm(binRoot, { recursive: true, force: true });+ await mkdir(binRoot, { recursive: true });++ for (const { name, target } of bins) {+ await symlink(target, join(binRoot, name));+ rewritten++;+ }+ }++ console.log(`[normalize-bun-binaries] rebuilt ${rewritten} links`);+}++if (import.meta.main) {+ await normalizeBunBinaries();+}diff --git a/.nix/scripts/normalize-install.ts b/.nix/scripts/normalize-install.tsnew file mode 100644index 00000000..ec8647ba--- /dev/null+++ b/.nix/scripts/normalize-install.ts@@ -0,0 +1,36 @@+/**+ * Make `bun install --linker=isolated` output bit-reproducible.+ *+ * Entry point for the nixpkgs FOD build — invoke this single script after+ * `bun install` to get a `node_modules/` tree that is byte-identical+ * across machines and runs with the same bun.lock.+ *+ * `bun install --linker=isolated` is not bit-reproducible out of the box,+ * even with a frozen lockfile, a pinned bun version, `--ignore-scripts`,+ * and a clean `BUN_INSTALL_CACHE_DIR`. Running it across a local NixOS+ * sandbox, a GitHub Actions ubuntu-latest runner, and a GitHub Actions+ * macos-latest runner produces subtly different `node_modules/` trees+ * from identical inputs. Two independent sources of drift show up:+ *+ * - Missing `.bin/<peer>` entries around circular peer dependencies,+ * caused by a timing race inside bun's isolated installer thread+ * pool. See `heal-peer-dep-bins.ts` for the full explanation and fix.+ *+ * - Non-deterministic symlink creation order in `.bun/node_modules/`+ * and in each per-package `.bin/` directory. NAR hashing sorts+ * entries during serialization, so this is usually harmless, but we+ * defensively rebuild both trees in canonical sorted order.+ * See `canonicalize-node-modules.ts` and `normalize-bun-binaries.ts`.+ *+ * Each sub-script is also runnable standalone for debugging:+ *+ * bun --bun .nix/scripts/<name>.ts+ */++import { canonicalizeNodeModules } from "./canonicalize-node-modules.ts";+import { healPeerDepBins } from "./heal-peer-dep-bins.ts";+import { normalizeBunBinaries } from "./normalize-bun-binaries.ts";++await canonicalizeNodeModules();+await healPeerDepBins();+await normalizeBunBinaries();