Skip to content

armgit / Handy

Public Imported from GitHub
0

A free, open source, and extensible speech-to-text application that works completely offline.

chore(nix): add bun node_modules normalization scripts (#1256)

Evgeny Khudobaaf6ec6c903e4
diff --git a/.nix/scripts/canonicalize-node-modules.ts b/.nix/scripts/canonicalize-node-modules.ts
new file mode 100644
index 00000000..32356967
--- /dev/null
+++ b/.nix/scripts/canonicalize-node-modules.ts
@@ -0,0 +1,97 @@
+/**
+ * Canonicalize bun's internal node_modules symlinks for reproducible FOD hashes.
+ *
+ * Isolated-install layout produced by `bun install --linker=isolated`:
+ *
+ * node_modules/
+ * ├── react → .bun/[email protected]/node_modules/react (symlink)
+ * ├── minimatch → .bun/[email protected]/node_modules/minimatch (symlink)
+ * └── .bun/
+ * ├── [email protected]/
+ * │ └── node_modules/
+ * │ └── react/ ← real package content
+ * ├── [email protected]/
+ * │ └── node_modules/
+ * │ └── minimatch/ ← real package content
+ * └── node_modules/ ← target of this script
+ * ├── react → ../[email protected]/node_modules/react
+ * ├── minimatch → ../[email protected]/node_modules/minimatch
+ * └── @babel/
+ * └── core → ../../@[email protected]+…/node_modules/@babel/core
+ *
+ * Real package content lives in .bun/<pkg>@<ver>/node_modules/<pkg>/.
+ * The .bun/node_modules/ directory (linkRoot) holds only symlinks — it acts
+ * as a fallback upward-resolution path for packages inside .bun/.
+ *
+ * Bun's creation order for those symlinks is not guaranteed to be stable
+ * across hosts or filesystems, which can break fixed-output derivation hashes.
+ * This script reads the existing symlinks, removes them, and recreates them
+ * in lexicographic order while preserving the exact targets bun picked.
+ */
+
+import { lstat, mkdir, readdir, readlink, rm, symlink } from "fs/promises";
+import { join } from "path";
+
+type LinkEntry = {
+ slug: string;
+ target: string;
+};
+
+async function isDirectory(path: string) {
+ try {
+ const info = await lstat(path);
+ return info.isDirectory();
+ } catch {
+ return false;
+ }
+}
+
+async function collectLinks(dir: string, prefix: string): Promise<LinkEntry[]> {
+ const result: LinkEntry[] = [];
+ const names = await readdir(dir);
+ for (const name of names) {
+ const full = join(dir, name);
+ const info = await lstat(full);
+ if (info.isSymbolicLink()) {
+ const target = await readlink(full);
+ const slug = prefix ? `${prefix}/${name}` : name;
+ result.push({ slug, target });
+ } else if (info.isDirectory() && !prefix && name.startsWith("@")) {
+ result.push(...(await collectLinks(full, name)));
+ }
+ }
+ return result;
+}
+
+export async function canonicalizeNodeModules(): Promise<void> {
+ const root = process.cwd();
+ const linkRoot = join(root, "node_modules/.bun/node_modules");
+
+ if (!(await isDirectory(linkRoot))) {
+ console.log(
+ "[canonicalize-node-modules] no .bun/node_modules directory, skipping",
+ );
+ return;
+ }
+
+ const entries = await collectLinks(linkRoot, "");
+ entries.sort((a, b) => a.slug.localeCompare(b.slug));
+
+ await rm(linkRoot, { recursive: true, force: true });
+ await mkdir(linkRoot, { recursive: true });
+
+ for (const { slug, target } of entries) {
+ const parts = slug.split("/");
+ const leaf = parts.pop();
+ if (!leaf) continue;
+ const parent = join(linkRoot, ...parts);
+ await mkdir(parent, { recursive: true });
+ await symlink(target, join(parent, leaf));
+ }
+
+ console.log(`[canonicalize-node-modules] rebuilt ${entries.length} links`);
+}
+
+if (import.meta.main) {
+ await canonicalizeNodeModules();
+}
diff --git a/.nix/scripts/heal-peer-dep-bins.ts b/.nix/scripts/heal-peer-dep-bins.ts
new file mode 100644
index 00000000..80eb8888
--- /dev/null
+++ b/.nix/scripts/heal-peer-dep-bins.ts
@@ -0,0 +1,229 @@
+/**
+ * Heal missing `.bin/<peer>` symlinks produced by bun's isolated installer.
+ *
+ * ## The bug
+ *
+ * Bun's `--linker=isolated` installer creates `.bin/<name>` symlinks inside
+ * each package's private node_modules/.bin/ for every dependency that has a
+ * `bin` field in its manifest — regular, optional, AND peer dependencies all
+ * go through the same code path (`Installer.zig::linkDependencyBins`), and
+ * the decision to link is made purely on whether the source file exists on
+ * disk at the moment the linker looks (`bin.zig`:
+ *
+ * if (!bun.sys.exists(abs_target)) {
+ * this.skipped_due_to_missing_bin = true;
+ * return;
+ * }
+ *
+ * ). For most dependencies the installer blocks the consuming package on the
+ * provider via `isTaskBlocked`, so by the time `linkDependencyBins` runs for
+ * package A the provider's file is guaranteed to be in place. But for
+ * circular peer dependency pairs — A declares B as a peer, B (transitively)
+ * depends on A — that blocking would deadlock, so bun's `Store.isCycle`
+ * detector explicitly bypasses it and lets both sides run in parallel.
+ *
+ * The consequence is a plain timing race between two worker threads. Which
+ * side wins depends on anything that shifts the relative scheduling of the
+ * two workers — CPU load, thread-pool size, filesystem write latency and
+ * caching, the kernel scheduler, NICE / cgroup limits — so the same bun
+ * version with the same bun.lock and the same install flags can produce
+ * different `.bin/` sets not just between different hosts but in principle
+ * between two consecutive runs on the same host. In practice we have
+ * observed divergence between a local NixOS sandbox, a GitHub Actions
+ * ubuntu-latest runner, and a GitHub Actions macos-latest runner, which is
+ * enough to break any single-hash FOD.
+ *
+ * Concretely, the Handy install hits this with
+ * - update-browserslist-db/.bin/browserslist (update-browserslist-db
+ * declares browserslist as a peer, browserslist has
+ * update-browserslist-db in its regular dependencies → cycle), and
+ * - @eslint-community/eslint-utils/.bin/eslint (eslint has eslint-utils
+ * as a regular dep, eslint-utils declares eslint as a peer → cycle).
+ *
+ * There is no bun configuration flag or env var that makes the outcome
+ * deterministic, and no upstream issue yet tracks this specific symptom
+ * (oven-sh/bun#28147 is the closest family match, different project). See
+ * the header of `canonicalize-node-modules.ts` for our sibling normalization
+ * pass that rebuilds `.bun/node_modules/` in sorted order.
+ *
+ * ## The fix this script applies
+ *
+ * For every package under `node_modules/.bun/` we walk its declared
+ * `peerDependencies`, find each resolved peer inside the package's private
+ * `node_modules/`, and create any `.bin/<name> → ../<peerName>/<path>`
+ * symlinks that bun's installer "intended" to create but may have skipped.
+ * Entries that already exist are left alone (the script is idempotent).
+ *
+ * This is the "fix by adding" approach — we produce the complete `.bin/`
+ * set that bun would have produced without the race, rather than stripping
+ * the inconsistent subset. Advantages:
+ *
+ * - Matches bun's intended behavior; if bun ever fixes the race upstream,
+ * this script becomes a no-op (every entry it would add already exists)
+ * and the FOD hash is unchanged.
+ * - Preserves `.bin/` entries that real code might depend on. We don't
+ * rely on the (true but brittle) argument that peer-dep `.bin/` entries
+ * are dead code in Tauri apps.
+ * - Easy to explain in review: we're patching a known upstream race bug
+ * with the exact output the upstream code is trying to produce.
+ */
+
+import { lstat, mkdir, readdir, readlink, symlink } from "fs/promises";
+import { join } from "path";
+
+type Manifest = {
+ name?: string;
+ bin?: string | Record<string, string>;
+ peerDependencies?: Record<string, string>;
+};
+
+async function isDirectory(path: string) {
+ try {
+ const info = await lstat(path);
+ return info.isDirectory();
+ } catch {
+ return false;
+ }
+}
+
+async function readManifest(path: string): Promise<Manifest | null> {
+ const file = Bun.file(path);
+ if (!(await file.exists())) return null;
+ return (await file.json()) as Manifest;
+}
+
+// Parse a .bun entry directory name (e.g. "@[email protected]+a1c3dd1b9adf390b")
+// back into an npm package name ("@babel/core"). Returns null for entries that
+// do not look like <name>@<version>[+<peer-hash>].
+function parsePkgName(bunEntry: string): string | null {
+ const at = bunEntry.startsWith("@")
+ ? bunEntry.indexOf("@", 1)
+ : bunEntry.indexOf("@");
+ if (at <= 0) return null;
+ return bunEntry.slice(0, at).replace(/\+/g, "/");
+}
+
+// Unscoped name used as the default bin name when `bin` is a bare string.
+// For "@scope/pkg" returns "pkg"; for "pkg" returns "pkg".
+function defaultBinName(pkgName: string): string {
+ const slash = pkgName.lastIndexOf("/");
+ return slash >= 0 ? pkgName.slice(slash + 1) : pkgName;
+}
+
+type BinSpec = { name: string; path: string };
+
+function parseBinField(pkgName: string, binField: Manifest["bin"]): BinSpec[] {
+ if (!binField) return [];
+ if (typeof binField === "string") {
+ return [{ name: defaultBinName(pkgName), path: binField }];
+ }
+ return Object.entries(binField).map(([name, path]) => ({
+ name: defaultBinName(name),
+ path,
+ }));
+}
+
+// Produce the relative symlink target that bun itself uses for a .bin entry
+// sitting inside `.bun/<containing>/node_modules/.bin/`, pointing to a file
+// under `.bun/<containing>/node_modules/<peerName>/...`.
+function binTarget(peerName: string, binPath: string): string {
+ const clean = binPath.replace(/^\.\//, "");
+ return `../${peerName}/${clean}`;
+}
+
+type HealedEntry = {
+ containingEntry: string;
+ containingPkg: string;
+ peerName: string;
+ binName: string;
+ target: string;
+};
+
+export async function healPeerDepBins(): Promise<void> {
+ const root = process.cwd();
+ const bunRoot = join(root, "node_modules/.bun");
+
+ if (!(await isDirectory(bunRoot))) {
+ console.log("[heal-peer-dep-bins] no .bun directory, skipping");
+ return;
+ }
+
+ const bunEntries = (await readdir(bunRoot)).sort();
+ const healed: HealedEntry[] = [];
+
+ for (const entry of bunEntries) {
+ const pkgName = parsePkgName(entry);
+ if (!pkgName) continue;
+ const containingNodeModules = join(bunRoot, entry, "node_modules");
+ if (!(await isDirectory(containingNodeModules))) continue;
+
+ const manifest = await readManifest(
+ join(containingNodeModules, pkgName, "package.json"),
+ );
+ if (!manifest) continue;
+
+ const peers = Object.keys(manifest.peerDependencies ?? {});
+ if (peers.length === 0) continue;
+
+ const binRoot = join(containingNodeModules, ".bin");
+
+ for (const peerName of peers) {
+ // Peer may be optional and unresolved, or may not even be a real package
+ // directory in this install layout (e.g. bundled peer). Skip anything we
+ // cannot verify as "the peer's package.json is reachable from here".
+ const peerManifest = await readManifest(
+ join(containingNodeModules, peerName, "package.json"),
+ );
+ if (!peerManifest) continue;
+
+ const bins = parseBinField(peerName, peerManifest.bin);
+ if (bins.length === 0) continue;
+
+ // Ensure the bin directory exists (it may be absent entirely if bun's
+ // race skipped every link it would have created for this package).
+ await mkdir(binRoot, { recursive: true });
+
+ for (const bin of bins) {
+ const linkPath = join(binRoot, bin.name);
+ const target = binTarget(peerName, bin.path);
+
+ // Idempotent: skip if anything already occupies this path. We do not
+ // overwrite entries bun created; if bun already wrote a .bin/<name>,
+ // that is either the correct target (race won) or a close variant we
+ // should not second-guess.
+ try {
+ await lstat(linkPath);
+ continue;
+ } catch {
+ // Does not exist — fall through to create.
+ }
+
+ await symlink(target, linkPath);
+ healed.push({
+ containingEntry: entry,
+ containingPkg: pkgName,
+ peerName,
+ binName: bin.name,
+ target,
+ });
+ }
+ }
+ }
+
+ if (healed.length > 0) {
+ console.log(
+ `[heal-peer-dep-bins] healed ${healed.length} missing peer .bin entries:`,
+ );
+ for (const h of healed) {
+ console.log(
+ ` ${h.containingEntry}/node_modules/.bin/${h.binName} → ${h.target} (peer ${h.peerName} of ${h.containingPkg})`,
+ );
+ }
+ } else {
+ console.log("[heal-peer-dep-bins] nothing to heal");
+ }
+}
+
+if (import.meta.main) {
+ await healPeerDepBins();
+}
diff --git a/.nix/scripts/normalize-bun-binaries.ts b/.nix/scripts/normalize-bun-binaries.ts
new file mode 100644
index 00000000..a5c2daf9
--- /dev/null
+++ b/.nix/scripts/normalize-bun-binaries.ts
@@ -0,0 +1,102 @@
+/**
+ * Normalize .bin symlinks inside bun's internal module directories.
+ *
+ * In an isolated install, every package under .bun/ gets its own private
+ * node_modules/ with a .bin/ directory holding symlinks to its dependencies'
+ * executables:
+ *
+ * node_modules/.bun/
+ * ├── @[email protected]+…/
+ * │ └── node_modules/
+ * │ ├── vite → ../../[email protected]+…/node_modules/vite (peer symlink)
+ * │ └── .bin/ ← target of this script
+ * │ └── vite → ../vite/bin/vite.js
+ * ├── [email protected]+…/
+ * │ └── node_modules/
+ * │ └── .bin/
+ * │ └── eslint → ../eslint/bin/eslint.js
+ * └── [email protected]+…/
+ * └── node_modules/
+ * └── vite/ ← real package content
+ * └── bin/vite.js
+ *
+ * Real executables live in .bun/<pkg>@<ver>/node_modules/<pkg>/…; every .bin/
+ * entry is just a relative symlink reached through the peer symlinks in the
+ * same node_modules/.
+ *
+ * Bun's creation order for those .bin/ symlinks is not guaranteed to be stable
+ * across hosts or filesystems, which can break fixed-output derivation hashes.
+ * This script reads the .bin/ symlinks bun produced, removes them, and
+ * recreates them in lexicographic order while preserving the exact targets
+ * bun picked.
+ */
+
+import { lstat, mkdir, readdir, readlink, rm, symlink } from "fs/promises";
+import { join } from "path";
+
+type BinEntry = {
+ name: string;
+ target: string;
+};
+
+async function isDirectory(path: string) {
+ try {
+ const info = await lstat(path);
+ return info.isDirectory();
+ } catch {
+ return false;
+ }
+}
+
+async function collectBinLinks(binRoot: string): Promise<BinEntry[]> {
+ const entries: BinEntry[] = [];
+ let names: string[];
+ try {
+ names = await readdir(binRoot);
+ } catch {
+ return entries;
+ }
+ for (const name of names) {
+ const full = join(binRoot, name);
+ const info = await lstat(full);
+ if (!info.isSymbolicLink()) continue;
+ entries.push({ name, target: await readlink(full) });
+ }
+ return entries;
+}
+
+export async function normalizeBunBinaries(): Promise<void> {
+ const root = process.cwd();
+ const bunRoot = join(root, "node_modules/.bun");
+
+ if (!(await isDirectory(bunRoot))) {
+ console.log("[normalize-bun-binaries] no .bun directory, skipping");
+ return;
+ }
+
+ const bunEntries = (await readdir(bunRoot)).sort();
+ let rewritten = 0;
+
+ for (const entry of bunEntries) {
+ const binRoot = join(bunRoot, entry, "node_modules", ".bin");
+ if (!(await isDirectory(binRoot))) continue;
+
+ const bins = await collectBinLinks(binRoot);
+ if (bins.length === 0) continue;
+ bins.sort((a, b) => a.name.localeCompare(b.name));
+
+ await rm(binRoot, { recursive: true, force: true });
+ await mkdir(binRoot, { recursive: true });
+
+ for (const { name, target } of bins) {
+ await symlink(target, join(binRoot, name));
+ rewritten++;
+ }
+ }
+
+ console.log(`[normalize-bun-binaries] rebuilt ${rewritten} links`);
+}
+
+if (import.meta.main) {
+ await normalizeBunBinaries();
+}
diff --git a/.nix/scripts/normalize-install.ts b/.nix/scripts/normalize-install.ts
new file mode 100644
index 00000000..ec8647ba
--- /dev/null
+++ b/.nix/scripts/normalize-install.ts
@@ -0,0 +1,36 @@
+/**
+ * Make `bun install --linker=isolated` output bit-reproducible.
+ *
+ * Entry point for the nixpkgs FOD build — invoke this single script after
+ * `bun install` to get a `node_modules/` tree that is byte-identical
+ * across machines and runs with the same bun.lock.
+ *
+ * `bun install --linker=isolated` is not bit-reproducible out of the box,
+ * even with a frozen lockfile, a pinned bun version, `--ignore-scripts`,
+ * and a clean `BUN_INSTALL_CACHE_DIR`. Running it across a local NixOS
+ * sandbox, a GitHub Actions ubuntu-latest runner, and a GitHub Actions
+ * macos-latest runner produces subtly different `node_modules/` trees
+ * from identical inputs. Two independent sources of drift show up:
+ *
+ * - Missing `.bin/<peer>` entries around circular peer dependencies,
+ * caused by a timing race inside bun's isolated installer thread
+ * pool. See `heal-peer-dep-bins.ts` for the full explanation and fix.
+ *
+ * - Non-deterministic symlink creation order in `.bun/node_modules/`
+ * and in each per-package `.bin/` directory. NAR hashing sorts
+ * entries during serialization, so this is usually harmless, but we
+ * defensively rebuild both trees in canonical sorted order.
+ * See `canonicalize-node-modules.ts` and `normalize-bun-binaries.ts`.
+ *
+ * Each sub-script is also runnable standalone for debugging:
+ *
+ * bun --bun .nix/scripts/<name>.ts
+ */
+
+import { canonicalizeNodeModules } from "./canonicalize-node-modules.ts";
+import { healPeerDepBins } from "./heal-peer-dep-bins.ts";
+import { normalizeBunBinaries } from "./normalize-bun-binaries.ts";
+
+await canonicalizeNodeModules();
+await healPeerDepBins();
+await normalizeBunBinaries();