A local-first multi-agent app development platform, built on PenguinHarness
/**
* The builtin plugins, shipped as the npm packages they are: every `plugins/*` package with
* a code entry is built by its own `build` script, packed by `pnpm pack` — exactly what
* `npm publish` would send — and installed by npm into a staging directory laid out as an npm
* prefix (`<out>/package.json` + `<out>/node_modules/<name>/…`, dependencies included). That
* prefix is the one shape both consumers resolve from: the hot push ships it under `plugins/`
* in its assets, the desktop build stages it beside `skills/`.
*
* Nothing about a package is rewritten. Its `package.json`, its `exports`, its `dist/` and its
* `README.md` reach the target as the package's own build produced them; a dependency it
* declares is installed beside it the way npm installs it anywhere. The SDK's runtime is not
* among those dependencies — a plugin compiles against `@adelie/core`'s types
* (a devDependency) and shares the host's copy at run time.
*
* A builtin plugin bundles what it runs. Every file in the prefix is a blob a push carries
* separately, so an npm dependency tree (a grammar collection, a web framework's CJS, ESM and
* type copies) turns one plugin into hundreds of small transfers. Its own build compiles its
* pure-JS dependencies into `dist/`; what remains a runtime dependency is a native module whose
* per-platform binaries cannot live inside a bundle, named in NATIVE_DEPENDENCIES — and a
* package declaring anything else fails this build before anything is packed.
*
* What npm installs for those dependencies is what pnpm-lock.yaml resolves, nothing newer: the
* prefix's manifest pins their whole locked closure as `overrides`, and a tree that differs from
* the lockfile by version or by tarball integrity fails the build (scripts/lib/locked-prefix.mjs).
* The licenses of those third-party packages ship beside them as `THIRD-PARTY-NOTICES.md` at the
* prefix's root (scripts/lib/third-party-notices.mjs); a package without license text fails it.
*
* Cached by content: the hash over every plugin's `src/`, `package.json`, `README.md` and
* `tsup.config.ts`, and over the lockfile entries the prefix is pinned to, names a directory
* under `node_modules/.cache/penguin-plugins/`, and an unchanged set is not built, packed or
* installed again — a push of an unrelated change costs nothing here. Installing needs the registry (for the dependencies) the first time only.
*
* Usage (a library for deploy.mjs / desktop build-assets.mjs, and a CLI):
* node scripts/build-plugins.mjs --out <dir> stage the prefix into <dir>
*/
import { execFileSync } from "node:child_process";
import { createHash } from "node:crypto";
import fs from "node:fs";
import fsp from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import {
integrityMismatches,
lockCacheInput,
lockedClosure,
platformSpecs,
readPnpmLock,
} from "./lib/locked-prefix.mjs";
import { readVendoredPackages, thirdPartyNotices } from "./lib/third-party-notices.mjs";
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const PLUGINS_SRC = path.join(ROOT, "plugins");
const CACHE = path.join(ROOT, "node_modules", ".cache", "penguin-plugins");
const COMPLETE = ".complete";
/** Folded into the cache key: bump when what this script WRITES changes, not only what it reads. */
const PACK_FORMAT = 14;
/** The prefix's own manifest: npm needs one above `node_modules`, and it is ours, never a package's. */
const PREFIX_MANIFEST = { name: "penguin-builtin-plugins", private: true, version: "0.0.0" };
/**
* The runtime dependencies a builtin plugin may declare: native modules only, each with a
* reason. Everything else is compiled into the plugin's `dist/` by its own build.
*/
const NATIVE_DEPENDENCIES = new Map([
["koffi", "FFI with per-platform prebuilt binaries (sandbox-dsh's Windows ACL runner)"],
[
"@deepseek-ai/dsh-sandbox-local",
"picks its per-platform rung by bare specifier at run time — bundling it makes the Windows one unresolvable (sandbox-dsh)",
],
["@deepseek-ai/cordis", "the context the DSH chain is mounted on, shared with it (sandbox-dsh)"],
[
"@deepseek-ai/node-addon-landlock-run",
"resolves its per-platform launcher binary package at run time (sandbox-dsh)",
],
]);
/**
* The hosts a pushed prefix may land on. npm installs a native module's binary for the machine
* doing the install, and this build runs wherever CI or a developer happens to be — so a prefix
* built on Linux carried no Windows binary, and sandbox-dsh's Windows runner failed there with
* "Cannot find the native Koffi module". The per-platform packages of every NATIVE_DEPENDENCIES
* entry are installed for each of these as well; they are small next to the plugins themselves,
* and one prefix then serves every target a push can reach.
*/
const TARGET_PLATFORMS = [
{ os: "linux", cpu: "x64" },
{ os: "linux", cpu: "arm64" },
{ os: "win32", cpu: "x64" },
{ os: "darwin", cpu: "arm64" },
];
/** The third-party license notices written at the prefix's root, beside its manifest. */
const NOTICES_FILE = "THIRD-PARTY-NOTICES.md";
/** What npm leaves in the prefix that is not a package: its hidden lockfile. Never shipped. */
const NOT_SHIPPED = new Set(["node_modules/.package-lock.json"]);
/** Files under `dir`, as sorted relative posix paths (symlinks — `.bin` shims — excluded). */
async function walk(dir, prefix = "") {
const out = [];
for (const e of await fsp.readdir(dir, { withFileTypes: true })) {
const rel = prefix === "" ? e.name : `${prefix}/${e.name}`;
if (e.isDirectory()) out.push(...(await walk(path.join(dir, e.name), rel)));
else if (e.isFile()) out.push(rel);
}
return out.sort();
}
/** A package manager's command, as the platform names it. */
function command(name) {
return process.platform === "win32" ? `${name}.cmd` : name;
}
// cmd.exe does not unquote spawn args by itself: under `shell: true` the args are joined
// into one command line, so a path with a space (the pack directory lives under the user's
// temp directory, i.e. their profile) splits into two. Quoted the way run-with-env.mjs quotes.
const quote = (a) => (/[\s"^&|<>;,()%!]/.test(a) ? `"${a.replace(/"/g, '\\"')}"` : a);
function run(name, args, cwd) {
const windows = process.platform === "win32";
try {
execFileSync(command(name), windows ? args.map(quote) : args, {
cwd,
stdio: ["ignore", "pipe", "pipe"],
shell: windows,
env: process.env,
});
} catch (err) {
const stderr = err instanceof Object && "stderr" in err ? String(err.stderr).trim() : "";
throw new Error(`${name} ${args.slice(0, 3).join(" ")} failed in ${cwd}\n${stderr}`);
}
}
/**
* What a plugin's pack depends on: its sources, its manifest, its README, its build config —
* and every other directory the package SHIPS.
*
* That last part is not decoration. A plugin may carry files its code never imports (the Windows
* backend ships the PowerShell script its setup runs), and hashing only `src/` meant editing one
* of them changed nothing the cache could see: the build happily served a stale pack, and the
* fix nobody could find on the host was a file that had never left this machine.
*/
async function sourceHash(dir, into) {
for (const rel of ["package.json", "README.md", "tsup.config.ts"]) {
const file = path.join(dir, rel);
if (fs.existsSync(file))
into
.update(rel)
.update("\0")
.update(await fsp.readFile(file))
.update("\0");
}
const manifest = JSON.parse(await fsp.readFile(path.join(dir, "package.json"), "utf8"));
// `dist` and `vendor` are built or fetched from what is hashed here, never edited by hand.
const shipped = (manifest.files ?? []).filter((f) => !["dist", "vendor"].includes(f));
for (const name of ["src", ...shipped]) {
const sub = path.join(dir, name);
if (!fs.existsSync(sub) || !fs.statSync(sub).isDirectory()) continue;
for (const rel of await walk(sub)) {
into
.update(`${name}/${rel}`)
.update("\0")
.update(await fsp.readFile(path.join(sub, rel)))
.update("\0");
}
}
}
/** Every plugin package under `plugins/`: a package.json that declares `penguin`. */
async function pluginPackages() {
const out = [];
const entries = fs.existsSync(PLUGINS_SRC) ? await fsp.readdir(PLUGINS_SRC) : [];
for (const dirName of entries.sort()) {
const dir = path.join(PLUGINS_SRC, dirName);
const manifestFile = path.join(dir, "package.json");
if (!fs.existsSync(manifestFile)) continue;
const pkg = JSON.parse(await fsp.readFile(manifestFile, "utf8"));
// A package with a code entry is built and packed; one without (skills, hooks) carries no code.
if (pkg.main === undefined && pkg.exports === undefined) continue;
const unbundled = Object.keys(pkg.dependencies ?? {}).filter(
(d) => !NATIVE_DEPENDENCIES.has(d),
);
if (unbundled.length > 0) {
throw new Error(
`${pkg.name} declares runtime dependencies ${unbundled.join(", ")}: a builtin plugin bundles ` +
"what it runs (tsup noExternal, the package a devDependency) — every file it would " +
"install is a separate blob on every push. Only native modules stay dependencies " +
"(NATIVE_DEPENDENCIES in scripts/build-plugins.mjs).",
);
}
out.push({ name: pkg.name, version: pkg.version, dir });
}
return out;
}
/**
* Builds, packs and installs every builtin plugin into one staged prefix (from cache when
* nothing changed) and returns `{ dir, files, plugins }`: the prefix directory, the relative
* paths to ship, and `[{ name, version }]` of what it holds.
*/
export async function buildBuiltinPlugins({ log = () => {} } = {}) {
const plugins = await pluginPackages();
// A native module publishes one package per `<os>-<cpu>` (koffi: `@koromix/koffi-win32-x64`)
// and depends on all of them optionally; the lockfile names them, so the closure holds the
// ones for TARGET_PLATFORMS and this build never hardcodes a platform triple.
const lock = readPnpmLock(ROOT);
const importers = plugins.map((p) => path.relative(ROOT, p.dir).split(path.sep).join("/"));
const closure = lockedClosure(lock, importers, NATIVE_DEPENDENCIES, TARGET_PLATFORMS);
const builtinNames = new Set(plugins.map((p) => p.name));
const h = createHash("sha256").update(`pack ${PACK_FORMAT}\0`);
h.update(lockCacheInput(lock, closure)).update("\0");
for (const plugin of plugins) {
h.update(plugin.name).update("\0");
await sourceHash(plugin.dir, h);
}
const hash = h.digest("hex").slice(0, 16);
const out = path.join(CACHE, hash);
if (fs.existsSync(path.join(out, COMPLETE))) {
log(`${plugins.length} builtin plugins: cached (${hash})`);
} else {
await fsp.rm(out, { recursive: true, force: true });
await fsp.mkdir(out, { recursive: true });
const packed = await fsp.mkdtemp(path.join(os.tmpdir(), "penguin-plugins-pack-"));
try {
const tarballs = [];
for (const plugin of plugins) {
// The package's own build, then the package as npm would publish it (`files` honored,
// `workspace:` ranges rewritten) — nothing this script decides.
run("pnpm", ["--filter", plugin.name, "run", "build"], ROOT);
run("pnpm", ["pack", "--pack-destination", packed], plugin.dir);
const tarball = (await fsp.readdir(packed)).find(
(f) => f.endsWith(".tgz") && !tarballs.some((t) => path.basename(t) === f),
);
if (tarball === undefined) throw new Error(`pnpm pack left no tarball for ${plugin.name}`);
tarballs.push(path.join(packed, tarball));
log(`plugin ${plugin.name}@${plugin.version}: packed`);
}
// The manifest names what is shipped — how the loader tells the plugins from what npm
// installs beside them — and --no-save below keeps npm from rewriting it.
// `overrides` pins every package of the native closure, transitive ones included, to the
// version pnpm-lock.yaml resolves; without it npm resolves each `^` range afresh.
const dependencies = Object.fromEntries(plugins.map((p) => [p.name, p.version]));
const overrides = Object.fromEntries(closure);
await fsp.writeFile(
path.join(out, "package.json"),
`${JSON.stringify({ ...PREFIX_MANIFEST, dependencies, overrides }, null, 2)}\n`,
);
if (tarballs.length > 0) {
// npm installs the packages and their dependencies into the prefix; --no-save keeps
// the prefix's manifest ours (no `file:` paths into a temp directory), --omit=dev
// leaves the SDK's types and the build tools behind.
run(
"npm",
[
"install",
"--no-save",
"--no-package-lock",
"--omit=dev",
"--no-audit",
"--no-fund",
"--ignore-scripts",
"--",
...tarballs,
],
out,
);
// One call for every target: npm reconciles the tree on each install, so a second
// install would prune the first target's package as extraneous. --force is what makes
// npm accept a package whose `os`/`cpu` is not this machine's — the point of the call.
const platformPackages = platformSpecs(lock, closure);
if (platformPackages.length > 0) {
run(
"npm",
[
"install",
"--no-save",
"--no-package-lock",
"--omit=dev",
"--no-audit",
"--no-fund",
"--ignore-scripts",
"--force",
"--",
...platformPackages,
],
out,
);
}
if (platformPackages.length > 0) {
log(`${platformPackages.length} per-platform native binaries: installed`);
}
const npmLock = JSON.parse(
await fsp.readFile(path.join(out, "node_modules", ".package-lock.json"), "utf8"),
);
const drift = integrityMismatches(lock, closure, npmLock, builtinNames);
if (drift.length > 0) {
throw new Error(
`npm's tree in the builtin prefix differs from pnpm-lock.yaml:\n ${drift.join("\n ")}`,
);
}
const thirdParty = readVendoredPackages(path.join(out, "node_modules")).filter(
(p) => !builtinNames.has(p.name),
);
if (thirdParty.length > 0) {
const notices = thirdPartyNotices(thirdParty, {
carrier: PREFIX_MANIFEST.name,
location: "node_modules/",
});
await fsp.writeFile(path.join(out, NOTICES_FILE), notices);
}
log(`${closure.size} locked third-party packages: verified`);
}
// npm installs a package's files with the mode it pleases, and a vendored program
// arrives without its exec bit — which no consumer of the prefix can guess back.
for (const rel of await walk(out)) {
if (/(^|\/)vendor\/[^/]+\/bin\/[^/]+$/.test(rel)) {
await fsp.chmod(path.join(out, rel), 0o755);
}
}
await fsp.writeFile(path.join(out, COMPLETE), hash);
log(`${plugins.length} builtin plugins: installed (${hash})`);
} catch (err) {
await fsp.rm(out, { recursive: true, force: true });
throw err;
} finally {
await fsp.rm(packed, { recursive: true, force: true });
}
}
const files = (await walk(out)).filter((f) => f !== COMPLETE && !NOT_SHIPPED.has(f));
return { dir: out, files, plugins: plugins.map(({ name, version }) => ({ name, version })) };
}
/** The prefix as a file map, relative to the prefix, each value an absolute source path. */
export function prefixLayout(built) {
return new Map(built.files.map((rel) => [rel, { path: path.join(built.dir, rel) }]));
}
/** Writes the prefix into `dest`, replacing what was there. */
export async function stagePrefix(built, dest) {
await fsp.rm(dest, { recursive: true, force: true });
for (const [rel, source] of prefixLayout(built)) {
const target = path.join(dest, ...rel.split("/"));
await fsp.mkdir(path.dirname(target), { recursive: true });
await fsp.copyFile(source.path, target);
}
}
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
const outIdx = process.argv.indexOf("--out");
const out = outIdx === -1 ? null : process.argv[outIdx + 1];
const built = await buildBuiltinPlugins({ log: (m) => console.log(`[build-plugins] ${m}`) });
if (out) {
await stagePrefix(built, path.resolve(out));
console.log(
`[build-plugins] staged ${built.plugins.length} plugins (${built.files.length} files) into ${path.resolve(out)}`,
);
}
}