A local-first multi-agent app development platform, built on PenguinHarness
/**
* HmrHost: the runtime side of the stop-the-world hot-update protocol.
*
* HMR LAYER — MECHANISM ONLY. Nothing here may encode what the product
* does; policy belongs in the platform, which ships by HTTP push in seconds
* while every line in this file costs a rebuild and a redeploy of every
* installation. Before adding anything, read ./README.md — in particular the
* "fix where the code is" trap, which is how this rule usually gets broken.
*
* The host owns everything that must survive a platform swap: the resource
* registry (live processes + their output buffers), the operation queue the
* HTTP layer gates on, and the committed on-disk state.
*
* Freeze semantics: requests arriving during a swap are ENQUEUED, not
* rejected — the HTTP layer awaits waitIdle() and proceeds once the swap
* completes, so a client never observes the stop-the-world window (it only
* sees latency). Concurrent upgrade requests serialize on the same queue.
*
* ONE atomic push, THREE independent artifacts: platform code, the CLI's
* command implementations, and the web dist are three separately compiled
* single-file products — there is no physical bundle that carries more than
* one of them — but they always move together in ONE request to POST
* /api/hmr/upgrade and land as ONE atomic version. The pushed `platform` and
* `cli` fields are each inline ESM source (delivered as bytes over HTTP —
* never a server-side path a remote client could not produce): `platform`
* must export `hotPlatform` (this runtime's business unit, imported and
* booted here); `cli` is never imported or executed by this process at all —
* it is only content-addressed into the store, for packages/cli's own thin
* loader to dynamically import later (see hmr/manifest.ts's
* resolveCliBundlePath). The web dist rides along in the same request as a
* { relPath: base64 } manifest. There is no way to update platform, cli, or
* web independently: doUpgradeAll only commits after the platform boots
* successfully AND the web manifest validates — a failure in either leaves
* the previous committed version untouched.
*
* Web dist pushes are held in memory (a relPath → bytes map), not written to
* disk file-by-file: a dist can be hundreds of small files, and syncing each
* one separately serializes on the destination filesystem's per-file
* overhead (observed as low as ~100KB/s writing 300 small files to a
* Windows/Defender-scanned disk). The pushed bytes are persisted as ONE
* artifact instead (see persistVersion).
*
* Persistence: artifacts are content-addressed under hmr/store/ (platform and
* cli each get their own subtree, since they are no longer the same file) and
* the whole version (platform + cli + web pointers) is promoted by ONE atomic
* rename of hmr/harness.json — committed only AFTER the live in-memory boot
* AND web install both succeeded, so a restart can never resume a version
* that failed to take effect. A restart resumes harness.json as one unit (see
* restore()): any failure — of any one of the three pieces, including a `cli`
* pointer whose file is missing — warns and falls back to the packaged
* default entirely (never a platform/web mismatch, never a brick). The store
* keeps at most STORE_KEEP versions (current + one rollback) per artifact.
*
* CODE persists; STATE does not. Only platform code, the CLI bundle, and the web
* dist are written to disk — never the parked context document a swap produces.
* A restart therefore always resumes the pushed CODE, but boots it against a
* FRESH initial context (bundle.context — see PlatformBundle below), not
* whatever state was live when the process last exited. This is deliberate: the
* live resources a parked doc's handles would reference (child processes, etc.)
* die with the process anyway, so a restored doc could only ever resume as
* "handles that fail to reclaim, nodes stuck degraded" — worse than a clean
* boot. State DOES survive a hot swap (park → migrate → boot, entirely
* in-memory, see kernel/upgrade.ts) — it just never survives a process restart.
*
* Reload is strictly request-driven: nothing watches, nothing auto-triggers.
*/
import crypto from "node:crypto";
import fs from "node:fs";
import fsp from "node:fs/promises";
import path from "node:path";
import zlib from "node:zlib";
import { pathToFileURL } from "node:url";
import type { Instance, Json, AnyIface, AnyImpl, Park } from "@adelie/core/kernel";
import { boot, initialDoc, upgrade } from "@adelie/core/kernel";
import { HotResources } from "./resources.js";
import type { Manifest } from "./manifest.js";
import { MATERIALIZED } from "./manifest.js";
/**
* The contract every platform bundle must satisfy — packaged or pushed. `context` is the
* initial context a fresh boot creates the tree with (see initialDoc): the runtime never
* hardcodes a business value of its own, so this is where a business platform's own
* starting state belongs (the server's hmr/platform.ts calls its own `packagedPlatform`).
*/
export interface PlatformBundle {
id: string;
iface: AnyIface;
impl: AnyImpl;
context: Json;
}
/** Optional provenance recorded with a pushed version (never executed here). */
export interface GitSource {
repo: string;
revision: string;
}
/**
* Files a push needs on DISK rather than in memory. The web dist is only ever served as
* bytes, so it stays in RAM; an asset is something the platform hands to the OS — a
* native `.node` whose loader resolves it by path, a helper binary it execs — and a
* bundle cannot carry those: it is imported from the data root, where neither a relative
* `build/Release/pty.node` nor a bare specifier resolves (verified: node-pty's binding
* fails to load from a bundle placed outside the server's own module graph).
*/
export interface UpgradeAssets {
/** relPath → base64 content. */
files: Record<string, string>;
/** relPaths that must land executable (a helper binary the platform spawns). */
exec?: string[];
}
/**
* One atomic push: the platform bundle and the cli bundle (each inline ESM source,
* independent single-file artifacts) plus the web dist as a { relPath: base64 }
* manifest. All three travel in the SAME request — there is no partial-target
* upgrade.
*/
/** One materialized assets directory's own record of what it was built from. */
interface AssetsRecord {
/** relPath → sha256, the same map the pusher named (or the one derived from inline files). */
files: Record<string, string>;
}
const ASSETS_RECORD = ".manifest.json";
export interface UpgradeAllTarget {
platform: string;
cli: string;
web: Record<string, string>;
assets?: UpgradeAssets;
source?: GitSource;
}
export type UpgradeOutcome =
| {
status: "ok";
mode: "silent" | "migrated";
impl: string;
source: GitSource | null;
web: { rev: string };
/**
* Whether persistVersion's disk commit succeeded. The live swap always applies
* on `status: "ok"` regardless (see the module doc: "never brick"); when this is
* false the new version is running but NOT durable — a restart would resume the
* previously committed version instead. Callers (routes.ts, scripts/deploy.mjs)
* surface this so a filesystem failure during persistence is visible rather than
* indistinguishable from a fully-durable push.
*/
persisted: boolean;
}
| { status: "blocked"; dropped: string[]; missing: string[]; invalid: string[] };
/**
* How many versions of each artifact (platform bundle / cli bundle / web dist / assets
* dir, each independently) the store keeps by recency — a rollback copy behind the newest.
* The version harness.json references is kept on top of these, so a store holds three
* entries whenever the committed version is not among the two most recently written.
*/
const STORE_KEEP = 2;
export class HmrHost<Api extends Park = Park> {
readonly resources = new HotResources();
private instance: Instance<Api> | null = null;
/**
* The bundle behind the RUNNING instance, held as the loaded object rather than a
* pointer to re-read: it is what boot-failure recovery re-boots (see recoverPrevious).
* A bundle's `id` cannot stand in for this — the packaged export IS what a push
* delivers (the server's hmr/entry.ts re-exports its packaged platform as `hotPlatform`), so every
* pushed bundle carries the packaged id and comparing ids cannot tell a pushed version
* from the compiled-in default. Nor can the manifest: a push whose disk commit failed
* (`persisted: false`) is running a version harness.json does not name.
*/
private current: PlatformBundle;
/** Current version's materialized native assets dir (see assetsDir()). */
private assets: string | null = null;
private readonly hmrDir: string;
private readonly storeDir: string;
private readonly manifestPath: string;
/**
* Single-flight guard for the first boot (see ensure()): every concurrent caller before the
* first successful boot awaits the SAME promise, so restore() and the packaged-default
* fallback can never run twice and race each other into assigning `this.instance` (a double
* boot, or a restored version clobbered by a concurrently-booted packaged default).
*/
private initPromise: Promise<Instance<Api>> | null = null;
/** The freeze, as a queue: everything the HTTP layer gates on chains here. */
private opQueue: Promise<unknown> = Promise.resolve();
/**
* `packaged` is the bundle compiled INTO the program — the starting state, and the
* fallback a failed restore lands on. Passed in rather than imported: this package is the
* mechanism, and it must not know what a platform is or what this product's default one
* contains. The server hands it its own (hmr/platform.ts's `packagedPlatform`).
*/
constructor(
private readonly root: string,
private readonly packaged: PlatformBundle,
) {
this.current = packaged;
this.hmrDir = path.join(root, "hmr");
this.storeDir = path.join(this.hmrDir, "store");
this.manifestPath = path.join(this.hmrDir, "harness.json");
}
private warn(msg: string): void {
process.stderr.write(`[hmr] ${msg}\n`);
}
/** The running version's id — derived from the running bundle, never tracked alongside it. */
currentImplId(): string {
return this.current.id;
}
/**
* The request gate: resolves once no swap is in flight. Requests arriving
* during an upgrade wait here — they observe latency, never an error.
*/
waitIdle(): Promise<void> {
return this.opQueue.then(
() => undefined,
() => undefined,
);
}
/**
* Lazy first boot. Resumes the last committed version (platform + cli + web)
* from disk when present; otherwise boots the packaged platform v1 with a
* fresh document. Concurrent callers before the first boot completes share one
* in-flight promise (see `initPromise`) rather than each racing their own
* restore()/packaged-boot.
*/
ensure(): Promise<Instance<Api>> {
if (this.instance !== null) return Promise.resolve(this.instance);
this.initPromise ??= this.initialize();
return this.initPromise;
}
/** Runs exactly once per process (guarded by `initPromise` in ensure()). */
private async initialize(): Promise<Instance<Api>> {
try {
await this.restore();
if (this.instance === null) {
const bundle = this.packaged;
this.instance = (await boot(
bundle.impl,
bundle.iface,
initialDoc(bundle.iface, bundle.context),
this.resources,
)) as Instance<Api>;
this.current = bundle;
}
return this.instance;
} catch (err) {
// A failed init must not permanently strand ensure(): clear the guard so the next
// caller gets a fresh attempt instead of the same rejection forever.
this.initPromise = null;
throw err;
}
}
/**
* Resume from harness.json, as ONE unit: the platform bundle, the cli bundle's own
* existence, and the web artifact are all read and validated BEFORE anything is
* committed to `this.instance` / `this.webMem` — so a failure partway through (a
* pruned bundle, a missing web artifact, a missing cli artifact) never leaves
* platform and web resumed from different versions. The cli bundle is never
* imported here (this process never runs it — only packages/cli's own loader
* does); its file just has to exist, so a restore can never leave `cli.bundle`
* pointing at nothing. Any failure is non-fatal: it warns and leaves the runtime
* to boot the packaged default — a bad persisted version must never brick the
* runtime. Only ever called once (from initialize(), itself single-flighted by
* ensure()'s `initPromise`).
*
* Boots the restored bundle against its OWN fresh initial context (bundle.context),
* not a resumed state — state is never written to disk (see the module doc), so a
* restart always resumes the pushed CODE with a clean slate, never last run's doc.
*/
private async restore(): Promise<void> {
// No file is the only silent case: nothing has ever been pushed to this root. A file
// that cannot be read or parsed is a FAULT — this root does hold a committed version
// and it is now unreachable — so it is named rather than served as "nothing pushed",
// which would boot the packaged platform and let the next push overwrite the record
// with no one ever having been told.
let raw: string;
try {
raw = await fsp.readFile(this.manifestPath, "utf8");
} catch (err) {
if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
this.warn(`harness.json could not be read (${errMsg(err)}); booting the packaged platform`);
}
return;
}
let manifest: Manifest;
try {
manifest = JSON.parse(raw) as Manifest;
} catch (err) {
this.warn(
`harness.json is not readable JSON (${errMsg(err)}); booting the packaged platform. ` +
`The committed version stays on disk but cannot be resumed until a push rewrites the record.`,
);
return;
}
if (manifest.platform === undefined && manifest.web === undefined) {
return; // nothing committed yet
}
try {
if (manifest.platform === undefined) {
throw new Error("harness.json has no `platform` entry");
}
if (manifest.cli === undefined) {
throw new Error("harness.json has no `cli` entry");
}
if (manifest.web === undefined) {
throw new Error("harness.json has no `web` entry");
}
const cliPath = path.join(this.hmrDir, manifest.cli.bundle);
if (!fs.existsSync(cliPath)) {
throw new Error(`cli bundle '${manifest.cli.bundle}' does not exist`);
}
// Republished before boot, same ordering as an upgrade: a resumed platform loads its
// native modules out of the assets the version was committed with.
if (manifest.assets !== undefined) {
const assetsDir = path.join(this.hmrDir, manifest.assets.dir);
if (!fs.existsSync(assetsDir)) {
throw new Error(`assets dir '${manifest.assets.dir}' does not exist`);
}
this.publishAssets(assetsDir);
}
const bundle = await this.importBundleFile(path.join(this.hmrDir, manifest.platform.bundle));
const gz = await fsp.readFile(path.join(this.hmrDir, manifest.web.manifest));
const webMem = filesMapFromGzip(gz);
// The same floor a push is held to (doUpgradeAll): a web version without its entry
// page would be committed as "restored" and then answer 404 to every navigation,
// with nothing in the log to say why. Failing here lands on the packaged default
// instead, and names the reason.
if (!webMem.has("index.html")) {
throw new Error(`web dist '${manifest.web.manifest}' has no index.html`);
}
// Everything validated: commit together. boot() runs last so a boot failure
// leaves nothing partially applied either.
const instance = (await boot(
bundle.impl,
bundle.iface,
initialDoc(bundle.iface, bundle.context),
this.resources,
)) as Instance<Api>;
this.instance = instance;
this.current = bundle;
this.webMem = webMem;
} catch (err) {
this.warn(
`persisted version failed to restore (platform+cli+web are committed as one unit); ` +
`using the packaged default: ${errMsg(err)}`,
);
}
}
/** Strictly request-driven; serialized on the op queue (never auto-triggered). */
upgradeAll(
target: UpgradeAllTarget,
admit?: (instance: Instance<Api>) => Promise<string | null> | string | null,
): Promise<UpgradeOutcome> {
const run = this.opQueue.then(() => this.doUpgradeAll(target, admit));
// The queue must survive a failed upgrade: swallow for chaining only.
this.opQueue = run.then(
() => undefined,
() => undefined,
);
return run;
}
private async doUpgradeAll(
target: UpgradeAllTarget,
admit?: (instance: Instance<Api>) => Promise<string | null> | string | null,
): Promise<UpgradeOutcome> {
const current = await this.ensure();
if (typeof target.web["index.html"] !== "string") {
throw new Error("web dist manifest has no index.html");
}
const webMem = new Map<string, Buffer>();
for (const [rel, b64] of Object.entries(target.web)) {
if (!isSafeRelPath(rel)) throw new Error(`unsafe path in web dist manifest: ${rel}`);
webMem.set(rel, Buffer.from(b64, "base64"));
}
const { file: platformPath, sha: platformSha } = await this.storePlatformBundle(
target.platform,
);
const bundle = await this.importBundleFile(platformPath);
const source = target.source ?? null;
// Assets land BEFORE the boot: the platform's create() may load a native module out of
// them. A failed boot puts the pointer back, so the surviving old platform keeps
// claiming the assets it booted with.
const previousAssets = this.assets;
const assetsDir = target.assets ? await this.materializeAssets(target.assets) : null;
if (assetsDir !== null) this.publishAssets(assetsDir);
let result;
try {
result = await upgrade({
current,
impl: bundle.impl,
iface: bundle.iface,
resources: this.resources,
...(admit !== undefined ? { admit: admit as never } : {}),
});
} catch (err) {
this.publishAssets(previousAssets);
throw err;
}
if (result.status === "blocked") {
this.publishAssets(previousAssets);
// Old instance + web untouched; the doc + path lists are the input for the
// upper upgrade-ladder rungs (auto-upgrader / agent / human).
return {
status: "blocked",
dropped: result.dropped,
missing: result.missing,
invalid: result.invalid,
};
}
if (result.status === "failed") {
// The pushed platform's boot threw AFTER the old tree was disposed
// (validate-then-swap disposes first so the new tree can adopt what the old one
// delivered). Without recovery the process is half-dead: `this.instance` still
// answers HTTP out of closures, but its manager is closed and the current-App
// pointer is released — until a restart. So re-boot the PREVIOUS platform from the
// parked document the kernel handed back. The park-by-inventory contract is what
// makes this an ordinary load: the old App's dispose suspended and detached
// everything, so the recovered App adopts the delivered resources and restarts the
// suspended machinery like any successor. The pusher still gets an error — the
// push DID fail — but the machine it failed on keeps working.
this.publishAssets(previousAssets);
await this.recoverPrevious(result.doc);
throw new Error(`the pushed platform failed to boot: ${errMsg(result.error)}`);
}
// Boot succeeded: commit web to memory too, then persist platform + cli + web
// as one atomic version — never a platform that's newer (or older) than the
// web or cli it's paired with. `result.doc` (the swap's parked+migrated state)
// is never written to disk — see the module doc: code persists, state does not.
this.instance = result.instance as Instance<Api>;
this.current = bundle;
this.webMem = webMem;
const digest = filesDigest(target.web);
const gz = zlib.gzipSync(Buffer.from(JSON.stringify({ files: target.web })));
// The parts by the names a pusher gives them (HMR_BLOBS_PATH), whether or not this push
// named them: what the sweep keeps alive so an unchanged part is never uploaded twice.
const blobs = [
sha256(Buffer.from(target.platform, "utf8")),
sha256(Buffer.from(target.cli, "utf8")),
...[...webMem.values()].map(sha256),
];
const persisted = await this.persistVersion(
platformSha,
target.cli,
gz,
digest.slice(0, 16),
assetsDir,
source,
blobs,
);
return {
status: "ok",
mode: result.mode,
impl: bundle.id,
source,
web: { rev: digest.slice(0, 12) },
persisted,
};
}
/**
* Boot-failure recovery: re-boot the version that was running before the failed
* upgrade, from its own parked document. The bundle is `this.current` — the object
* that was running, still loaded — rather than anything re-read from disk: the
* manifest names the last DURABLE version, which is a different version whenever the
* running one could not be persisted, and a bundle's `id` cannot distinguish pushed
* from packaged at all (see the field's doc). Best-effort by design: a double fault
* only warns and leaves the disposed instance in place, because /api/hmr is
* runtime-owned and therefore still reachable for a follow-up push, and a process
* restart restores the committed version regardless.
*/
private async recoverPrevious(doc: Json): Promise<void> {
try {
const bundle = this.current;
this.instance = (await boot(bundle.impl, bundle.iface, doc, this.resources)) as Instance<Api>;
// `current` unchanged: the previous version is the running version again.
} catch (err) {
this.warn(
`boot-failure recovery failed too — the process serves a half-stopped App until ` +
`a successful push or a restart: ${errMsg(err)}`,
);
}
}
/** Layer (a) core: import one JS file, cache-busted so re-imports load fresh code. */
private async importBundleFile(file: string): Promise<PlatformBundle> {
const resolved = path.resolve(file);
if (!fs.existsSync(resolved)) throw new Error(`bundle file '${file}' does not exist`);
const url = `${pathToFileURL(resolved).href}?v=${Date.now()}`;
const mod = (await import(url)) as { hotPlatform?: PlatformBundle };
if (mod.hotPlatform === undefined) {
throw new Error(`${file} does not export 'hotPlatform'`);
}
if (mod.hotPlatform.context === undefined) {
// Every bundle carries its own initial context now (see PlatformBundle above):
// state is never persisted, so a boot with no context to fall back on would have
// nothing to boot a fresh tree with — reject it the same way a missing
// `hotPlatform` export is rejected, rather than booting with `undefined`.
throw new Error(`${file}'s 'hotPlatform' has no 'context'`);
}
return mod.hotPlatform;
}
/**
* Materializes the inline platform bundle (the single-file ESM sent in the
* request body) to disk and returns its path and sha — how a push reaches a
* runtime over HTTP alone: the bytes travel in the request, the server writes
* them, then loads them via importBundleFile. Only the platform artifact needs
* this treatment: it is the only one of the three this process ever imports
* and boots. The cli artifact is never imported here (see persistVersion) — it
* goes straight from request body to content-addressed store.
*
* It lands at its FINAL content-addressed path, before the boot that decides
* whether it will be committed, so the bytes exist on disk exactly once: an
* upload area separate from the store would hold a second copy of every bundle
* ever pushed, under no sweep (pruneStore only knows the store). The cost is
* that a push which fails to boot leaves an unreferenced file behind; the next
* successful push's prune collects it, and the committed version is force-kept
* regardless of how many failures sit between them.
*
* The write goes through writeStoreFile, which matters more here than for the other
* two artifacts: this path is taken before the commit, so an idempotent re-push aims
* at the exact file the current manifest points at — and the running version boots
* from it.
*/
private async storePlatformBundle(content: string): Promise<{ file: string; sha: string }> {
const sha = sha1(content).slice(0, 16);
const dir = path.join(this.storeDir, "platform");
await fsp.mkdir(dir, { recursive: true });
const file = path.join(dir, `${sha}.mjs`);
await writeStoreFile(file, Buffer.from(content, "utf8"));
return { file, sha };
}
// -- Web platform (the frontend package's built dist) ---------------------
/**
* The pushed web dist held in memory (relPath → bytes) — always installed
* together with a platform + cli version (see doUpgradeAll); never written or
* read file-by-file (see persistVersion). Null before anything has ever been
* pushed or restored.
*/
private webMem: Map<string, Buffer> | null = null;
/** Static hosting's resolution: an in-memory pushed/restored dist, or null (the caller falls back to the configured webDist). */
resolveWebSource(): { kind: "mem"; files: Map<string, Buffer> } | null {
return this.webMem !== null ? { kind: "mem", files: this.webMem } : null;
}
// -- Persistence ----------------------------------------------------------
/**
* Unpacks a push's assets under store/assets/<sha>/, content-addressed like every other
* artifact so an unchanged set reuses its directory. Modes are restored from the push's
* `exec` list: an asset arrives as base64 with no mode of its own, and a helper binary
* without its exec bit is exactly the failure this whole path exists to avoid.
*
* An identical set is NOT re-written. That is a correctness requirement, not a saving:
* these assets are native modules, and on Windows the copies from the last push are
* mapped into this very process — reopening one for writing fails with EBUSY and takes
* the whole upgrade down with it. The directory name already proves the content
* matches; MATERIALIZED, written last, is what proves the directory is complete, so a
* push interrupted halfway is repaired rather than trusted.
*/
private async materializeAssets(assets: UpgradeAssets): Promise<string> {
// Every file goes through the blob store first, so a set is one map of relPath → sha256
// whether its content arrived in this push or was already here.
const files: Record<string, string> = {};
for (const [rel, b64] of Object.entries(assets.files)) {
if (!isSafeRelPath(rel)) throw new Error(`unsafe path in assets manifest: ${rel}`);
files[rel] = await this.storeBlob(Buffer.from(b64, "base64"));
}
const sha = recordDigest(files).slice(0, 16);
const dir = path.join(this.storeDir, "assets", sha);
const marker = path.join(dir, MATERIALIZED);
if (fs.existsSync(marker)) return dir;
const exec = new Set(assets.exec ?? []);
for (const [rel, blob] of Object.entries(files)) {
const file = path.join(dir, rel);
await fsp.mkdir(path.dirname(file), { recursive: true });
// Repairing an incomplete directory: whatever already matches is left alone. The
// content comes from the blob store, never from the payload — after the first push
// of a given file the payload does not carry it any more.
const content = await fsp.readFile(this.blobPath(blob));
if (!(await sameFileContent(file, content))) await fsp.writeFile(file, content);
// Explicit chmod: writeFile's mode is masked by umask, and ignored outright when
// the file already exists (a reused, content-addressed directory).
await fsp.chmod(file, exec.has(rel) ? 0o755 : 0o644);
}
const record: AssetsRecord = { files };
await fsp.writeFile(path.join(dir, ASSETS_RECORD), JSON.stringify(record));
await fsp.writeFile(marker, sha);
return dir;
}
/**
* `store/blobs/<first two hex digits>/<sha256>`: one file per distinct content, shared by
* every assets set, spread over 256 subdirectories — a store holds thousands of blobs, and
* one directory that size is slow to create in, list and sweep.
*/
private blobPath(sha: string): string {
return path.join(this.storeDir, "blobs", sha.slice(0, 2), sha);
}
/** Writes a blob under its hash (a no-op when it is already there) and returns the hash. */
async storeBlob(content: Buffer): Promise<string> {
const sha = sha256(content);
const file = this.blobPath(sha);
if (!fs.existsSync(file)) {
await fsp.mkdir(path.dirname(file), { recursive: true });
// Written beside and renamed in: a crash mid-write must not leave a blob whose name
// promises content it does not have.
const tmp = `${file}.${process.pid}.tmp`;
await fsp.writeFile(tmp, content);
await fsp.rename(tmp, file);
}
return sha;
}
/** The bytes of a blob the store holds, or null: what a push that names its parts by hash is resolved from. */
readBlob(sha: string): Buffer | null {
if (!isBlobName(sha)) return null;
try {
return fs.readFileSync(this.blobPath(sha));
} catch {
return null;
}
}
/**
* Which of these blobs the store does NOT hold — what a pusher asks before it sends, so
* an unchanged native module or plugin never crosses the wire twice.
*/
missingBlobs(hashes: readonly string[]): string[] {
return hashes.filter((sha) => !isBlobName(sha) || !fs.existsSync(this.blobPath(sha)));
}
/** Points the registry at this version's assets (or clears it when a push has none). */
private publishAssets(dir: string | null): void {
this.assets = dir;
}
/**
* Where the current version's native-module assets live, or null when none were
* pushed. A declared member of the hmr capability (see HMR_INTERFACES), read by
* the bundle's pty loader — not a registry key: the host is already the claimed
* object, so its per-push state belongs on it.
*/
assetsDir(): string | null {
return this.assets;
}
/**
* Content-addresses the cli bundle (never imported — just stored, for packages/cli's
* own loader to pick up) and the web gzip artifact next to the platform bundle the
* boot already stored (its CODE only — see the module doc: state is never
* persisted), then flips harness.json ONCE — `platform`, `cli`, and `web` all land in the SAME atomic
* rename, never three separate commits that could leave one pointer ahead of the
* others. `platform.bundle` and `cli.bundle` are genuinely independent files
* (distinct content, distinct sha) rather than the same physical bundle under two
* manifest keys.
*
* Returns whether the commit succeeded — the caller surfaces this to clients
* (`persisted` in UpgradeOutcome) so a live swap that could not be written to disk
* is visibly at risk of reverting on the next restart, rather than silently ok.
*/
private async persistVersion(
platformSha: string,
cliContent: string,
webGz: Buffer,
webSha: string,
assetsDir: string | null,
source: GitSource | null,
blobs: string[],
): Promise<boolean> {
try {
// The platform bundle is already in the store: storePlatformBundle put it at its
// content-addressed path so the boot could import it from there, through the same
// writeStoreFile the other two artifacts go through.
const cliSha = sha1(cliContent).slice(0, 16);
const cliDir = path.join(this.storeDir, "cli");
await fsp.mkdir(cliDir, { recursive: true });
await writeStoreFile(path.join(cliDir, `${cliSha}.mjs`), Buffer.from(cliContent, "utf8"));
const webDir = path.join(this.storeDir, "web");
await fsp.mkdir(webDir, { recursive: true });
await writeStoreFile(path.join(webDir, `${webSha}.webz`), webGz);
await this.commitManifest(() => ({
platform: { bundle: `store/platform/${platformSha}.mjs` },
cli: { bundle: `store/cli/${cliSha}.mjs` },
web: { manifest: `store/web/${webSha}.webz` },
...(assetsDir === null
? {}
: { assets: { dir: path.relative(this.hmrDir, assetsDir).split(path.sep).join("/") } }),
// Provenance travels with the version it describes, so `penguin version` on this
// machine can name the revision a pushed harness came from — the bundles are
// content-addressed and say nothing about their origin on their own.
...(source === null ? {} : { source }),
pushedAt: new Date().toISOString(),
blobs,
}));
return true;
} catch (err) {
this.warn(`update not persisted (filesystem unavailable?): ${errMsg(err)}`);
return false;
}
}
/** Writes and atomically replaces harness.json (the single commit point for a whole version). */
private async commitManifest(next: () => Manifest): Promise<void> {
await fsp.mkdir(this.hmrDir, { recursive: true });
const manifest = next();
const tmp = `${this.manifestPath}.${process.pid}.tmp`;
await fsp.writeFile(tmp, JSON.stringify(manifest, null, 2));
// Atomic within the same directory/filesystem (libuv maps this to
// MoveFileEx REPLACE_EXISTING on Windows, rename(2) on POSIX).
await fsp.rename(tmp, this.manifestPath);
await this.pruneStore(manifest);
}
/**
* Store GC: keep the STORE_KEEP most recently written of each artifact, plus the one
* harness.json references, which is kept whether or not recency would have. Best-effort;
* ordered after the manifest flip so nothing referenced can be pruned. `platform`, `cli`,
* `web` and the assets directories are independent subtrees (no shared file to piggyback a
* sweep on), so each gets its own pass.
*/
private async pruneStore(manifest: Manifest): Promise<void> {
const keepNewest = async (
dir: string,
keys: (name: string) => string | null,
referenced: string | null,
remove: (key: string) => Promise<void>,
): Promise<void> => {
let names: string[];
try {
names = await fsp.readdir(dir);
} catch {
return;
}
const byKey = new Map<string, number>();
for (const name of names) {
const key = keys(name);
if (key === null) continue;
try {
const mtime = (await fsp.stat(path.join(dir, name))).mtimeMs;
byKey.set(key, Math.max(byKey.get(key) ?? 0, mtime));
} catch {
// raced with a concurrent prune
}
}
const ranked = [...byKey.entries()].sort((a, b) => b[1] - a[1]).map(([k]) => k);
const kept = new Set(ranked.slice(0, STORE_KEEP));
if (referenced !== null) kept.add(referenced);
for (const key of ranked) {
if (!kept.has(key)) await remove(key).catch(() => undefined);
}
};
const platformDir = path.join(this.storeDir, "platform");
const platformBundleRef = manifest.platform?.bundle.match(/([0-9a-f]+)\.mjs$/)?.[1] ?? null;
await keepNewest(
platformDir,
(name) => /^([0-9a-f]+)\.mjs$/.exec(name)?.[1] ?? null,
platformBundleRef,
(sha) => fsp.rm(path.join(platformDir, `${sha}.mjs`), { force: true }),
);
const cliDir = path.join(this.storeDir, "cli");
const cliRef = manifest.cli?.bundle.match(/([0-9a-f]+)\.mjs$/)?.[1] ?? null;
await keepNewest(
cliDir,
(name) => /^([0-9a-f]+)\.mjs$/.exec(name)?.[1] ?? null,
cliRef,
(sha) => fsp.rm(path.join(cliDir, `${sha}.mjs`), { force: true }),
);
const webDir = path.join(this.storeDir, "web");
const webRef = manifest.web?.manifest.match(/([0-9a-f]+)\.webz$/)?.[1] ?? null;
await keepNewest(
webDir,
(name) => /^([0-9a-f]+)\.webz$/.exec(name)?.[1] ?? null,
webRef,
(sha) => fsp.rm(path.join(webDir, `${sha}.webz`), { force: true }),
);
// Pre-store pushes staged the platform bundle in `hmr/uploads/` before importing it
// and then wrote a second copy into the store; nothing ever read the staged copy
// again and no sweep covered that directory. Removing it here is one line in the
// sweep that already owns disk reclamation, rather than a migration step.
await fsp
.rm(path.join(this.hmrDir, "uploads"), { recursive: true, force: true })
.catch(() => undefined);
// Assets are directories, not single files; otherwise the same keep-newest rule.
const assetsRoot = path.join(this.storeDir, "assets");
const assetsRef = manifest.assets?.dir.match(/([0-9a-f]+)$/)?.[1] ?? null;
await keepNewest(
assetsRoot,
(name) => (/^[0-9a-f]+$/.test(name) ? name : null),
assetsRef,
(sha) => fsp.rm(path.join(assetsRoot, sha), { recursive: true, force: true }),
);
// Blobs are shared by every assets set and by the committed version's own parts, so
// they are collected LAST, against what the sweep above left: a blob neither a
// remaining set nor the version records is unreachable. A set without a record
// (materialized before records existed) keeps nothing alive through the blob store
// because it never read from it, and its own files stay untouched.
const live = new Set<string>(manifest.blobs ?? []);
for (const name of await fsp.readdir(assetsRoot).catch(() => [] as string[])) {
try {
const record = JSON.parse(
await fsp.readFile(path.join(assetsRoot, name, ASSETS_RECORD), "utf8"),
) as AssetsRecord;
for (const sha of Object.values(record.files)) live.add(sha);
} catch {
// No record, or an unreadable one: nothing to keep alive from here.
}
}
const blobsDir = path.join(this.storeDir, "blobs");
for (const shard of await fsp.readdir(blobsDir).catch(() => [] as string[])) {
const shardDir = path.join(blobsDir, shard);
if (!/^[0-9a-f]{2}$/.test(shard)) {
// A blob from the flat layout this store used before (or a stray temp file): nothing
// resolves it any more, so it goes; a push that needs its content uploads it again.
if (isBlobName(shard) || shard.endsWith(".tmp")) {
await fsp.rm(shardDir, { force: true }).catch(() => undefined);
}
continue;
}
const names = await fsp.readdir(shardDir).catch(() => [] as string[]);
let left = names.length;
for (const name of names) {
if ((isBlobName(name) && !live.has(name)) || name.endsWith(".tmp")) {
await fsp.rm(path.join(shardDir, name), { force: true }).catch(() => undefined);
left -= 1;
}
}
if (left === 0) await fsp.rmdir(shardDir).catch(() => undefined);
}
}
/** Process-exit sweep only; never part of an upgrade. */
dispose(): void {
this.instance?.dispose();
this.instance = null;
this.resources.disposeAll();
}
}
function errMsg(err: unknown): string {
return err instanceof Error ? err.message : String(err);
}
/** A blob is named by the lowercase hex sha256 of its content, nothing else. */
export function isBlobName(name: string): boolean {
return /^[0-9a-f]{64}$/.test(name);
}
function sha256(content: Buffer): string {
return crypto.createHash("sha256").update(content).digest("hex");
}
function sha1(content: string): string {
return crypto.createHash("sha1").update(content).digest("hex");
}
/**
* The name of a materialized assets directory: a hash over relPath → sha256, so the same
* files name the same directory whether they arrived inline or by manifest, and a set is
* never written twice.
*/
function recordDigest(files: Record<string, string>): string {
const hash = crypto.createHash("sha1");
for (const rel of Object.keys(files).sort()) hash.update(rel).update("\0").update(files[rel]!);
return hash.digest("hex");
}
/** Content hash over a web dist manifest: stable across re-pushes of identical content. */
function filesDigest(files: Record<string, string>): string {
const hash = crypto.createHash("sha1");
for (const rel of Object.keys(files).sort()) hash.update(rel).update("\0").update(files[rel]!);
return hash.digest("hex");
}
/**
* Writes one content-addressed artifact. A file that already holds these exact bytes is left
* untouched — the store is keyed by content hash, so re-pushing a version (an idempotent push, a
* rollback to something installed before) otherwise rewrites the very file the committed manifest
* points at, and a crash during that rewrite truncates a bundle the runtime still needs to boot.
* Anything else is written to a temp file and renamed into place, so an interrupted write leaves
* the store's previous state rather than a half-written artifact under a hash that promises
* complete content.
*/
async function writeStoreFile(file: string, content: Buffer): Promise<void> {
if (await sameFileContent(file, content)) return;
const tmp = `${file}.${process.pid}.tmp`;
try {
await fsp.writeFile(tmp, content, { flush: true });
await fsp.rename(tmp, file);
} catch (err) {
await fsp.rm(tmp, { force: true }).catch(() => undefined);
throw err;
}
}
/** Whether `file` already holds exactly these bytes (missing/unreadable counts as no). */
async function sameFileContent(file: string, content: Buffer): Promise<boolean> {
try {
const stat = await fsp.stat(file);
if (!stat.isFile() || stat.size !== content.length) return false;
return (await fsp.readFile(file)).equals(content);
} catch {
return false;
}
}
/** No absolute paths, no `..` segments — the map is looked up by exact key, but a malformed key must never be stored. */
function isSafeRelPath(rel: string): boolean {
if (rel === "" || rel.startsWith("/") || rel.includes("\\")) return false;
return rel.split("/").every((seg) => seg !== "" && seg !== "." && seg !== "..");
}
/** Decodes a gzip(JSON.stringify({ files })) artifact into its manifest. */
function filesFromGzip(gz: Buffer): Record<string, string> {
let parsed: { files?: unknown };
try {
parsed = JSON.parse(zlib.gunzipSync(gz).toString("utf8")) as { files?: unknown };
} catch (err) {
throw new Error(`invalid gzip web dist artifact: ${errMsg(err)}`);
}
if (typeof parsed.files !== "object" || parsed.files === null) {
throw new Error("gzip web dist artifact has no `files`");
}
return parsed.files as Record<string, string>;
}
/** Decodes a gzip artifact straight into the in-memory relPath → bytes map (the restore path). */
function filesMapFromGzip(gz: Buffer): Map<string, Buffer> {
const files = filesFromGzip(gz);
const mem = new Map<string, Buffer>();
for (const [rel, b64] of Object.entries(files)) {
if (!isSafeRelPath(rel)) throw new Error(`unsafe path in web dist manifest: ${rel}`);
mem.set(rel, Buffer.from(b64, "base64"));
}
return mem;
}