A local-first multi-agent app development platform, built on PenguinHarness
/**
* Installing THIS server's build onto another machine, from inside the server process —
* platform code, so the whole capability travels by hot push (see packages/hmr/README.md).
*
* Nothing installable is produced here. The far side runs the ordinary release installer
* ONLINE, pinned to this server's own base release, so the program tree — launchers, libs,
* web assets, bundled runtime — is the standard artifact downloaded from the release
* sources, exactly as a person installing by hand would get it. What this code adds is only
* what makes the remote THIS machine's peer rather than a stock install: the hmr state
* (harness.json + store/) is streamed across afterwards, so the remote's next boot runs the
* same pushed platform, web and CLI this server runs (hmr/host.ts's restore).
*
* The remote therefore needs its own route to the release sources (GitHub or the OSS
* mirror); the ssh channel carries only the installer script and the store.
*
* The remote is left with exactly what a local install plus a push leaves: the program
* directory, the `~/.local/bin/penguin` symlink on POSIX, and the data root's hmr/ state.
* No sudo, no service units, no profile edits, and the rest of the data root is untouched.
*/
import { randomBytes } from "node:crypto";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { runInstallScriptCommand, unpackStoreCommand } from "./commands.js";
import type { RemoteTarget } from "./commands.js";
import { parseProbeOutput, posixProbe, windowsProbe } from "./detect.js";
import type { RemoteIdentity, RemotePlatform } from "./detect.js";
import type { RemoteLayout } from "./layout.js";
import { connectionTo, looksLikeAuthFailure, runBytes } from "./transport/index.js";
import type { MachineChannel } from "./transport/index.js";
/** Which installer runs the far side; also the asset keys deploy.mjs pushes. */
const installerFileFor = (platform: RemotePlatform): string =>
platform === "win32" ? "install.ps1" : "install.sh";
/** The version out of a lib/package.json path, or null when it is not a manifest. */
function versionOfManifest(manifestPath: string): string | null {
try {
const parsed: unknown = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
if (typeof parsed === "object" && parsed !== null) {
const version = (parsed as { version?: unknown }).version;
if (typeof version === "string" && version !== "") return version;
}
} catch {
/* absent or damaged: not an install */
}
return null;
}
/**
* What an install would put on the remote: the release this server stands on, plus the hmr
* state to replicate over it (null when nothing was ever pushed here). `version` is the
* display form the page and the install records use.
*/
export interface PushPlan {
/** The base release's version as lib/package.json spells it (no `v`). */
baseVersion: string;
/** Raw harness.json text of this server's own hmr state, or null when none exists. */
harness: string | null;
/** The hmr directory the store is streamed from; null exactly when `harness` is. */
hmrDir: string | null;
version: string;
}
/**
* The pushed-state suffix for a display version: the platform bundle's content sha out of a
* harness.json (`store/platform/<sha>.mjs`), shortened. Falls back to a bare marker for a
* manifest whose shape this build does not recognize — the suffix is display, not identity;
* equality checks compare the harness text itself.
*/
function harnessSuffix(harnessText: string): string {
try {
const parsed = JSON.parse(harnessText) as { platform?: { bundle?: string } };
const sha = /([0-9a-f]{8,})\.mjs$/.exec(parsed.platform?.bundle ?? "")?.[1];
if (sha !== undefined) return `+hmr.${sha.slice(0, 12)}`;
} catch {
/* fall through */
}
return "+hmr";
}
/**
* Resolves what this server would install elsewhere.
*
* The base release is read from the running install's own tree — the tarball layout
* (`<root>/lib/dist/penguin.js` under a `lib/`) or the desktop app's staged payload — the
* same way `penguin --version` would answer. A development checkout has neither, and
* answers null: it stands on no release the remote could download.
*/
export function resolvePushPlan(
dataRoot: string | null,
argv1: string | undefined = process.argv[1],
): PushPlan | null {
const baseVersion = baseReleaseVersion(argv1);
if (baseVersion === null) return null;
let harness: string | null = null;
let hmrDir: string | null = null;
if (dataRoot !== null) {
const dir = path.join(dataRoot, "hmr");
try {
const text = fs.readFileSync(path.join(dir, "harness.json"), "utf8").trim();
if (text !== "") {
harness = text;
hmrDir = dir;
}
} catch {
/* never pushed to: the plan is the bare release */
}
}
return {
baseVersion,
harness,
hmrDir,
version: harness === null ? baseVersion : baseVersion + harnessSuffix(harness),
};
}
/**
* The base release version around the process entry, or null when this process stands on no
* published release. Read from disk rather than from the running artifact's own build info,
* because a hot-pushed server must still report the BASE it was installed from.
*/
function baseReleaseVersion(argv1: string | undefined): string | null {
if (!argv1 || path.basename(path.dirname(argv1)) !== "dist") return null;
const parent = path.dirname(path.dirname(argv1));
// Tarball install: <root>/lib/dist/<entry>.js. lib/package.json is the CLI package's own
// manifest, and keeps naming the base release while this process runs a pushed bundle.
if (path.basename(parent) === "lib") {
return versionOfManifest(path.join(parent, "package.json"));
}
// Packaged desktop app: <resources>/app/dist/server.js — the app forks the server as one
// bundled file (packages/desktop/tsup.config.ts), and asar is off, so these are real files.
// The app's own manifest names the release it was published under, app and tarballs
// shipping from one tag. A source run sits at packages/desktop rather than under
// resources/, and is deliberately unmatched: it stands on no release a remote could fetch.
if (path.basename(parent) === "app" && path.basename(path.dirname(parent)) === "resources") {
return versionOfManifest(path.join(parent, "package.json"));
}
return null;
}
/**
* Asks the machine what it is. POSIX first, over the session — the only round trip a POSIX
* host ever costs. A cmd.exe host has no `sh` to hold a session on, so the session dies
* unopened and the Windows form is asked on a connection of its own. Two round trips at
* worst, once per connect.
*/
export async function detectRemote(
target: RemoteTarget,
layout: RemoteLayout,
channel?: MachineChannel,
): Promise<{ identity: RemoteIdentity } | { error: string }> {
const conn = channel ?? connectionTo(target);
const posix = await conn.exec(posixProbe(layout));
const identity = parseProbeOutput(posix.stdout);
if (identity) return { identity };
// The session's output is merged, so ssh's own words arrive as stdout.
const said = posix.stdout.trim();
if (posix.code !== 0 && looksLikeAuthFailure({ ...posix, stderr: said })) {
return {
error: `${said}\n\nConnections use BatchMode: set up key or agent authentication for that host first.`,
};
}
const windows = await conn.oneShot(windowsProbe(layout), { timeoutMs: 30_000 });
const identityWin = parseProbeOutput(windows.stdout);
if (identityWin) return { identity: identityWin };
if (windows.code !== 0 && looksLikeAuthFailure(windows)) {
return {
error: `${windows.stderr.trim()}\n\nConnections use BatchMode: set up key or agent authentication for that host first.`,
};
}
const words = said || windows.stderr.trim();
return {
error:
"Could not tell what that machine is: neither the POSIX nor the Windows probe answered." +
(words === "" ? "" : ` It said: ${words}`),
};
}
export type RemoteInstallOutcome =
| { kind: "already-installed"; version: string; identity: RemoteIdentity }
/**
* The base release over there already matches; only the pushed state differs. Nothing to
* INSTALL — this is a hot update, and the machine has a channel for it that answers
* (machines/upgrade.ts). The caller routes it there rather than this path copying the
* store over and restarting the process, which replaces a running server without ever
* asking whether it can run what it was handed.
*/
| { kind: "state-only"; identity: RemoteIdentity }
| { kind: "installed"; output: string; identity: RemoteIdentity }
| { kind: "failed"; step: string; detail: string };
/**
* Installs the plan. Steps are sequential and each failure stops the run with the far side's
* own message; the scratch directory is removed on the way out either way, since a leftover
* script in someone's temp directory is litter we created.
*/
export async function installOnRemote(opts: {
target: RemoteTarget;
plan: PushPlan;
onProgress?: (line: string) => void;
/** Identity from an earlier probe in the same flow, to save the round trips. */
identity?: RemoteIdentity;
/** The hmr capability's assetsDir accessor: where a pushed bundle's assets were unpacked. */
assets?: () => string | null;
/** The channel to the machine; a test hands in a scripted one. */
channel?: MachineChannel;
/**
* Run the installer even when the release over there already matches.
*
* For the case the short-circuit below gets wrong: a machine whose PROGRAM is the right
* version and whose hmr store this server cannot reach — an empty store, or one holding a
* build that cannot receive an update. Nothing about the versions says so, so this is asked
* for and never inferred. The installer replicates the store on its way through, which is
* what puts the machine back within reach.
*/
forceInstaller?: boolean;
/** Which installation on that machine this is: the profile's program directory and root. */
layout: RemoteLayout;
}): Promise<RemoteInstallOutcome> {
const { target, plan, layout } = opts;
const conn = opts.channel ?? connectionTo(target);
const say = opts.onProgress ?? (() => {});
let identity = opts.identity;
if (identity === undefined) {
say("Asking what that machine is…");
const detected = await detectRemote(target, layout, conn);
if ("error" in detected) return { kind: "failed", step: "connect", detail: detected.error };
identity = detected.identity;
say(`${identity.platform}-${identity.arch}.`);
}
const baseCurrent =
identity.installedVersion === plan.baseVersion && opts.forceInstaller !== true;
if (baseCurrent && identity.harness === plan.harness) {
return { kind: "already-installed", version: plan.version, identity };
}
if (baseCurrent) {
// Same release, different pushed state: a hot update, not an install. Handing it over
// as files and restarting the process would swap the code under a server that may not
// be able to claim it — a runtime older than the pushed platform warns, falls back to
// its packaged default, and carries on serving, so the restart looks like a success
// from here and the machine is recorded at a version it is not running. The update
// channel asks the machine itself and comes back with its answer, refusals included.
return { kind: "state-only", identity };
}
// Release tags are v-prefixed semver; a base that does not spell one cannot be pinned —
// notably 0.0.0-hmr.* trees, which stand on no published release.
if (
!/^\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?$/.test(plan.baseVersion) ||
plan.baseVersion.startsWith("0.0.0")
) {
return {
kind: "failed",
step: "resolve the release",
detail: `this install's own version (${plan.baseVersion}) does not name a published release.`,
};
}
let windowsTmp: { local: string; remote: string } | null = null;
try {
const output: string[] = [];
if (!baseCurrent) {
// The ordinary installer. Where it sits follows from what this server is: a hot-pushed
// bundle has it among the assets published with that same version, anything else is a
// packaged install and it is beside this module (dist/ after a build; this package's
// tsup.config.ts copies it there).
const installerFile = installerFileFor(identity.platform);
const installerHome = opts.assets?.() ?? path.dirname(fileURLToPath(import.meta.url));
let installer: Buffer;
try {
installer = fs.readFileSync(path.join(installerHome, installerFile));
} catch (err) {
return {
kind: "failed",
step: "prepare the installer",
detail: err instanceof Error ? err.message : String(err),
};
}
say(`Installing release ${plan.baseVersion} (downloaded on the remote)…`);
// Where the script ends up decides how it is invoked, so resolve that first and let the
// command itself say whether stdin has to carry it.
let where: Parameters<typeof runInstallScriptCommand>[1];
if (identity.platform === "win32") {
// A name of our own making: hex only, so it needs no quoting on either side.
const name = `penguin-${randomBytes(6).toString("hex")}.ps1`;
const local = path.join(os.tmpdir(), name);
fs.writeFileSync(local, installer);
windowsTmp = { local, remote: `%USERPROFILE%\\${name}` };
const copy = await conn.copyTo([local], ".");
if (copy.code !== 0) {
return { kind: "failed", step: "copy", detail: copy.stderr.trim() || "scp failed" };
}
where = { platform: "win32", scriptPath: windowsTmp.remote };
} else {
where = { platform: identity.platform };
}
const step = runInstallScriptCommand(`v${plan.baseVersion}`, where, layout);
// The script rides the session's stdin as a heredoc, and the far side's own progress
// is relayed as it arrives rather than after the minutes an install can take. A Windows
// host runs its copied script on a connection of its own (no session to ride).
const install = step.scriptOnStdin
? await conn.stream(step.command, { input: installer, onLine: say })
: await conn.oneShot(step.command);
if (install.code !== 0) {
return {
kind: "failed",
step: "install",
detail: `${install.stdout.trim()}\n${install.stderr.trim()}`.trim(),
};
}
output.push(install.stdout.trim());
}
if (plan.hmrDir !== null && identity.harness !== plan.harness) {
say("Replicating the pushed version…");
// harness.json and store/ only: uploads/ is this machine's scratch, not state. Packed
// here, then handed to the machine's tar on the session's stdin.
const packed = await runBytes("tar", [
"-czf",
"-",
"-C",
plan.hmrDir,
"harness.json",
"store",
]);
if (packed.code !== 0) {
return {
kind: "failed",
step: "replicate the pushed version",
detail: packed.stderr.trim() || `tar exited ${packed.code}`,
};
}
const unpack = unpackStoreCommand(identity.platform, layout);
const sync =
identity.platform === "win32"
? await conn.oneShot(unpack, { input: packed.stdout })
: await conn.stream(unpack, { input: packed.stdout });
if (sync.code !== 0) {
return {
kind: "failed",
step: "replicate the pushed version",
detail:
`${sync.stdout.trim()}\n${sync.stderr.trim()}`.trim() || `tar exited ${sync.code}`,
};
}
output.push(`Pushed version replicated (${plan.version}).`);
}
// ASK THE MACHINE what it now has, rather than reporting what we meant to put there.
// Every step above answers for itself — the installer exited 0, the store unpacked — and
// none of them answers the only question that matters, which is whether the thing on
// disk over there is now this version. An install that ran cleanly and changed nothing
// (wrong home, a package manager that declined, a path the installer did not own) would
// otherwise be recorded as a success at OUR version, and that record is what
// syncOutOfDate filters on: the machine is then excluded from the very sweep that would
// have tried again. A false success here does not just mislead, it seals itself in.
say("Checking what it ended up with…");
const after = await detectRemote(target, layout, conn);
if ("error" in after) {
return {
kind: "failed",
step: "verify the install",
detail: `the install ran, but the machine could not be asked what it now has: ${after.error}`,
};
}
if (after.identity.installedVersion !== plan.baseVersion) {
return {
kind: "failed",
step: "verify the install",
detail:
`the install reported success, but the machine still has ` +
`${after.identity.installedVersion ?? "no install"} where ${plan.baseVersion} was expected.`,
};
}
// The base is only half of what gets recorded. A plan carrying a pushed state is recorded
// at `plan.version`, which is the base plus that state's content sha — so a store whose
// unpack exited 0 without landing (a partial tarball, a data root somewhere else, a
// harness.json the far side could not replace) would seal the machine in at a version it
// is not running. This is the comparison the entry gate above already makes; the machine
// has to still make it true afterwards. Scoped to a plan that HAD a pushed state: a
// base-only install neither carries nor removes one, and must not be failed for a remote
// hmr directory it was never asked to touch.
if (plan.hmrDir !== null && after.identity.harness !== plan.harness) {
return {
kind: "failed",
step: "verify the install",
detail:
"the install reported success, but the pushed version is not what the machine ended " +
`up with: it reports ${after.identity.harness === null ? "no pushed state" : "a different one"}.`,
};
}
return { kind: "installed", output: output.join("\n").trim(), identity: after.identity };
} finally {
// Nothing to clean on a POSIX remote: the installer was never a file there. A Windows
// one deletes its own copy as part of the install command; this is the local original.
if (windowsTmp !== null) fs.rmSync(windowsTmp.local, { force: true });
}
}