A local-first multi-agent app development platform, built on PenguinHarness
/**
* The machines API: who may use it, what the list answers, and the install job's lifecycle
* as the Machines page sees it — start, poll, finish, and every way a start is refused
* before an ssh runs.
*
* The service's three reaching-out effects are faked (see MachinesEffects): the real ones
* read the developer's own ~/.ssh/config and spawn ssh against whatever it names. What the
* push actually does over ssh is covered by machines-push.test.ts, against a fake ssh
* binary and the real installOnRemote.
*
* One App serves the file; each case builds its own service (data root, store, effects),
* which the App's machines node forwards to. The one case about the App's own boot boots
* an App of its own.
*
* Scenarios:
* - Given a member, a stranger or an admin, only the admin reaches the routes and a machine's
* proxied API.
* - Given an ssh config, the list is its aliases with the version this server would push; an
* empty or unreadable config leaves this machine alone, and no image is said as such.
* - Given an install, the job narrates, finishes and is remembered per Project; every failure
* (far-side words, a refused build, a throw, an unwritten version) ends the job and records
* nothing; a machine already on this build is a result, and still gets pushed state.
* - Given installed machines, status is probed only when asked, once per round, and a stopped
* server or an unreachable machine is an answer; a restart follows an install only where a
* server was running.
* - Given a connect, the remembered port is the one used, a dead server is started, a live one
* is left alone, and a held session is re-held at start() and at the App's boot — never
* after a disconnect.
* - Given a model edit during a sync, it is written once afterwards, never dropped.
* - Given a connected machine, its directories are listed through the alias holding it.
* - Given a probe, the machine's own id is learned and kept; this machine's id is minted once.
* - Given a batch to use, machines work side by side and each ends connected; stop using keeps
* the install.
* - Given an ssh-config edit, a new block is appended, this app's block is rewritten in place,
* and a hand-written one is refused.
* - Given a refusal decidable without ssh (self-install, a second install, an unknown host, no
* image), nothing runs.
*/
import fs from "node:fs";
import http from "node:http";
import type { AddressInfo } from "node:net";
import path from "node:path";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
import type { MachinesUseResponse, MachinesResponse } from "../src/api/types.js";
import type { DatabaseSync } from "node:sqlite";
import { openDatabase } from "../src/db/database.js";
import { MachinesRepo } from "../src/db/repos/machines.js";
import { MachinesService } from "../src/machines/service.js";
import type { MachinesEffects } from "../src/machines/service.js";
import { remoteLayoutFor } from "../src/machines/layout.js";
import type { RemoteLayout } from "../src/machines/layout.js";
import type { RemoteInstallOutcome } from "../src/machines/install-server.js";
import type { RemoteIdentity } from "../src/machines/detect.js";
import {
apiClient,
createTestApp,
loginAdmin,
makeTempRoot,
provisionUser,
waitFor,
} from "./helpers.js";
import type { TestApp } from "./helpers.js";
import { forwardingTo } from "./fixtures/forwarding.js";
/** Every route is under a Project; the seeded one every server has. */
const PROJECT = "default_project";
/** This machine's id. Minted from the `machine` table in the real server; fixed here. */
const LOCAL_ID = "TESTlocalID00000";
const IDENTITY: RemoteIdentity = {
platform: "linux",
arch: "x64",
installedVersion: null,
harness: null,
};
/** Addresses the fake transport holds a session to. */
const connected = new Set<string>();
/**
* A plan with pushed state of its own: what a hot-pushed controller carries, and the only
* shape under which an install can come back `state-only` — the release matches, the pushed
* state does not — and the hand-over has something to hand over.
*/
const PUSHED_PLAN = {
baseVersion: "9.9.9",
harness: '{"platform":{"bundle":"store/platform/cafe.mjs"}}',
hmrDir: "/nonexistent",
version: "9.9.9+hmr.cafe",
};
/** An effects set that names two hosts and installs successfully, with the parts a test cares about overridden. */
function effects(over: Partial<MachinesEffects> = {}): Partial<MachinesEffects> {
return {
listAliases: () => ["build-box", "nas"],
resolvePlan: () => ({ baseVersion: "9.9.9", harness: null, hmrDir: null, version: "9.9.9" }),
now: () => new Date("2026-08-24T12:00:00.000Z"),
startServer: async () => ({ ok: true }),
stopServer: async () => ({ ok: true as const }),
upgrade: async () => ({ kind: "upgraded", detail: "", persisted: true }),
runOn: async () => ({
code: 0,
stdout: "---penguin-auth-token---\nremote-token\n",
stderr: "",
timedOut: false,
}),
// The connection: a fake registry this suite marks machines connected in, the way the
// transport's own registry answers `session` for a held ssh session.
hold: async (target) => {
connected.add(`ssh:${target.alias}`);
return { ok: true, session: { pid: process.pid, socksPort: 1 } };
},
session: (address) => (connected.has(address) ? { pid: process.pid, socksPort: 1 } : null),
agent: () => new http.Agent(),
probe: async () => ({ state: { kind: "running", port: 7364, pid: 4242 }, machineId: null }),
install: async (opts): Promise<RemoteInstallOutcome> => {
opts.onProgress?.("Pushing…");
return { kind: "installed", output: "done", identity: IDENTITY };
},
...over,
};
}
describe("machines API", () => {
let t: TestApp;
let admin: ReturnType<typeof apiClient>;
/** The service's own data root — where it writes the install records this suite reads back. */
let machinesRoot: string;
let machinesRepo: MachinesRepo;
let store: DatabaseSync;
/** The case's service: what the App's routes and `t.deps.machines` reach. */
let service: MachinesService;
/** Every service built so far (the App's first included), retired after the case. */
const built: Array<{ service: MachinesService; store: DatabaseSync; root: string }> = [];
/** Builds the case's service over a fresh data root and store, with its effects overridden. */
const boot = async (over: Partial<MachinesEffects> = {}, layout?: RemoteLayout) => {
connected.clear();
machinesRoot = await makeTempRoot();
// The store is the service's own here, not the App's, so it needs the Project row the
// membership table's foreign key points at — a membership follows its Project out.
store = openDatabase(":memory:");
store
.prepare(
"INSERT INTO users (user_id, password_hash, is_admin, created_at) VALUES (?, ?, 1, ?)",
)
.run("admin", "x", "2026-08-24T00:00:00.000Z");
store
.prepare("INSERT INTO projects (project_id, owner_user_id, created_at) VALUES (?, ?, ?)")
.run("default_project", "admin", "2026-08-24T00:00:00.000Z");
machinesRepo = new MachinesRepo(store);
service = new MachinesService(
machinesRoot,
LOCAL_ID,
machinesRepo,
effects(over),
undefined,
layout,
);
built.push({ service, store, root: machinesRoot });
};
beforeAll(async () => {
await boot();
t = await createTestApp({ machines: forwardingTo(() => service) });
admin = apiClient(t.app, (await loginAdmin(t.app)).cookie);
});
afterAll(async () => {
await t.cleanup();
});
afterEach(async () => {
for (const retired of built.splice(0)) {
// Its jobs finish before the next case's service takes over, so nothing a job does
// lands in another case.
await waitFor(() => retired.service.jobs().every((job) => !job.running && !job.queued));
retired.service.stop();
retired.store.close();
fs.rmSync(retired.root, { recursive: true, force: true });
}
});
/** What the store holds for each machine, as the service left it. */
const recordsInStore = (): Record<string, { version: string; at: string }> =>
Object.fromEntries(
machinesRepo
.all()
.filter((row) => row.version !== null)
.map((row) => [row.address, { version: row.version!, at: row.installedAt! }]),
);
describe("permission", () => {
let member: ReturnType<typeof apiClient>;
beforeAll(async () => {
member = apiClient(t.app, (await provisionUser(t.app, "member")).cookie);
});
beforeEach(() => boot());
it("is admin-only: a provisioned user gets 403", async () => {
expect((await member.get("/api/projects/default_project/machines")).status).toBe(403);
expect(
(await member.post("/api/projects/default_project/machines/ssh:nas/install")).status,
).toBe(403);
});
it("needs a session at all", async () => {
expect((await t.app.request("/api/projects/default_project/machines")).status).toBe(401);
});
it("gates a machine's proxied API the same way, under its own prefix", async () => {
// /server/<machineId>/api/… is a protected group outside /api: the gate is what puts
// the user on the context, and an ungated prefix left the handler reading isAdmin off
// nothing — a 500, for the admin who was allowed and the stranger who was not.
expect((await t.app.request("/server/tXIvjrl0pgKa5_dD/api/version")).status).toBe(401);
expect((await member.get("/server/tXIvjrl0pgKa5_dD/api/version")).status).toBe(403);
// An admin reaches the proxy itself, which answers for a machine it does not hold.
const res = await admin.get("/server/tXIvjrl0pgKa5_dD/api/version");
expect(res.status).toBe(503);
expect(((await res.json()) as { error: { code: string } }).error.code).toBe("not_connected");
});
});
describe("the list", () => {
it("is the ssh config's aliases, with the version this server would push", async () => {
await boot();
const body = (await (
await admin.get("/api/projects/default_project/machines")
).json()) as MachinesResponse;
expect(body.machines[0]).toMatchObject({
id: "local",
local: true,
status: { state: "running" },
// This process's own root, resolved — not the layout's template, which names
// where a server would run on some OTHER machine.
root: machinesRoot,
});
expect(body.machines[0]?.installed).not.toBeNull();
expect(body.machines.slice(1)).toEqual([
{
id: "ssh:build-box",
alias: "build-box",
machineId: null,
installed: null,
local: false,
connection: null,
api: null,
// The release profile's root over there, in the shell a machine of unknown
// platform is assumed to speak.
root: "$HOME/.penguin/data",
status: null,
},
{
id: "ssh:nas",
alias: "nas",
machineId: null,
installed: null,
local: false,
connection: null,
api: null,
// The release profile's root over there, in the shell a machine of unknown
// platform is assumed to speak.
root: "$HOME/.penguin/data",
status: null,
},
]);
expect(body.imageVersion).toBe("9.9.9");
expect(body.job).toBeNull();
});
it("an empty or unreadable ssh config leaves this machine alone in the list, not an error", async () => {
await boot({ listAliases: () => [] });
const res = await admin.get("/api/projects/default_project/machines");
expect(res.status).toBe(200);
const machines = ((await res.json()) as MachinesResponse).machines;
expect(machines.map((m) => m.id)).toEqual(["local"]);
});
it("reports no image when this server has none to push", async () => {
await boot({ resolvePlan: () => null });
const body = (await (
await admin.get("/api/projects/default_project/machines")
).json()) as MachinesResponse;
expect(body.imageVersion).toBeNull();
});
});
describe("installing", () => {
it("starts a job, narrates it, and finishes", async () => {
let release = () => {};
const held = new Promise<void>((resolve) => {
release = resolve;
});
await boot({
install: async (opts) => {
opts.onProgress?.("Pushing…");
await held;
return { kind: "installed", output: "done", identity: IDENTITY };
},
});
const started = await admin.post("/api/projects/default_project/machines/ssh:nas/install");
expect(started.status).toBe(202);
expect(((await started.json()) as MachinesResponse).job).toMatchObject({
machineId: "ssh:nas",
alias: "nas",
running: true,
result: null,
});
release();
await waitFor(() => t.deps.machines.job()?.running === false);
const body = (await (
await admin.get("/api/projects/default_project/machines")
).json()) as MachinesResponse;
expect(body.job?.result).toEqual({ ok: true, installed: "installed", version: "9.9.9" });
expect(body.job?.log[0]).toBe("Installing 9.9.9 on nas…");
expect(body.job?.log).toContain("Pushing…");
});
it("reports the far side's own words when the push fails", async () => {
await boot({
install: async () => ({
kind: "failed",
step: "connect",
detail: "Permission denied (publickey).",
}),
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(t.deps.machines.job()?.result).toEqual({
ok: false,
step: "connect",
message: "Permission denied (publickey).",
// Every failed install offers installing the program anyway.
canReplaceProgram: true,
});
});
it("a machine already on this version is a result, not a failure", async () => {
await boot({
install: async () => ({ kind: "already-installed", version: "9.9.9", identity: IDENTITY }),
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(t.deps.machines.job()?.result).toEqual({
ok: true,
installed: "already-installed",
version: "9.9.9",
});
});
it("an alias that points back at this machine is refused by the machine's own id, before anything is written", async () => {
// `Host localhost`, or a second alias for this host: the address is new to the store,
// so nothing recorded says "that is here" — only the machine can, and it does.
let installs = 0;
await boot({
probe: async () => ({
state: { kind: "running" as const, port: 7364, pid: 1 },
machineId: LOCAL_ID,
}),
install: async () => {
installs++;
return { kind: "installed", output: "", identity: IDENTITY };
},
});
await admin.post(`/api/projects/${PROJECT}/machines/ssh:nas/install`);
await waitFor(() => t.deps.machines.job()?.running === false);
expect(t.deps.machines.job()?.result).toMatchObject({
ok: false,
message: expect.stringContaining("this very machine"),
});
expect(installs).toBe(0);
expect(recordsInStore()).toEqual({});
});
it("records the platform the install found, and probes the machine in that dialect", async () => {
const dialects: (string | null)[] = [];
await boot({
install: async () => ({
kind: "installed",
output: "",
identity: { ...IDENTITY, platform: "win32" as const },
}),
probe: async (_target, _layout, _run, platform) => {
dialects.push(platform ?? null);
return { state: { kind: "stopped" as const }, machineId: null };
},
});
await admin.post(`/api/projects/${PROJECT}/machines/ssh:nas/install`);
await waitFor(() => t.deps.machines.job()?.running === false);
expect(machinesRepo.get("ssh:nas")?.platform).toBe("win32");
// Before the install nothing was known (null). The install records what it found
// before the restart asks the machine again, so that probe and the explicit one after
// both speak cmd.exe.
await admin.post(`/api/projects/${PROJECT}/machines/probe`);
expect(dialects).toEqual([null, "win32", "win32"]);
});
it("re-running an install keeps the machine id the record already carried", async () => {
await boot({
install: async () => ({ kind: "already-installed", version: "9.9.9", identity: IDENTITY }),
});
machinesRepo.patch("ssh:nas", {
version: "9.9.9",
installedAt: "2026-08-01T00:00:00.000Z",
machineId: "kUkIyqU-1GOfXgKD",
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(machinesRepo.get("ssh:nas")?.machineId).toBe("kUkIyqU-1GOfXgKD");
});
it("a machine that refuses this build says so, instead of reporting an install", async () => {
await boot({
resolvePlan: () => PUSHED_PLAN,
install: async () => ({ kind: "state-only", identity: IDENTITY }),
upgrade: async () => ({
kind: "refused",
detail: "this runtime publishes no business capabilities this platform can claim",
}),
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(t.deps.machines.job()?.result).toMatchObject({
ok: false,
step: "hand over the pushed build",
// The release over there matches, so the installer is skipped and nothing else can
// change that machine: installing anyway is the one step left, and it needs a yes.
canReplaceProgram: true,
});
expect((t.deps.machines.job()?.result as { message: string }).message).toContain(
"no business capabilities",
);
});
it("a machine with nothing running on it is done, not failed — its disk already has this", async () => {
// A hot swap replaces the code a RUNNING server is serving. With none there is nothing
// to swap and nothing wrong: the files are in place for its next start.
let asked = false;
await boot({
probe: async () => ({ state: { kind: "stopped" }, machineId: null }),
resolvePlan: () => PUSHED_PLAN,
install: async () => ({ kind: "state-only", identity: IDENTITY }),
upgrade: async () => {
asked = true;
return { kind: "upgraded", detail: "", persisted: true };
},
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(asked).toBe(false);
expect(t.deps.machines.job()?.result).toMatchObject({ ok: true });
expect(t.deps.machines.job()?.log.join(" ")).toContain("next starts");
});
it("a machine already carrying this build still gets it handed over: files are not the process", async () => {
// `already-installed` is decided from what is on its disk. What runs there is what it
// loaded at start — so a machine whose files were brought forward while it ran reports
// a version it is not running until the update channel makes the two agree.
let asked = false;
await boot({
// This server carries a pushed state of its own: there is something to hand over.
resolvePlan: () => ({
baseVersion: "9.9.9",
harness: '{"platform":{"bundle":"store/platform/cafe.mjs"}}',
hmrDir: "/nonexistent",
version: "9.9.9+hmr.cafe",
}),
install: async () => ({
kind: "already-installed",
version: "9.9.9+hmr.cafe",
identity: IDENTITY,
}),
upgrade: async () => {
asked = true;
return { kind: "upgraded", detail: "", persisted: true };
},
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(asked).toBe(true);
expect(t.deps.machines.job()?.result).toMatchObject({
ok: true,
installed: "already-installed",
});
});
it("a server with nothing pushed to it hands nothing over: a matching machine is simply done", async () => {
// A packaged install has no pushed state (plan.harness === null) and readPushedBuild
// would answer no-build. The former no-op must stay one, not become a failed job with
// an offer to reinstall a machine that already matches.
let asked = false;
await boot({
install: async () => ({ kind: "already-installed", version: "9.9.9", identity: IDENTITY }),
upgrade: async () => {
asked = true;
return { kind: "no-build" };
},
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(asked).toBe(false);
expect(t.deps.machines.job()?.result).toMatchObject({
ok: true,
installed: "already-installed",
});
expect(t.deps.machines.job()?.log.join(" ")).toContain("Nothing pushed here");
});
it("a build the machine took but could not write down is not recorded as installed", async () => {
// `persisted: false` is the machine's own word that the swap is live and will revert at
// its next restart. Recording the version would be true until then and false after.
await boot({
resolvePlan: () => PUSHED_PLAN,
install: async () => ({ kind: "state-only", identity: IDENTITY }),
upgrade: async () => ({ kind: "upgraded", detail: "", persisted: false }),
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(t.deps.machines.job()?.result).toMatchObject({
ok: false,
step: "hand over the pushed build",
canReplaceProgram: true,
});
expect((t.deps.machines.job()?.result as { message: string }).message).toContain("revert");
expect(machinesRepo.get("ssh:nas")?.version ?? null).toBeNull();
});
it("answering that offer runs the installer the version check would have skipped", async () => {
let forced: boolean | undefined;
await boot({
install: async (opts) => {
forced = opts.forceInstaller;
return { kind: "installed", output: "", identity: IDENTITY };
},
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install", {
replaceProgram: true,
});
await waitFor(() => t.deps.machines.job()?.running === false);
expect(forced).toBe(true);
});
it("a throw from the push path still ends the job", async () => {
await boot({
install: () => {
throw new Error("scp vanished");
},
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(t.deps.machines.job()?.result).toMatchObject({ ok: false, message: "scp vanished" });
});
});
describe("what stays installed", () => {
/** GET /api/machines, as the page reads it. */
const listed = async () =>
(
(await (
await admin.get(`/api/projects/default_project/machines`)
).json()) as MachinesResponse
).machines;
it("a successful install is remembered, and is already visible on the first settled poll", async () => {
await boot();
await admin.post(`/api/projects/default_project/machines/ssh:nas/install`);
await waitFor(() => t.deps.machines.job()?.running === false);
const machines = await listed();
expect(machines.find((m) => m.id === "ssh:nas")?.installed).toEqual({
version: "9.9.9",
at: "2026-08-24T12:00:00.000Z",
});
// The machine that was never installed to is untouched.
expect(machines.find((m) => m.id === "ssh:build-box")?.installed).toBeNull();
});
it("survives the process: a fresh service over the same data root reads it back", async () => {
await boot();
await admin.post(`/api/projects/default_project/machines/ssh:nas/install`);
await waitFor(() => t.deps.machines.job()?.running === false);
// A new instance has no job and no memory — only the file. This is the restart case,
// and the hot-push case: the App is rebuilt, the data root is not.
const reborn = new MachinesService(machinesRoot, LOCAL_ID, machinesRepo, effects());
expect(reborn.job()).toBeNull();
expect(reborn.list("default_project").find((m) => m.id === "ssh:nas")?.installed).toEqual({
version: "9.9.9",
at: "2026-08-24T12:00:00.000Z",
});
});
it("belongs to the Project that installed it, and to no other", async () => {
await boot();
await admin.post(`/api/projects/default_project/machines/ssh:nas/install`);
await waitFor(() => t.deps.machines.job()?.running === false);
// Another Project sees the same host as installed-but-not-mine: an adoption, not an
// install. The record is one machine's, the membership is per Project.
const other = t.deps.machines.list("other_project").find((m) => m.id === "ssh:nas");
expect(other?.installed).toBeNull();
expect(other?.elsewhere).toEqual({ version: "9.9.9", at: "2026-08-24T12:00:00.000Z" });
});
it("release drops the membership and leaves the install alone", async () => {
await boot();
await admin.post(`/api/projects/default_project/machines/ssh:nas/install`);
await waitFor(() => t.deps.machines.job()?.running === false);
const body = (await (
await admin.post(`/api/projects/default_project/machines/ssh:nas/release`)
).json()) as MachinesResponse;
expect(body.machines.find((m) => m.id === "ssh:nas")?.installed).toBeNull();
// Still installed — the program is on that host whoever uses it.
expect(recordsInStore()["ssh:nas"]).toEqual({
version: "9.9.9",
at: "2026-08-24T12:00:00.000Z",
});
});
it("installing elsewhere does not erase the first machine", async () => {
// The job is one slot, so before the records file existed this is exactly what made an
// installed machine disappear: the second install overwrote the only evidence.
await boot();
await admin.post(`/api/projects/default_project/machines/ssh:nas/install`);
await waitFor(() => t.deps.machines.job()?.running === false);
await admin.post(`/api/projects/default_project/machines/ssh:build-box/install`);
await waitFor(() => t.deps.machines.job()?.machineId === "ssh:build-box");
await waitFor(() => t.deps.machines.job()?.running === false);
const machines = await listed();
expect(machines.every((m) => m.installed !== null)).toBe(true);
expect(recordsInStore()).toEqual({
"ssh:nas": { version: "9.9.9", at: "2026-08-24T12:00:00.000Z" },
"ssh:build-box": { version: "9.9.9", at: "2026-08-24T12:00:00.000Z" },
});
});
it("records what the REMOTE already had when nothing needed sending", async () => {
await boot({
install: async () => ({ kind: "already-installed", version: "8.8.8", identity: IDENTITY }),
});
await admin.post(`/api/projects/default_project/machines/ssh:nas/install`);
await waitFor(() => t.deps.machines.job()?.running === false);
expect((await listed()).find((m) => m.id === "ssh:nas")?.installed?.version).toBe("8.8.8");
});
it("a deleted Project takes its machine list with it, and leaves the install alone", async () => {
await boot();
await admin.post(`/api/projects/${PROJECT}/machines/ssh:nas/install`);
await waitFor(() => t.deps.machines.job()?.running === false);
expect(machinesRepo.members(PROJECT)).toEqual(["ssh:nas"]);
// The membership row is the Project's and follows it out (ON DELETE CASCADE), so a
// Project later recreated under the same id does not inherit a dead one's machines.
// The install record is the machine's, and stays.
store.prepare("DELETE FROM projects WHERE project_id = ?").run(PROJECT);
expect(machinesRepo.members(PROJECT)).toBeNull();
expect(recordsInStore()["ssh:nas"]).toEqual({
version: "9.9.9",
at: "2026-08-24T12:00:00.000Z",
});
});
it("a failed install records nothing", async () => {
await boot({
install: async () => ({ kind: "failed", step: "connect", detail: "no route to host" }),
});
await admin.post(`/api/projects/default_project/machines/ssh:nas/install`);
await waitFor(() => t.deps.machines.job()?.running === false);
expect((await listed()).filter((m) => !m.local).every((m) => m.installed === null)).toBe(
true,
);
expect(recordsInStore()).toEqual({});
});
});
describe("server status", () => {
const listed = async () =>
(
(await (
await admin.get("/api/projects/default_project/machines")
).json()) as MachinesResponse
).machines;
const byId = async (id: string) => (await listed()).find((machine) => machine.id === id);
it("this machine reports itself running without any probe", async () => {
await boot({
probe: () => {
throw new Error("the local entry must not be probed");
},
});
expect(await byId("local")).toMatchObject({ local: true, status: { state: "running" } });
});
it("is null until probed — listing costs no ssh", async () => {
let probes = 0;
await boot({
probe: async () => {
probes++;
return { state: { kind: "running", port: 7364, pid: 1 }, machineId: null };
},
});
await admin.get("/api/projects/default_project/machines");
await admin.get("/api/projects/default_project/machines");
expect(probes).toBe(0);
expect((await byId("ssh:nas"))?.status).toBeNull();
});
it("probes only the machines something was installed on", async () => {
const probed: string[] = [];
await boot({
probe: async (target) => {
probed.push(target.alias);
return { state: { kind: "running", port: 7364, pid: 1 }, machineId: null };
},
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
probed.length = 0;
const res = await admin.post("/api/projects/default_project/machines/probe");
expect(res.status).toBe(200);
expect(probed).toEqual(["nas"]);
expect((await byId("ssh:nas"))?.status).toMatchObject({ state: "running", port: 7364 });
expect((await byId("ssh:build-box"))?.status).toBeNull();
});
it("a stopped server and an unreachable machine are both answers, not errors", async () => {
await boot({ probe: async () => ({ state: { kind: "stopped" }, machineId: null }) });
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
await admin.post("/api/projects/default_project/machines/probe");
expect((await byId("ssh:nas"))?.status).toMatchObject({ state: "stopped" });
expect((await byId("ssh:nas"))?.status?.port).toBeUndefined();
await boot({
probe: async () => ({
state: { kind: "unreachable", detail: "Permission denied (publickey)." },
machineId: null,
}),
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
await admin.post("/api/projects/default_project/machines/probe");
expect((await byId("ssh:nas"))?.status).toMatchObject({
state: "unreachable",
detail: "Permission denied (publickey).",
});
});
it("two probe requests at once share one round: each machine is asked once", async () => {
let probes = 0;
await boot({
probe: async () => {
probes++;
await new Promise((resolve) => setTimeout(resolve, 30));
return { state: { kind: "stopped" as const }, machineId: null };
},
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
probes = 0;
// Two tabs, one Project: the second asker joins the round the first started.
await Promise.all([
admin.post("/api/projects/default_project/machines/probe"),
admin.post("/api/projects/default_project/machines/probe"),
]);
expect(probes).toBe(1);
// A round that has settled is not reused: the next ask is a fresh one.
await admin.post("/api/projects/default_project/machines/probe");
expect(probes).toBe(2);
});
it("refuses to install onto this very machine", async () => {
await boot();
const res = await admin.post("/api/projects/default_project/machines/local/install");
expect(res.status).toBe(409);
expect(((await res.json()) as { error: { code: string } }).error.code).toBe("self_install");
expect(t.deps.machines.job()).toBeNull();
});
});
describe("putting a new build into service", () => {
it("restarts a machine whose server was running, so the installed build is the running one", async () => {
const calls: string[] = [];
await boot({
probe: async () => ({ state: { kind: "running", port: 7364, pid: 99 }, machineId: null }),
stopServer: async () => {
calls.push("stop");
return { ok: true as const };
},
startServer: async (_t, port) => {
calls.push(`start:${port}`);
return { ok: true };
},
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(t.deps.machines.job()?.result).toMatchObject({ ok: true, installed: "installed" });
expect(calls).toEqual(["stop", "start:7364"]);
expect(t.deps.machines.job()?.log.join(" ")).toContain("Stopping its server");
});
it("leaves a machine that was NOT running down — installing is not a decision to serve", async () => {
const calls: string[] = [];
await boot({
probe: async () => ({ state: { kind: "stopped" }, machineId: null }),
stopServer: async () => {
calls.push("stop");
return { ok: true as const };
},
startServer: async () => {
calls.push("start");
return { ok: true };
},
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(calls).toEqual([]);
expect(t.deps.machines.job()?.log.join(" ")).toContain("was not running");
});
it("does not restart when nothing was sent — an unchanged version is a no-op", async () => {
const calls: string[] = [];
await boot({
install: async () => ({ kind: "already-installed", version: "9.9.9", identity: IDENTITY }),
probe: async () => ({ state: { kind: "running", port: 7364, pid: 99 }, machineId: null }),
stopServer: async () => {
calls.push("stop");
return { ok: true as const };
},
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(calls).toEqual([]);
});
it("a restart that fails is the job's outcome, not a line in its log", async () => {
// The files landed — the record says so — but the old process is still serving, and
// an API reporting a clean install over that is exactly the disagreement every guard on
// this path exists to prevent.
await boot({
probe: async () => ({ state: { kind: "running", port: 7364, pid: 99 }, machineId: null }),
startServer: async () => ({ ok: false, detail: "port 7364 already in use" }),
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(t.deps.machines.job()?.result).toMatchObject({
ok: false,
step: "restart its server",
});
expect((t.deps.machines.job()?.result as { message: string }).message).toContain(
"did not come back up",
);
// On disk it IS this version; the record says what is there, the job says what runs.
expect(machinesRepo.get("ssh:nas")?.version).toBe("9.9.9");
});
});
describe("connecting", () => {
/** An install this server did earlier, so the machine is one connect() accepts. */
const installed = (version: string) =>
machinesRepo.patch("ssh:nas", { version, installedAt: "2026-08-01T00:00:00.000Z" });
it("a live connection to a dead server is not 'connected': connect asks the machine and starts it", async () => {
// The loop this guards: the connection is an ssh process on THIS side and outlives the
// far server. Connect used to answer "Already connected" on the connection's word alone,
// so the one job that could have started the dead server never did — and every caller
// that found the machine silent asked for another connect, forever.
const starts: number[] = [];
let up = false;
await boot({
probe: async () =>
up
? { state: { kind: "running" as const, port: 7364, pid: 4242 }, machineId: null }
: { state: { kind: "stopped" as const }, machineId: null },
startServer: async (_t, port) => {
starts.push(port);
up = true;
return { ok: true };
},
});
installed("9.9.9");
// A connection that is up, to a server that probes as stopped.
connected.add("ssh:nas");
machinesRepo.patch("ssh:nas", { remotePort: 7364 });
await admin.post("/api/projects/default_project/machines/ssh:nas/connect");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(t.deps.machines.job()?.result).toEqual({ ok: true, connected: true });
expect(starts).toEqual([7364]);
expect(t.deps.machines.job()?.log.join(" ")).toContain("Starting its server");
});
it("a remembered port that does not take is the failure, not a cue to try another", async () => {
// The row's port is the only one tried: this side does not guess at numbers on a
// machine other people use, and the far side's own words are what a person reads.
const starts: number[] = [];
await boot({
probe: async () => ({ state: { kind: "stopped" as const }, machineId: null }),
startServer: async (_t, port) => {
starts.push(port);
return { ok: false, detail: "EADDRINUSE" };
},
});
installed("9.9.9");
machinesRepo.patch("ssh:nas", { remotePort: 7376 });
await admin.post("/api/projects/default_project/machines/ssh:nas/connect");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(t.deps.machines.job()?.result).toMatchObject({
ok: false,
step: "start its server",
message: "EADDRINUSE",
});
expect(starts).toEqual([7376]);
expect(machinesRepo.get("ssh:nas")?.remotePort).toBe(7376);
});
it("a machine never reached before starts on the profile's own default port", async () => {
const starts: number[] = [];
let up = false;
await boot(
{
probe: async () =>
up
? { state: { kind: "running" as const, port: 7371, pid: 4242 }, machineId: null }
: { state: { kind: "stopped" as const }, machineId: null },
startServer: async (_t, port) => {
starts.push(port);
up = true;
return { ok: true };
},
},
remoteLayoutFor("dev"),
);
installed("9.9.9");
await admin.post("/api/projects/default_project/machines/ssh:nas/connect");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(t.deps.machines.job()?.result).toEqual({ ok: true, connected: true });
expect(starts).toEqual([7371]);
expect(machinesRepo.get("ssh:nas")?.remotePort).toBe(7371);
});
it("remembers the port the machine says it serves on, not the one that was asked for", async () => {
// The data root admits one server, so the server that answers after a start can be an
// earlier one on another port. The connection has to go where it actually listens.
let up = false;
await boot({
probe: async () =>
up
? { state: { kind: "running" as const, port: 7376, pid: 4242 }, machineId: null }
: { state: { kind: "stopped" as const }, machineId: null },
startServer: async () => {
up = true;
return { ok: true };
},
});
installed("9.9.9");
await admin.post("/api/projects/default_project/machines/ssh:nas/connect");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(t.deps.machines.job()?.result).toEqual({ ok: true, connected: true });
expect(machinesRepo.get("ssh:nas")?.remotePort).toBe(7376);
});
it("reconnecting over a live connection to an answering server starts nothing new", async () => {
// hold() is idempotent in the transport: it promotes the session in place. What must
// not happen is a start — the server is up — and the record must name the held session.
const starts: number[] = [];
let holds = 0;
await boot({
startServer: async (_t, port) => {
starts.push(port);
return { ok: true };
},
hold: async () => {
holds++;
return { ok: true, session: { pid: process.pid, socksPort: 1 } };
},
});
installed("9.9.9");
connected.add("ssh:nas");
machinesRepo.patch("ssh:nas", { remotePort: 7364 });
await admin.post("/api/projects/default_project/machines/ssh:nas/connect");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(t.deps.machines.job()?.result).toEqual({ ok: true, connected: true });
expect(starts).toEqual([]);
expect(holds).toBe(1);
expect(machinesRepo.get("ssh:nas")?.sessionPid).toBe(process.pid);
});
it("a connect's own probes are the machine's status: it does not read as stopped until the next round", async () => {
let up = false;
await boot({
probe: async () =>
up
? {
state: { kind: "running" as const, port: 7364, pid: 4242 },
machineId: "LNrJdHAZJ91G58i0",
}
: { state: { kind: "stopped" as const }, machineId: null },
startServer: async () => {
up = true;
return { ok: true };
},
});
installed("9.9.9");
await admin.post("/api/projects/default_project/machines/ssh:nas/connect");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(t.deps.machines.job()?.result).toEqual({ ok: true, connected: true });
const nas = (
(await (await admin.get(`/api/projects/${PROJECT}/machines`)).json()) as MachinesResponse
).machines.find((m) => m.id === "ssh:nas");
expect(nas?.status).toMatchObject({ state: "running", port: 7364 });
expect(nas?.machineId).toBe("LNrJdHAZJ91G58i0");
expect(nas?.connection).not.toBeNull();
});
it("refuses, from what it just heard, an alias that answers this server's own id", async () => {
// Never probed before, so the record could not refuse it; the probe inside the job can.
let holds = 0;
await boot({
probe: async () => ({
state: { kind: "running" as const, port: 7364, pid: 1 },
machineId: LOCAL_ID,
}),
hold: async () => {
holds++;
return { ok: true, session: { pid: process.pid, socksPort: 1 } };
},
});
installed("9.9.9");
await admin.post("/api/projects/default_project/machines/ssh:nas/connect");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(t.deps.machines.job()?.result).toMatchObject({ ok: false, step: "connect" });
expect((t.deps.machines.job()?.result as { message: string }).message).toContain(
"this server's own id",
);
expect(holds).toBe(0);
});
it("409s a Windows machine up front: there is no shell to hold a session on", async () => {
await boot();
installed("9.9.9");
machinesRepo.patch("ssh:nas", { platform: "win32" });
const res = await admin.post("/api/projects/default_project/machines/ssh:nas/connect");
expect(res.status).toBe(409);
expect(((await res.json()) as { error: { code: string } }).error.code).toBe(
"connect_unsupported",
);
expect(t.deps.machines.job()).toBeNull();
});
it("start() re-holds what the record says was held, and only that", async () => {
const heldNow: string[] = [];
await boot({
hold: async (target) => {
heldNow.push(target.alias);
connected.add(`ssh:${target.alias}`);
return { ok: true, session: { pid: process.pid, socksPort: 1 } };
},
});
// nas was held by the generation before; build-box was installed on but never asked
// for, and a Windows host has nothing to hold.
machinesRepo.patch("ssh:nas", {
version: "9.9.9",
installedAt: "2026-08-01T00:00:00.000Z",
sessionPid: 424242,
remotePort: 7364,
});
machinesRepo.patch("ssh:build-box", {
version: "9.9.9",
installedAt: "2026-08-01T00:00:00.000Z",
sessionPid: null,
});
await t.deps.machines.start();
expect(heldNow).toEqual(["nas"]);
expect(machinesRepo.get("ssh:nas")?.sessionPid).toBe(process.pid);
expect(t.deps.machines.list(PROJECT).find((m) => m.id === "ssh:nas")?.connection).toEqual({
pid: process.pid,
});
});
it("the App's boot re-holds what the record says was held, without being asked", async () => {
// What a hot push or a restart finds: a record written by the generation before. Nobody
// calls start() here — the Startup component does, as the App comes up.
const heldNow: string[] = [];
await boot({
hold: async (target) => {
heldNow.push(target.alias);
connected.add(`ssh:${target.alias}`);
return { ok: true, session: { pid: process.pid, socksPort: 1 } };
},
});
machinesRepo.patch("ssh:nas", {
version: "9.9.9",
installedAt: "2026-08-01T00:00:00.000Z",
sessionPid: 424242,
remotePort: 7364,
});
// An App of its own, booted over that record: its boot is the behaviour.
const booted = await createTestApp({ machines: service });
try {
await waitFor(() => heldNow.length > 0);
expect(heldNow).toEqual(["nas"]);
expect(machinesRepo.get("ssh:nas")?.sessionPid).toBe(process.pid);
} finally {
await booted.cleanup();
}
});
it("disconnect clears the record, so a later boot leaves the machine alone", async () => {
const heldNow: string[] = [];
await boot({
hold: async (target) => {
heldNow.push(target.alias);
connected.add(`ssh:${target.alias}`);
return { ok: true, session: { pid: process.pid, socksPort: 1 } };
},
});
installed("9.9.9");
await admin.post("/api/projects/default_project/machines/ssh:nas/connect");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(machinesRepo.get("ssh:nas")?.sessionPid).toBe(process.pid);
await admin.post("/api/projects/default_project/machines/ssh:nas/disconnect");
connected.delete("ssh:nas");
expect(machinesRepo.get("ssh:nas")?.sessionPid).toBeNull();
heldNow.length = 0;
await t.deps.machines.start();
expect(heldNow).toEqual([]);
});
});
describe("syncing models outward", () => {
/** A machine's server: counts what it is asked, and can be made slow. */
const machineServer = async (opts: { delayMs: number }) => {
const asked: string[] = [];
const server = http.createServer((req, res) => {
asked.push(`${req.method} ${req.url}`);
setTimeout(() => {
res.writeHead(200, { "content-type": "application/json" });
res.end(
req.url === "/api/projects"
? JSON.stringify({ projects: [{ projectId: PROJECT }] })
: JSON.stringify({ models: [] }),
);
}, opts.delayMs);
});
const port = await new Promise<number>((resolve) =>
server.listen(0, "127.0.0.1", () => resolve((server.address() as AddressInfo).port)),
);
return { asked, port, close: () => server.close() };
};
it("an edit that lands while a sync is in flight is written afterwards, once — not dropped", async () => {
const remote = await machineServer({ delayMs: 60 });
try {
await boot({
loadConfig: async () =>
({
models: [{ provider: "deepseek", model_id: "deepseek-v4-flash", api_key: "sk-x" }],
}) as never,
});
machinesRepo.patch("ssh:nas", {
version: "9.9.9",
installedAt: "2026-08-01T00:00:00.000Z",
remotePort: remote.port,
});
machinesRepo.setMembers(PROJECT, ["ssh:nas"]);
connected.add("ssh:nas");
// Three edits in a burst: the first starts a sync, the next two land during it.
const first = t.deps.machines.syncModelsEverywhere(PROJECT);
await new Promise((resolve) => setTimeout(resolve, 10));
const second = t.deps.machines.syncModelsEverywhere(PROJECT);
const third = t.deps.machines.syncModelsEverywhere(PROJECT);
await Promise.all([first, second, third]);
// Give the trailing run its own round trips.
await waitFor(() => remote.asked.filter((a) => a.startsWith("PUT")).length >= 2);
const puts = remote.asked.filter((a) => a.startsWith("PUT"));
// Exactly two writes: the one in flight, and ONE trailing run for everything that
// arrived during it — the config is read fresh then, so the last edit is what lands.
expect(puts).toHaveLength(2);
} finally {
remote.close();
}
});
});
describe("browsing a machine's directories", () => {
const ID = "LNrJdHAZJ91G58i0";
/** A machine this server holds a connection to, whose id a probe has heard. */
const connectedMachine = () => {
machinesRepo.patch("ssh:nas", {
version: "9.9.9",
installedAt: "2026-08-01T00:00:00.000Z",
machineId: ID,
remotePort: 7364,
});
connected.add("ssh:nas");
};
const listing = (stdout: string) => async (_t: unknown, command: string) =>
command.includes("---penguin-dirs---")
? { code: 0, stdout, stderr: "", timedOut: false }
: {
code: 0,
stdout: "---penguin-auth-token---\nremote-token\n",
stderr: "",
timedOut: false,
};
it("lists the subdirectories at the path the machine resolved, parent included", async () => {
const asked: string[] = [];
await boot({
runOn: async (_t, command) => {
asked.push(command);
return listing("/home/deploy/work\n---penguin-dirs---\nsrc\nnotes\n")(_t, command);
},
});
connectedMachine();
const res = await admin.get(`/api/projects/${PROJECT}/machines/${ID}/dirs?path=~/work`);
expect(res.status).toBe(200);
expect(await res.json()).toEqual({
path: "/home/deploy/work",
parent: "/home/deploy",
entries: [
{ name: "notes", path: "/home/deploy/work/notes" },
{ name: "src", path: "/home/deploy/work/src" },
],
});
// The path went to the machine quoted, so a name with a space is one argument there.
expect(asked.find((c) => c.includes("---penguin-dirs---"))).toContain("'~/work'");
});
it("404s a directory the machine does not have", async () => {
await boot({
runOn: async (_t, command) =>
command.includes("---penguin-dirs---")
? { code: 3, stdout: "", stderr: "", timedOut: false }
: { code: 0, stdout: "", stderr: "", timedOut: false },
});
connectedMachine();
const res = await admin.get(`/api/projects/${PROJECT}/machines/${ID}/dirs?path=/nope`);
expect(res.status).toBe(404);
});
it("403s a directory the machine has but refuses to list, with its own code", async () => {
await boot({
runOn: async (_t, command) =>
command.includes("---penguin-dirs---")
? { code: 4, stdout: "", stderr: "", timedOut: false }
: { code: 0, stdout: "", stderr: "", timedOut: false },
});
connectedMachine();
const res = await admin.get(`/api/projects/${PROJECT}/machines/${ID}/dirs?path=/root`);
expect(res.status).toBe(403);
expect(((await res.json()) as { error: { code: string } }).error.code).toBe(
"dir_permission_denied",
);
});
it("404s a machine that is not connected, and asks it nothing — a read never opens ssh", async () => {
const asked: string[] = [];
await boot({
runOn: async (_t, command) => {
asked.push(command);
return { code: 0, stdout: "", stderr: "", timedOut: false };
},
});
connectedMachine();
connected.delete("ssh:nas");
const res = await admin.get(`/api/projects/${PROJECT}/machines/${ID}/dirs`);
expect(res.status).toBe(404);
expect(asked).toEqual([]);
});
it("speaks through the alias that holds the session when two aliases share one machine", async () => {
const asked: string[] = [];
await boot({
runOn: async (target, command) => {
asked.push(target.alias);
return listing("/home/deploy\n---penguin-dirs---\n")(target, command);
},
});
// Both aliases answered the same id; only build-box is connected.
machinesRepo.patch("ssh:nas", {
machineId: ID,
version: "9.9.9",
installedAt: "2026-08-02T00:00:00.000Z",
});
machinesRepo.patch("ssh:build-box", {
machineId: ID,
version: "9.9.9",
installedAt: "2026-08-01T00:00:00.000Z",
});
connected.add("ssh:build-box");
const res = await admin.get(`/api/projects/${PROJECT}/machines/${ID}/dirs`);
expect(res.status).toBe(200);
expect(asked).toEqual(["build-box"]);
});
});
describe("the api sighting", () => {
const REMOTE = "kUkIyqU-1GOfXgKD";
const nas = async () =>
(
(await (
await admin.get("/api/projects/default_project/machines")
).json()) as MachinesResponse
).machines.find((m) => m.id === "ssh:nas");
it("is stamped by the proxy's report and served with the list, by machine id", async () => {
await boot();
machinesRepo.patch("ssh:nas", { machineId: REMOTE });
expect((await nas())?.api).toBeNull();
t.deps.machines.noteApiSeen(REMOTE, { ok: true });
expect((await nas())?.api).toEqual({ answeredAt: "2026-08-24T12:00:00.000Z" });
t.deps.machines.noteApiSeen(REMOTE, { ok: false, detail: "connect ECONNREFUSED" });
expect((await nas())?.api).toEqual({
failedAt: "2026-08-24T12:00:00.000Z",
detail: "connect ECONNREFUSED",
});
});
it("a sighting of a machine no row names lands nowhere, and does not throw", async () => {
await boot();
t.deps.machines.noteApiSeen("nobody-knows-this", { ok: true });
expect((await nas())?.api).toBeNull();
});
});
describe("machine identity", () => {
const ID = "LNrJdHAZJ91G58i0";
const listed = async () =>
(
(await (
await admin.get("/api/projects/default_project/machines")
).json()) as MachinesResponse
).machines;
const byId = async (id: string) => (await listed()).find((machine) => machine.id === id);
it("mints this machine's id once and keeps answering with it", async () => {
await boot();
const db = openDatabase(path.join(machinesRoot, "machine-id.db"));
try {
const minted = new MachinesRepo(db).ownId();
expect(minted).toMatch(/^[A-Za-z0-9_-]{16}$/);
expect(new MachinesRepo(db).ownId()).toBe(minted);
} finally {
db.close();
}
});
it("reports it as this machine's own, alongside the machines it reaches", async () => {
await boot();
expect((await byId("local"))?.machineId).toBe(LOCAL_ID);
});
it("a remote's id is null until a probe has heard it", async () => {
await boot();
expect((await byId("ssh:nas"))?.machineId).toBeNull();
});
it("a probe learns it, and it is remembered without another round trip", async () => {
await boot({
probe: async () => ({ state: { kind: "running", port: 7364, pid: 1 }, machineId: ID }),
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
await admin.post("/api/projects/default_project/machines/probe");
expect((await byId("ssh:nas"))?.machineId).toBe(ID);
const reborn = new MachinesService(machinesRoot, LOCAL_ID, machinesRepo, effects());
expect(reborn.list("default_project").find((m) => m.id === "ssh:nas")?.machineId).toBe(ID);
});
it("a restart records the id the machine minted when it came back", async () => {
// A machine mints its id when a server that mints one starts there, so a restart can be
// the moment an identity comes into existence. Until this side has heard it the machine
// is addressable by nothing — the proxy cannot route to it and the workspace picker
// will not offer it — so the restart asks rather than leaving it to the next probe.
let id: string | null = null;
await boot({
// Already carrying this build, so the install path never restarts it: the explicit
// restart below is the only thing that brings a new process — and its id — up.
install: async () => ({ kind: "already-installed", version: "9.9.9", identity: IDENTITY }),
probe: async () => ({ state: { kind: "running", port: 7364, pid: 1 }, machineId: id }),
stopServer: async () => {
id = ID; // It minted one on the way back up.
return { ok: true as const };
},
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
expect((await byId("ssh:nas"))?.machineId).toBeNull();
await admin.post("/api/projects/default_project/machines/ssh:nas/restart");
await waitFor(() => t.deps.machines.job()?.kind === "restart");
await waitFor(() => t.deps.machines.job()?.running === false);
expect(t.deps.machines.job()?.result).toMatchObject({ ok: true });
expect((await byId("ssh:nas"))?.machineId).toBe(ID);
});
it("a machine whose server never started stays without one", async () => {
await boot({ probe: async () => ({ state: { kind: "stopped" }, machineId: null }) });
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
await admin.post("/api/projects/default_project/machines/probe");
expect((await byId("ssh:nas"))?.status).toMatchObject({ state: "stopped" });
expect((await byId("ssh:nas"))?.machineId).toBeNull();
});
it("an alias repointed at a different machine takes the newer id, and forgets the old one", async () => {
let id = ID;
await boot({
probe: async () => ({ state: { kind: "running", port: 7364, pid: 1 }, machineId: id }),
});
await admin.post("/api/projects/default_project/machines/ssh:nas/install");
await waitFor(() => t.deps.machines.job()?.running === false);
await admin.post("/api/projects/default_project/machines/probe");
expect((await byId("ssh:nas"))?.machineId).toBe(ID);
expect(machinesRepo.get("ssh:nas")?.version).not.toBeNull();
id = "PO_VCwpQrw1hQLV-";
await admin.post("/api/projects/default_project/machines/probe");
expect((await byId("ssh:nas"))?.machineId).toBe(id);
// What the row held was learned from the machine no longer at this address. Kept, the
// newcomer would read as already installed at a build it has never run — and an
// "already installed" record is what suppresses the install that would correct it.
const row = machinesRepo.get("ssh:nas");
expect(row?.version).toBeNull();
expect(row?.installedAt).toBeNull();
expect(row?.platform).toBeNull();
expect(row?.remotePort).toBeNull();
expect((await byId("ssh:nas"))?.installed).toBeNull();
});
});
describe("using machines", () => {
const useBody = (machines: string[], replaceProgram = false) =>
admin.post("/api/projects/default_project/machines/use", {
machines,
...(replaceProgram ? { replaceProgram } : {}),
});
const jobsOf = () => t.deps.machines.jobs();
const settled = () => jobsOf().every((job) => !job.queued && !job.running);
it("a batch works its machines side by side, and each ends connected", async () => {
let release = () => {};
const held = new Promise<void>((resolve) => {
release = resolve;
});
const installs: string[] = [];
await boot({
install: async (opts) => {
installs.push(opts.target.alias);
if (installs.length === 1) await held;
return { kind: "installed", output: "done", identity: IDENTITY };
},
});
const started = await useBody(["ssh:build-box", "ssh:nas"]);
expect(started.status).toBe(202);
const body = (await started.json()) as MachinesUseResponse;
expect(body.refused).toEqual([]);
// Two hosts, two jobs at once: the second does not wait for the first's held install.
expect(body.jobs.map((job) => [job.machineId, job.queued, job.running])).toEqual([
["ssh:build-box", false, true],
["ssh:nas", false, true],
]);
await waitFor(() => jobsOf().some((job) => job.machineId === "ssh:nas" && !job.running));
expect(jobsOf().find((job) => job.machineId === "ssh:build-box")?.running).toBe(true);
release();
await waitFor(settled);
expect([...installs].sort()).toEqual(["build-box", "nas"]);
for (const job of jobsOf()) {
expect(job.kind).toBe("use");
expect(job.result).toEqual({ ok: true, connected: true });
}
expect(connected).toEqual(new Set(["ssh:build-box", "ssh:nas"]));
expect(Object.keys(recordsInStore()).sort()).toEqual(["ssh:build-box", "ssh:nas"]);
});
it("a machine already on this build is not reinstalled: use goes straight to connecting", async () => {
let installs = 0;
await boot({
install: async () => {
installs += 1;
return { kind: "installed", output: "done", identity: IDENTITY };
},
});
machinesRepo.patch("ssh:nas", { version: "9.9.9", installedAt: "2026-08-01T00:00:00.000Z" });
machinesRepo.setMembers("default_project", ["ssh:nas"]);
await useBody(["ssh:nas"]);
await waitFor(settled);
expect(installs).toBe(0);
expect(jobsOf()[0]?.log[0]).toBe("Already on 9.9.9.");
expect(jobsOf()[0]?.result).toEqual({ ok: true, connected: true });
});
it("a machine behind this build is brought forward first, then connected", async () => {
let installs = 0;
await boot({
install: async () => {
installs += 1;
return { kind: "installed", output: "done", identity: IDENTITY };
},
});
machinesRepo.patch("ssh:nas", { version: "9.9.8", installedAt: "2026-08-01T00:00:00.000Z" });
machinesRepo.setMembers("default_project", ["ssh:nas"]);
await useBody(["ssh:nas"]);
await waitFor(settled);
expect(installs).toBe(1);
expect(jobsOf()[0]?.result).toEqual({ ok: true, connected: true });
expect(recordsInStore()["ssh:nas"]?.version).toBe("9.9.9");
});
it("an install that fails stops that machine's job there, with the forced install on offer, and the next machine still runs", async () => {
await boot({
install: async (opts) =>
opts.target.alias === "build-box"
? { kind: "failed", step: "connect", detail: "Permission denied (publickey)." }
: { kind: "installed", output: "done", identity: IDENTITY },
});
await useBody(["ssh:build-box", "ssh:nas"]);
await waitFor(settled);
const [box, nas] = jobsOf();
expect(box?.result).toEqual({
ok: false,
step: "connect",
message: "Permission denied (publickey).",
canReplaceProgram: true,
});
expect(nas?.result).toEqual({ ok: true, connected: true });
expect(connected).toEqual(new Set(["ssh:nas"]));
});
it("refusals that need no ssh come back by id, and the rest of the batch is still queued", async () => {
await boot();
const body = (await (
await useBody(["ssh:nope", "local", "ssh:nas"])
).json()) as MachinesUseResponse;
expect(body.refused).toEqual([
{ machineId: "ssh:nope", why: "unknown-machine" },
{ machineId: "local", why: "self" },
]);
expect(body.jobs.map((job) => job.machineId)).toEqual(["ssh:nas"]);
await waitFor(settled);
});
it("an empty batch is a bad request", async () => {
await boot();
expect((await useBody([])).status).toBe(400);
});
it("stop using drops the connection and the membership, and keeps the install", async () => {
await boot();
await useBody(["ssh:nas"]);
await waitFor(settled);
expect(connected.has("ssh:nas")).toBe(true);
const stopped = await admin.post("/api/projects/default_project/machines/stop-using", {
machines: ["ssh:nas"],
});
expect(stopped.status).toBe(200);
const body = (await stopped.json()) as MachinesResponse;
const nas = body.machines.find((machine) => machine.id === "ssh:nas");
// Released from the Project, the install itself remembered as somebody else's.
expect(nas?.installed).toBeNull();
expect(nas?.elsewhere?.version).toBe("9.9.9");
// Nothing re-holds a machine nobody uses: the held mark is gone from the record. (The
// fake transport's own registry is not the record's; the real one closes with the ssh.)
expect(machinesRepo.get("ssh:nas")?.sessionPid ?? null).toBeNull();
});
it("a machine whose re-hold failed waits out a backoff before the next try", async () => {
// A host that is down costs one ssh per try; without a backoff the standing intent
// would knock on it every minute forever.
let holds = 0;
let clock = new Date("2026-08-24T12:00:00.000Z");
await boot({
now: () => clock,
hold: async () => {
holds += 1;
return { ok: false, detail: "Connection refused" };
},
});
machinesRepo.patch("ssh:nas", {
version: "9.9.9",
installedAt: "2026-08-01T00:00:00.000Z",
sessionPid: 4242,
});
machinesRepo.setMembers("default_project", ["ssh:nas"]);
await t.deps.machines.autoConnect();
expect(holds).toBe(1);
// Within the first minute after a failure: not tried again.
await t.deps.machines.autoConnect();
expect(holds).toBe(1);
// Once the wait has passed, it is.
clock = new Date(clock.getTime() + 61_000);
await t.deps.machines.autoConnect();
expect(holds).toBe(2);
// The second failure doubles the wait: two minutes now, so one minute later is too soon.
clock = new Date(clock.getTime() + 61_000);
await t.deps.machines.autoConnect();
expect(holds).toBe(2);
});
});
describe("adding a host to the ssh config", () => {
const post = (body: Record<string, unknown>) =>
admin.post("/api/projects/default_project/machines/ssh-hosts", body);
it("appends the block, and the list names the new host at once", async () => {
const written: string[] = [];
const aliases = ["build-box", "nas"];
await boot({
listAliases: () => aliases,
appendHost: (block) => {
written.push(block);
aliases.push("orchid-2");
},
});
const res = await post({ alias: "orchid-2", hostName: "10.0.0.9", user: "k", port: "2222" });
expect(res.status).toBe(201);
expect(written).toEqual([
[
"# Added by Adelie on 2026-08-24T12:00:00.000Z",
"Host orchid-2",
" HostName 10.0.0.9",
" User k",
" Port 2222",
"",
].join("\n"),
]);
const body = (await res.json()) as MachinesResponse;
expect(body.machines.some((m) => m.id === "ssh:orchid-2")).toBe(true);
});
it("refuses an alias the config already declares, writing nothing", async () => {
const written: string[] = [];
await boot({ appendHost: (block) => void written.push(block) });
const res = await post({ alias: "nas", hostName: "10.0.0.9" });
expect(res.status).toBe(409);
expect(((await res.json()) as { error: { code: string } }).error.code).toBe(
"ssh_host_exists",
);
expect(written).toEqual([]);
});
const CONFIG = [
"# Added by Adelie on 2026-08-01T00:00:00.000Z",
"Host nas",
" HostName 10.0.0.2",
"",
"Host build-box",
" HostName box.example.net",
" ProxyJump bastion",
].join("\n");
it("reads a block back, saying whether this app wrote it", async () => {
await boot({ readConfig: () => CONFIG });
const ours = await admin.get("/api/projects/default_project/machines/ssh-hosts/nas");
expect(ours.status).toBe(200);
expect(await ours.json()).toEqual({ alias: "nas", hostName: "10.0.0.2", editable: true });
const theirs = await admin.get("/api/projects/default_project/machines/ssh-hosts/build-box");
expect(((await theirs.json()) as { editable: boolean }).editable).toBe(false);
const none = await admin.get("/api/projects/default_project/machines/ssh-hosts/nope");
expect(none.status).toBe(404);
});
it("rewrites a block this app wrote in place, and refuses one written by hand", async () => {
const written: string[] = [];
await boot({ readConfig: () => CONFIG, writeConfig: (text) => void written.push(text) });
const ok = await admin.put("/api/projects/default_project/machines/ssh-hosts/nas", {
hostName: "10.0.0.3",
user: "deploy",
});
expect(ok.status).toBe(200);
expect(written).toHaveLength(1);
expect(written[0]!.split("\n").slice(0, 5)).toEqual([
"# Added by Adelie on 2026-08-24T12:00:00.000Z",
"Host nas",
" HostName 10.0.0.3",
" User deploy",
"",
]);
expect(written[0]!.endsWith(" ProxyJump bastion")).toBe(true);
const foreign = await admin.put(
"/api/projects/default_project/machines/ssh-hosts/build-box",
{ hostName: "x" },
);
expect(foreign.status).toBe(409);
expect(((await foreign.json()) as { error: { code: string } }).error.code).toBe(
"ssh_host_foreign",
);
expect(written).toHaveLength(1);
});
it("names the field that would not survive as one config line", async () => {
const written: string[] = [];
await boot({ appendHost: (block) => void written.push(block) });
const res = await post({ alias: "new box", hostName: "h" });
expect(res.status).toBe(400);
const body = (await res.json()) as { error: { code: string; message: string } };
expect(body.error.code).toBe("ssh_host_invalid");
expect(body.error.message).toContain("alias");
expect(written).toEqual([]);
});
});
describe("refusals decided before any ssh runs", () => {
it("409s an install through an alias a probe has already heard this machine's id from", async () => {
await boot();
machinesRepo.patch("ssh:nas", { machineId: LOCAL_ID });
const res = await admin.post(`/api/projects/${PROJECT}/machines/ssh:nas/install`);
expect(res.status).toBe(409);
expect(((await res.json()) as { error: { code: string } }).error.code).toBe("self_install");
expect(t.deps.machines.job()).toBeNull();
});
it("409s a second install while one is running", async () => {
let release = () => {};
const held = new Promise<void>((resolve) => {
release = resolve;
});
await boot({
install: async () => {
await held;
return { kind: "installed", output: "", identity: IDENTITY };
},
});
expect(
(await admin.post("/api/projects/default_project/machines/ssh:nas/install")).status,
).toBe(202);
const second = await admin.post(
"/api/projects/default_project/machines/ssh:build-box/install",
);
expect(second.status).toBe(409);
expect(((await second.json()) as { error: { code: string } }).error.code).toBe(
"install_running",
);
release();
await waitFor(() => t.deps.machines.job()?.running === false);
});
it("404s a host this server's config does not declare", async () => {
await boot();
const res = await admin.post(
"/api/projects/default_project/machines/ssh:not-in-config/install",
);
expect(res.status).toBe(404);
expect(((await res.json()) as { error: { code: string } }).error.code).toBe(
"unknown_machine",
);
expect(t.deps.machines.job()).toBeNull();
});
it("409s when this server carries no install image", async () => {
await boot({ resolvePlan: () => null });
const res = await admin.post("/api/projects/default_project/machines/ssh:nas/install");
expect(res.status).toBe(409);
expect(((await res.json()) as { error: { code: string } }).error.code).toBe(
"no_install_image",
);
});
});
});