Skip to content

A local-first multi-agent app development platform, built on PenguinHarness

17.8 KB
Raw Blame
/**
 * One-shot build-and-deploy to a running runtime: `node scripts/deploy.mjs <port>`.
 *
 * Aimed at one ad-hoc target and run once — for deploying to a machine reached through an
 * ssh tunnel (`ssh -L <port>:127.0.0.1:<remote port> …`) as much as to a local runtime.
 *
 * Builds the web dist, compiles the platform and cli entries, and pushes all three as ONE
 * atomic version to POST /api/hmr/upgrade. Authentication is an admin session established
 * per run: ADELIE_ADMIN_PASSWORD is read from the environment and exchanged for a cookie,
 * so no credential of any kind is written to disk — a file holding an admin-equivalent
 * secret is readable by everything running as this user, agent shells included, which makes
 * the file itself the vulnerability.
 *
 * Usage:
 *   ADELIE_ADMIN_PASSWORD=… node scripts/deploy.mjs 53531
 *   ADELIE_ADMIN_PASSWORD=… node scripts/deploy.mjs 53531 --skip-web-build
 *   ADELIE_ADMIN_PASSWORD=… node scripts/deploy.mjs https://box.example.com
 */
import { createRequire } from "node:module";
import { execFileSync } from "node:child_process";
import fs from "node:fs";
import fsp from "node:fs/promises";
import http from "node:http";
import https from "node:https";
import os from "node:os";
import path from "node:path";
import zlib from "node:zlib";
import { fileURLToPath } from "node:url";
import { unsafePlaintextTarget } from "./deploy-target-safety.mjs";
import { buildGitDefine, checkoutFacts, originUrl } from "./build-git-stamp.mjs";
import { ESM_CJS_BANNER } from "./esm-cjs-banner.mjs";
import { FAR_SIDE_SCRIPTS } from "./far-side-scripts.mjs";
import { buildBuiltinPlugins } from "./build-plugins.mjs";
import { archiveName, packArchive, prefixPackages } from "./asset-archives.mjs";
import { typescriptPayload } from "./typescript-payload.mjs";
import { createHash } from "node:crypto";

const require = createRequire(import.meta.url);
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const WEB_DIST = path.join(ROOT, "packages", "web", "dist");
const PLATFORM_ENTRY = path.join(ROOT, "packages", "server", "src", "hmr", "entry.ts");
const CLI_ENTRY = path.join(ROOT, "packages", "cli", "src", "index.ts");
const PLATFORM_BUNDLE = path.join(os.tmpdir(), `penguin-deploy-platform-${process.pid}.mjs`);
const CLI_BUNDLE = path.join(os.tmpdir(), `penguin-deploy-cli-${process.pid}.mjs`);

const log = (msg) => console.log(`[deploy] ${msg}`);

/**
 * Provenance for this push, recorded with the version so `penguin version --json` on the
 * target can name where its harness came from — harness.json's copy, alongside the same
 * revision inlined into the bundles themselves.
 *
 * `revision` is spelled exactly as core's BuildInfo.describe, `-dirty` included: a deploy
 * from an uncommitted tree is normal here and the record has to admit it, since the sha
 * alone would name code that never existed. Null when this is not a checkout of this
 * repository, in which case the push carries no `source` rather than a fabricated one.
 */
function pushSource() {
  const facts = checkoutFacts();
  const repo = originUrl();
  if (facts?.described == null || repo === null) return null;
  return { repo, revision: facts.described };
}

function usage(problem) {
  console.error(
    `${problem}\n\n` +
      "Usage: ADELIE_ADMIN_PASSWORD=… node scripts/deploy.mjs <port|url> [--skip-web-build]\n" +
      "       ADELIE_API_TOKEN=$(cat <root>/api-token) node scripts/deploy.mjs <port|url>\n" +
      "  <port>  a port on this machine (an ssh -L tunnel to the target runtime, or a local server)\n" +
      "  <url>   a full origin, when the target is not reached over loopback\n",
  );
  process.exit(1);
}

const args = process.argv.slice(2);
const skipWebBuild = args.includes("--skip-web-build");
const target = args.find((a) => !a.startsWith("--"));
if (target === undefined) usage("[deploy] no target given.");
// Two credentials, either one: the admin password (exchanged for a cookie), or the
// runtime's own local API token (`<root>/api-token`, admin-equivalent — see
// server/src/auth/api-token.ts), sent as a Bearer. A local push needs no password.
const ADMIN_PASSWORD = process.env.ADELIE_ADMIN_PASSWORD;
const API_TOKEN = process.env.ADELIE_API_TOKEN;
if (!ADMIN_PASSWORD && !API_TOKEN)
  usage("[deploy] set ADELIE_ADMIN_PASSWORD or ADELIE_API_TOKEN (the runtime's <root>/api-token).");

/** A bare port means this machine's loopback (typically an ssh -L tunnel to the real target). */
const baseUrl = /^\d+$/.test(target) ? `http://127.0.0.1:${target}` : target.replace(/\/+$/, "");

const plaintextProblem = unsafePlaintextTarget(baseUrl);
if (plaintextProblem) usage(`[deploy] ${plaintextProblem}`);

/**
 * On a loopback bind 127.0.0.1 is the PREVIEW host, where /api answers 401; the API is
 * served under the canonical app host. A tunnel lands on 127.0.0.1:<port> at this end,
 * so the request must still be addressed to `localhost` by name.
 */
const hostOverride = new URL(baseUrl).hostname === "127.0.0.1" ? "localhost" : undefined;

/**
 * node:http rather than the global fetch: fetch (undici) silently derives Host from the
 * URL and ignores an explicit `headers.host`, which breaks the override above.
 */
function request(urlStr, { method = "GET", headers = {}, body } = {}) {
  return new Promise((resolve, reject) => {
    const url = new URL(urlStr);
    const lib = url.protocol === "https:" ? https : http;
    const req = lib.request(
      {
        hostname: url.hostname,
        port: url.port || (url.protocol === "https:" ? 443 : 80),
        path: `${url.pathname}${url.search}`,
        method,
        headers: { ...headers, ...(hostOverride ? { host: hostOverride } : {}) },
      },
      (res) => {
        const chunks = [];
        res.on("data", (c) => chunks.push(c));
        res.on("end", () =>
          resolve({
            status: res.statusCode ?? 0,
            headers: res.headers,
            body: Buffer.concat(chunks),
          }),
        );
      },
    );
    req.on("error", reject);
    if (body !== undefined) req.write(body);
    req.end();
  });
}

/** The request headers that authenticate as admin: a Bearer token, or a cookie from a password login. */
async function authHeaders() {
  if (API_TOKEN) return { authorization: `Bearer ${API_TOKEN}` };
  return { cookie: await login() };
}

/** Signs in as `admin` and returns the session cookie. */
async function login() {
  const res = await request(`${baseUrl}/api/auth/login`, {
    method: "POST",
    headers: { "content-type": "application/json" },
    body: JSON.stringify({ userId: "admin", password: ADMIN_PASSWORD }),
  });
  if (res.status !== 200) {
    throw new Error(`admin login failed (${res.status}): ${res.body.toString("utf8")}`);
  }
  const setCookie = res.headers["set-cookie"];
  if (!setCookie?.length) throw new Error("login succeeded but set no session cookie");
  return setCookie.map((c) => c.split(";")[0]).join("; ");
}

/**
 * Compiles one entry to a self-contained ESM file. The banner is load-bearing: bundled CJS
 * deps reference `require`, `__filename` and `__dirname` inside their own wrapper, and an
 * ESM bundle supplies none of the three — see scripts/esm-cjs-banner.mjs.
 */
async function compileEntry(entry, outfile) {
  if (!fs.existsSync(entry)) throw new Error(`compile entry missing: ${entry}`);
  // The platform imports its generated interface table; make sure it is current.
  const { execFileSync } = await import("node:child_process");
  execFileSync(
    process.execPath,
    [
      path.join(ROOT, "scripts/gen-ifaces.mjs"),
      "--project",
      path.join(ROOT, "packages/server/tsconfig.json"),
      "--out",
      path.join(ROOT, "packages/server/src/ifaces.json"),
    ],
    { stdio: "inherit" },
  );
  const esbuild = await import("esbuild");
  await esbuild.build({
    entryPoints: [entry],
    bundle: true,
    format: "esm",
    platform: "node",
    // The Node that runs it (package.json engines). Without a target esbuild compiles for
    // `esnext`, where it takes standard decorators to be supported and emits them as they
    // were written — and no Node parses those, so the pushed bundle fails to import with
    // "Invalid or unexpected token" and the target keeps the generation it had. The
    // packaged build does not hit this because tsup passes a target of its own.
    target: "node24",
    outfile,
    logLevel: "silent",
    banner: { js: ESM_CJS_BANNER },
    alias: {
      "@adelie/core/kernel": require.resolve("@adelie/core/kernel"),
    },
    // The pushed bundle lands under `<root>/hmr/store/`, outside any checkout, so its own
    // revision has to be inlined here or it can never be recovered: `penguin version` from
    // a hot-loaded CLI would otherwise report the bare version it was compiled from. This is
    // the same value the push records as `source`, reaching the target by a second route —
    // in the artifact itself rather than in harness.json.
    define: buildGitDefine(),
  });
}

/** The built web dist as a { relPath: base64 } manifest. */
async function readWebManifest() {
  const files = {};
  for (const entry of await fsp.readdir(WEB_DIST, { recursive: true, withFileTypes: true })) {
    if (!entry.isFile()) continue;
    const abs = path.join(entry.parentPath, entry.name);
    files[path.relative(WEB_DIST, abs).split(path.sep).join("/")] = await fsp.readFile(abs);
  }
  return files;
}

/**
 * Files the pushed platform needs as real files rather than bundled code. A bundle cannot carry one: it is
 * imported from the runtime's data root, where node-pty's own relative `build/Release/
 * pty.node` does not resolve. So the package ships whole — its JS, its prebuilds and its
 * darwin `spawn-helper` — and the runtime unpacks it next to the bundle (hmr/host.ts's
 * UpgradeAssets), where node-pty's normal resolution works again.
 *
 * `exec` carries the files whose exec bit must survive the trip: base64 has no mode, and a
 * spawn-helper without it makes every terminal fail to start on macOS.
 */
async function readNativeAssets() {
  const ptyDir = path.dirname(
    require.resolve("node-pty/package.json", {
      paths: [path.join(ROOT, "packages", "server")],
    }),
  );
  const files = {};
  const exec = [];
  // Only what the loader reads at runtime. `.pdb` files are Windows debug symbols that
  // nothing ever loads and are 90% of the package's bytes (58 MB → 3 MB without them),
  // which is the difference between a push that crosses an ssh tunnel and one that does
  // not; `.lib` are link-time import libraries, equally never read at runtime.
  const wanted = (rel) =>
    !rel.endsWith(".pdb") &&
    !rel.endsWith(".lib") &&
    (rel === "package.json" ||
      rel.startsWith("lib/") ||
      rel.startsWith("build/Release/") ||
      rel.startsWith("prebuilds/"));
  const pty = [];
  for (const entry of await fsp.readdir(ptyDir, { recursive: true, withFileTypes: true })) {
    if (!entry.isFile()) continue;
    const abs = path.join(entry.parentPath, entry.name);
    const rel = path.relative(ptyDir, abs).split(path.sep).join("/");
    if (!wanted(rel)) continue;
    // node-pty ships its prebuilt spawn-helper as 0644; the archive records it executable
    // regardless of how it looks on this machine.
    const executable = rel.endsWith("spawn-helper") || ((await fsp.stat(abs)).mode & 0o111) !== 0;
    pty.push({ rel: `node_modules/node-pty/${rel}`, abs, exec: executable });
  }
  // Every package travels as ONE archive (scripts/asset-archives.mjs): a push of hundreds of
  // small files is hundreds of blobs, probes and transfers, and stalls. The platform unpacks
  // `archives/*.tgz` before resolving anything from its assets (hmr/asset-archives.ts).
  files[`archives/${archiveName("", "node-pty")}`] = await packArchive(pty);
  // The compiler plugin interfaces and workflows are checked with: the target's program may
  // predate it being a dependency (or be a desktop bundle with no node_modules), and a
  // platform that cannot find it compares no plugin interface and loads no workflow.
  // Content-addressed like everything here, so it crosses once.
  files[`archives/${archiveName("", "typescript")}`] = await packArchive(
    typescriptPayload(path.join(ROOT, "packages", "server")).map(({ rel, abs }) => ({
      rel: `node_modules/typescript/${rel}`,
      abs,
      exec: false,
    })),
  );
  // The scripts that run on the FAR side — the release installers a remote install feeds
  // over, the one thing that has to arrive before the CLI does. A pushed bundle resolves them
  // from its own assets directory, so a push that omits one leaves a server that cannot
  // install a machine at all. Same set the packaged build copies into dist/; see the module.
  for (const { name, from } of FAR_SIDE_SCRIPTS) {
    files[name] = await fsp.readFile(path.join(ROOT, from));
  }
  // The builtin plugins, as the npm prefix the loader resolves from (`plugins/package.json`
  // + `plugins/node_modules/<name>/…`, see scripts/build-plugins.mjs), from cache when
  // unchanged — one archive per package, so an unchanged plugin is an unchanged blob.
  const built = await buildBuiltinPlugins({ log });
  const { byPackage, loose } = await prefixPackages(built.dir, built.files, "plugins");
  files["archives/plugins.tgz"] = await packArchive(loose);
  for (const [pkg, entries] of byPackage) {
    files[`archives/${archiveName("plugins.", pkg)}`] = await packArchive(entries);
  }
  return { files, exec };
}

async function main() {
  if (skipWebBuild) {
    if (!fs.existsSync(path.join(WEB_DIST, "index.html"))) {
      throw new Error(`--skip-web-build given but ${WEB_DIST} has no index.html`);
    }
    log("reusing the existing web dist");
  } else {
    log("building the web dist…");
    execFileSync("pnpm", ["--filter", "@adelie/web", "build"], {
      cwd: ROOT,
      stdio: "inherit",
    });
  }

  log("compiling platform + cli…");
  await compileEntry(PLATFORM_ENTRY, PLATFORM_BUNDLE);
  await compileEntry(CLI_ENTRY, CLI_BUNDLE);

  const files = await readWebManifest();
  const assets = await readNativeAssets();
  const source = pushSource();

  const auth = await authHeaders();
  const platform = await fsp.readFile(PLATFORM_BUNDLE);
  const cli = await fsp.readFile(CLI_BUNDLE);
  const mapValues = (o, f) => Object.fromEntries(Object.entries(o).map(([k, v]) => [k, f(v)]));
  const body = (part) => ({
    platform: part(platform, "utf8"),
    cli: part(cli, "utf8"),
    web: { files: mapValues(files, (b) => part(b, "base64")) },
    assets: { files: mapValues(assets.files, (b) => part(b, "base64")), exec: assets.exec },
    ...(source === null ? {} : { source }),
  });
  // Content-addressed transfer, the way git pushes: name every part by its sha256, ask the
  // target which blobs it lacks, PUT only those (raw), then push a body of names. A target
  // without the probe (an older runtime) answers 404 and gets every part inline.
  const blobs = new Map();
  let payload = body((bytes) => {
    const sha = createHash("sha256").update(bytes).digest("hex");
    blobs.set(sha, bytes);
    return { sha };
  });
  const probe = await request(`${baseUrl}/api/hmr/assets/probe`, {
    method: "POST",
    headers: { "content-type": "application/json", ...auth },
    body: JSON.stringify({ hashes: [...blobs.keys()] }),
  });
  if (probe.status === 200) {
    let sent = 0;
    for (const sha of JSON.parse(probe.body.toString("utf8")).missing) {
      const bytes = blobs.get(sha);
      if (bytes === undefined) continue;
      const put = await request(`${baseUrl}/api/hmr/blobs/${sha}`, {
        method: "PUT",
        headers: { "content-type": "application/octet-stream", ...auth },
        body: bytes,
      });
      if (put.status !== 200) {
        throw new Error(`PUT /api/hmr/blobs/${sha} → ${put.status}: ${put.body.toString("utf8")}`);
      }
      sent += bytes.length;
    }
    log(`${blobs.size} blobs; ${(sent / 1048576).toFixed(1)} MB were new to the target`);
  } else {
    log("target has no probe: pushing everything inline");
    payload = body((bytes, encoding) => bytes.toString(encoding));
  }
  const gz = zlib.gzipSync(Buffer.from(JSON.stringify(payload)));
  if (source !== null) log(`provenance: ${source.revision}`);
  log(
    `pushing ${Object.keys(files).length} web files + ${Object.keys(assets.files).length} assets + 2 bundles (${(gz.length / 1048576).toFixed(1)} MB body) to ${baseUrl}…`,
  );

  const started = Date.now();
  const res = await request(`${baseUrl}/api/hmr/upgrade`, {
    method: "POST",
    headers: { "content-type": "application/gzip", ...auth },
    body: gz,
  });
  const seconds = ((Date.now() - started) / 1000).toFixed(1);
  const text = res.body.toString("utf8");
  if (res.status < 200 || res.status >= 300) {
    throw new Error(`/api/hmr/upgrade → ${res.status}: ${text}`);
  }
  const outcome = JSON.parse(text);
  if (outcome.status === "blocked") {
    // A blocked upgrade is a first-class outcome, not an HTTP error: the running
    // version keeps serving and these paths say what would have been discarded.
    log(`BLOCKED after ${seconds}s — the target kept its current version.`);
    console.error(JSON.stringify(outcome, null, 2));
    process.exitCode = 1;
    return;
  }
  log(
    `ok in ${seconds}s — impl ${outcome.impl}, mode ${outcome.mode}, web rev ${outcome.web?.rev}`,
  );
  if (outcome.persisted === false) {
    // The live swap took effect, but the server could not write it to disk (see
    // host.ts's persistVersion) — a restart on the target reverts to the previously
    // committed version, silently undoing this push.
    console.warn(
      "[deploy] WARNING: the target could not persist this version to disk (persisted: false). " +
        "It is live now but will REVERT on the target's next restart — check the target's disk/permissions.",
    );
  }
}

main()
  .catch((err) => {
    console.error(`[deploy] ${err instanceof Error ? err.message : String(err)}`);
    process.exitCode = 1;
  })
  .finally(() => {
    for (const f of [PLATFORM_BUNDLE, CLI_BUNDLE]) fs.rmSync(f, { force: true });
  });