Python
// Node 20+, no npm dependencies. Owns native child handles, never arbitrary PIDs.
import { mkdir, readFile, writeFile, open, stat } from "node:fs/promises";
import { resolve, dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { randomBytes, randomUUID } from "node:crypto";
import { spawn } from "node:child_process";
import { once } from "node:events";
import net from "node:net";
import { write as writeOperations } from "./operations.mjs";
const here = dirname(fileURLToPath(import.meta.url));
export const repo = resolve(here, "../..");
export const workerLifecycle = Object.freeze({
drainSeconds: 300,
cleanupSeconds: 45,
marginSeconds: 15,
});
export const stopBudgetMs = (role) =>
role.startsWith("worker-")
? (workerLifecycle.drainSeconds +
workerLifecycle.cleanupSeconds +
workerLifecycle.marginSeconds) *
1000
: 35000;
const secret = () => randomBytes(32).toString("hex");
export const json = async (p) => JSON.parse(await readFile(p, "utf8"));
export const delay = (ms) => new Promise((r) => setTimeout(r, ms));
export async function command(program, args, options = {}) {
const child = spawn(program, args, {
stdio: ["ignore", "pipe", "pipe"],
...options,
});
let out = "",
error = "";
child.stdout?.on("data", (b) => {
if (out.length < 262144) out += b;
});
child.stderr?.on("data", (b) => {
if (error.length < 262144) error += b;
});
const [code] = await once(child, "close");
if (code !== 0)
throw Error(
`${program} failed (${code}); inspect the private service logs. ${options.safeOutput ? error : ""}`,
);
return out;
}
export async function init(root, connection, observability = true, offset = 0) {
root = resolve(root);
if (/[\r\n$#]/.test(root)) throw Error("Choose a simple state path");
if (!Number.isInteger(offset) || offset < 0 || offset > 7000)
throw Error("Port offset must be 0–7000");
const gateway = await json(resolve(connection));
if (!gateway.runner || !gateway.openshell)
throw Error("Connection file needs runner and openshell");
await mkdir(root, { mode: 0o700 }); // Refuse to overwrite an existing deployment.
await mkdir(join(root, "private"), { mode: 0o700 });
await mkdir(join(root, "logs"), { mode: 0o700 });
const ports = {
pg: 55461,
s3: 58333,
otlp: 54318,
grafana: 53000,
apiA: 58181,
apiB: 58182,
console: 54340,
};
for (const name of Object.keys(ports)) ports[name] += offset;
const id = randomUUID().replaceAll("-", "");
const manifest = {
version: 1,
project: `owref-${id.slice(0, 12)}`,
id,
ports,
observability,
observer_request_id: "local-console-observer",
};
const write = (name, data) =>
writeFile(join(root, name), data, { mode: 0o600, flag: "wx" });
const pg = secret(),
access = secret().slice(0, 20),
key = secret();
await write("private/postgres-password", pg);
await write("private/api-token", secret());
await write("private/observer-token", secret());
await write(
"private/observer-spec.json",
JSON.stringify({
principal: "console-observer",
role: "observer",
scope: { kind: "all" },
}),
);
await write("private/console-key", secret());
await write("private/grafana-password", secret());
await write(
"private/s3.json",
JSON.stringify({
identities: [
{
name: "local-app",
credentials: [{ accessKey: access, secretKey: key }],
actions: [
"Read:openworker-artifacts",
"Write:openworker-artifacts",
"List:openworker-artifacts",
],
},
],
}),
);
await write(
"private/runtime-env.json",
JSON.stringify({
OW_SERVER_DATABASE_URL: `postgresql://openworker:${pg}@127.0.0.1:${ports.pg}/openworker`,
AWS_ACCESS_KEY_ID: access,
AWS_SECRET_ACCESS_KEY: key,
}),
);
const config = {
database_env: "OW_SERVER_DATABASE_URL",
schema: "ow_reference",
database_loopback_plaintext: true,
state_root: join(root, "controllers"),
bundle: join(repo, "harness/examples/assistant"),
runner: resolve(gateway.runner),
model: "fixture",
scripted: true,
api_token_file: join(root, "private/api-token"),
actor: "local-operator",
openshell: gateway.openshell,
artifacts: {
kind: "s3",
deployment: `local-${id}`,
bucket: "openworker-artifacts",
region: "us-east-1",
endpoint: `http://127.0.0.1:${ports.s3}`,
},
telemetry: { endpoint: `http://127.0.0.1:${ports.otlp}`, deployment: id },
operational_metrics: {},
};
if (!observability) {
delete config.telemetry;
delete config.operational_metrics;
}
await writeOperations(root, { deployment: id, expectedSlots: 2 });
await write("reference.json", JSON.stringify(config, null, 2));
await write(
"console.json",
JSON.stringify(
{
port: ports.console,
apis: [ports.apiA, ports.apiB].map((p) => `http://127.0.0.1:${p}`),
api_token_file: join(root, "private/observer-token"),
access_key_file: join(root, "private/console-key"),
grafana_url: observability
? `http://127.0.0.1:${ports.grafana}/d/openworker-harness`
: null,
},
null,
2,
),
);
await write(
"compose.env",
`STATE_DIR=${root}\nPG_PORT=${ports.pg}\nS3_PORT=${ports.s3}\nOTLP_PORT=${ports.otlp}\nGRAFANA_PORT=${ports.grafana}\n`,
);
await write("deployment.json", JSON.stringify(manifest, null, 2));
return manifest;
}
export async function load(root) {
root = resolve(root);
const m = await json(join(root, "deployment.json"));
if (m.version !== 1 || !/^owref-[a-f0-9]{12}$/.test(m.project))
throw Error("Not a reference deployment");
if ((await stat(join(root, "private"))).mode & 0o077)
throw Error("Private state directory must be mode 0700");
return { ...m, root };
}
export async function compose(m, args) {
const endpoint =
process.env.DOCKER_HOST ||
(
await command("docker", [
"context",
"inspect",
"--format",
"{{.Endpoints.docker.Host}}",
])
).trim();
if (!endpoint.startsWith("unix://"))
throw Error(
"This reference requires a local Unix-socket Docker daemon; remote contexts are out of scope",
);
return command(
"docker",
[
"compose",
"--project-name",
m.project,
"--env-file",
join(m.root, "compose.env"),
"-f",
join(here, "compose.yaml"),
...(m.observability ? ["--profile", "observability"] : []),
...args,
],
{ safeOutput: true },
);
}
export async function up(m) {
await compose(m, ["up", "-d", "--wait", "--wait-timeout", "120"]);
await until(
async () =>
(
await fetch(`http://127.0.0.1:${m.ports.s3}/openworker-artifacts`, {
signal: AbortSignal.timeout(2000),
})
).status === 403,
);
if (m.observability)
await until(
async () =>
(
await fetch(`http://127.0.0.1:${m.ports.grafana}/api/health`, {
signal: AbortSignal.timeout(2000),
})
).ok,
);
}
export async function until(fn, ms = 60000, signal) {
const end = Date.now() + ms;
while (Date.now() < end) {
signal?.throwIfAborted();
try {
const r = await fn();
if (r) {
signal?.throwIfAborted();
return r;
}
} catch {}
await delay(300);
}
throw Error("Readiness deadline exceeded");
}
async function available(port) {
const s = net.createServer();
s.listen(port, "127.0.0.1");
await once(s, "listening");
await new Promise((r) => s.close(r));
}
async function provisionAuthorization(m, bin, config, env, upgrade = false) {
if (upgrade) {
// Explicit upgrade only: preserve all existing credentials, including revoked keys.
for (const [name, value] of [
["observer-token", secret()],
[
"observer-spec.json",
JSON.stringify({
principal: "console-observer",
role: "observer",
scope: { kind: "all" },
}),
],
]) {
try {
await writeFile(join(m.root, "private", name), value, {
mode: 0o600,
flag: "wx",
});
} catch (e) {
if (e.code !== "EEXIST") throw e;
}
}
}
await command(bin, ["--config", config, "auth-bootstrap"], { env });
const observer = JSON.parse(
await command(
bin,
[
"--config",
config,
"auth-issue",
"--spec",
join(m.root, "private/observer-spec.json"),
"--token-file",
join(m.root, "private/observer-token"),
"--request-id",
m.observer_request_id || "local-console-observer",
],
{ env },
),
);
if (observer.revoked)
throw Error("Console credential revoked; explicitly issue a replacement");
if (upgrade) {
const path = join(m.root, "console.json"),
c = await json(path);
c.api_token_file = join(m.root, "private/observer-token");
await writeFile(path, JSON.stringify(c, null, 2), { mode: 0o600 });
}
}
export async function launch(m, signal) {
const bin =
process.env.OW_SERVER_BINARY ||
join(repo, "harness/target/debug/openworker-server-reference");
const env = {
...process.env,
...(await json(join(m.root, "private/runtime-env.json"))),
};
// Never forward ambient cloud credential/config overrides into this reference.
for (const k of Object.keys(env))
if (
k.startsWith("AWS_") &&
!["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"].includes(k)
)
delete env[k];
const config = join(m.root, "reference.json"),
children = new Map();
const token = (
await readFile(join(m.root, "private/api-token"), "utf8")
).trim();
for (const p of [m.ports.apiA, m.ports.apiB, m.ports.console])
await available(p);
await command(bin, ["--config", config, "migrate"], { env });
await provisionAuthorization(m, bin, config, env);
async function start(role) {
signal?.throwIfAborted();
if (!["api-a", "api-b", "worker-a", "worker-b", "console"].includes(role))
throw Error("Unknown native role");
if (children.has(role)) throw Error("Role already running");
const api = role.startsWith("api");
const args =
role === "console"
? [
join(repo, "harness/console/server.mjs"),
join(m.root, "console.json"),
]
: [
"--config",
config,
api ? "api" : "worker",
"--node",
role,
...(api
? [
"--listen",
`127.0.0.1:${role === "api-a" ? m.ports.apiA : m.ports.apiB}`,
]
: [
"--slots",
"1",
"--drain-timeout-seconds",
String(workerLifecycle.drainSeconds),
"--shutdown-timeout-seconds",
String(workerLifecycle.cleanupSeconds),
]),
];
const log = await open(join(m.root, "logs", `${role}.log`), "a", 0o600);
if (signal?.aborted) {
await log.close();
signal.throwIfAborted();
}
const childEnv =
role === "console"
? Object.fromEntries(
["PATH", "LANG", "LC_ALL", "TZ"]
.filter((k) => process.env[k])
.map((k) => [k, process.env[k]]),
)
: { ...env };
if (!api)
for (const k of Object.keys(childEnv))
if (k.startsWith("AWS_")) delete childEnv[k];
const child = spawn(role === "console" ? process.execPath : bin, args, {
env: childEnv,
stdio: ["ignore", log.fd, log.fd],
});
await log.close();
children.set(role, child);
child.on("error", () => {});
if (api)
await until(
async () =>
(
await fetch(
`http://127.0.0.1:${role === "api-a" ? m.ports.apiA : m.ports.apiB}/readyz`,
{
headers: { Authorization: `Bearer ${token}` },
signal: AbortSignal.timeout(2000),
},
)
).ok,
20000,
signal,
);
return child;
}
async function stop(role, signal = "SIGTERM") {
const child = children.get(role);
if (!child) return;
children.delete(role);
if (child.exitCode !== null || child.signalCode) return;
const done = once(child, "exit");
child.kill(signal);
const timer = setTimeout(() => child.kill("SIGKILL"), stopBudgetMs(role));
try {
await done;
} finally {
clearTimeout(timer);
}
}
async function close() {
await Promise.all([...children.keys()].map((r) => stop(r)));
}
try {
for (const role of ["api-a", "api-b", "worker-a", "worker-b", "console"])
await start(role);
await until(
async () => {
for (const p of [m.ports.apiA, m.ports.apiB])
if (
!(
await fetch(`http://127.0.0.1:${p}/readyz`, {
headers: { Authorization: `Bearer ${token}` },
signal: AbortSignal.timeout(2000),
})
).ok
)
return false;
return true;
},
60000,
signal,
);
await until(
async () =>
(
await fetch(`http://127.0.0.1:${m.ports.console}/`, {
signal: AbortSignal.timeout(2000),
})
).ok,
60000,
signal,
);
} catch (e) {
await close();
throw e;
}
return { children, start, stop, close, token, env, bin, config };
}
if (
process.argv[1] &&
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
) {
const [action, root, connection] = process.argv.slice(2);
if (
!root ||
!["init", "up", "run", "down", "status", "authorize"].includes(action)
)
throw Error(
"Usage: node stack.mjs init STATE CONNECTION.json | up/run/down/status/authorize STATE",
);
if (action === "init") {
const n = process.argv.indexOf("--port-offset");
await init(
root,
connection,
!process.argv.includes("--no-observability"),
n < 0 ? 0 : Number(process.argv[n + 1]),
);
console.log("Private configuration created. Run up, then run.");
} else {
const m = await load(root);
if (action === "authorize") {
const bin =
process.env.OW_SERVER_BINARY ||
join(repo, "harness/target/debug/openworker-server-reference"),
config = join(m.root, "reference.json"),
env = {
...process.env,
...(await json(join(m.root, "private/runtime-env.json"))),
};
await command(bin, ["--config", config, "migrate"], { env });
await provisionAuthorization(m, bin, config, env, true);
console.log(
"Authorization provisioned; console uses an observer. No saved sessions or credentials deleted.",
);
}
if (action === "up") {
await up(m);
console.log("Backing services started.");
}
if (action === "down") {
await compose(m, ["down"]);
console.log(
"Backing services stopped; named volumes and private state retained.",
);
}
if (action === "status") console.log(await compose(m, ["ps"]));
if (action === "run") {
const startup = new AbortController();
const interrupted = Promise.race([
once(process, "SIGINT"),
once(process, "SIGTERM"),
]);
interrupted.then(() => startup.abort());
const running = await launch(m, startup.signal);
console.log(
`Console: http://127.0.0.1:${m.ports.console}/\nAccess key file: ${join(m.root, "private/console-key")}${m.observability ? `\nGrafana: http://127.0.0.1:${m.ports.grafana}/ (admin; password in private/grafana-password)` : ""}\nCtrl-C stops native children; backing services/data remain.`,
);
await interrupted;
await running.close();
}
}
}