Skip to content
Python
7.2 KB
Raw Blame
import { dockerObjectAbsent } from "./contract.mjs";
// Each fault case owns a fresh database and Linux process tree. A failed case
// cannot leave actors, catalog state or signal handlers in the next case.
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { mkdir, writeFile, unlink } from "node:fs/promises";
import { join, resolve } from "node:path";
import { randomBytes, randomUUID } from "node:crypto";
const exec = promisify(execFile);
const [directory, image, postgres] = process.argv.slice(2);
assert.match(image ?? "", /^sha256:[a-f0-9]{64}$/);
assert.match(postgres ?? "", /^postgres:16-alpine@sha256:[a-f0-9]{64}$/);
const root = resolve(directory, "worker-cases");
await mkdir(root, { mode: 0o700 });
let stopping = false,
  active;
process.once("SIGTERM", () => {
  stopping = true;
  active?.kill("SIGTERM");
});
const docker = async (args, timeout = 120000) => {
  const pending = exec("docker", args, { timeout, maxBuffer: 4194304 });
  active = pending.child;
  try {
    return await pending;
  } finally {
    active = null;
  }
};
const inspect = async (name) => {
  try {
    return JSON.parse((await docker(["inspect", name])).stdout)[0];
  } catch (e) {
    if (dockerObjectAbsent(e)) return null;
    throw e; // An unavailable daemon is not evidence of deletion.
  }
};
const listed = await docker([
  "run",
  "--pull=never",
  "--rm",
  "--network",
  "none",
  "--entrypoint",
  "/out/worker-recovery",
  image,
  "--list",
  "--ignored",
]);
const cases = listed.stdout
  .split("\n")
  .filter((s) => s.endsWith(": test"))
  .map((s) => s.slice(0, -6))
  .filter(
    (s) =>
      !s.includes(
        "mixed_versions_keep_native_identity_pinned_and_old_pool_can_resume",
      ) && !s.includes("bounded_load_matrix"),
  );
assert.equal(
  cases.length,
  15,
  "Review the worker catalog before changing fault coverage",
);
assert(
  cases.includes(
    "resilience_streaming_worker_death_preserves_budget_and_cursor_without_tool_replay",
  ),
);
const report = {
  passed: false,
  cases: [],
  cleanup: {
    allOwnedContainersRemoved: true,
    allGeneratedSecretsRemoved: true,
  },
};
try {
  for (const [index, test] of cases.entries()) {
    assert(
      !stopping,
      "Qualification cancelled; no new fixture will be created",
    );
    const dir = join(root, String(index).padStart(2, "0"));
    await mkdir(dir, { mode: 0o700 });
    const id = randomUUID().replaceAll("-", ""),
      pg = "ow-resilience-pg-case-" + id,
      worker = "ow-resilience-worker-case-" + id;
    const label = "openworker.resilience",
      password = randomBytes(24).toString("hex");
    const secret = join(dir, "password"),
      env = join(dir, "worker-env");
    const row = { name: test, passed: false, cleanup: false };
    report.cases.push(row);
    await writeFile(secret, password, { mode: 0o600, flag: "wx" });
    await writeFile(
      env,
      `OW_PG_TEST_URL=postgresql://openworker:${password}@127.0.0.1:5432/execution\n`,
      { mode: 0o600, flag: "wx" },
    );
    await writeFile(
      join(dir, "intent.json"),
      JSON.stringify({ id, pg, worker, image, postgres }),
      { mode: 0o600, flag: "wx" },
    );
    try {
      await docker([
        "run",
        "--pull=never",
        "--rm",
        "-d",
        "--name",
        pg,
        "--label",
        `${label}=${id}`,
        "--memory",
        "1g",
        "--tmpfs",
        "/var/lib/postgresql/data:rw,size=1g",
        "-e",
        "POSTGRES_USER=openworker",
        "-e",
        "POSTGRES_DB=execution",
        "-e",
        "POSTGRES_PASSWORD_FILE=/run/secrets/password",
        "--mount",
        `type=bind,source=${secret},target=/run/secrets/password,readonly`,
        postgres,
        "-c",
        "shared_buffers=32MB",
      ]);
      let ready = false;
      for (let n = 0; n < 120; n++) {
        assert(!stopping, "Qualification cancelled before worker admission");
        try {
          await docker([
            "exec",
            pg,
            "pg_isready",
            "-h",
            "127.0.0.1",
            "-U",
            "openworker",
            "-d",
            "execution",
          ]);
          ready = true;
          break;
        } catch {
          await new Promise((r) => setTimeout(r, 250));
        }
      }
      assert(ready, "Owned PostgreSQL listener did not become ready");
      assert(!stopping, "Qualification cancelled before worker admission");
      let output,
        code = 0;
      try {
        output = await docker(
          [
            "run",
            "--pull=never",
            "--rm",
            "--name",
            worker,
            "--label",
            `${label}=${id}`,
            "--network",
            `container:${pg}`,
            "--memory",
            "2g",
            "--env-file",
            env,
            "--entrypoint",
            "/out/worker-recovery",
            image,
            "--ignored",
            "--exact",
            test,
            "--nocapture",
          ],
          180000,
        );
      } catch (e) {
        code = e.code;
        output = { stdout: e.stdout ?? "", stderr: e.stderr ?? "" };
      }
      const log = output.stdout + output.stderr;
      await writeFile(join(dir, "worker.log"), log, {
        mode: 0o600,
        flag: "wx",
      });
      // Preserve Rust's count/name evidence for the parent runner, without keys.
      process.stdout.write(log.replaceAll(password, "[redacted]"));
      row.passed = code === 0 && /test result: ok\. 1 passed/.test(log);
    } finally {
      try {
        const pgState = await inspect(pg);
        if (pgState) {
          assert.equal(pgState.Config.Labels[label], id);
          const log = await docker(["logs", pg]);
          await writeFile(join(dir, "postgres.log"), log.stdout + log.stderr, {
            mode: 0o600,
            flag: "wx",
          });
          await writeFile(
            join(dir, "postgres-state.json"),
            JSON.stringify({
              state: pgState.State,
              memoryLimit: pgState.HostConfig.Memory,
            }),
            { mode: 0o600, flag: "wx" },
          );
          row.diagnosticsComplete = true;
        }
      } catch {
        row.diagnosticsComplete = false;
      } // Never skip owned cleanup because log capture failed.
      for (const name of [worker, pg]) {
        const v = await inspect(name);
        if (v) {
          assert.equal(v.Config.Labels[label], id);
          await docker(["rm", "-f", v.Id]);
        }
        assert.equal(await inspect(name), null);
      }
      await Promise.all([secret, env].map((p) => unlink(p)));
      row.cleanup = true;
    }
    assert(
      row.diagnosticsComplete,
      "Owned database diagnostics unavailable; inspect private fixture intent",
    );
    assert(
      row.passed,
      `Isolated worker fault case failed: ${test}; inspect private evidence`,
    );
  }
  report.passed = true;
} finally {
  report.cleanup.allOwnedContainersRemoved = report.cases.every(
    (c) => c.cleanup,
  );
  report.cleanup.allGeneratedSecretsRemoved = report.cases.every(
    (c) => c.cleanup,
  );
  await writeFile(join(root, "report.json"), JSON.stringify(report, null, 2), {
    mode: 0o600,
    flag: "wx",
  });
}