Skip to content
Python
6.1 KB
Raw Blame
// Explicit early teardown of the approved disposable experiment, using the same
// cloud-side guard as expiry. No wildcard account cleanup and no target override.
import { readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { connect, account } from "./client.mjs";
import { name } from "./templates.mjs";
import { repo, kubectl } from "../helm/local.mjs";
const root = join(repo, ".ow-aws"),
  { aws } = await connect(),
  action = process.argv[2];
const stack = async (target) => {
  try {
    return (
      await aws(["cloudformation", "describe-stacks", "--stack-name", target])
    ).Stacks[0];
  } catch (e) {
    if (String(e.stderr).includes("does not exist")) return null;
    throw e;
  }
};
const own = (s) => {
  if (s && !s.Tags?.some((t) => t.Key === "Project" && t.Value === name))
    throw Error("Refusing unowned stack");
};
if (action === "begin" || action === "abort") {
  const main = await stack(name),
    guard = await stack(name + "-guard");
  own(main);
  own(guard);
  if (
    !main ||
    (guard?.StackStatus !== "UPDATE_COMPLETE" &&
      guard?.StackStatus !== "CREATE_COMPLETE")
  )
    throw Error("Expected live experiment and ready cleanup guard");
  // Failure must never trap paid resources behind a success-only gate.
  // abort is explicit, same exact scope/guard, and records no qualification claim.
  if (action === "begin") {
    const proof = JSON.parse(
      await readFile(join(root, "current-qualification.json"), "utf8"),
    );
    if (proof.status !== "completed")
      throw Error(
        "Save qualification result before normal teardown; use abort after failure",
      );
  }
  let workloadsStopped = false;
  try {
    const m = JSON.parse(await readFile(join(root, "kubernetes.json"), "utf8"));
    if (m.context !== name) throw Error("Wrong cluster");
    const ns = JSON.parse(
      await kubectl(m, ["get", "namespace", "ow-app", "-o", "json"]),
    );
    if (ns.metadata.labels?.["openworker.dev/experiment"] !== name)
      throw Error("Unowned namespace");
    await kubectl(m, [
      "-n",
      "ow-app",
      "scale",
      "deployment",
      "-l",
      "app.kubernetes.io/name=openworker,app.kubernetes.io/component=worker",
      "--replicas=0",
    ]);
    await kubectl(m, [
      "-n",
      "ow-app",
      "delete",
      "secret",
      "openai-model",
      "--ignore-not-found=true",
    ]);
    workloadsStopped = true;
  } catch (error) {
    if (action !== "abort") throw error;
    console.log(
      "Workload cleanup unavailable; owned infrastructure teardown will remove it. No successful qualification claimed.",
    );
  }
  const expiry = new Date(Date.now() - 1000).toISOString();
  await aws([
    "lambda",
    "update-function-configuration",
    "--function-name",
    name + "-cleanup",
    "--environment",
    JSON.stringify({
      Variables: {
        EXPERIMENT: name,
        ACCOUNT: account,
        EXPIRY: expiry,
        BUCKET: `${name}-${account}`,
      },
    }),
  ]);
  await writeFile(
    join(root, "teardown-start.json"),
    JSON.stringify(
      {
        started: new Date().toISOString(),
        mainStack: main.StackId,
        guardStack: guard.StackId,
        providerSecretRemoved: workloadsStopped,
        aborted: action === "abort",
      },
      null,
      2,
    ),
    { mode: 0o600 },
  );
  console.log(
    "Owned cloud-side cleanup deadline advanced to now. Invoke after the function update finishes; workload cleanup status is recorded privately.",
  );
} else if (action === "invoke") {
  const guard = await stack(name + "-guard");
  own(guard);
  if (!guard) throw Error("No cleanup guard");
  const config = await aws([
    "lambda",
    "get-function-configuration",
    "--function-name",
    name + "-cleanup",
  ]);
  if (
    config.LastUpdateStatus !== "Successful" ||
    Date.parse(config.Environment.Variables.EXPIRY) > Date.now()
  )
    throw Error("Guard update not ready or deadline not reached");
  const output = join(root, "cleanup-invocation.json");
  const result = await aws([
    "lambda",
    "invoke",
    "--function-name",
    name + "-cleanup",
    output,
  ]);
  if (result.FunctionError) {
    console.log(await readFile(output, "utf8"));
    throw Error("Cleanup invocation failed");
  }
  console.log(await readFile(output, "utf8"));
} else if (action === "verify") {
  const main = await stack(name);
  own(main);
  const volumes = await aws([
    "ec2",
    "describe-volumes",
    "--filters",
    `Name=tag:Project,Values=${name}`,
    "--query",
    "Volumes[].{id:VolumeId,state:State}",
  ]);
  const instances = await aws([
    "ec2",
    "describe-instances",
    "--filters",
    `Name=tag:Project,Values=${name}`,
    "--query",
    "Reservations[].Instances[].{id:InstanceId,state:State.Name}",
  ]);
  const result = {
    time: new Date().toISOString(),
    stack: main?.StackStatus || "absent",
    volumes,
    instances,
    complete:
      !main &&
      volumes.length === 0 &&
      instances.every((i) => i.state === "terminated"),
  };
  await writeFile(
    join(root, "teardown-verification.json"),
    JSON.stringify(result, null, 2),
    { mode: 0o600 },
  );
  console.log(JSON.stringify(result));
  if (main?.StackStatus === "DELETE_FAILED")
    console.log(
      JSON.stringify(
        await aws([
          "cloudformation",
          "describe-stack-events",
          "--stack-name",
          name,
          "--query",
          "StackEvents[?ResourceStatus==`DELETE_FAILED`].{resource:LogicalResourceId,reason:ResourceStatusReason}",
        ]),
      ),
    );
} else if (action === "remove-guard") {
  const main = await stack(name);
  if (main) throw Error("Main stack remains");
  const volumes = await aws([
    "ec2",
    "describe-volumes",
    "--filters",
    `Name=tag:Project,Values=${name}`,
  ]);
  if (volumes.Volumes.length) throw Error("Owned volumes remain");
  const guard = await stack(name + "-guard");
  own(guard);
  if (guard)
    await aws([
      "cloudformation",
      "delete-stack",
      "--stack-name",
      guard.StackId,
    ]);
  console.log(
    "Owned guard stack removal requested after main resource verification.",
  );
} else throw Error("Use begin | abort | invoke | verify | remove-guard");