Skip to content
Python
30.2 KB
Raw Blame
// Real local Kubernetes + OpenShell, deterministic model, production service SDK.
// No tool requests are retried after uncertain delivery. Originals are inspected.
import assert from "node:assert/strict";
import { spawn, execFile } from "node:child_process";
import { promisify } from "node:util";
import { readFile, writeFile } from "node:fs/promises";
import { resolve, join } from "node:path";
import { createRequire } from "node:module";
import { Harness } from "../../harness/typescript-sdk/dist/index.js";
import { start as startUI } from "../../harness/service-ui/server.mjs";
import { run as command } from "../service/stack.mjs";
import { ownedLocalFixture, cursorTail, noReplay } from "./contract.mjs";
import { collectorRecords, attributes } from "../reference/export-contract.mjs";
const [directory, confirmation, output] = process.argv.slice(2),
  root = resolve(output),
  state = resolve(directory);
const m = JSON.parse(await readFile(join(state, "kubernetes.json")));
const kub = (args, input) =>
  command(
    "kubectl",
    ["--kubeconfig", m.kubeconfig, "--context", m.context, ...args],
    { input, timeout: 240000 },
  );
const kube = (args, input) => kub(["-n", m.namespace, ...args], input);
const json = async (args) => JSON.parse(await kube([...args, "-o", "json"]));
ownedLocalFixture(
  m,
  JSON.parse(await kub(["get", "namespace", m.namespace, "-o", "json"])),
  confirmation,
);
const installedConfig = JSON.parse(
  (await json(["get", "configmap", m.release + "-config"])).data["config.json"],
);
assert.equal(
  installedConfig.scripted,
  true,
  "Actual installed service must use the deterministic model",
);
assert.equal(installedConfig.service.id, m.id);
for (const role of ["api", "worker"]) {
  const d = await json(["get", "deployment", m.release + "-" + role]);
  assert.equal(
    d.spec.template.spec.containers.find((c) => c.name === "harness").image,
    m.imageRef,
    "Qualify the exact declared service image",
  );
}
const save = (name, v) =>
  writeFile(join(root, name + ".json"), JSON.stringify(v, null, 2) + "\n", {
    mode: 0o600,
    flag: "wx",
  });
const report = {
  version: 1,
  passed: false,
  checks: [],
  cleanup: {},
  fixture: {
    serviceId: m.id,
    context: m.context,
    namespace: m.namespace,
    serviceImage: m.imageRef,
    runnerImage: m.runnerImage,
  },
};
await writeFile(
  join(state, "resilience-qualification.json"),
  JSON.stringify({ ...report.fixture, output: root }),
  { mode: 0o600, flag: "wx" },
);
// One destructive qualification per prepared service. Failure requires inspecting
// the saved original operation, not re-running this program against uncertain work.
await save("cluster-intent", report.fixture);
const pause = (ms) => new Promise((r) => setTimeout(r, ms));
async function eventually(f, description, ms = 150000) {
  const end = Date.now() + ms;
  let last;
  while (Date.now() < end) {
    try {
      const v = await f();
      if (v) return v;
    } catch (e) {
      if (e.name === "AssertionError") throw e;
      last = e;
    }
    await pause(300);
  }
  throw Error(
    `Timed out: ${description}${last ? " (inspect scoped logs)" : ""}`,
  );
}
const checks = (name, details = {}) => {
  report.checks.push({ name, ...details });
  console.log("PASS " + name);
};
let h, other, ui, browser;
const children = [],
  sessions = [];
const token = String(await readFile(join(state, "input/admin-token"))).trim();
const credentials = { getToken: async () => token };
const readerImage = m.runnerImage;
const pins = JSON.parse(
  await readFile(
    new URL("../reference/export-qualification-images.json", import.meta.url),
  ),
);
// Test-only central collector with a separate read-only observer of its output.
// All worker OTLP requests use an uncredentialed loopback relay, never HTTPS off.
async function collector() {
  const config = `receivers:\n  otlp:\n    protocols:\n      http:\n        endpoint: 0.0.0.0:4318\nprocessors:\n  batch:\n    timeout: 100ms\nexporters:\n  file:\n    path: /out/traces.jsonl\n    format: json\n    flush_interval: 100ms\nservice:\n  telemetry:\n    logs:\n      level: warn\n  pipelines:\n    traces:\n      receivers: [otlp]\n      processors: [batch]\n      exporters: [file]\n`;
  const items = [
    {
      apiVersion: "v1",
      kind: "ConfigMap",
      metadata: { name: "resilience-collector" },
      data: { "collector.yaml": config },
    },
    {
      apiVersion: "v1",
      kind: "Service",
      metadata: { name: "resilience-collector" },
      spec: {
        selector: { app: "resilience-collector" },
        ports: [{ port: 4318 }],
      },
    },
    {
      apiVersion: "apps/v1",
      kind: "Deployment",
      metadata: { name: "resilience-collector" },
      spec: {
        replicas: 1,
        selector: { matchLabels: { app: "resilience-collector" } },
        template: {
          metadata: { labels: { app: "resilience-collector" } },
          spec: {
            automountServiceAccountToken: false,
            securityContext: { fsGroup: 1000 },
            containers: [
              {
                name: "collector",
                image: pins.collector,
                args: ["--config=/config/collector.yaml"],
                securityContext: {
                  runAsUser: 1000,
                  runAsGroup: 1000,
                  allowPrivilegeEscalation: false,
                  readOnlyRootFilesystem: true,
                },
                resources: {
                  requests: { memory: "64Mi", cpu: "20m" },
                  limits: { memory: "128Mi", cpu: "500m" },
                },
                volumeMounts: [
                  { name: "config", mountPath: "/config", readOnly: true },
                  { name: "out", mountPath: "/out" },
                ],
              },
              {
                name: "reader",
                image: readerImage,
                command: ["sleep", "infinity"],
                securityContext: {
                  runAsUser: 1000,
                  runAsGroup: 1000,
                  allowPrivilegeEscalation: false,
                  readOnlyRootFilesystem: true,
                },
                resources: {
                  requests: { memory: "8Mi", cpu: "1m" },
                  limits: { memory: "32Mi", cpu: "100m" },
                },
                volumeMounts: [
                  { name: "out", mountPath: "/out", readOnly: true },
                ],
              },
            ],
            volumes: [
              { name: "config", configMap: { name: "resilience-collector" } },
              { name: "out", emptyDir: { sizeLimit: "32Mi" } },
            ],
          },
        },
      },
    },
  ];
  await kube(
    ["apply", "-f", "-"],
    JSON.stringify({ apiVersion: "v1", kind: "List", items }),
  );
  await kube([
    "rollout",
    "status",
    "deployment/resilience-collector",
    "--timeout=120s",
  ]);
  const relay = `const h=require('http');h.createServer((q,s)=>{const r=h.request({host:'resilience-collector',port:4318,path:q.url,method:q.method,headers:q.headers},v=>{s.writeHead(v.statusCode,v.headers);v.pipe(s)});r.on('error',()=>{if(!s.headersSent)s.writeHead(503);s.end()});q.pipe(r)}).listen(4318,'127.0.0.1')`;
  for (const role of ["worker", "api"]) {
    const patch = {
      spec: {
        template: {
          spec: {
            containers: [
              {
                name: "resilience-relay",
                image: readerImage,
                command: ["node", "-e", relay],
                securityContext: {
                  runAsUser: 1000,
                  runAsGroup: 1000,
                  allowPrivilegeEscalation: false,
                  readOnlyRootFilesystem: true,
                },
                resources: {
                  requests: { memory: "24Mi", cpu: "10m" },
                  limits: { memory: "64Mi", cpu: "200m" },
                },
              },
            ],
          },
        },
      },
    };
    await kube([
      "patch",
      "deployment",
      m.release + "-" + role,
      "--type=strategic",
      "-p",
      JSON.stringify(patch),
    ]);
  }
  await kube(["scale", "deployment/" + m.release + "-api", "--replicas=2"]);
  await kube([
    "rollout",
    "status",
    "deployment/" + m.release + "-worker",
    "--timeout=180s",
  ]);
  await kube([
    "rollout",
    "status",
    "deployment/" + m.release + "-api",
    "--timeout=180s",
  ]);
}
async function forward(pod, port) {
  const p = spawn(
    "kubectl",
    [
      "--kubeconfig",
      m.kubeconfig,
      "--context",
      m.context,
      "-n",
      m.namespace,
      "port-forward",
      "pod/" + pod,
      `${port}:8443`,
      "--address",
      "127.0.0.1",
    ],
    { stdio: ["ignore", "ignore", "ignore"] },
  );
  children.push(p);
  const url = `https://localhost:${port}`;
  return eventually(
    async () => {
      try {
        return await Harness.connect({ url, namespace: "local", credentials });
      } catch {
        return false;
      }
    },
    "authenticated API forward",
    30000,
  );
}
const events = async (s, after = 0) =>
  (await h.json("GET", `/v1/sessions/${s.id}/events/page?after=${after}`))
    .events;
async function terminal(run, expected = "completed") {
  return eventually(
    async () => {
      const r = await run.snapshot();
      if (r.status === "running") return false;
      assert.equal(r.status, expected);
      return r;
    },
    "terminal run " + expected,
    180000,
  );
}
function plan(steps) {
  return (
    "TEST_RESOURCE_GRANTS " +
    JSON.stringify(steps.map(([name, args]) => ({ name, arguments: args })))
  );
}
async function send(s, label, steps, expected = "completed") {
  await save(label + "-intent", {
    sessionId: s.id,
    text: typeof steps === "string" ? steps : plan(steps),
  });
  const r = await s.send(typeof steps === "string" ? steps : plan(steps), {
    idempotencyKey: label,
  });
  await save(label + "-run", { runId: r.id, submissionId: r.submissionId });
  await terminal(r, expected);
  const e = (await events(s)).filter((e) => e.turn_id === r.id);
  await save(label + "-events", e);
  return { run: r, events: e };
}
function result(e, tool) {
  const r = e
    .filter((e) => e.type === "tool_finished" && e.data.name === tool)
    .at(-1);
  assert(r, `Expected result for ${tool}`);
  const envelope = r.data.result;
  assert(
    envelope?.isError !== true && envelope?.denied !== true,
    `Tool ${tool} failed`,
  );
  const data = envelope?.content
    ? JSON.parse(envelope.content[0].text)
    : envelope;
  assert(
    data && data.isError !== true && data.executed !== false,
    `Tool ${tool} was not executed`,
  );
  return data;
}
async function killWorker() {
  const workers = (
    await json(["get", "pods", "-l", "app=" + m.release + "-worker"])
  ).items.filter(
    (p) => p.status.phase === "Running" && !p.metadata.deletionTimestamp,
  );
  assert.equal(
    workers.length,
    1,
    "Dedicated single-worker fault fixture required",
  );
  const p = workers[0];
  await save("fault-" + report.checks.length, {
    pod: p.metadata.name,
    uid: p.metadata.uid,
    action: "zero-grace deletion of owned worker pod",
  });
  // A zero-grace deletion forces an abrupt container stop from the kubelet;
  // do not rely on an unprivileged process being able to kill namespace PID 1.
  // Issue once, then reconcile pod identity and readiness without repeating it.
  await kube([
    "delete",
    "pod",
    p.metadata.name,
    "--grace-period=0",
    "--force",
    "--wait=false",
  ]);
  await eventually(async () => {
    const pods = (
      await json(["get", "pods", "-l", "app=" + m.release + "-worker"])
    ).items;
    return (
      !pods.some((n) => n.metadata.uid === p.metadata.uid) &&
      pods.some(
        (n) =>
          !n.metadata.deletionTimestamp &&
          n.status.containerStatuses?.every((c) => c.ready),
      )
    );
  }, "worker pod replacement");
}
async function recording(s, run, required) {
  return eventually(
    async () => {
      const t = await h.json(
        "GET",
        `/v1/sessions/${s.id}/observability/${run.id}/timeline?limit=200`,
      );
      return required.every((op) =>
        t.page.steps.some((x) => x.step.operation === op),
      )
        ? t
        : false;
    },
    "recorded operations " + required.join(","),
    20000,
  );
}
async function collectWire() {
  // The shared installer helper intentionally caps stdout at 2 MB. Collector
  // output can exceed that during fault/recovery queries, so read the actual
  // bounded 32 MiB volume explicitly instead of silently parsing a prefix.
  const { stdout: raw } = await promisify(execFile)(
    "kubectl",
    [
      "--kubeconfig",
      m.kubeconfig,
      "--context",
      m.context,
      "-n",
      m.namespace,
      "exec",
      "deployment/resilience-collector",
      "-c",
      "reader",
      "--",
      "cat",
      "/out/traces.jsonl",
    ],
    { timeout: 30000, maxBuffer: 40 * 1024 * 1024 },
  );
  return collectorRecords(
    raw.trim().split("\n").filter(Boolean).map(JSON.parse),
  );
}
try {
  await collector();
  const apis = (
    await json(["get", "pods", "-l", "app=" + m.release + "-api"])
  ).items.filter(
    (p) => p.status.phase === "Running" && !p.metadata.deletionTimestamp,
  );
  assert.equal(apis.length, 2);
  h = await forward(apis[0].metadata.name, m.port);
  other = await forward(apis[1].metadata.name, m.port + 1);
  assert.equal(h.capabilities.serviceId, m.id);
  assert.equal(other.capabilities.serviceId, m.id);
  const baseline = JSON.parse(await kub(["get", "pods", "-A", "-o", "json"]))
    .items.filter(
      (p) =>
        p.metadata.namespace !== m.namespace &&
        (p.metadata.name.includes("--owr-") ||
          p.metadata.name.startsWith("os-supervisor-")),
    )
    .map((p) => p.metadata.uid);
  await save("baseline-environment-pods", baseline);
  await h.registerRuntimeProfile(
    {
      id: "resilience@1",
      provider: "openshell",
      cpu: 1,
      memoryMB: 512,
      sharing: "shared",
      approvalMode: "bypass",
      autoSkipQuestions: true,
    },
    { idempotencyKey: "resilience-profile" },
  );
  await h.activateRuntimeProfile("resilience@1", {
    alias: "resilience-runtime",
    idempotencyKey: "resilience-profile-activate",
  });
  await h.deployAgent(
    {
      id: "resilience@1",
      instructions:
        "Run the deterministic qualification plan. Inspect results; never replay uncertain effects.",
      tools: [
        "read_file",
        "write_file",
        "run_shell",
        "process_start",
        "process_status",
        "process_logs",
        "process_stop",
      ],
      runtimeProfiles: ["resilience@1"],
      defaultRuntimeProfile: "resilience@1",
    },
    { idempotencyKey: "resilience-release" },
  );
  await h.activateAgent("resilience@1", {
    alias: "resilience",
    idempotencyKey: "resilience-activate",
  });
  const agent = await h.agent("resilience");
  await save("coder-admission-intent", {
    idempotencyKey: "session-coder",
    agent: "resilience",
  });
  const a = await agent.newSession({
    project: "resilience",
    name: "Resilience coding",
    idempotencyKey: "session-coder",
    waitTimeoutMs: 180000,
  });
  sessions.push(a);
  await save("coder", a.reference());
  const b = await agent.newSession({
    environmentId: a.environmentId,
    project: "resilience",
    name: "Resilience verifier",
    idempotencyKey: "session-verifier",
    waitTimeoutMs: 180000,
  });
  sessions.push(b);
  assert.equal(b.environmentId, a.environmentId);
  await save("verifier", b.reference());
  const env = await h.json("GET", "/v1/environments/" + a.environmentId);
  await save("environment", env);
  const remote = env.environment.binding.remote_name;
  assert(remote.startsWith("owr-"));
  const outside = JSON.parse(
    await kub(["get", "pods", "-A", "-o", "json"]),
  ).items.filter((p) => p.metadata.name.endsWith("--" + remote));
  assert.equal(outside.length, 1);
  const sandboxPod = outside[0];
  const allEnvironmentPods = JSON.parse(
    await kub(["get", "pods", "-A", "-o", "json"]),
  )
    .items.filter(
      (p) =>
        p.metadata.uid === sandboxPod.metadata.uid ||
        p.metadata.name ===
          "os-supervisor-" + env.environment.binding.remote_id,
    )
    .map((p) => p.metadata.uid);
  assert.equal(
    allEnvironmentPods.length,
    2,
    "Observe both the original sandbox and its supervisor before testing deletion",
  );
  await save("original-environment-pods", allEnvironmentPods);
  // Genuine independent sessions, same environment. Deliberately cooperative
  // directories/ports; sharing is not per-session OS isolation.
  const start = async (s, label, port) =>
    send(s, "start-" + label, [
      [
        "run_shell",
        {
          command: `mkdir -p ${label}; printf ${label} > ${label}/index.txt`,
          yield_ms: 10000,
        },
      ],
      [
        "process_start",
        {
          command: `node -e 'const fs=require("fs"),h=require("http");h.createServer((q,s)=>s.end(fs.readFileSync("${label}/index.txt","utf8"))).listen(${port},"127.0.0.1")'`,
          timeout_seconds: 900,
        },
      ],
    ]);
  const [startedA, startedB] = await Promise.all([
    start(a, "coder", 4000),
    start(b, "verifier", 4001),
  ]);
  const pa = result(startedA.events, "process_start").process_id,
    pb = result(startedB.events, "process_start").process_id;
  assert(pa && pb && pa !== pb);
  await save("processes", { coder: pa, verifier: pb });
  await recording(a, startedA.run, ["process.started"]);
  await recording(b, startedB.run, ["process.started"]);
  const health = () =>
    send(b, "health-" + report.checks.length, [
      [
        "run_shell",
        {
          command:
            'test "$(curl -fsS --max-time 3 http://127.0.0.1:4000)" = coder && test "$(curl -fsS --max-time 3 http://127.0.0.1:4001)" = verifier && printf both-healthy',
          yield_ms: 10000,
        },
      ],
    ]);
  assert.equal(result((await health()).events, "run_shell").exit_code, 0);
  checks(
    "two sessions share environment with separate files, handles and ports",
  );
  const stream = await a.send("TEST_RESILIENCE_STREAM", {
    idempotencyKey: "stream-fault",
    limits: { maxModelCalls: 7, maxOutputTokens: 20000, summaryTokens: 512 },
  });
  await save("stream-run", {
    runId: stream.id,
    submissionId: stream.submissionId,
  });
  await eventually(
    async () =>
      (await events(a)).some(
        (e) => e.turn_id === stream.id && e.type === "assistant_delta",
      ),
    "partial stream barrier",
  );
  const before = await events(a),
    snapshot = await a.snapshot();
  const budget = snapshot.turn.budget;
  await recording(a, stream, ["model.requested"]);
  await killWorker();
  const rebound = await other.resume(a.reference());
  await terminal(await rebound.resumeRun(stream.submissionId), "interrupted");
  assert.deepEqual((await rebound.snapshot()).turn.budget, budget);
  const tail = (
    await other.json(
      "GET",
      `/v1/sessions/${a.id}/events/page?after=${before.at(-1).event_id}`,
    )
  ).events;
  cursorTail(before, tail);
  assert.deepEqual(
    [...before, ...tail].map((e) => e.event_id),
    (await events(a)).map((e) => e.event_id),
    "Reconnect must recover the complete durable tail",
  );
  assert(!tail.some((e) => e.type === "tool_started"));
  await save("stream-recovered", { snapshot: await rebound.snapshot(), tail });
  assert.equal(result((await health()).events, "run_shell").exit_code, 0);
  checks(
    "worker killed during streaming: budgets and cursor survive, no partial tool accepted",
  );
  const effect = await a.send(
    plan([
      [
        "run_shell",
        {
          command:
            "printf once >> effects.txt; sleep 8; printf finished > effect-finished",
          yield_ms: 10000,
          timeout_seconds: 30,
        },
      ],
    ]),
    { idempotencyKey: "tool-fault" },
  );
  await save("effect-run", {
    runId: effect.id,
    submissionId: effect.submissionId,
  });
  await eventually(
    async () =>
      (await events(a)).some(
        (e) => e.turn_id === effect.id && e.type === "tool_started",
      ),
    "tool admission",
  );
  // Read-only observation of a real effect in the exact original sandbox. This
  // cannot launch/replay the side effect; wait until it actually happened.
  await eventually(async () => {
    try {
      return (
        (await kub([
          "-n",
          sandboxPod.metadata.namespace,
          "exec",
          sandboxPod.metadata.name,
          "--",
          "cat",
          "/sandbox/effects.txt",
        ])) === "once"
      );
    } catch {
      return false;
    }
  }, "durable effect before result");
  await killWorker();
  await terminal(await rebound.resumeRun(effect.submissionId), "interrupted");
  const recovered = await rebound.snapshot();
  assert.equal(
    recovered.turn.queue.length,
    0,
    "Started uncertain action must not remain replayable",
  );
  assert.match(a.id, /^s_[a-f0-9]{32}$/);
  const unknownCount = await kube([
    "exec",
    "deployment/postgres",
    "--",
    "psql",
    "-U",
    "openworker",
    "-d",
    "openworker",
    "-Atc",
    `SELECT count(*) FROM ow_service.history WHERE session_id='${a.id}' AND data->>'type'='function_call_output' AND data->>'output' LIKE '%tool_outcome_unknown%'`,
  ]);
  assert.equal(unknownCount.trim(), "1");
  const effectEvents = (await events(a)).filter((e) => e.turn_id === effect.id);
  const cid = effectEvents.find((e) => e.type === "tool_started").data.call_id;
  noReplay(effectEvents, cid);
  await save("unknown-effect", { snapshot: recovered, events: effectEvents });
  const inspected = await send(b, "inspect-effect", [
    [
      "run_shell",
      {
        command:
          'test "$(cat effects.txt)" = once && test -f effect-finished && printf inspected-once',
        yield_ms: 10000,
      },
    ],
  ]);
  assert.equal(result(inspected.events, "run_shell").exit_code, 0);
  await recording(a, effect, ["session.recovery", "tool.unknown_outcome"]);
  checks(
    "worker killed after real tool effect: unknown outcome persisted once, explicit read-back, no replay",
  );
  const waiting = await a.send("TEST_RESILIENCE_STREAM", {
    idempotencyKey: "cancel-stream",
  });
  await eventually(
    async () =>
      (await events(a)).some(
        (e) => e.turn_id === waiting.id && e.type === "assistant_delta",
      ),
    "cancellation barrier",
  );
  await rebound.control(
    waiting.id,
    { kind: "cancel" },
    { idempotencyKey: "cancel-via-other-api" },
  );
  await terminal(waiting, "cancelled");
  assert.equal(result((await health()).events, "run_shell").exit_code, 0);
  await recording(a, waiting, ["turn.cancellation"]);
  checks(
    "cross-API cancellation stops turn and preserves environment-owned servers",
  );
  const exp = await send(a, "expire-process", [
    [
      "process_start",
      {
        command: "(sleep 3; echo orphan > expiry-orphan.txt) & wait",
        timeout_seconds: 1,
      },
    ],
  ]);
  const px = result(exp.events, "process_start").process_id;
  await pause(3300);
  const expiry = await send(a, "observe-expiry", [
    ["process_status", { process_id: px }],
    ["run_shell", { command: "test ! -e expiry-orphan.txt", yield_ms: 10000 }],
  ]);
  assert.equal(result(expiry.events, "process_status").status, "timed_out");
  assert.equal(result(expiry.events, "run_shell").exit_code, 0);
  const stop = await send(a, "stop-coder", [
    ["process_stop", { process_id: pa }],
    ["process_status", { process_id: pb }],
    [
      "run_shell",
      {
        command:
          '! curl -fsS --max-time 2 http://127.0.0.1:4000 && test "$(curl -fsS --max-time 2 http://127.0.0.1:4001)" = verifier',
        yield_ms: 10000,
      },
    ],
  ]);
  assert.equal(result(stop.events, "process_stop").status, "stopped");
  await recording(a, stop.run, ["process.stopped"]);
  assert.equal(result(stop.events, "process_status").status, "running");
  assert.equal(result(stop.events, "run_shell").exit_code, 0);
  checks(
    "deadline kills descendant without status polling; stopping coder leaves verifier healthy",
  );
  const budgetRun = await a.send(
    plan([
      [
        "write_file",
        { path: "must-not-exist", content: "PRIVATE_RESILIENCE_CONTENT" },
      ],
    ]),
    {
      idempotencyKey: "budget-stop",
      limits: { maxModelCalls: 1, maxOutputTokens: 1024, summaryTokens: 256 },
    },
  );
  await terminal(budgetRun);
  const proof = await send(b, "budget-proof", [
    ["run_shell", { command: "test ! -e must-not-exist", yield_ms: 10000 }],
  ]);
  assert.equal(result(proof.events, "run_shell").exit_code, 0);
  await recording(a, budgetRun, ["turn.budget_exceeded"]);
  checks("budget exhaustion refuses queued file effect and records the stop");
  const normal = await send(b, "observability-proof", [
    ["process_status", { process_id: pb }],
    ["process_logs", { process_id: pb }],
  ]);
  const recorded = await recording(b, normal.run, [
    "model.attempt",
    "tool.execute",
  ]);
  const base = `/v1/sessions/${b.id}/observability/${normal.run.id}`;
  const summary = (
    await h.json("GET", base + "/summary?through=" + recorded.page.through)
  ).summary;
  const rows = recorded.page.steps.map((x) => x.step);
  await save("recording-before-export", { summary, timeline: recorded });
  assert.equal(
    summary.models.reduce((n, g) => n + g.calls, 0),
    rows.filter((r) => r.operation === "model.attempt").length,
  );
  assert.equal(
    summary.tools.reduce((n, g) => n + g.calls, 0),
    rows.filter((r) => r.operation === "tool.execute").length,
  );
  // This scripted model supplies no usage. Missing counters must stay unknown.
  assert.equal(summary.usage.output.total, null);
  assert(summary.usage.output.unknown > 0);
  const wire = await eventually(
    async () => {
      const w = await collectWire();
      return rows.every((r) =>
        w.spans.some((s) => s.spanId === r.span_id && s.traceId === r.trace_id),
      )
        ? w
        : false;
    },
    "same run exported and recorded",
    30000,
  );
  for (const r of rows) {
    const matches = wire.spans.filter((s) => s.spanId === r.span_id);
    assert.equal(matches.length, 1);
    assert.equal(
      attributes(matches[0].attributes)["openworker.operation"],
      r.operation,
    );
  }
  const serialized = JSON.stringify(wire);
  assert(!serialized.includes(token));
  assert(!serialized.includes("PRIVATE_RESILIENCE_CONTENT"));
  assert(!serialized.includes("TEST_RESOURCE_GRANTS"));
  await save("recording-export-proof", {
    summary,
    timeline: recorded,
    exportedSpans: wire.spans.filter((s) =>
      rows.some((r) => r.span_id === s.spanId),
    ),
  });
  checks(
    "management summaries agree with timeline and actual OTLP collector by span/trace; content redacted",
  );
  const require = createRequire(import.meta.url);
  assert(
    process.env.OW_PLAYWRIGHT_MODULE,
    "Set OW_PLAYWRIGHT_MODULE to the installed playwright package",
  );
  const { chromium } = require(process.env.OW_PLAYWRIGHT_MODULE);
  ui = await startUI({
    port: 0,
    serviceUrl: m.url,
    serviceId: m.id,
    namespace: "local",
  });
  browser = await chromium.launch({ headless: true });
  const page = await browser.newPage({
    viewport: { width: 1440, height: 1000 },
  });
  const errors = [];
  page.on("pageerror", (e) => errors.push(e.message));
  await page.goto(ui.url + "/#observability");
  await page.locator("#token").fill(token);
  await page.getByRole("button", { name: "Sign in", exact: true }).click();
  await page.locator("#obs-run-search").fill(normal.run.id);
  await page
    .locator(".obs-run-option")
    .filter({ hasText: normal.run.id })
    .first()
    .click();
  await page.locator(".obs-stats").waitFor();
  // Selecting a run can open Timeline. Explicitly exercise each distinct view,
  // rather than labelling two screenshots of the default tab as separate checks.
  await page.getByRole("tab", { name: "Usage", exact: true }).click();
  await page
    .getByRole("heading", { name: "Accumulated work", exact: true })
    .waitFor();
  assert.equal(
    await page.locator("#obs-usage-tab").getAttribute("aria-selected"),
    "true",
  );
  await page.screenshot({
    path: join(root, "management-usage.png"),
    fullPage: true,
  });
  await page.getByRole("tab", { name: "Timeline", exact: true }).click();
  await page.locator(".obs-lanes").waitFor();
  assert.equal(
    await page.locator("#obs-timeline-tab").getAttribute("aria-selected"),
    "true",
  );
  await page.screenshot({
    path: join(root, "management-timeline.png"),
    fullPage: true,
  });
  assert.deepEqual(errors, []);
  checks(
    "browser finds exact run and renders grouped usage and timeline without page errors",
  );
  await browser.close();
  browser = null;
  await ui.close();
  ui = null;
  for (const s of sessions)
    await h.json("POST", `/v1/sessions/${s.id}/close`, {}, "close-" + s.id);
  await h.json(
    "POST",
    `/v1/environments/${a.environmentId}/delete`,
    {},
    "delete-original-environment",
  );
  await eventually(
    async () =>
      (await h.json("GET", "/v1/environments/" + a.environmentId)).environment
        .phase === "deleted",
    "original environment deletion",
  );
  await eventually(
    async () =>
      !JSON.parse(await kub(["get", "pods", "-A", "-o", "json"])).items.some(
        (p) => allEnvironmentPods.includes(p.metadata.uid),
      ),
    "original sandbox and supervisor pod disappearance",
  );
  const after = JSON.parse(
    await kub(["get", "pods", "-A", "-o", "json"]),
  ).items;
  assert(
    baseline.every((uid) => after.some((p) => p.metadata.uid === uid)),
    "Unrelated sandbox pods must remain",
  );
  report.cleanup.originalEnvironmentDeleted = true;
  report.cleanup.unrelatedSandboxesUnchanged = true;
  checks(
    "environment deletion removes remaining server and original pod; other sandboxes retained",
  );
  report.passed = true;
} catch (error) {
  report.failure = {
    type: error.name,
    code: error.code ?? null,
    operationId: error.operationId ?? null,
  };
  if (error.operationId && h) {
    const original = await h.operation(error.operationId).catch(() => null);
    if (original) await save("original-uncertain-operation", original);
  }
  throw error;
} finally {
  await browser?.close();
  await ui?.close();
  h?.disconnect();
  other?.disconnect();
  for (const p of children) p.kill("SIGTERM");
  report.cleanup.serviceNamespaceRetained = true; // Entry command destroys it only after successful environment cleanup.
  await save("cluster-report", report);
}