Skip to content
Python
19.8 KB
Raw Blame
// Browser -> UI pod -> Rust API -> PostgreSQL -> worker -> real OpenShell tools.
// Scripted model, no paid inference. Deliberately do not auto-retry mutations.
import assert from "node:assert/strict";
import { readFile, writeFile, mkdir } from "node:fs/promises";
import { resolve, join } from "node:path";
import { spawn } from "node:child_process";
import { createRequire } from "node:module";
import {
  randomBytes,
  randomUUID,
  createHash,
  X509Certificate,
} from "node:crypto";
import https from "node:https";
import { run } from "./stack.mjs";
const root = resolve(process.argv[2] || ""),
  m = JSON.parse(await readFile(join(root, "kubernetes.json"), "utf8"));
assert(m.context.startsWith("kind-") && !m.destroyed);
const ca = await readFile(join(root, "input/api-ca.crt")),
  cert = new X509Certificate(await readFile(join(root, "input/api.crt")));
const spki = createHash("sha256")
  .update(cert.publicKey.export({ type: "spki", format: "der" }))
  .digest("base64");
const admin = (await readFile(join(root, "input/admin-token"), "utf8")).trim();
const resumeCancellation = process.argv[3] === "--resume-cancellation";
const resume = process.argv[3] === "--resume" || resumeCancellation;
const report = join(root, "qualification-ui.json");
const runner = resume
  ? (await readFile(join(root, "ui-runner.token"), "utf8")).trim()
  : randomBytes(32).toString("hex");
const record = resume
  ? JSON.parse(await readFile(report, "utf8"))
  : {
      started: new Date().toISOString(),
      passed: false,
      serviceId: m.id,
      namespace: m.namespace,
      checks: [],
    };
if (resume) {
  assert.equal(record.serviceId, m.id);
  assert(record.sessionId && record.environmentId && !record.passed);
  assert.equal(
    record.phase,
    resumeCancellation
      ? "Cancelling a real in-flight sandbox command"
      : "Restarting UI while real shell work is running",
  );
  delete record.error;
} else
  try {
    await readFile(report);
    throw Error(
      "Qualification already has a journal; inspect it before another run.",
    );
  } catch (e) {
    if (e.code !== "ENOENT") throw e;
  }
const checkpoint = async (label) => {
  record.phase = label;
  await writeFile(report, JSON.stringify(record, null, 2), { mode: 0o600 });
  console.log(label);
};
const kube = (args) =>
  run("kubectl", [
    "--kubeconfig",
    m.kubeconfig,
    "--context",
    m.context,
    "-n",
    m.namespace,
    ...args,
  ]);
let browser, apiForward, uiForward;
async function forward(service, port) {
  return new Promise((ok, no) => {
    const p = spawn(
      "kubectl",
      [
        "--kubeconfig",
        m.kubeconfig,
        "--context",
        m.context,
        "-n",
        m.namespace,
        "port-forward",
        "service/" + service,
        port + ":443",
        "--address",
        "127.0.0.1",
      ],
      { stdio: ["ignore", "pipe", "pipe"] },
    );
    let ready = false;
    const timer = setTimeout(() => {
      p.kill();
      no(Error("Port-forward timeout"));
    }, 30000);
    p.stdout.on("data", (b) => {
      if (!ready && b.toString().includes("Forwarding from")) {
        ready = true;
        clearTimeout(timer);
        ok(p);
      }
    });
    p.stderr.resume();
    p.on("exit", () => {
      clearTimeout(timer);
      if (!ready) no(Error("Port-forward stopped before readiness"));
    });
    p.on("error", no);
  });
}
async function stopForward(p) {
  if (!p || p.exitCode !== null) return;
  await new Promise((r) => {
    p.once("exit", r);
    p.kill("SIGTERM");
  });
}
async function api(
  path,
  method = "GET",
  body,
  token = admin,
  key = randomUUID(),
) {
  return new Promise((ok, no) => {
    const q = https.request(
      m.url + path,
      {
        method,
        ca,
        headers: {
          Authorization: "Bearer " + token,
          "x-openworker-namespace": "local",
          "x-openworker-service-id": m.id,
          "content-type": "application/json",
          "idempotency-key": key,
        },
      },
      (r) => {
        let data = "";
        r.on("data", (b) => (data += b));
        r.on("end", () => {
          try {
            const value = JSON.parse(data);
            if (!r.statusCode || r.statusCode >= 300)
              throw Error(
                `API ${method} ${path}: ${r.statusCode} ${JSON.stringify(value)}`,
              );
            assert.equal(r.headers["x-openworker-service-id"], m.id);
            ok(value);
          } catch (e) {
            no(e);
          }
        });
      },
    );
    q.on("error", no);
    q.setTimeout(10000, () => q.destroy(Error("API timeout")));
    q.end(body === undefined ? undefined : JSON.stringify(body));
  });
}
async function until(fn, seconds = 120) {
  let last;
  const end = Date.now() + seconds * 1000;
  while (Date.now() < end) {
    try {
      const value = await fn();
      if (value) return value;
    } catch (e) {
      last = e;
    }
    await new Promise((r) => setTimeout(r, 500));
  }
  throw last || Error("Timed out: " + record.phase);
}
const snapshot = () =>
  api("/v1/sessions/" + record.sessionId, "GET", undefined, runner);
const events = async () => {
  const out = [];
  let after = 0;
  for (let n = 0; n < 10; n++) {
    const p = await api(
      `/v1/sessions/${record.sessionId}/events/page?after=${after}`,
      "GET",
      undefined,
      runner,
    );
    if (!p.events.length) break;
    out.push(...p.events);
    after = p.events.at(-1).event_id;
    if (p.events.length < 500) break;
  }
  return out;
};
try {
  await checkpoint("Starting verified local TLS connections");
  apiForward = await forward("ows-api", 58444);
  uiForward = await forward("ow-ui-ingress", 58644);
  await api("/v1/auth/me");
  let credential = { id: record.runnerCredentialId };
  if (!resume) {
    credential = await api(
      "/v1/credentials",
      "POST",
      {
        token: runner,
        spec: {
          principal: "ui-runner",
          role: "operator",
          scope: { kind: "owned" },
          expires_at: Math.floor(Date.now() / 1000) + 3600,
        },
      },
      admin,
      "ui-runner-credential",
    );
    record.runnerCredentialId = credential.id;
    await writeFile(join(root, "ui-runner.token"), runner, { mode: 0o600 });
  }
  const { chromium, expect } = createRequire(import.meta.url)(
    process.env.OW_PLAYWRIGHT_MODULE,
  );
  browser = await chromium.launch({
    headless: true,
    args: ["--ignore-certificate-errors-spki-list=" + spki],
  });
  const page = await browser.newPage({
    viewport: { width: 1440, height: 1000 },
  });
  page.setDefaultTimeout(20000);
  const errors = [];
  page.on("pageerror", (e) => errors.push(e.message));
  await mkdir(join(root, "screenshots"), { recursive: true });
  async function login(token) {
    await page.getByLabel("Service token", { exact: true }).fill(token);
    await page.getByRole("button", { name: "Sign in", exact: true }).click();
    await expect(page.locator("#app")).toBeVisible();
  }
  async function logout() {
    await page.locator("#profile-button").click();
    await page.getByRole("button", { name: "Sign out", exact: true }).click();
    await expect(page.locator("#login")).toBeVisible();
  }
  async function save() {
    await page.locator("#save").click();
    await expect(page.locator("#panel")).not.toBeVisible();
  }
  await page.goto(
    "https://localhost:58644" + (resume ? "#sessions/" + record.sessionId : ""),
  );
  await login(resume ? runner : admin);
  const cookie = (await page.context().cookies()).find(
    (c) => c.name === "__Host-ow_ui",
  );
  assert(cookie?.secure && cookie.httpOnly && cookie.sameSite === "Strict");
  if (!resume) {
    await checkpoint("Publishing RuntimeProfile and agent through the browser");
    await page.locator("[data-nav=infrastructure]").click();
    await page
      .getByRole("button", { name: "New profile", exact: true })
      .click();
    const profile = {
      id: "ui-ask@1",
      provider: "openshell",
      cpu: 1,
      memoryMB: 512,
      sharing: "exclusive",
      approvalMode: "ask",
      allowedPrincipals: ["ui-runner"],
    };
    await page
      .getByLabel("Definition", { exact: true })
      .fill(JSON.stringify(profile));
    await save();
    await page.locator('[data-action="activate-profile:ui-ask@1"]').click();
    await save();
    await page.locator("[data-nav=agents]").click();
    await page
      .getByRole("button", { name: "Publish release", exact: true })
      .click();
    const agent = {
      id: "ui-builder@1",
      instructions: "Complete the requested task using the provided tools.",
      tools: ["read_file", "write_file", "run_shell"],
      runtimeProfiles: ["ui-ask@1"],
      defaultRuntimeProfile: "ui-ask@1",
      allowedPrincipals: ["ui-runner"],
    };
    await page
      .getByLabel("Definition", { exact: true })
      .fill(JSON.stringify(agent));
    await save();
    await page.locator('[data-action="activate-release:ui-builder@1"]').click();
    await save();
    record.checks.push("Profile/release publication and activation through UI");
    await logout();
    await login(runner);
    await expect(
      page.getByRole("button", { name: "Publish release", exact: true }),
    ).toHaveCount(0);
    const task =
      "TEST_RESOURCE_GRANTS " +
      JSON.stringify([
        {
          name: "run_shell",
          arguments: {
            command:
              "printf 'M5_UI_PROOF\\n' >> ui-proof.txt; sleep 12; cat ui-proof.txt",
          },
        },
        { name: "read_file", arguments: { path: "ui-proof.txt" } },
      ]);
    await checkpoint(
      "Submitting task and provisioning real OpenShell environment",
    );
    await page.locator('[data-action="run:ui-builder@1"]').click();
    await page.getByLabel("Task", { exact: true }).fill(task);
    await save();
    await page.waitForURL(/#operation\//);
    record.operationId = page.url().split("#operation/")[1];
    await checkpoint("Waiting for worker admission");
    const op = await until(async () => {
      const o = await api(
        "/v1/operations/" + record.operationId,
        "GET",
        undefined,
        runner,
      );
      if (o.status === "failed") throw Error(o.error);
      return o.status === "complete" && o;
    }, 180);
    record.sessionId = op.sessionId;
    record.environmentId = op.environmentId;
    await page.waitForURL(/#sessions\//, { timeout: 30000 });
    const waiting = await until(async () => {
      const s = await snapshot();
      return s.interactions.length && s;
    });
    assert.equal(waiting.session.sandbox, "openshell");
    assert.equal(waiting.session.permissions.mode, "ask");
    assert.equal(waiting.mode, "scripted-test");
    record.firstRunId = waiting.turn.id;
    record.approvalId = waiting.interactions[0].id;
    record.environmentBinding = waiting.session.environment;
    assert.equal(waiting.session.environment.spec.provider, "openshell-k8s");
    await page.reload();
    await expect(page.locator("#interactions")).toContainText(
      "Approval needed",
    );
    assert.equal((await snapshot()).interactions[0].id, record.approvalId);
    await checkpoint("Restarting API while the approval is parked");
    await stopForward(apiForward);
    apiForward = null;
    await kube(["rollout", "restart", "deployment/ows-api"]);
    await kube(["rollout", "status", "deployment/ows-api", "--timeout=120s"]);
    apiForward = await forward("ows-api", 58444);
    assert.equal((await snapshot()).interactions[0].id, record.approvalId);
    await expect(page.locator("#stream-state")).toHaveText("Live", {
      timeout: 30000,
    });
    await page
      .getByRole("button", { name: "Review request", exact: true })
      .click();
    await page.getByLabel("Decision", { exact: true }).selectOption("once");
    await save();
    await until(async () =>
      (await events()).some(
        (e) => e.turn_id === record.firstRunId && e.type === "tool_started",
      ),
    );
    await checkpoint("Restarting UI while real shell work is running");
    await kube(["rollout", "restart", "deployment/ows-ui"]);
    await kube(["rollout", "status", "deployment/ows-ui", "--timeout=120s"]);
    await page.reload();
    await expect(page.locator("#login")).toBeVisible();
    await login(runner);
    await expect(page.locator("main h1")).not.toHaveText("Unable to load");
  }
  if (!resumeCancellation) {
    const done = await until(async () => {
      const s = await snapshot();
      if (s.turn.status === "waiting" && s.interactions.length) {
        const i = s.interactions[0];
        assert.equal(i.call.name, "read_file");
        assert.equal(JSON.parse(i.call.arguments).path, "ui-proof.txt");
        if (!(s.controls || []).some((c) => c.target === "response:" + i.id)) {
          await expect(page.locator("#interactions")).toContainText(
            "Approval needed",
          );
          await page
            .getByRole("button", { name: "Review request", exact: true })
            .click();
          await save();
        }
        return false;
      }
      assert.notEqual(s.turn.status, "failed");
      return s.turn.status === "completed" && s;
    });
    assert.equal(done.session.id, record.sessionId);
    assert.deepEqual(done.session.environment, record.environmentBinding);
    let ev = await events();
    const first = ev.filter((e) => e.turn_id === record.firstRunId);
    assert.equal(
      first.filter(
        (e) => e.type === "tool_started" && e.data.name === "run_shell",
      ).length,
      1,
    );
    assert(
      first.some(
        (e) =>
          e.type === "tool_finished" &&
          JSON.stringify(e.data.result).includes("M5_UI_PROOF"),
      ),
    );
    record.checks.push(
      "Real OpenShell shell/file effects; approval identity survives API rollout; UI restart requires login and preserves execution",
    );
    await page.getByRole("button", { name: "Refresh", exact: true }).click();
    await expect(page.locator(".session-meta .status")).toHaveText("completed");
    await page.screenshot({
      path: join(root, "screenshots", "completed.png"),
      fullPage: true,
    });
    async function sendTask(command) {
      await page
        .getByRole("button", { name: "Send message", exact: true })
        .click();
      await page
        .getByLabel("Message", { exact: true })
        .fill(
          "TEST_RESOURCE_GRANTS " +
            JSON.stringify([{ name: "run_shell", arguments: { command } }]),
        );
      await save();
      const s = await until(async () => {
        const s = await snapshot();
        return s.interactions.length && s;
      });
      await expect(page.locator("#interactions")).toContainText(
        "Approval needed",
      );
      await page
        .getByRole("button", { name: "Review request", exact: true })
        .click();
      await save();
      return s.turn.id;
    }
    await checkpoint("Verifying the persisted file and exactly one append");
    const checkRun = await sendTask(
      "printf 'ui-lines='; wc -l < ui-proof.txt; cat ui-proof.txt",
    );
    await until(async () => (await snapshot()).turn.status === "completed");
    ev = await events();
    assert(
      ev.some(
        (e) =>
          e.turn_id === checkRun &&
          e.type === "tool_finished" &&
          JSON.stringify(e.data.result).includes("ui-lines=1"),
      ),
    );
    record.checks.push(
      "Independent sandbox read confirms one persisted append",
    );
    await checkpoint("Cancelling a real in-flight sandbox command");
    const cancelRun = await sendTask(
      "printf started > cancel-proof.txt; sleep 30; printf done >> cancel-proof.txt",
    );
    record.cancelRunId = cancelRun;
    await until(async () =>
      (await events()).some(
        (e) => e.turn_id === cancelRun && e.type === "tool_started",
      ),
    );
    await page.getByRole("button", { name: "Cancel run", exact: true }).click();
    await save();
  }
  const cancelled = await until(async () => {
    const s = await snapshot();
    return ["cancelled", "interrupted"].includes(s.turn.status) && s;
  });
  assert.equal(cancelled.turn.id, record.cancelRunId);
  assert(
    cancelled.controls.some(
      (c) =>
        c.target === "cancel:" + record.cancelRunId && c.status === "applied",
    ),
  );
  const cancelEvents = (await events()).filter(
    (e) => e.turn_id === record.cancelRunId,
  );
  assert.equal(
    cancelEvents.filter((e) => e.type === "tool_started").length,
    1,
    "Cancelled remote action must not be replayed",
  );
  if (cancelled.turn.status === "interrupted")
    assert(
      cancelEvents.some(
        (e) => e.type === "turn_completed" && e.data.outcome_unknown_count > 0,
      ),
      "Interrupted cancellation must expose tool-outcome uncertainty",
    );
  record.cancellationStatus = cancelled.turn.status;
  if (cancelled.turn.status === "interrupted") {
    const values = JSON.parse(
      await readFile(join(root, "values.json"), "utf8"),
    );
    const schema = values.config.schema;
    assert(/^ow_[a-z0-9_]+$/.test(schema));
    assert(/^s_[a-f0-9]+$/.test(record.sessionId));
    const count = await kube([
      "exec",
      "deployment/postgres",
      "--",
      "psql",
      "-U",
      "openworker",
      "-d",
      "openworker",
      "-Atc",
      `SELECT count(*) FROM ${schema}.history WHERE session_id='${record.sessionId}' AND data->>'type'='function_call_output' AND data->>'output' LIKE '%outcome_unknown%'`,
    ]);
    assert(Number(count.trim()) > 0);
    record.unknownOutcomeInModelHistory = true;
  }

  await page.getByRole("button", { name: "Refresh", exact: true }).click();
  await expect(page.locator(".session-meta .status")).toHaveText(
    cancelled.turn.status,
  );
  await page.screenshot({
    path: join(root, "screenshots", "cancelled.png"),
    fullPage: true,
  });
  record.checks.push(
    "Browser cancellation applied; unknown remote outcome explicitly interrupted, no tool replay",
  );
  await checkpoint("Scaling workers through the management UI");
  await logout();
  await login(admin);
  await page.locator("[data-nav=infrastructure]").click();
  await page.getByRole("button", { name: "Scale", exact: true }).click();
  await page.getByLabel("Desired workers").fill("2");
  await save();
  await until(async () => {
    const p = (await api("/v1/worker-pools")).pools[0];
    return (
      p.desiredReplicas === 2 &&
      p.observed.readyReplicas === 2 &&
      p.observed.status === "applied"
    );
  });
  await page.getByRole("button", { name: "Refresh", exact: true }).click();
  await page.screenshot({
    path: join(root, "screenshots", "workers.png"),
    fullPage: true,
  });
  record.checks.push(
    "UI scale request reconciled to two ready Kubernetes workers",
  );
  assert.deepEqual(errors, []);
  await checkpoint(
    "Closing session and deleting only its OpenShell environment",
  );
  await api(
    `/v1/sessions/${record.sessionId}/close`,
    "POST",
    {},
    runner,
    "ui-close",
  );
  await api(
    `/v1/environments/${record.environmentId}/delete`,
    "POST",
    {},
    admin,
    "ui-delete",
  );
  await until(
    async () =>
      (await api("/v1/environments/" + record.environmentId)).environment
        .phase === "deleted",
  );
  await api(
    "/v1/credentials/" + credential.id + "/revoke",
    "POST",
    {},
    admin,
    "ui-revoke",
  );
  record.checks.push(
    "Session close, remote environment deletion, and runner credential revocation",
  );
  record.passed = true;
  record.finished = new Date().toISOString();
  await checkpoint("Complete-stack UI qualification passed");
  console.log(
    JSON.stringify({
      passed: true,
      sessionId: record.sessionId,
      environmentId: record.environmentId,
      report,
    }),
  );
} catch (e) {
  record.error = String(e.message)
    .replaceAll(admin, "[credential]")
    .replaceAll(runner, "[credential]");
  await writeFile(report, JSON.stringify(record, null, 2), { mode: 0o600 });
  console.error(record.error);
  process.exitCode = 1;
} finally {
  await browser?.close();
  await stopForward(apiForward);
  await stopForward(uiForward);
}