Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

11.5 KB
Raw Blame
import { test } from "node:test";
import assert from "node:assert/strict";
import {
  checkDeployedHealth,
  checkLiveSession,
  checkSlackCredentials,
  deployedHealthUrls,
  firstAdminPrincipal,
  INVALID_INDEX_QUERY,
  PARALLEL_EXCEPTION_QUERY,
  stagingApiHeaders,
} from "../src/deployment/postdeploy-smoke.ts";
import { PORTAL_IDENTITY_HEADER, verifyPortalIdentity } from "../src/auth/portal-identity.ts";

test("deployed database smoke rejects the parallel exception-handler failure class", () => {
  assert.match(PARALLEL_EXCEPTION_QUERY, /p\.proparallel = 's'/);
  assert.ok(PARALLEL_EXCEPTION_QUERY.includes("p.prosrc ~* '\\mEXCEPTION\\M'"));
  assert.match(PARALLEL_EXCEPTION_QUERY, /l\.lanname = 'plpgsql'/);
});

test("deployed database smoke rejects invalid or unready indexes", () => {
  assert.match(INVALID_INDEX_QUERY, /NOT i\.indisvalid OR NOT i\.indisready/);
});

test("deployed API smoke uses a configured real org admin", () => {
  assert.equal(firstAdminPrincipal("[email protected]:org_admin,[email protected]:org_admin"), "[email protected]");
  assert.throws(() => firstAdminPrincipal(undefined), /requires ADMIN_GRANTS/);
  assert.throws(() => firstAdminPrincipal("[email protected]:viewer"), /requires ADMIN_GRANTS/);
});

test("deployed API smoke proves the production portal-identity boundary", async () => {
  const now = Date.now();
  const headers = await stagingApiHeaders(
    "[email protected]",
    "source-secret",
    "portal-secret",
    "GET",
    "/v1/admin/sessions",
    "",
    { "x-admin-actor": "[email protected]@acme" },
    now,
  );
  assert.equal(headers["x-admin-actor"], "[email protected]@acme");
  const actor = await verifyPortalIdentity(headers[PORTAL_IDENTITY_HEADER]!, "portal-secret", now);
  assert.equal(actor?.p, "[email protected]");
});

test("deployed staging smoke reaches every Fly service and the public portal", () => {
  assert.deepEqual(deployedHealthUrls("qm-core", "https://qm-portal.fly.dev"), [
    "http://127.0.0.1:8080/healthz",
    "http://qm-admin.internal:8080/healthz",
    "http://qm-web-ui.flycast/healthz",
    "http://qm-portal.internal:8080/healthz",
    "https://qm-portal.fly.dev/healthz",
  ]);
  assert.throws(() => deployedHealthUrls("wrong-app", "https://example.com"), /FLY_APP_NAME ending in -core/);
});

test("deployed staging health rejects an unhealthy service", async () => {
  await assert.rejects(
    checkDeployedHealth(["http://core/healthz", "http://portal/healthz"], async (input) => {
      return new Response(null, { status: String(input).includes("portal") ? 503 : 200 });
    }),
    /http:\/\/portal\/healthz returned 503/,
  );
});

test("deployed staging smoke verifies its own Slack bot and Socket Mode credentials", async () => {
  const calls: Array<{ url: string; authorization: string | null }> = [];
  await checkSlackCredentials("xoxb-secret", "xapp-secret", "https://slack.example/api", async (input, init) => {
    calls.push({
      url: String(input),
      authorization: new Headers(init?.headers).get("authorization"),
    });
    return Response.json({ ok: true });
  });
  assert.deepEqual(calls, [
    { url: "https://slack.example/api/auth.test", authorization: "Bearer xoxb-secret" },
    { url: "https://slack.example/api/apps.connections.open", authorization: "Bearer xapp-secret" },
  ]);
  await assert.rejects(
    checkSlackCredentials("xoxb-secret", "xapp-secret", undefined, async () =>
      Response.json({ ok: false, error: "invalid_auth" }),
    ),
    /Slack auth\.test failed: invalid_auth/,
  );
});

test("live session smoke proves a model turn, persistence, title, error log, and cleanup", async () => {
  const calls: Array<{ method: string; path: string; body: string; portalIdentity: string | null }> = [];
  const config = {
    adminGrants: "[email protected]:org_admin",
    orgId: "acme",
    portalIdentitySecret: "portal-secret",
    signingSecret: "source-secret",
  };
  await checkLiveSession(config, "http://core.internal:8080", async (input, init) => {
    const url = new URL(String(input));
    const method = init?.method ?? "GET";
    calls.push({
      method,
      path: `${url.pathname}${url.search}`,
      body: String(init?.body ?? ""),
      portalIdentity: new Headers(init?.headers).get(PORTAL_IDENTITY_HEADER),
    });
    if (url.pathname === "/v1/turns")
      return Response.json({ status: "ok", sessionId: "sess-1", reply: "QM deployment canary passed." });
    if (url.pathname === "/v1/admin/errors") return Response.json({ errors: [] });
    if (method === "POST") return Response.json({ session: { id: "sess-1", archived: true } });
    return Response.json({
      session: { id: "sess-1", title: "Deployment canary" },
      entries: [{ type: "user" }, { type: "assistant" }],
    });
  });
  assert.deepEqual(
    calls.map(({ method, path }) => [method, path]),
    [
      ["POST", "/v1/turns"],
      ["GET", "/v1/sessions/sess-1?viewer=josh%40example.com&tailTurns=1"],
      ["GET", "/v1/admin/errors?scope=personal%3Ajosh%40example.com&sessionId=sess-1"],
      ["POST", "/v1/sessions/sess-1"],
    ],
  );
  assert.equal(JSON.parse(calls[0]!.body).readOnly, true);
  assert.equal(JSON.parse(calls[0]!.body).skipMemory, true);
  assert.deepEqual(JSON.parse(calls[3]!.body), { principalId: "[email protected]", archived: true });
  for (const call of calls) {
    const identity = await verifyPortalIdentity(call.portalIdentity ?? "", "portal-secret", Date.now());
    assert.equal(identity?.p, "[email protected]", `${call.method} ${call.path} carries the canary identity`);
  }

  let archivedFailedSession = false;
  let titleWait = 0;
  await assert.rejects(
    checkLiveSession(
      config,
      "http://core.internal:8080",
      async (input, init) => {
        const path = new URL(String(input)).pathname;
        if (path === "/v1/turns")
          return Response.json({ status: "ok", sessionId: "sess-2", reply: "QM deployment canary passed." });
        if (path === "/v1/sessions/sess-2" && init?.method === "POST") archivedFailedSession = true;
        return Response.json({ session: { id: "sess-2" }, entries: [{ type: "user" }, { type: "assistant" }] });
      },
      {
        now: () => titleWait,
        sleep: async (ms) => {
          titleWait += ms;
        },
      },
    ),
    /generated title/,
  );
  assert.equal(archivedFailedSession, true);
  assert.equal(titleWait, 60_000);

  await assert.rejects(
    checkLiveSession(config, "http://core.internal:8080", async (input) => {
      const path = new URL(String(input)).pathname;
      if (path === "/v1/turns") return Response.json({ status: "ok", sessionId: "sess-3", reply: "Looks good" });
      return Response.json({ session: { id: "sess-3", archived: true } });
    }),
    /unexpected model reply/,
  );

  let failedThreadRef = "";
  let archivedFailedRequest = false;
  await assert.rejects(
    checkLiveSession(config, "http://core.internal:8080", async (input, init) => {
      const url = new URL(String(input));
      if (url.pathname === "/v1/turns") {
        failedThreadRef = JSON.parse(String(init?.body)).conversation.threadRef as string;
        return new Response("model failed", { status: 500 });
      }
      if (url.pathname === "/v1/admin/sessions") {
        return Response.json({ sessions: [{ id: "sess-500", threadRef: failedThreadRef }] });
      }
      if (url.pathname === "/v1/sessions/sess-500" && init?.method === "POST") archivedFailedRequest = true;
      return Response.json({ session: { id: "sess-500", archived: true } });
    }),
    /returned 500/,
  );
  assert.equal(archivedFailedRequest, true);
});

test("live session smoke waits for a background worker title before checking errors and archiving", async () => {
  let elapsed = 0;
  let reads = 0;
  let checkedErrors = false;
  let archived = false;
  await checkLiveSession(
    {
      adminGrants: "[email protected]:org_admin",
      orgId: "acme",
      portalIdentitySecret: "portal-secret",
      signingSecret: "source-secret",
    },
    "http://core.internal:8080",
    async (input, init) => {
      const path = new URL(String(input)).pathname;
      if (path === "/v1/turns")
        return Response.json({ status: "ok", sessionId: "delayed-title", reply: "QM deployment canary passed." });
      if (path === "/v1/admin/errors") {
        assert.equal(reads, 3);
        checkedErrors = true;
        return Response.json({ errors: [] });
      }
      if (init?.method === "POST") {
        assert.equal(checkedErrors, true);
        archived = true;
        return Response.json({});
      }
      assert.equal(path, "/v1/sessions/delayed-title");
      reads += 1;
      return Response.json({
        session: { id: "delayed-title", ...(reads >= 3 ? { title: "Verify deployment canary" } : {}) },
        entries: [{ type: "user" }, { type: "assistant" }],
      });
    },
    {
      now: () => elapsed,
      sleep: async (ms) => {
        elapsed += ms;
      },
    },
  );
  assert.equal(elapsed, 1000);
  assert.equal(archived, true);
});

test("live session smoke aborts a hung first title read and still archives", async () => {
  const controller = new AbortController();
  let archived = false;
  await assert.rejects(
    checkLiveSession(
      {
        adminGrants: "[email protected]:org_admin",
        orgId: "acme",
        portalIdentitySecret: "portal-secret",
        signingSecret: "source-secret",
      },
      "http://core.internal:8080",
      async (input, init) => {
        const path = new URL(String(input)).pathname;
        if (path === "/v1/turns") {
          assert.equal(init?.signal, undefined);
          return Response.json({ status: "ok", sessionId: "hung-title", reply: "QM deployment canary passed." });
        }
        if (init?.method === "POST") {
          assert.equal(init.signal, undefined);
          archived = true;
          return Response.json({});
        }
        assert.equal(path, "/v1/sessions/hung-title");
        assert.equal(init?.signal, controller.signal);
        return new Promise((_resolve, reject) => {
          controller.signal.addEventListener("abort", () => reject(controller.signal.reason), { once: true });
          queueMicrotask(() => controller.abort(new DOMException("Title deadline", "TimeoutError")));
        });
      },
      {
        now: Date.now,
        sleep: async () => {
          throw new Error("Unexpected sleep");
        },
        timeout: (ms) => {
          assert.equal(ms, 60_000);
          return controller.signal;
        },
      },
    ),
    { name: "TimeoutError" },
  );
  assert.equal(archived, true);
});

test("live session smoke rejects a title returned after the deadline", async () => {
  let elapsed = 0;
  let archived = false;
  await assert.rejects(
    checkLiveSession(
      {
        adminGrants: "[email protected]:org_admin",
        orgId: "acme",
        portalIdentitySecret: "portal-secret",
        signingSecret: "source-secret",
      },
      "http://core.internal:8080",
      async (input, init) => {
        const path = new URL(String(input)).pathname;
        if (path === "/v1/turns")
          return Response.json({ status: "ok", sessionId: "late-title", reply: "QM deployment canary passed." });
        if (init?.method === "POST") {
          archived = true;
          return Response.json({});
        }
        assert.equal(path, "/v1/sessions/late-title");
        elapsed = 60_001;
        return Response.json({ session: { title: "Late title" }, entries: [{ type: "user" }, { type: "assistant" }] });
      },
      {
        now: () => elapsed,
        sleep: async () => {
          throw new Error("Unexpected sleep");
        },
      },
    ),
    /generated title/,
  );
  assert.equal(archived, true);
});