Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Run deployment session checks inside the ready core (#1294)

Regan Bell8c5eaa2618c7
diff --git a/docs/background-ownership.md b/docs/background-ownership.md
index d9dc16d97..f3f4edde8 100644
--- a/docs/background-ownership.md
+++ b/docs/background-ownership.md
@@ -63,3 +63,33 @@ When a process cannot acknowledge, the trusted operator must independently prove
```
Retirement preserves the ownership generation, updates `lastRequestId`, and marks only the matching members retired and drained. Retired task identities cannot enroll again. An unreachable HTTP endpoint, old heartbeat, or elapsed timeout is not termination evidence.
+
+## Live deployment session check
+
+A ready, active deployment accepts `POST /v1/deployment/live-session` with the
+same source signature and distinct deployment bearer credential. The exact body
+contains `requestId` (a fresh UUID), `expectedDeploymentId`, `expectedGeneration`,
+and `expectedTaskArns` (the exact healthy task cohort). The endpoint rejects
+unknown fields, inactive ownership, incomplete readiness, stale generations,
+and mismatched membership before starting work.
+
+The check runs the same fixed session command used by the standalone deployment
+smoke: a real model reply, persisted turns, generated title, session error log,
+session archival, then the configured PostgreSQL catalog checks. The request
+cannot select a principal, URL, model, prompt, or command. Model HTTP requests
+have a five-minute limit; other HTTP requests and database operations have
+30-second limits. These are failure bounds, not a release-duration claim.
+
+The response is newline-delimited JSON with an initial newline and five-second
+whitespace heartbeats, followed by one final object containing `ok`, `requestId`,
+`deploymentId`, `instanceId`, `taskArn`, and `generation`. A failed result adds a
+fixed error code without raw model, database, or credential details. Ownership
+and cohort readiness are checked again before success. Heartbeats alone never
+prove success; the caller must receive and verify the complete final object.
+
+Request IDs are durably consumed across replicas and cannot be replayed. Each
+process permits one check at a time, including its cleanup. Disconnecting the
+caller does not cancel the check or release that guard before cleanup finishes.
+The deployment lease still serializes operator requests across the cohort.
+An uncertain result must fail the release; do not retry automatically or fall
+back to a second canary execution.
diff --git a/plugins/portal/src/index.ts b/plugins/portal/src/index.ts
index da32b1f3d..e5775f8de 100644
--- a/plugins/portal/src/index.ts
+++ b/plugins/portal/src/index.ts
@@ -1132,7 +1132,10 @@ async function handle(req: IncomingMessage, res: ServerResponse): Promise<void>
});
}
- if ((method === "GET" || method === "POST") && pathname === "/v1/background-work") {
+ if (
+ ((method === "GET" || method === "POST") && pathname === "/v1/background-work") ||
+ (method === "POST" && pathname === "/v1/deployment/live-session")
+ ) {
return proxyToUpstream(req, res, { baseUrl: CORE, path: pathname, search: url.search }, [
...FORWARD_DEPLOYMENT_LAYER_HEADERS,
"authorization",
diff --git a/plugins/portal/test/router.test.ts b/plugins/portal/test/router.test.ts
index 063105a76..650082fc5 100644
--- a/plugins/portal/test/router.test.ts
+++ b/plugins/portal/test/router.test.ts
@@ -12,7 +12,11 @@ let deploymentLayerRequests = 0;
const VALID_SOURCE_SIGNATURE = "v0=valid-source-signature";
const upstream = createServer((req: IncomingMessage, res) => {
- if (req.url?.startsWith("/v1/deployment-layer") || req.url?.startsWith("/v1/background-work")) {
+ if (
+ req.url?.startsWith("/v1/deployment-layer") ||
+ req.url?.startsWith("/v1/background-work") ||
+ req.url?.startsWith("/v1/deployment/live-session")
+ ) {
deploymentLayerRequests++;
if (req.headers["x-timestamp"] !== "123" || req.headers["x-signature"] !== VALID_SOURCE_SIGNATURE) {
res.writeHead(401, { "content-type": "application/json" });
@@ -631,8 +635,12 @@ test("impersonate: an admin starts it; the web-ui hop carries target + impersona
});
test("background ownership forwards both credentials only on its exact control routes", async () => {
- for (const method of ["GET", "POST"]) {
- const response = await fetch(`${base}/v1/background-work`, {
+ for (const [method, path] of [
+ ["GET", "/v1/background-work"],
+ ["POST", "/v1/background-work"],
+ ["POST", "/v1/deployment/live-session"],
+ ]) {
+ const response = await fetch(`${base}${path}`, {
method,
headers: {
"x-timestamp": "123",
@@ -655,6 +663,8 @@ test("background ownership forwards both credentials only on its exact control r
for (const [method, path] of [
["PUT", "/v1/background-work"],
["GET", "/v1/background-work/nearby"],
+ ["GET", "/v1/deployment/live-session"],
+ ["POST", "/v1/deployment/live-session/nearby"],
]) {
const response = await fetch(`${base}${path}`, {
method,
diff --git a/src/api/deps.ts b/src/api/deps.ts
index ce88b5fdc..d54df9cb5 100644
--- a/src/api/deps.ts
+++ b/src/api/deps.ts
@@ -82,6 +82,7 @@ export interface ServerDeps {
requireSignedPortalIdentity?: boolean;
control: ControlService;
replayDedupe?: ReplayDedupe;
+ deploymentLiveSmoke?: () => Promise<void>;
brokerSessions?: BrokerSessionStore;
connectorTokens?: ConnectorTokenStore;
managedSlack?: ManagedSlack;
diff --git a/src/api/routes/background-work.ts b/src/api/routes/background-work.ts
index eaab81115..81ea72f67 100644
--- a/src/api/routes/background-work.ts
+++ b/src/api/routes/background-work.ts
@@ -31,18 +31,26 @@ function status(ctx: ApiCtx, state: BackgroundOwnership): void {
});
}
-async function backgroundWork(ctx: ApiCtx): Promise<void> {
+export function requireDeploymentControl(ctx: ApiCtx): boolean {
const control = ctx.deps.backgroundOwnership;
const secret = ctx.deps.deploymentControlSecret;
if (!control || !ctx.secret || !ctx.auth || !isStrongSigningSecret(secret) || secret === ctx.secret) {
- return sendJson(ctx.res, 503, { error: "background_control_unavailable" });
+ sendJson(ctx.res, 503, { error: "background_control_unavailable" });
+ return false;
}
const bearer = ctx.req.headers.authorization;
const expected = Buffer.from(`Bearer ${secret}`);
const supplied = Buffer.from(typeof bearer === "string" ? bearer : "");
if (supplied.length !== expected.length || !timingSafeEqual(supplied, expected)) {
- return sendJson(ctx.res, 401, { error: "unauthorized" });
+ sendJson(ctx.res, 401, { error: "unauthorized" });
+ return false;
}
+ return true;
+}
+
+async function backgroundWork(ctx: ApiCtx): Promise<void> {
+ if (!requireDeploymentControl(ctx)) return;
+ const control = ctx.deps.backgroundOwnership!;
if (ctx.method === "GET") return status(ctx, await control.store.get());
const body = ctx.body;
if (
diff --git a/src/api/routes/deployment-live-smoke.ts b/src/api/routes/deployment-live-smoke.ts
new file mode 100644
index 000000000..0bc9b33f0
--- /dev/null
+++ b/src/api/routes/deployment-live-smoke.ts
@@ -0,0 +1,101 @@
+import { sendJson } from "../http.ts";
+import type { BackgroundOwnership } from "../../runs/background-ownership.ts";
+import { requireDeploymentControl } from "./background-work.ts";
+import { isObj } from "./shared.ts";
+import type { ApiCtx, Route } from "./route.ts";
+
+const running = new WeakSet<object>();
+const REQUEST_EXPIRY = Date.UTC(9999, 0, 1);
+
+async function deploymentLiveSmoke(ctx: ApiCtx): Promise<void> {
+ if (!requireDeploymentControl(ctx)) return;
+ const control = ctx.deps.backgroundOwnership!;
+ const run = ctx.deps.deploymentLiveSmoke;
+ const replay = ctx.deps.replayDedupe;
+ if (!run || !replay?.durable) return sendJson(ctx.res, 503, { error: "deployment_smoke_unavailable" });
+ const body = ctx.body;
+ if (
+ !isObj(body) ||
+ !Object.keys(body).every((key) =>
+ ["requestId", "expectedDeploymentId", "expectedGeneration", "expectedTaskArns"].includes(key),
+ ) ||
+ typeof body.requestId !== "string" ||
+ !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(body.requestId) ||
+ typeof body.expectedDeploymentId !== "string" ||
+ body.expectedDeploymentId !== control.deploymentId ||
+ !Number.isSafeInteger(body.expectedGeneration) ||
+ (body.expectedGeneration as number) < 1 ||
+ !Array.isArray(body.expectedTaskArns) ||
+ !body.expectedTaskArns.length ||
+ body.expectedTaskArns.length > 1000 ||
+ !body.expectedTaskArns.every((arn) => typeof arn === "string" && arn.length > 0 && arn.length <= 2048) ||
+ new Set(body.expectedTaskArns).size !== body.expectedTaskArns.length
+ )
+ return sendJson(ctx.res, 400, { error: "invalid_deployment_smoke_request" });
+ if (running.has(control)) return sendJson(ctx.res, 409, { error: "deployment_smoke_running" });
+ const expectedTasks = new Set(body.expectedTaskArns as string[]);
+ const valid = (state: BackgroundOwnership): boolean => {
+ const members = state.members.filter((member) => !member.retired && member.state === "admitted");
+ return (
+ state.enabled &&
+ state.generation === body.expectedGeneration &&
+ state.desiredDeploymentId === control.deploymentId &&
+ members.some((member) => member.instanceId === control.instanceId) &&
+ members.length === expectedTasks.size &&
+ new Set(members.map((member) => member.taskArn)).size === expectedTasks.size &&
+ members.every(
+ (member) =>
+ member.deploymentId === control.deploymentId &&
+ member.generation === state.generation &&
+ member.state === "admitted" &&
+ member.ready &&
+ member.taskArn !== null &&
+ expectedTasks.has(member.taskArn),
+ )
+ );
+ };
+ running.add(control);
+ let heartbeat: ReturnType<typeof setInterval> | undefined;
+ try {
+ const state = await control.store.get();
+ if (!valid(state)) return sendJson(ctx.res, 409, { error: "deployment_smoke_ownership_conflict" });
+ if (!(await replay.claim(`deployment-live-smoke:${body.requestId.toLowerCase()}`, REQUEST_EXPIRY))) {
+ return sendJson(ctx.res, 409, { error: "deployment_smoke_replay" });
+ }
+ const member = state.members.find((entry) => entry.instanceId === control.instanceId)!;
+ const identity = {
+ requestId: body.requestId,
+ deploymentId: control.deploymentId,
+ instanceId: control.instanceId,
+ taskArn: member.taskArn,
+ generation: state.generation,
+ };
+ ctx.res.writeHead(200, {
+ "content-type": "application/x-ndjson",
+ "cache-control": "no-store, no-transform",
+ "x-accel-buffering": "no",
+ });
+ ctx.res.write("\n");
+ heartbeat = setInterval(() => {
+ if (!ctx.res.destroyed) ctx.res.write("\n");
+ }, 5000);
+ heartbeat.unref();
+ ctx.res.once("close", () => clearInterval(heartbeat));
+ let error: string | undefined;
+ try {
+ await run();
+ if (!valid(await control.store.get())) error = "deployment_smoke_ownership_changed";
+ } catch {
+ error = "deployment_smoke_failed";
+ }
+ if (!ctx.res.destroyed)
+ ctx.res.end(`${JSON.stringify({ ok: !error, ...identity, ...(error ? { error } : {}) })}\n`);
+ } finally {
+ clearInterval(heartbeat);
+ running.delete(control);
+ }
+}
+
+export const deploymentLiveSmokeRoutes: ReadonlyArray<Route<ApiCtx>> = [
+ { method: "POST", path: "/v1/deployment/live-session", auth: "source", handle: deploymentLiveSmoke },
+];
diff --git a/src/api/routes/index.ts b/src/api/routes/index.ts
index 6407d4925..baaae17ab 100644
--- a/src/api/routes/index.ts
+++ b/src/api/routes/index.ts
@@ -1,3 +1,4 @@
+import { deploymentLiveSmokeRoutes } from "./deployment-live-smoke.ts";
import { backgroundWorkRoutes } from "./background-work.ts";
import { composioRoutes } from "./composio.ts";
import { sendJson } from "../http.ts";
@@ -60,6 +61,7 @@ export const apiRoutes: ReadonlyArray<Route<ApiCtx>> = [
...searchRoutes,
...deploymentLayerRoutes,
...backgroundWorkRoutes,
+ ...deploymentLiveSmokeRoutes,
...turnRoutes,
...runEventRoutes,
...credentialRoutes,
diff --git a/src/api/user-scoped-routes.ts b/src/api/user-scoped-routes.ts
index d649a0ac6..8ed33ad8b 100644
--- a/src/api/user-scoped-routes.ts
+++ b/src/api/user-scoped-routes.ts
@@ -140,6 +140,7 @@ const SYSTEM: Rule[] = [
pat("POST", "/v1/auth/broker/sessions/use"),
pat("PUT", "/v1/deployment-layer"),
pat("POST", "/v1/background-work"),
+ pat("POST", "/v1/deployment/live-session"),
pat("POST", "/v1/session-cap"),
pat("POST", "/v1/keychain/drops/:id"),
pat("POST", "/v1/keychain/asks"),
diff --git a/src/deployment/postdeploy-smoke.ts b/src/deployment/postdeploy-smoke.ts
index 6af19dd81..0911b241e 100644
--- a/src/deployment/postdeploy-smoke.ts
+++ b/src/deployment/postdeploy-smoke.ts
@@ -285,6 +285,9 @@ async function checkDatabase(config: PostdeployConfig): Promise<void> {
const pg = (await import("pg")).default;
const client = new pg.Client({
connectionString: databaseUrl,
+ connectionTimeoutMillis: 30_000,
+ query_timeout: 30_000,
+ statement_timeout: 30_000,
...resolvePgCaTrust({
...(config.databaseCaCert ? { cert: config.databaseCaCert } : {}),
...(config.databaseCaCertFile ? { certFile: config.databaseCaCertFile } : {}),
@@ -323,11 +326,20 @@ async function runPostdeploySmoke(config: PostdeployConfig): Promise<void> {
console.log("deployed staging database, API, services, public route, and Slack smoke passed");
}
+export async function runSessionSmoke(config: PostdeployConfig, baseUrl: string): Promise<void> {
+ await checkLiveSession(config, baseUrl, (input, init) => {
+ const timeout = AbortSignal.timeout(
+ init?.method === "POST" && new URL(String(input)).pathname === "/v1/turns" ? 300_000 : 30_000,
+ );
+ return fetch(input, { ...init, signal: init?.signal ? AbortSignal.any([timeout, init.signal]) : timeout });
+ });
+ if (config.databaseUrl) await checkDatabase(config);
+}
+
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
const config = loadConfig();
if (process.argv[2] === "session") {
- await checkLiveSession(config, process.argv[3] ?? `http://127.0.0.1:${config.port}`);
- if (config.databaseUrl) await checkDatabase(config);
+ await runSessionSmoke(config, process.argv[3] ?? `http://127.0.0.1:${config.port}`);
console.log("database and live session smoke passed");
} else {
await runPostdeploySmoke(config);
diff --git a/src/wiring.ts b/src/wiring.ts
index 18c015316..0e7cb7f47 100644
--- a/src/wiring.ts
+++ b/src/wiring.ts
@@ -1,3 +1,4 @@
+import { runSessionSmoke } from "./deployment/postdeploy-smoke.ts";
import {
createBackgroundOwnershipStore,
type BackgroundOwnershipStore,
@@ -2503,7 +2504,11 @@ export function serverDeps(
return {
production: config.production,
...(built.backgroundOwnership
- ? { backgroundOwnership: built.backgroundOwnership, deploymentControlSecret: config.deploymentControlSecret }
+ ? {
+ backgroundOwnership: built.backgroundOwnership,
+ deploymentControlSecret: config.deploymentControlSecret,
+ deploymentLiveSmoke: () => runSessionSmoke(config, `http://127.0.0.1:${config.port}`),
+ }
: {}),
allowUnauthenticatedCore: config.allowUnauthenticatedCore,
...(config.signingSecret ? { signingSecret: config.signingSecret } : {}),
diff --git a/test/deployment-live-smoke-routes.test.ts b/test/deployment-live-smoke-routes.test.ts
new file mode 100644
index 000000000..8d517c42c
--- /dev/null
+++ b/test/deployment-live-smoke-routes.test.ts
@@ -0,0 +1,188 @@
+import "./support/auto-fake-sprites.ts";
+import assert from "node:assert/strict";
+import { randomUUID } from "node:crypto";
+import type { AddressInfo } from "node:net";
+import test from "node:test";
+import { createServer } from "../src/api/server.ts";
+import { mintCapabilityToken, CAPABILITY_TTL_MS } from "../src/auth/capability-token.ts";
+import { scopeId } from "../src/types.ts";
+import { signRequest } from "../src/auth/source-auth.ts";
+import { buildApp } from "../src/wiring.ts";
+import { createMemoryMap } from "../src/persistence/durable-map.ts";
+import { createBackgroundOwnershipStore, type BackgroundOwnership } from "../src/runs/background-ownership.ts";
+import { createMemoryReplayDedupe } from "../src/auth/replay-dedupe.ts";
+import { testConfig } from "./support/test-config.ts";
+
+const sourceSecret = "source-only-secret".repeat(3);
+const controlSecret = "deployment-only-secret".repeat(3);
+const path = "/v1/deployment/live-session";
+
+async function fixture(run: () => Promise<void> = async () => {}) {
+ const store = createBackgroundOwnershipStore(createMemoryMap<BackgroundOwnership>());
+ await store.register({ instanceId: "instance-a", deploymentId: "cohort-a", taskArn: "task-a" });
+ await store.transition({
+ expectedGeneration: 0,
+ requestId: randomUUID(),
+ desiredDeploymentId: "cohort-a",
+ bootstrapTaskArns: ["task-a"],
+ });
+ await store.admit("instance-a", 1, false);
+ await store.markReady("instance-a", 1);
+ const dedupe = createMemoryReplayDedupe();
+ const built = buildApp(testConfig({ signingSecret: sourceSecret }));
+ const server = createServer(built.app, {
+ signingSecret: sourceSecret,
+ portalIdentitySecret: "portal-identity-secret".repeat(3),
+ capabilitySecret: "capability-only-secret".repeat(3),
+ requireSignedPortalIdentity: true,
+ backgroundOwnership: { store, instanceId: "instance-a", deploymentId: "cohort-a" },
+ deploymentControlSecret: controlSecret,
+ deploymentLiveSmoke: run,
+ replayDedupe: { durable: true, claim: (...args) => dedupe.claim(...args) },
+ });
+ server.listen(0);
+ const url = `http://localhost:${(server.address() as AddressInfo).port}${path}`;
+ let nonce = 0;
+ const request = async (method: string, body?: unknown, overrides: Record<string, string> = {}) => {
+ const raw = body === undefined ? "" : JSON.stringify(body);
+ const ts = Math.floor(Date.now() / 1000) + nonce++;
+ return fetch(url, {
+ method,
+ headers: {
+ "content-type": "application/json",
+ "x-timestamp": String(ts),
+ "x-signature": signRequest(sourceSecret, ts, `${method}\n${path}\n${raw}`),
+ authorization: `Bearer ${controlSecret}`,
+ ...overrides,
+ },
+ ...(raw ? { body: raw } : {}),
+ });
+ };
+ return { store, request, close: () => new Promise<void>((resolve) => server.close(() => resolve())) };
+}
+
+const body = () => ({
+ requestId: randomUUID(),
+ expectedDeploymentId: "cohort-a",
+ expectedGeneration: 1,
+ expectedTaskArns: ["task-a"],
+});
+
+test("live smoke requires distinct credentials, rejects capabilities and arbitrary inputs", async () => {
+ let calls = 0;
+ const srv = await fixture(async () => {
+ calls++;
+ });
+ try {
+ const invalidCredentials: Record<string, string>[] = [
+ { authorization: "" },
+ { authorization: `Bearer ${sourceSecret}` },
+ { "x-signature": "invalid" },
+ ];
+ for (const credentials of invalidCredentials) {
+ assert.equal((await srv.request("POST", body(), credentials)).status, 401);
+ }
+ const capability = await mintCapabilityToken(
+ { actorId: "U1", scopeId: scopeId("personal", "U1"), exp: Date.now() + CAPABILITY_TTL_MS },
+ "capability-only-secret".repeat(3),
+ );
+ assert.equal((await srv.request("POST", body(), { "x-agent-capability": capability })).status, 403);
+ for (const request of [
+ null,
+ { ...body(), url: "https://untrusted.invalid" },
+ { ...body(), expectedTaskArns: [] },
+ { ...body(), expectedTaskArns: ["task-a", "task-a"] },
+ { ...body(), expectedDeploymentId: "other" },
+ ]) {
+ assert.equal((await srv.request("POST", request)).status, 400);
+ }
+ for (const request of [
+ { ...body(), expectedGeneration: 2 },
+ { ...body(), expectedTaskArns: ["different"] },
+ ]) {
+ assert.equal((await srv.request("POST", request)).status, 409);
+ }
+ assert.equal(calls, 0);
+ } finally {
+ await srv.close();
+ }
+});
+
+test("live smoke returns exact identity and rejects request replay", async () => {
+ let calls = 0;
+ const srv = await fixture(async () => {
+ calls++;
+ });
+ try {
+ const request = body();
+ const response = await srv.request("POST", request);
+ assert.equal(response.status, 200);
+ assert.equal(response.headers.get("content-type"), "application/x-ndjson");
+ assert.deepEqual(await response.json(), {
+ ok: true,
+ requestId: request.requestId,
+ deploymentId: "cohort-a",
+ instanceId: "instance-a",
+ taskArn: "task-a",
+ generation: 1,
+ });
+ assert.equal((await srv.request("POST", request)).status, 409);
+ assert.equal(calls, 1);
+ } finally {
+ await srv.close();
+ }
+});
+
+test("live smoke keeps singleflight through client disconnect and cleanup", async () => {
+ let release!: () => void;
+ let finished!: () => void;
+ const done = new Promise<void>((resolve) => {
+ finished = resolve;
+ });
+ let calls = 0;
+ const srv = await fixture(async () => {
+ calls++;
+ await new Promise<void>((resolve) => {
+ release = resolve;
+ });
+ finished();
+ });
+ try {
+ const response = await srv.request("POST", body());
+ const reader = response.body!.getReader();
+ assert.equal(new TextDecoder().decode((await reader.read()).value).trim(), "");
+ assert.equal(new TextDecoder().decode((await reader.read()).value).trim(), "");
+ await reader.cancel();
+ assert.equal((await srv.request("POST", body())).status, 409);
+ assert.equal(calls, 1);
+ release();
+ await done;
+ } finally {
+ release?.();
+ await srv.close();
+ }
+});
+
+test("live smoke fails when ownership changes and never returns raw errors", async () => {
+ const srv = await fixture(async () => {
+ await srv.store.transition({ expectedGeneration: 1, requestId: randomUUID(), desiredDeploymentId: null });
+ });
+ try {
+ const result = (await (await srv.request("POST", body())).json()) as { ok: boolean; error: string };
+ assert.equal(result.ok, false);
+ assert.equal(result.error, "deployment_smoke_ownership_changed");
+ } finally {
+ await srv.close();
+ }
+ const failed = await fixture(async () => {
+ throw new Error("secret-value-do-not-expose");
+ });
+ try {
+ const response = await failed.request("POST", body());
+ const raw = await response.text();
+ assert.equal(JSON.parse(raw).error, "deployment_smoke_failed");
+ assert.ok(!raw.includes("secret-value"));
+ } finally {
+ await failed.close();
+ }
+});
diff --git a/test/postdeploy-session-command.test.ts b/test/postdeploy-session-command.test.ts
new file mode 100644
index 000000000..5e75ec59f
--- /dev/null
+++ b/test/postdeploy-session-command.test.ts
@@ -0,0 +1,84 @@
+import assert from "node:assert/strict";
+import { mock, test } from "node:test";
+import { testConfig } from "./support/test-config.ts";
+
+const events: string[] = [];
+let invalidIndexes = false;
+let databaseOptions: Record<string, unknown> = {};
+mock.module("pg", {
+ defaultExport: {
+ Client: class {
+ constructor(options: Record<string, unknown>) {
+ databaseOptions = options;
+ }
+ async connect() {
+ events.push("database-connect");
+ }
+ async query(sql: string) {
+ const index = sql.includes("pg_index");
+ events.push(index ? "index-check" : "parallel-check");
+ return { rows: index && invalidIndexes ? [{ index_name: "invalid" }] : [] };
+ }
+ async end() {
+ events.push("database-close");
+ }
+ },
+ },
+});
+const { runSessionSmoke } = await import("../src/deployment/postdeploy-smoke.ts");
+
+test("session command preserves archive then database checks with bounded requests and database operations", async (t) => {
+ const timeouts: number[] = [];
+ const timeout = AbortSignal.timeout;
+ t.mock.method(AbortSignal, "timeout", (ms: number) => {
+ timeouts.push(ms);
+ return timeout(ms);
+ });
+ t.mock.method(globalThis, "fetch", async (input: string | URL | Request, init?: RequestInit) => {
+ const path = new URL(String(input)).pathname;
+ assert.ok(init?.signal);
+ if (path === "/v1/turns") {
+ events.push("model");
+ return Response.json({ status: "ok", sessionId: "session", reply: "QM deployment canary passed." });
+ }
+ if (path === "/v1/admin/errors") {
+ events.push("errors");
+ return Response.json({ errors: [] });
+ }
+ if (init?.method === "POST") {
+ events.push("archive");
+ return Response.json({});
+ }
+ events.push("persisted");
+ return Response.json({ session: { title: "Canary" }, entries: [{ type: "user" }, { type: "assistant" }] });
+ });
+ const config = testConfig({
+ databaseUrl: "postgres://unused",
+ adminGrants: "admin:org_admin",
+ portalIdentitySecret: "portal-secret",
+ signingSecret: "source-secret",
+ });
+ await runSessionSmoke(config, "http://127.0.0.1:8080");
+ assert.deepEqual(events, [
+ "model",
+ "persisted",
+ "errors",
+ "archive",
+ "database-connect",
+ "parallel-check",
+ "index-check",
+ "database-close",
+ ]);
+ assert.deepEqual(timeouts, [300_000, 60_000, 30_000, 30_000, 30_000]);
+ for (const option of ["connectionTimeoutMillis", "query_timeout", "statement_timeout"])
+ assert.equal(databaseOptions[option], 30_000);
+ invalidIndexes = true;
+ events.length = 0;
+ try {
+ await assert.rejects(runSessionSmoke(config, "http://127.0.0.1:8080"), /invalid PostgreSQL indexes/);
+ assert.equal(events.at(-1), "database-close");
+ assert.ok(events.indexOf("archive") < events.indexOf("database-connect"));
+ } finally {
+ invalidIndexes = false;
+ }
+});