Multiplayer agent harness for work.
Run deployment session checks inside the ready core (#1294)
Regan Bell8c5eaa2618c7
diff --git a/docs/background-ownership.md b/docs/background-ownership.mdindex d9dc16d97..f3f4edde8 100644--- a/docs/background-ownership.md+++ b/docs/background-ownership.md@@ -63,3 +63,33 @@ When a process cannot acknowledge, the trusted operator must independently prove```Retirement preserves the ownership generation, updates `lastRequestId`, and marks only the matching members retired and drained. Retired task identities cannot enroll again. An unreachable HTTP endpoint, old heartbeat, or elapsed timeout is not termination evidence.++## Live deployment session check++A ready, active deployment accepts `POST /v1/deployment/live-session` with the+same source signature and distinct deployment bearer credential. The exact body+contains `requestId` (a fresh UUID), `expectedDeploymentId`, `expectedGeneration`,+and `expectedTaskArns` (the exact healthy task cohort). The endpoint rejects+unknown fields, inactive ownership, incomplete readiness, stale generations,+and mismatched membership before starting work.++The check runs the same fixed session command used by the standalone deployment+smoke: a real model reply, persisted turns, generated title, session error log,+session archival, then the configured PostgreSQL catalog checks. The request+cannot select a principal, URL, model, prompt, or command. Model HTTP requests+have a five-minute limit; other HTTP requests and database operations have+30-second limits. These are failure bounds, not a release-duration claim.++The response is newline-delimited JSON with an initial newline and five-second+whitespace heartbeats, followed by one final object containing `ok`, `requestId`,+`deploymentId`, `instanceId`, `taskArn`, and `generation`. A failed result adds a+fixed error code without raw model, database, or credential details. Ownership+and cohort readiness are checked again before success. Heartbeats alone never+prove success; the caller must receive and verify the complete final object.++Request IDs are durably consumed across replicas and cannot be replayed. Each+process permits one check at a time, including its cleanup. Disconnecting the+caller does not cancel the check or release that guard before cleanup finishes.+The deployment lease still serializes operator requests across the cohort.+An uncertain result must fail the release; do not retry automatically or fall+back to a second canary execution.diff --git a/plugins/portal/src/index.ts b/plugins/portal/src/index.tsindex da32b1f3d..e5775f8de 100644--- a/plugins/portal/src/index.ts+++ b/plugins/portal/src/index.ts@@ -1132,7 +1132,10 @@ async function handle(req: IncomingMessage, res: ServerResponse): Promise<void>});}- if ((method === "GET" || method === "POST") && pathname === "/v1/background-work") {+ if (+ ((method === "GET" || method === "POST") && pathname === "/v1/background-work") ||+ (method === "POST" && pathname === "/v1/deployment/live-session")+ ) {return proxyToUpstream(req, res, { baseUrl: CORE, path: pathname, search: url.search }, [...FORWARD_DEPLOYMENT_LAYER_HEADERS,"authorization",diff --git a/plugins/portal/test/router.test.ts b/plugins/portal/test/router.test.tsindex 063105a76..650082fc5 100644--- a/plugins/portal/test/router.test.ts+++ b/plugins/portal/test/router.test.ts@@ -12,7 +12,11 @@ let deploymentLayerRequests = 0;const VALID_SOURCE_SIGNATURE = "v0=valid-source-signature";const upstream = createServer((req: IncomingMessage, res) => {- if (req.url?.startsWith("/v1/deployment-layer") || req.url?.startsWith("/v1/background-work")) {+ if (+ req.url?.startsWith("/v1/deployment-layer") ||+ req.url?.startsWith("/v1/background-work") ||+ req.url?.startsWith("/v1/deployment/live-session")+ ) {deploymentLayerRequests++;if (req.headers["x-timestamp"] !== "123" || req.headers["x-signature"] !== VALID_SOURCE_SIGNATURE) {res.writeHead(401, { "content-type": "application/json" });@@ -631,8 +635,12 @@ test("impersonate: an admin starts it; the web-ui hop carries target + impersona});test("background ownership forwards both credentials only on its exact control routes", async () => {- for (const method of ["GET", "POST"]) {- const response = await fetch(`${base}/v1/background-work`, {+ for (const [method, path] of [+ ["GET", "/v1/background-work"],+ ["POST", "/v1/background-work"],+ ["POST", "/v1/deployment/live-session"],+ ]) {+ const response = await fetch(`${base}${path}`, {method,headers: {"x-timestamp": "123",@@ -655,6 +663,8 @@ test("background ownership forwards both credentials only on its exact control rfor (const [method, path] of [["PUT", "/v1/background-work"],["GET", "/v1/background-work/nearby"],+ ["GET", "/v1/deployment/live-session"],+ ["POST", "/v1/deployment/live-session/nearby"],]) {const response = await fetch(`${base}${path}`, {method,diff --git a/src/api/deps.ts b/src/api/deps.tsindex ce88b5fdc..d54df9cb5 100644--- a/src/api/deps.ts+++ b/src/api/deps.ts@@ -82,6 +82,7 @@ export interface ServerDeps {requireSignedPortalIdentity?: boolean;control: ControlService;replayDedupe?: ReplayDedupe;+ deploymentLiveSmoke?: () => Promise<void>;brokerSessions?: BrokerSessionStore;connectorTokens?: ConnectorTokenStore;managedSlack?: ManagedSlack;diff --git a/src/api/routes/background-work.ts b/src/api/routes/background-work.tsindex eaab81115..81ea72f67 100644--- a/src/api/routes/background-work.ts+++ b/src/api/routes/background-work.ts@@ -31,18 +31,26 @@ function status(ctx: ApiCtx, state: BackgroundOwnership): void {});}-async function backgroundWork(ctx: ApiCtx): Promise<void> {+export function requireDeploymentControl(ctx: ApiCtx): boolean {const control = ctx.deps.backgroundOwnership;const secret = ctx.deps.deploymentControlSecret;if (!control || !ctx.secret || !ctx.auth || !isStrongSigningSecret(secret) || secret === ctx.secret) {- return sendJson(ctx.res, 503, { error: "background_control_unavailable" });+ sendJson(ctx.res, 503, { error: "background_control_unavailable" });+ return false;}const bearer = ctx.req.headers.authorization;const expected = Buffer.from(`Bearer ${secret}`);const supplied = Buffer.from(typeof bearer === "string" ? bearer : "");if (supplied.length !== expected.length || !timingSafeEqual(supplied, expected)) {- return sendJson(ctx.res, 401, { error: "unauthorized" });+ sendJson(ctx.res, 401, { error: "unauthorized" });+ return false;}+ return true;+}++async function backgroundWork(ctx: ApiCtx): Promise<void> {+ if (!requireDeploymentControl(ctx)) return;+ const control = ctx.deps.backgroundOwnership!;if (ctx.method === "GET") return status(ctx, await control.store.get());const body = ctx.body;if (diff --git a/src/api/routes/deployment-live-smoke.ts b/src/api/routes/deployment-live-smoke.tsnew file mode 100644index 000000000..0bc9b33f0--- /dev/null+++ b/src/api/routes/deployment-live-smoke.ts@@ -0,0 +1,101 @@+import { sendJson } from "../http.ts";+import type { BackgroundOwnership } from "../../runs/background-ownership.ts";+import { requireDeploymentControl } from "./background-work.ts";+import { isObj } from "./shared.ts";+import type { ApiCtx, Route } from "./route.ts";++const running = new WeakSet<object>();+const REQUEST_EXPIRY = Date.UTC(9999, 0, 1);++async function deploymentLiveSmoke(ctx: ApiCtx): Promise<void> {+ if (!requireDeploymentControl(ctx)) return;+ const control = ctx.deps.backgroundOwnership!;+ const run = ctx.deps.deploymentLiveSmoke;+ const replay = ctx.deps.replayDedupe;+ if (!run || !replay?.durable) return sendJson(ctx.res, 503, { error: "deployment_smoke_unavailable" });+ const body = ctx.body;+ if (+ !isObj(body) ||+ !Object.keys(body).every((key) =>+ ["requestId", "expectedDeploymentId", "expectedGeneration", "expectedTaskArns"].includes(key),+ ) ||+ typeof body.requestId !== "string" ||+ !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(body.requestId) ||+ typeof body.expectedDeploymentId !== "string" ||+ body.expectedDeploymentId !== control.deploymentId ||+ !Number.isSafeInteger(body.expectedGeneration) ||+ (body.expectedGeneration as number) < 1 ||+ !Array.isArray(body.expectedTaskArns) ||+ !body.expectedTaskArns.length ||+ body.expectedTaskArns.length > 1000 ||+ !body.expectedTaskArns.every((arn) => typeof arn === "string" && arn.length > 0 && arn.length <= 2048) ||+ new Set(body.expectedTaskArns).size !== body.expectedTaskArns.length+ )+ return sendJson(ctx.res, 400, { error: "invalid_deployment_smoke_request" });+ if (running.has(control)) return sendJson(ctx.res, 409, { error: "deployment_smoke_running" });+ const expectedTasks = new Set(body.expectedTaskArns as string[]);+ const valid = (state: BackgroundOwnership): boolean => {+ const members = state.members.filter((member) => !member.retired && member.state === "admitted");+ return (+ state.enabled &&+ state.generation === body.expectedGeneration &&+ state.desiredDeploymentId === control.deploymentId &&+ members.some((member) => member.instanceId === control.instanceId) &&+ members.length === expectedTasks.size &&+ new Set(members.map((member) => member.taskArn)).size === expectedTasks.size &&+ members.every(+ (member) =>+ member.deploymentId === control.deploymentId &&+ member.generation === state.generation &&+ member.state === "admitted" &&+ member.ready &&+ member.taskArn !== null &&+ expectedTasks.has(member.taskArn),+ )+ );+ };+ running.add(control);+ let heartbeat: ReturnType<typeof setInterval> | undefined;+ try {+ const state = await control.store.get();+ if (!valid(state)) return sendJson(ctx.res, 409, { error: "deployment_smoke_ownership_conflict" });+ if (!(await replay.claim(`deployment-live-smoke:${body.requestId.toLowerCase()}`, REQUEST_EXPIRY))) {+ return sendJson(ctx.res, 409, { error: "deployment_smoke_replay" });+ }+ const member = state.members.find((entry) => entry.instanceId === control.instanceId)!;+ const identity = {+ requestId: body.requestId,+ deploymentId: control.deploymentId,+ instanceId: control.instanceId,+ taskArn: member.taskArn,+ generation: state.generation,+ };+ ctx.res.writeHead(200, {+ "content-type": "application/x-ndjson",+ "cache-control": "no-store, no-transform",+ "x-accel-buffering": "no",+ });+ ctx.res.write("\n");+ heartbeat = setInterval(() => {+ if (!ctx.res.destroyed) ctx.res.write("\n");+ }, 5000);+ heartbeat.unref();+ ctx.res.once("close", () => clearInterval(heartbeat));+ let error: string | undefined;+ try {+ await run();+ if (!valid(await control.store.get())) error = "deployment_smoke_ownership_changed";+ } catch {+ error = "deployment_smoke_failed";+ }+ if (!ctx.res.destroyed)+ ctx.res.end(`${JSON.stringify({ ok: !error, ...identity, ...(error ? { error } : {}) })}\n`);+ } finally {+ clearInterval(heartbeat);+ running.delete(control);+ }+}++export const deploymentLiveSmokeRoutes: ReadonlyArray<Route<ApiCtx>> = [+ { method: "POST", path: "/v1/deployment/live-session", auth: "source", handle: deploymentLiveSmoke },+];diff --git a/src/api/routes/index.ts b/src/api/routes/index.tsindex 6407d4925..baaae17ab 100644--- a/src/api/routes/index.ts+++ b/src/api/routes/index.ts@@ -1,3 +1,4 @@+import { deploymentLiveSmokeRoutes } from "./deployment-live-smoke.ts";import { backgroundWorkRoutes } from "./background-work.ts";import { composioRoutes } from "./composio.ts";import { sendJson } from "../http.ts";@@ -60,6 +61,7 @@ export const apiRoutes: ReadonlyArray<Route<ApiCtx>> = [...searchRoutes,...deploymentLayerRoutes,...backgroundWorkRoutes,+ ...deploymentLiveSmokeRoutes,...turnRoutes,...runEventRoutes,...credentialRoutes,diff --git a/src/api/user-scoped-routes.ts b/src/api/user-scoped-routes.tsindex d649a0ac6..8ed33ad8b 100644--- a/src/api/user-scoped-routes.ts+++ b/src/api/user-scoped-routes.ts@@ -140,6 +140,7 @@ const SYSTEM: Rule[] = [pat("POST", "/v1/auth/broker/sessions/use"),pat("PUT", "/v1/deployment-layer"),pat("POST", "/v1/background-work"),+ pat("POST", "/v1/deployment/live-session"),pat("POST", "/v1/session-cap"),pat("POST", "/v1/keychain/drops/:id"),pat("POST", "/v1/keychain/asks"),diff --git a/src/deployment/postdeploy-smoke.ts b/src/deployment/postdeploy-smoke.tsindex 6af19dd81..0911b241e 100644--- a/src/deployment/postdeploy-smoke.ts+++ b/src/deployment/postdeploy-smoke.ts@@ -285,6 +285,9 @@ async function checkDatabase(config: PostdeployConfig): Promise<void> {const pg = (await import("pg")).default;const client = new pg.Client({connectionString: databaseUrl,+ connectionTimeoutMillis: 30_000,+ query_timeout: 30_000,+ statement_timeout: 30_000,...resolvePgCaTrust({...(config.databaseCaCert ? { cert: config.databaseCaCert } : {}),...(config.databaseCaCertFile ? { certFile: config.databaseCaCertFile } : {}),@@ -323,11 +326,20 @@ async function runPostdeploySmoke(config: PostdeployConfig): Promise<void> {console.log("deployed staging database, API, services, public route, and Slack smoke passed");}+export async function runSessionSmoke(config: PostdeployConfig, baseUrl: string): Promise<void> {+ await checkLiveSession(config, baseUrl, (input, init) => {+ const timeout = AbortSignal.timeout(+ init?.method === "POST" && new URL(String(input)).pathname === "/v1/turns" ? 300_000 : 30_000,+ );+ return fetch(input, { ...init, signal: init?.signal ? AbortSignal.any([timeout, init.signal]) : timeout });+ });+ if (config.databaseUrl) await checkDatabase(config);+}+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {const config = loadConfig();if (process.argv[2] === "session") {- await checkLiveSession(config, process.argv[3] ?? `http://127.0.0.1:${config.port}`);- if (config.databaseUrl) await checkDatabase(config);+ await runSessionSmoke(config, process.argv[3] ?? `http://127.0.0.1:${config.port}`);console.log("database and live session smoke passed");} else {await runPostdeploySmoke(config);diff --git a/src/wiring.ts b/src/wiring.tsindex 18c015316..0e7cb7f47 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -1,3 +1,4 @@+import { runSessionSmoke } from "./deployment/postdeploy-smoke.ts";import {createBackgroundOwnershipStore,type BackgroundOwnershipStore,@@ -2503,7 +2504,11 @@ export function serverDeps(return {production: config.production,...(built.backgroundOwnership- ? { backgroundOwnership: built.backgroundOwnership, deploymentControlSecret: config.deploymentControlSecret }+ ? {+ backgroundOwnership: built.backgroundOwnership,+ deploymentControlSecret: config.deploymentControlSecret,+ deploymentLiveSmoke: () => runSessionSmoke(config, `http://127.0.0.1:${config.port}`),+ }: {}),allowUnauthenticatedCore: config.allowUnauthenticatedCore,...(config.signingSecret ? { signingSecret: config.signingSecret } : {}),diff --git a/test/deployment-live-smoke-routes.test.ts b/test/deployment-live-smoke-routes.test.tsnew file mode 100644index 000000000..8d517c42c--- /dev/null+++ b/test/deployment-live-smoke-routes.test.ts@@ -0,0 +1,188 @@+import "./support/auto-fake-sprites.ts";+import assert from "node:assert/strict";+import { randomUUID } from "node:crypto";+import type { AddressInfo } from "node:net";+import test from "node:test";+import { createServer } from "../src/api/server.ts";+import { mintCapabilityToken, CAPABILITY_TTL_MS } from "../src/auth/capability-token.ts";+import { scopeId } from "../src/types.ts";+import { signRequest } from "../src/auth/source-auth.ts";+import { buildApp } from "../src/wiring.ts";+import { createMemoryMap } from "../src/persistence/durable-map.ts";+import { createBackgroundOwnershipStore, type BackgroundOwnership } from "../src/runs/background-ownership.ts";+import { createMemoryReplayDedupe } from "../src/auth/replay-dedupe.ts";+import { testConfig } from "./support/test-config.ts";++const sourceSecret = "source-only-secret".repeat(3);+const controlSecret = "deployment-only-secret".repeat(3);+const path = "/v1/deployment/live-session";++async function fixture(run: () => Promise<void> = async () => {}) {+ const store = createBackgroundOwnershipStore(createMemoryMap<BackgroundOwnership>());+ await store.register({ instanceId: "instance-a", deploymentId: "cohort-a", taskArn: "task-a" });+ await store.transition({+ expectedGeneration: 0,+ requestId: randomUUID(),+ desiredDeploymentId: "cohort-a",+ bootstrapTaskArns: ["task-a"],+ });+ await store.admit("instance-a", 1, false);+ await store.markReady("instance-a", 1);+ const dedupe = createMemoryReplayDedupe();+ const built = buildApp(testConfig({ signingSecret: sourceSecret }));+ const server = createServer(built.app, {+ signingSecret: sourceSecret,+ portalIdentitySecret: "portal-identity-secret".repeat(3),+ capabilitySecret: "capability-only-secret".repeat(3),+ requireSignedPortalIdentity: true,+ backgroundOwnership: { store, instanceId: "instance-a", deploymentId: "cohort-a" },+ deploymentControlSecret: controlSecret,+ deploymentLiveSmoke: run,+ replayDedupe: { durable: true, claim: (...args) => dedupe.claim(...args) },+ });+ server.listen(0);+ const url = `http://localhost:${(server.address() as AddressInfo).port}${path}`;+ let nonce = 0;+ const request = async (method: string, body?: unknown, overrides: Record<string, string> = {}) => {+ const raw = body === undefined ? "" : JSON.stringify(body);+ const ts = Math.floor(Date.now() / 1000) + nonce++;+ return fetch(url, {+ method,+ headers: {+ "content-type": "application/json",+ "x-timestamp": String(ts),+ "x-signature": signRequest(sourceSecret, ts, `${method}\n${path}\n${raw}`),+ authorization: `Bearer ${controlSecret}`,+ ...overrides,+ },+ ...(raw ? { body: raw } : {}),+ });+ };+ return { store, request, close: () => new Promise<void>((resolve) => server.close(() => resolve())) };+}++const body = () => ({+ requestId: randomUUID(),+ expectedDeploymentId: "cohort-a",+ expectedGeneration: 1,+ expectedTaskArns: ["task-a"],+});++test("live smoke requires distinct credentials, rejects capabilities and arbitrary inputs", async () => {+ let calls = 0;+ const srv = await fixture(async () => {+ calls++;+ });+ try {+ const invalidCredentials: Record<string, string>[] = [+ { authorization: "" },+ { authorization: `Bearer ${sourceSecret}` },+ { "x-signature": "invalid" },+ ];+ for (const credentials of invalidCredentials) {+ assert.equal((await srv.request("POST", body(), credentials)).status, 401);+ }+ const capability = await mintCapabilityToken(+ { actorId: "U1", scopeId: scopeId("personal", "U1"), exp: Date.now() + CAPABILITY_TTL_MS },+ "capability-only-secret".repeat(3),+ );+ assert.equal((await srv.request("POST", body(), { "x-agent-capability": capability })).status, 403);+ for (const request of [+ null,+ { ...body(), url: "https://untrusted.invalid" },+ { ...body(), expectedTaskArns: [] },+ { ...body(), expectedTaskArns: ["task-a", "task-a"] },+ { ...body(), expectedDeploymentId: "other" },+ ]) {+ assert.equal((await srv.request("POST", request)).status, 400);+ }+ for (const request of [+ { ...body(), expectedGeneration: 2 },+ { ...body(), expectedTaskArns: ["different"] },+ ]) {+ assert.equal((await srv.request("POST", request)).status, 409);+ }+ assert.equal(calls, 0);+ } finally {+ await srv.close();+ }+});++test("live smoke returns exact identity and rejects request replay", async () => {+ let calls = 0;+ const srv = await fixture(async () => {+ calls++;+ });+ try {+ const request = body();+ const response = await srv.request("POST", request);+ assert.equal(response.status, 200);+ assert.equal(response.headers.get("content-type"), "application/x-ndjson");+ assert.deepEqual(await response.json(), {+ ok: true,+ requestId: request.requestId,+ deploymentId: "cohort-a",+ instanceId: "instance-a",+ taskArn: "task-a",+ generation: 1,+ });+ assert.equal((await srv.request("POST", request)).status, 409);+ assert.equal(calls, 1);+ } finally {+ await srv.close();+ }+});++test("live smoke keeps singleflight through client disconnect and cleanup", async () => {+ let release!: () => void;+ let finished!: () => void;+ const done = new Promise<void>((resolve) => {+ finished = resolve;+ });+ let calls = 0;+ const srv = await fixture(async () => {+ calls++;+ await new Promise<void>((resolve) => {+ release = resolve;+ });+ finished();+ });+ try {+ const response = await srv.request("POST", body());+ const reader = response.body!.getReader();+ assert.equal(new TextDecoder().decode((await reader.read()).value).trim(), "");+ assert.equal(new TextDecoder().decode((await reader.read()).value).trim(), "");+ await reader.cancel();+ assert.equal((await srv.request("POST", body())).status, 409);+ assert.equal(calls, 1);+ release();+ await done;+ } finally {+ release?.();+ await srv.close();+ }+});++test("live smoke fails when ownership changes and never returns raw errors", async () => {+ const srv = await fixture(async () => {+ await srv.store.transition({ expectedGeneration: 1, requestId: randomUUID(), desiredDeploymentId: null });+ });+ try {+ const result = (await (await srv.request("POST", body())).json()) as { ok: boolean; error: string };+ assert.equal(result.ok, false);+ assert.equal(result.error, "deployment_smoke_ownership_changed");+ } finally {+ await srv.close();+ }+ const failed = await fixture(async () => {+ throw new Error("secret-value-do-not-expose");+ });+ try {+ const response = await failed.request("POST", body());+ const raw = await response.text();+ assert.equal(JSON.parse(raw).error, "deployment_smoke_failed");+ assert.ok(!raw.includes("secret-value"));+ } finally {+ await failed.close();+ }+});diff --git a/test/postdeploy-session-command.test.ts b/test/postdeploy-session-command.test.tsnew file mode 100644index 000000000..5e75ec59f--- /dev/null+++ b/test/postdeploy-session-command.test.ts@@ -0,0 +1,84 @@+import assert from "node:assert/strict";+import { mock, test } from "node:test";+import { testConfig } from "./support/test-config.ts";++const events: string[] = [];+let invalidIndexes = false;+let databaseOptions: Record<string, unknown> = {};+mock.module("pg", {+ defaultExport: {+ Client: class {+ constructor(options: Record<string, unknown>) {+ databaseOptions = options;+ }+ async connect() {+ events.push("database-connect");+ }+ async query(sql: string) {+ const index = sql.includes("pg_index");+ events.push(index ? "index-check" : "parallel-check");+ return { rows: index && invalidIndexes ? [{ index_name: "invalid" }] : [] };+ }+ async end() {+ events.push("database-close");+ }+ },+ },+});+const { runSessionSmoke } = await import("../src/deployment/postdeploy-smoke.ts");++test("session command preserves archive then database checks with bounded requests and database operations", async (t) => {+ const timeouts: number[] = [];+ const timeout = AbortSignal.timeout;+ t.mock.method(AbortSignal, "timeout", (ms: number) => {+ timeouts.push(ms);+ return timeout(ms);+ });+ t.mock.method(globalThis, "fetch", async (input: string | URL | Request, init?: RequestInit) => {+ const path = new URL(String(input)).pathname;+ assert.ok(init?.signal);+ if (path === "/v1/turns") {+ events.push("model");+ return Response.json({ status: "ok", sessionId: "session", reply: "QM deployment canary passed." });+ }+ if (path === "/v1/admin/errors") {+ events.push("errors");+ return Response.json({ errors: [] });+ }+ if (init?.method === "POST") {+ events.push("archive");+ return Response.json({});+ }+ events.push("persisted");+ return Response.json({ session: { title: "Canary" }, entries: [{ type: "user" }, { type: "assistant" }] });+ });+ const config = testConfig({+ databaseUrl: "postgres://unused",+ adminGrants: "admin:org_admin",+ portalIdentitySecret: "portal-secret",+ signingSecret: "source-secret",+ });+ await runSessionSmoke(config, "http://127.0.0.1:8080");+ assert.deepEqual(events, [+ "model",+ "persisted",+ "errors",+ "archive",+ "database-connect",+ "parallel-check",+ "index-check",+ "database-close",+ ]);+ assert.deepEqual(timeouts, [300_000, 60_000, 30_000, 30_000, 30_000]);+ for (const option of ["connectionTimeoutMillis", "query_timeout", "statement_timeout"])+ assert.equal(databaseOptions[option], 30_000);+ invalidIndexes = true;+ events.length = 0;+ try {+ await assert.rejects(runSessionSmoke(config, "http://127.0.0.1:8080"), /invalid PostgreSQL indexes/);+ assert.equal(events.at(-1), "database-close");+ assert.ok(events.indexOf("archive") < events.indexOf("database-connect"));+ } finally {+ invalidIndexes = false;+ }+});