Multiplayer agent harness for work.
import { resolvePgCaTrust } from "../persistence/pg-pool.ts";
import { randomUUID } from "node:crypto";
import { pathToFileURL } from "node:url";
import { PORTAL_IDENTITY_HEADER } from "../auth/portal-identity.ts";
import { mintSignedPayload } from "../auth/signed-token.ts";
import { signedRequestHeaders } from "../../plugins/chassis/src/source-auth-sign.ts";
import { loadConfig, type Config } from "../config.ts";
import { errMessage } from "../util/errors.ts";
import { sleep } from "../util/async.ts";
export const PARALLEL_EXCEPTION_QUERY = `
SELECT n.nspname AS schema_name, p.proname AS function_name
FROM pg_proc p
JOIN pg_namespace n ON n.oid = p.pronamespace
JOIN pg_language l ON l.oid = p.prolang
WHERE l.lanname = 'plpgsql'
AND p.proparallel = 's'
AND p.prosrc ~* '\\mEXCEPTION\\M'
AND n.nspname NOT IN ('pg_catalog', 'information_schema')
`;
export const INVALID_INDEX_QUERY = `
SELECT n.nspname AS schema_name, c.relname AS index_name
FROM pg_index i
JOIN pg_class c ON c.oid = i.indexrelid
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE (NOT i.indisvalid OR NOT i.indisready)
AND n.nspname NOT IN ('pg_catalog', 'information_schema')
`;
export function firstAdminPrincipal(raw: string | undefined): string {
const principal = raw
?.split(",")
.map((entry) => entry.trim())
.find((entry) => entry.endsWith(":org_admin"))
?.slice(0, -":org_admin".length)
.trim();
if (!principal) throw new Error("postdeploy smoke requires ADMIN_GRANTS with an org_admin");
return principal;
}
type FetchLike = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
export function deployedHealthUrls(flyAppName: string | undefined, publicWebUrl: string | undefined): string[] {
if (!flyAppName?.endsWith("-core")) throw new Error("postdeploy smoke requires FLY_APP_NAME ending in -core");
if (!publicWebUrl) throw new Error("postdeploy smoke requires PUBLIC_WEB_URL");
const prefix = flyAppName.slice(0, -"-core".length);
return [
"http://127.0.0.1:8080/healthz",
`http://${prefix}-admin.internal:8080/healthz`,
`http://${prefix}-web-ui.flycast/healthz`,
`http://${prefix}-portal.internal:8080/healthz`,
new URL("/healthz", publicWebUrl).toString(),
];
}
export async function checkDeployedHealth(urls: string[], fetchImpl: FetchLike = fetch): Promise<void> {
for (const url of urls) {
const response = await fetchImpl(url, { redirect: "manual" });
if (!response.ok) throw new Error(`deployed staging health ${url} returned ${response.status}`);
}
}
export async function checkSlackCredentials(
botToken: string | undefined,
appToken: string | undefined,
apiUrl: string | undefined,
fetchImpl: FetchLike = fetch,
): Promise<void> {
if (!botToken) throw new Error("postdeploy smoke requires SLACK_BOT_TOKEN");
if (!appToken) throw new Error("postdeploy smoke requires SLACK_APP_TOKEN");
const root = (apiUrl ?? "https://slack.com").replace(/\/+$/, "");
const base = root.endsWith("/api") ? `${root}/` : `${root}/api/`;
for (const [method, token] of [
["auth.test", botToken],
["apps.connections.open", appToken],
] as const) {
const response = await fetchImpl(`${base}${method}`, {
method: "POST",
headers: {
authorization: `Bearer ${token}`,
"content-type": "application/x-www-form-urlencoded",
},
});
const result = (await response.json()) as { ok?: boolean; error?: string };
if (!response.ok || result.ok !== true) {
throw new Error(`staging Slack ${method} failed: ${result.error ?? `HTTP ${response.status}`}`);
}
}
}
export async function stagingApiHeaders(
principalId: string,
sourceSecret: string,
portalIdentitySecret: string,
method: string,
path: string,
body = "",
base: Record<string, string> = {},
nowMs = Date.now(),
): Promise<Record<string, string>> {
const portalIdentity = await mintSignedPayload({ p: principalId, exp: nowMs + 60_000 }, portalIdentitySecret);
return signedRequestHeaders(
sourceSecret,
method,
path,
body,
{
...base,
[PORTAL_IDENTITY_HEADER]: portalIdentity,
},
Math.floor(nowMs / 1000),
);
}
type LiveSessionConfig = Pick<Config, "adminGrants" | "orgId" | "portalIdentitySecret" | "signingSecret">;
export async function checkLiveSession(
config: LiveSessionConfig,
baseUrl: string,
fetchImpl: FetchLike = fetch,
timing: { now: () => number; sleep: (ms: number) => Promise<void>; timeout?: (ms: number) => AbortSignal } = {
now: Date.now,
sleep,
},
): Promise<void> {
const { orgId, portalIdentitySecret, signingSecret: sourceSecret } = config;
if (!orgId) throw new Error("live session smoke requires ORG_ID");
if (!sourceSecret) throw new Error("live session smoke requires CORE_SIGNING_SECRET");
if (!portalIdentitySecret) throw new Error("live session smoke requires PORTAL_IDENTITY_SECRET");
const principalId = firstAdminPrincipal(config.adminGrants);
const root = baseUrl.replace(/\/+$/, "");
const request = async (
method: "GET" | "POST",
path: string,
body?: unknown,
admin = false,
signal?: AbortSignal,
): Promise<unknown> => {
const raw = body === undefined ? "" : JSON.stringify(body);
const headers = await stagingApiHeaders(
principalId,
sourceSecret,
portalIdentitySecret,
method,
path,
raw,
admin ? { "x-admin-actor": `${principalId}@${orgId}` } : {},
);
const response = await fetchImpl(`${root}${path}`, {
method,
headers: { ...headers, ...(raw ? { "content-type": "application/json" } : {}) },
...(raw ? { body: raw } : {}),
...(signal ? { signal } : {}),
});
const text = await response.text();
if (!response.ok)
throw new Error(`live session ${method} ${path} returned ${response.status}: ${text.slice(0, 500)}`);
try {
return JSON.parse(text) as unknown;
} catch {
throw new Error(`live session ${method} ${path} returned unparseable JSON`);
}
};
const nonce = randomUUID();
const expectedReply = "QM deployment canary passed.";
const threadRef = `web:${principalId}:deployment-canary-${nonce}`;
let turn: { status?: string; sessionId?: string; reply?: string } | undefined;
let failure: unknown;
try {
turn = (await request("POST", "/v1/turns", {
surface: "web",
actor: { externalId: principalId },
conversation: { kind: "dm", threadRef },
text: `Reply with exactly: ${expectedReply}`,
origin: { kind: "human" },
addressed: true,
readOnly: true,
skipMemory: true,
idempotencyKey: nonce,
})) as { status?: string; sessionId?: string; reply?: string };
if (!turn.sessionId) throw new Error(`live session model turn failed with status ${turn.status ?? "missing"}`);
if (turn.status !== "ok") throw new Error(`live session model turn failed with status ${turn.status ?? "missing"}`);
if (turn.reply?.trim() !== expectedReply) throw new Error("live session received an unexpected model reply");
const sessionPath = `/v1/sessions/${encodeURIComponent(turn.sessionId)}?viewer=${encodeURIComponent(principalId)}&tailTurns=1`;
const titleDeadline = timing.now() + 60_000;
const titleSignal = (timing.timeout ?? AbortSignal.timeout)(60_000);
let persisted = (await request("GET", sessionPath, undefined, false, titleSignal)) as {
session?: { title?: string | null };
entries?: Array<{ type?: string }>;
};
while (!persisted.session?.title?.trim() && timing.now() < titleDeadline && !titleSignal.aborted) {
await timing.sleep(Math.min(500, titleDeadline - timing.now()));
if (timing.now() >= titleDeadline || titleSignal.aborted) break;
persisted = (await request("GET", sessionPath, undefined, false, titleSignal)) as typeof persisted;
}
if (timing.now() >= titleDeadline || titleSignal.aborted || !persisted.session?.title?.trim())
throw new Error("live session has no generated title");
if (!persisted.entries?.some((entry) => entry.type === "user"))
throw new Error("live session has no persisted user turn");
if (!persisted.entries.some((entry) => entry.type === "assistant")) {
throw new Error("live session has no persisted assistant turn");
}
const errorsPath = `/v1/admin/errors?scope=${encodeURIComponent(`personal:${principalId}`)}&sessionId=${encodeURIComponent(turn.sessionId)}`;
const logged = (await request("GET", errorsPath, undefined, true)) as { errors?: unknown[] };
if (!Array.isArray(logged.errors)) throw new Error("live session error log response has no errors array");
if (logged.errors.length) throw new Error(`live session recorded ${logged.errors.length} error event(s)`);
} catch (error) {
failure = error;
}
let sessionId = turn?.sessionId;
if (!sessionId) {
try {
const sessionsPath = `/v1/admin/sessions?scope=${encodeURIComponent(`personal:${principalId}`)}&category=all&limit=200`;
const listed = (await request("GET", sessionsPath, undefined, true)) as {
sessions?: Array<{ id?: string; threadRef?: string }>;
};
sessionId = listed.sessions?.find((session) => session.threadRef === threadRef)?.id;
} catch (error) {
if (failure)
throw new AggregateError(
[failure, error],
`${errMessage(failure)}; session recovery failed: ${errMessage(error)}`,
{ cause: error },
);
throw error;
}
}
if (sessionId) {
try {
await request("POST", `/v1/sessions/${encodeURIComponent(sessionId)}`, { principalId, archived: true });
} catch (error) {
if (failure)
throw new AggregateError(
[failure, error],
`${errMessage(failure)}; session archive failed: ${errMessage(error)}`,
{ cause: error },
);
throw error;
}
}
if (failure) throw failure;
}
async function checkApi(
orgId: string,
principalId: string,
sourceSecret: string,
portalIdentitySecret: string,
port: string,
): Promise<void> {
const path = `/v1/admin/sessions?scope=${encodeURIComponent(`org:${orgId}`)}&limit=5&_smoke=${randomUUID()}`;
const response = await fetch(`http://127.0.0.1:${port}${path}`, {
headers: await stagingApiHeaders(principalId, sourceSecret, portalIdentitySecret, "GET", path, "", {
"x-admin-actor": `${principalId}@${orgId}`,
}),
});
const body = await response.text();
if (!response.ok) throw new Error(`staging session API returned ${response.status}: ${body.slice(0, 500)}`);
const parsed = JSON.parse(body) as { sessions?: unknown[] };
if (!Array.isArray(parsed.sessions)) throw new Error("staging session API response has no sessions array");
}
type PostdeployConfig = Pick<
Config,
| "adminGrants"
| "databaseUrl"
| "databaseCaCert"
| "databaseCaCertFile"
| "flyAppName"
| "orgId"
| "port"
| "portalIdentitySecret"
| "publicWebUrl"
| "signingSecret"
| "slack"
>;
async function checkDatabase(config: PostdeployConfig): Promise<void> {
const { databaseUrl } = config;
if (!databaseUrl) throw new Error("postdeploy smoke requires DATABASE_URL");
const pg = (await import("pg")).default;
const client = new pg.Client({
connectionString: databaseUrl,
connectionTimeoutMillis: 30_000,
query_timeout: 30_000,
statement_timeout: 30_000,
...resolvePgCaTrust({
...(config.databaseCaCert ? { cert: config.databaseCaCert } : {}),
...(config.databaseCaCertFile ? { certFile: config.databaseCaCertFile } : {}),
}),
});
await client.connect();
try {
const unsafe = await client.query(PARALLEL_EXCEPTION_QUERY);
if (unsafe.rows.length) {
throw new Error(`parallel-safe PL/pgSQL exception handlers: ${JSON.stringify(unsafe.rows)}`);
}
const invalid = await client.query(INVALID_INDEX_QUERY);
if (invalid.rows.length) throw new Error(`invalid PostgreSQL indexes: ${JSON.stringify(invalid.rows)}`);
} finally {
await client.end();
}
}
async function runPostdeploySmoke(config: PostdeployConfig): Promise<void> {
const { orgId, portalIdentitySecret, signingSecret: sourceSecret } = config;
if (!orgId) throw new Error("postdeploy smoke requires ORG_ID");
if (!sourceSecret) throw new Error("postdeploy smoke requires CORE_SIGNING_SECRET");
if (!portalIdentitySecret) throw new Error("postdeploy smoke requires PORTAL_IDENTITY_SECRET");
await checkDatabase(config);
await checkApi(
orgId,
firstAdminPrincipal(config.adminGrants),
sourceSecret,
portalIdentitySecret,
String(config.port),
);
await checkDeployedHealth(deployedHealthUrls(config.flyAppName, config.publicWebUrl));
await checkSlackCredentials(config.slack?.botToken, config.slack?.appToken, config.slack?.apiUrl);
console.log("deployed staging database, API, services, public route, and Slack smoke passed");
}
export async function runSessionSmoke(config: PostdeployConfig, baseUrl: string): Promise<void> {
await checkLiveSession(config, baseUrl, (input, init) => {
const timeout = AbortSignal.timeout(
init?.method === "POST" && new URL(String(input)).pathname === "/v1/turns" ? 300_000 : 30_000,
);
return fetch(input, { ...init, signal: init?.signal ? AbortSignal.any([timeout, init.signal]) : timeout });
});
if (config.databaseUrl) await checkDatabase(config);
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
const config = loadConfig();
if (process.argv[2] === "session") {
await runSessionSmoke(config, process.argv[3] ?? `http://127.0.0.1:${config.port}`);
console.log("database and live session smoke passed");
} else {
await runPostdeploySmoke(config);
}
}