Multiplayer agent harness for work.
Stage items cutover until deployment promotion
Regan Bell4ee45b57a8e5
diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.tsindex 4ce355a93..10a9bbe51 100644--- a/cli/src/backends/aws.ts+++ b/cli/src/backends/aws.ts@@ -3126,6 +3126,26 @@ export async function awsBackgroundWorkStatus(return { status, deploymentId: cohort.deploymentId, taskArns: cohort.taskArns, manifestId: cohort.manifest.id };}+export async function awsItemsPromotion(+ config: QmConfig,+ configDir: string,+ options: { writersStopped?: boolean; candidate?: string } = {},+): Promise<{ deploymentId: string; ready: boolean }> {+ assertAwsCallerAccount(requireAws(config));+ const cohort = await awsBackgroundCohort(config, configDir, options.candidate);+ const transport = awsBackgroundWorkTransport(config, "/v1/items-promotion", 120_000);+ const response = await transport(+ options.writersStopped ? "POST" : "GET",+ options.writersStopped ? JSON.stringify({ deploymentId: cohort.deploymentId, writersStopped: true }) : "",+ );+ if (response.status !== 200) throw new CliError(`items promotion failed with HTTP ${response.status}`);+ const result = JSON.parse(response.body) as { deploymentId?: string; ready?: boolean };+ if (result.deploymentId !== cohort.deploymentId || typeof result.ready !== "boolean")+ throw new CliError("items promotion returned an unexpected deployment");+ if (options.writersStopped && !result.ready) throw new CliError("items promotion has not completed");+ return { deploymentId: cohort.deploymentId, ready: result.ready };+}+export interface AwsBackgroundWorkCapacity {manifestId: string;deploymentId: string;diff --git a/deploy/core/Dockerfile b/deploy/core/Dockerfileindex d35660561..b8ca1ad75 100644--- a/deploy/core/Dockerfile+++ b/deploy/core/Dockerfile@@ -24,6 +24,7 @@ RUN npm ci --omit=dev && npm run audit:prod \COPY --from=connector-sdk-build /app/.generated/connector-sdk ./.generated/connector-sdkCOPY src ./src+COPY scripts/ops/items-copy.ts ./scripts/ops/items-copy.tsCOPY cli/src/canonical-json.ts cli/src/credential-paths.ts cli/src/scope-storage-key.ts cli/src/tool-descriptor.ts ./cli/src/COPY cli/templates/slack-manifest.json ./cli/templates/slack-manifest.jsonCOPY skills-seed ./skills-seeddiff --git a/package.json b/package.jsonindex c946db626..967552409 100644--- a/package.json+++ b/package.json@@ -48,7 +48,7 @@"sandbox:local:build": "bash scripts/local-sandbox-build.sh","sandbox:porter:build": "docker build --platform linux/amd64 -t \"${PORTER_SANDBOX_IMAGE:?Set PORTER_SANDBOX_IMAGE to the registry tag to publish}\" porter && docker push \"$PORTER_SANDBOX_IMAGE\"","deploy:porter-runner-image": "docker build --platform linux/amd64 -t \"${PORTER_DEPLOY_RUNNER_IMAGE:?Set PORTER_DEPLOY_RUNNER_IMAGE to the registry tag to publish}\" -f deploy/porter-runner/Dockerfile . && docker push \"$PORTER_DEPLOY_RUNNER_IMAGE\"",- "test:pg": "node scripts/run-pg-tests.mjs test/batch-paths.test.ts test/session-append-many.test.ts test/cron-queue.test.ts test/postgres-migration-startup.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-tapeless-store.test.ts test/postgres-legacy-tape-steps.test.ts test/postgres-items-copy.test.ts test/semantic-memory-pg.test.ts test/semantic-memory-http.test.ts test/semantic-memory-large-pg.test.ts test/postgres-instance-registry.test.ts test/readiness.test.ts test/postgres-grant-store.test.ts test/postgres-acl-file-id-migration.test.ts test/postgres-admin-grants.test.ts test/postgres-teams.test.ts test/postgres-decisions.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-linked-identity-membership.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-cron-run-as-migration.test.ts test/postgres-trigger-recipient-consent-migration.test.ts test/postgres-legacy-harness-migration.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-app-page-views.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/run-retry-delay.test.ts test/run-priority.test.ts test/postgres-run-stream.test.ts test/postgres-items-migration.test.ts test/postgres-items-contract.test.ts test/pi-harness-replay-fidelity.test.ts test/migrate-main.test.ts",+ "test:pg": "node scripts/run-pg-tests.mjs test/batch-paths.test.ts test/session-append-many.test.ts test/cron-queue.test.ts test/postgres-migration-startup.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-tapeless-store.test.ts test/postgres-legacy-tape-steps.test.ts test/postgres-items-copy.test.ts test/semantic-memory-pg.test.ts test/semantic-memory-http.test.ts test/semantic-memory-large-pg.test.ts test/postgres-instance-registry.test.ts test/readiness.test.ts test/postgres-grant-store.test.ts test/postgres-acl-file-id-migration.test.ts test/postgres-admin-grants.test.ts test/postgres-teams.test.ts test/postgres-decisions.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-linked-identity-membership.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-cron-run-as-migration.test.ts test/postgres-trigger-recipient-consent-migration.test.ts test/postgres-legacy-harness-migration.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-app-page-views.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/run-retry-delay.test.ts test/run-priority.test.ts test/postgres-run-stream.test.ts test/postgres-items-migration.test.ts test/postgres-items-promotion.test.ts test/postgres-items-contract.test.ts test/pi-harness-replay-fidelity.test.ts test/migrate-main.test.ts","livetest": "node scripts/api-livetest.ts","bench:memory": "node --env-file-if-exists=.env scripts/memory-bench.ts","prepare": "node .husky/install.mjs",diff --git a/scripts/ops/items-copy.ts b/scripts/ops/items-copy.tsindex 8df114067..e3dc06f48 100644--- a/scripts/ops/items-copy.ts+++ b/scripts/ops/items-copy.ts@@ -7,7 +7,7 @@ import { sleep } from "../../src/util/async.ts";import { ITEMS_TABLES_SQL, ITEM_INDEXES, itemIndexSql, itemsMigration } from "../../src/sessions/items-schema.ts";import { itemsCopySelectedSql, itemsSelectedRowsSql, ITEMS_COPY_REPAIR_SQL } from "../../src/sessions/items-copy.ts";-const fingerprint = createHash("sha256")+export const fingerprint = createHash("sha256").update(JSON.stringify([ITEMS_TABLES_SQL, ITEM_INDEXES, ITEMS_COPY_REPAIR_SQL, itemsCopySelectedSql()])).digest("hex");const state = "items_copy_ops.state";@@ -312,7 +312,7 @@ export class ItemsCopyOperator {}}- async finish(writersStopped: boolean) {+ async finish(writersStopped: boolean, options: { retireLeases?: boolean } = {}) {if (!writersStopped)throw new Error("finish requires --writers-stopped; keep old writers stopped through the items cutover");await this.transaction(async () => {@@ -321,6 +321,7 @@ export class ItemsCopyOperator {"LOCK TABLE public.session_entries,public.participants,public.directory_members,public.legacy_tape_steps IN SHARE ROW EXCLUSIVE MODE",);await this.assertState(["verified", "ready"]);+ if (options.retireLeases) await this.client.query("DELETE FROM public.session_leases");const {rows: [leases],} = await this.client.query(diff --git a/src/api/deps.ts b/src/api/deps.tsindex 499019f69..c8160d046 100644--- a/src/api/deps.ts+++ b/src/api/deps.ts@@ -1,4 +1,5 @@import type { SlackWebClientFactory } from "../slack/web-client.ts";+import type { ItemsPromotion } from "../sessions/items-promotion.ts";import type { TeamStore } from "../teams/teams.ts";import type { DecisionStore } from "../decisions/store.ts";import type { ExternalSlackPolicies } from "../resolution/external-slack.ts";@@ -85,6 +86,7 @@ import type { SlackInstallationStore, SlackSocketAppIdReader } from "../surfacesimport type { SlackAccountLink, ComposioReturn } from "./routes/composio.ts";export interface ServerDeps {+ itemsPromotion?: ItemsPromotion;externalSlackPolicies?: ExternalSlackPolicies;checkReadiness?: (signal: AbortSignal) => Promise<void>;browserModelGateway?: ModelGatewayTransportConfig;diff --git a/src/api/routes/index.ts b/src/api/routes/index.tsindex e9dd0d3b9..a79a14ed0 100644--- a/src/api/routes/index.ts+++ b/src/api/routes/index.ts@@ -1,3 +1,4 @@+import { itemsPromotionRoutes } from "./items-promotion.ts";import { browserModelRoutes } from "./browser-model.ts";import { deploymentLiveSmokeRoutes } from "./deployment-live-smoke.ts";import { backgroundWorkRoutes } from "./background-work.ts";@@ -87,6 +88,7 @@ export const apiRoutes: ReadonlyArray<Route<ApiCtx>> = [...searchRoutes,...deploymentLayerRoutes,...backgroundWorkRoutes,+ ...itemsPromotionRoutes,...deploymentLiveSmokeRoutes,...turnRoutes,...runEventRoutes,diff --git a/src/api/routes/items-promotion.ts b/src/api/routes/items-promotion.tsnew file mode 100644index 000000000..3fcf8912c--- /dev/null+++ b/src/api/routes/items-promotion.ts@@ -0,0 +1,30 @@+import { requireDeploymentControl } from "./background-work.ts";+import { sendJson } from "../http.ts";+import { isObj } from "./shared.ts";+import type { ApiCtx, Route } from "./route.ts";++async function itemsPromotion(ctx: ApiCtx): Promise<void> {+ if (!requireDeploymentControl(ctx)) return;+ const promotion = ctx.deps.itemsPromotion;+ if (!promotion) return sendJson(ctx.res, 409, { error: "items_promotion_not_staged" });+ if (ctx.method === "GET") await promotion.preflight();+ if (ctx.method === "POST") {+ if (+ !isObj(ctx.body) ||+ ctx.body.writersStopped !== true ||+ ctx.body.deploymentId !== ctx.deps.backgroundOwnership!.deploymentId ||+ Object.keys(ctx.body).some((key) => !["writersStopped", "deploymentId"].includes(key))+ )+ return sendJson(ctx.res, 400, { error: "invalid_items_promotion" });+ await promotion.promote(true);+ }+ sendJson(ctx.res, 200, {+ deploymentId: ctx.deps.backgroundOwnership!.deploymentId,+ ready: await promotion.refresh(),+ });+}++export const itemsPromotionRoutes: ReadonlyArray<Route<ApiCtx>> = [+ { method: "GET", path: "/v1/items-promotion", auth: "source", handle: itemsPromotion },+ { method: "POST", path: "/v1/items-promotion", actor: "system", auth: "source", handle: itemsPromotion },+];diff --git a/src/api/server.ts b/src/api/server.tsindex e396962ad..6d0eba043 100644--- a/src/api/server.ts+++ b/src/api/server.ts@@ -566,6 +566,30 @@ function buildServer(app: App, deps: ServerOptions, allowUnsignedSourceAuth: booserver.maxConnections = 1024;async function front(req: IncomingMessage, res: ServerResponse): Promise<void> {+ const promotion = deps.itemsPromotion;+ const path = new URL(req.url ?? "/", "http://core").pathname;+ if (promotion && !promotion.ready()) {+ if (req.method === "GET" && path === "/healthz") return sendJson(res, 200, { status: "staged" });+ if (req.method === "GET" && path === "/readyz") return sendJson(res, 503, { ok: false });+ const controlRequest =+ path === "/v1/items-promotion" ||+ (path === "/v1/background-work" && req.method === "GET") ||+ path === "/v1/deployment-layer";+ if (!controlRequest) {+ const disconnected = new AbortController();+ const abort = () => disconnected.abort();+ res.once("close", abort);+ try {+ await promotion.wait(AbortSignal.any([disconnected.signal, AbortSignal.timeout(30_000)]));+ } catch {+ if (res.destroyed) return;+ res.setHeader("Retry-After", "1");+ return sendJson(res, 503, { error: "items_promotion_pending" });+ } finally {+ res.removeListener("close", abort);+ }+ }+ }armBodyDeadline(req, 30_000);const base = baseCtx(req, res, wiring);if (await proxyDeploymentSubdomain(base)) {diff --git a/src/config.ts b/src/config.tsindex 0b97d112c..f428417c2 100644--- a/src/config.ts+++ b/src/config.ts@@ -128,6 +128,7 @@ export interface Config {backgroundJobTtlMaxMs: number;backgroundWorkEnabled: boolean;backgroundDeploymentId?: string;+ stageItemsCutover?: boolean;deploymentControlSecret?: string;spendExportToken?: string;buildSha?: string;@@ -1486,6 +1487,7 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {: {}),sessionStore: env.SESSION_STORE === "postgres" ? "postgres" : "memory",...(env.DATABASE_URL ? { databaseUrl: env.DATABASE_URL } : {}),+ stageItemsCutover: env.QM_STAGE_ITEMS_CUTOVER === "1",...(env.DATABASE_POOL_URL ? { databasePoolUrl: env.DATABASE_POOL_URL } : {}),...(env.DATABASE_POOL_CA_CERT ? { databasePoolCaCert: env.DATABASE_POOL_CA_CERT } : {}),...(env.DATABASE_POOL_MAX ? { databasePoolMax: numEnvStrict("DATABASE_POOL_MAX", env.DATABASE_POOL_MAX) } : {}),diff --git a/src/index.ts b/src/index.tsindex 92e586639..c5f7c6512 100644--- a/src/index.ts+++ b/src/index.ts@@ -1,4 +1,5 @@import "./instrument.ts";+import { createItemsPromotion } from "./sessions/items-promotion.ts";import { createBackgroundController } from "./runs/background-controller.ts";import { createManagedSlack } from "./surfaces/slack-managed.ts";import { randomBytes } from "node:crypto";@@ -15,7 +16,13 @@ import { migrateRegisteredPgSchemas } from "./persistence/pg-pool.ts";const config = loadConfig();+if (+ config.stageItemsCutover &&+ (config.sessionStore !== "postgres" || !config.databaseUrl || !config.backgroundDeploymentId)+)+ throw new Error("Staged items promotion requires Postgres and controlled background ownership");const built = buildApp(config);+const itemsPromotion = config.stageItemsCutover ? await createItemsPromotion(config.databaseUrl!) : undefined;await migrateRegisteredPgSchemas(config.databaseUrl);await built.sandboxResources.initialize();const backfilledFires = await built.crons.backfillFires();@@ -39,6 +46,7 @@ const managedSlack = process.env.QM_SLACK_SERVICE_URLconst server = createServer(built.app, {...serverDeps(config, built, slackEnvironmentState, envSlackConfig?.botToken),managedSlack,+ itemsPromotion,});await built.config.hydrate?.();@@ -46,7 +54,7 @@ await built.refreshCustomProviders();await built.identity.hydrate();await built.deploymentLayerReady;built.deploymentLayerRefresh.start();-built.runtime.start();+if (!itemsPromotion) built.runtime.start();server.listen(config.port, () => {console.log(@@ -55,6 +63,20 @@ server.listen(config.port, () => {);});+if (itemsPromotion) {+ const poll = setInterval(() => {+ void itemsPromotion+ .refresh()+ .catch((error) => console.error("[qm] items promotion check failed:", errMessage(error)));+ }, 500);+ try {+ await itemsPromotion.wait();+ } finally {+ clearInterval(poll);+ }+ built.runtime.start();+}+if (config.deployAppsDomain) {const domain = config.deployAppsDomain;const probe = `qm-probe-${randomBytes(4).toString("hex")}.${domain}`;@@ -192,6 +214,7 @@ function shutdown(signal: string): void {async stop() {await backgroundController?.stop();await built.runtime.stop();+ await itemsPromotion?.close();},releaseInFlightRuns: () => built.runtime.releaseInFlightRuns(),},diff --git a/src/migrate-main.ts b/src/migrate-main.tsindex 88b1cdaf5..8c6a8ef2c 100644--- a/src/migrate-main.ts+++ b/src/migrate-main.ts@@ -1,3 +1,4 @@+import { createItemsPromotion } from "./sessions/items-promotion.ts";import { loadConfig } from "./config.ts";import { migrateRegisteredPgSchemas } from "./persistence/pg-pool.ts";import { buildApp } from "./wiring.ts";@@ -5,6 +6,12 @@ import { buildApp } from "./wiring.ts";const config = loadConfig();const { runtime } = buildApp(config);try {+ if (config.stageItemsCutover) {+ if (!config.databaseUrl || config.sessionStore !== "postgres")+ throw new Error("Staged items promotion requires Postgres");+ const promotion = await createItemsPromotion(config.databaseUrl, { analyze: true });+ await promotion.close();+ }await migrateRegisteredPgSchemas(config.databaseUrl);console.log("[qm:migrate] database migrations applied");} finally {diff --git a/src/sessions/items-cutover.ts b/src/sessions/items-cutover.tsindex aec1b32df..6fdc51f94 100644--- a/src/sessions/items-cutover.ts+++ b/src/sessions/items-cutover.ts@@ -61,5 +61,4 @@ export const ITEMS_CUTOVER_SQL: readonly string[] = [`DROP FUNCTION IF EXISTS participant_activity_on_insert_v3(), participant_activity_on_reopen_v3(),participant_activity_on_entry_v3(), session_entry_boundary_v1()`,`ALTER TABLE session_entries RENAME TO legacy_session_entries`,- `ANALYZE items`,];diff --git a/src/sessions/items-promotion.ts b/src/sessions/items-promotion.tsnew file mode 100644index 000000000..45fea0a3c--- /dev/null+++ b/src/sessions/items-promotion.ts@@ -0,0 +1,138 @@+import pg from "pg";+import { applyPgMigrations, definePgMigration, pgMigrationChecksum, pgCaOptions } from "../persistence/pg-pool.ts";+import { ItemsCopyOperator, fingerprint as itemsCopyFingerprint } from "../../scripts/ops/items-copy.ts";+import { ITEM_INDEXES, itemsMigration } from "./items-schema.ts";+import { ITEMS_CUTOVER_SQL } from "./items-cutover.ts";++export const ITEMS_CUTOVER_ID = "sessions/store/0036-items-cutover";++export async function createItemsPromotion(connectionString: string, options: { analyze?: boolean } = {}) {+ const pool = new pg.Pool({ connectionString, ...pgCaOptions(), max: 2 });+ let ready = false;+ let running: Promise<void> | undefined;+ let refreshing: Promise<boolean> | undefined;+ const waiters = new Set<() => void>();+ async function completed(): Promise<boolean> {+ const { rows } = await pool.query("SELECT id,checksum FROM qm_schema_migrations WHERE id=ANY($1)", [+ [itemsMigration.id, ITEMS_CUTOVER_ID],+ ]);+ const expected = new Map([+ [itemsMigration.id, pgMigrationChecksum(itemsMigration.statements)],+ [ITEMS_CUTOVER_ID, pgMigrationChecksum(ITEMS_CUTOVER_SQL)],+ ]);+ for (const row of rows)+ if (expected.get(row.id) !== row.checksum) throw new Error("Items promotion migration checksum mismatch");+ return rows.length === expected.size;+ }+ async function validate(): Promise<void> {+ if (await completed()) return;+ const { rows } = await pool.query(+ `SELECT phase,source_oid='public.session_entries'::regclass AS source_valid,+ target_oid='public.items'::regclass AS target_valid, fingerprint=$1 AS mapping_valid,+ tenant_id=(SELECT id FROM public.qm_tenant) AS tenant_valid FROM items_copy_ops.state`,+ [itemsCopyFingerprint],+ );+ if (+ rows.length !== 1 ||+ !["verified", "ready"].includes(rows[0].phase) ||+ !rows[0].source_valid ||+ !rows[0].target_valid ||+ !rows[0].mapping_valid ||+ !rows[0].tenant_valid+ )+ throw new Error("Items promotion requires a verified staged copy");+ const indexes = await pool.query(+ `SELECT count(*)::int AS n FROM pg_index i JOIN pg_class c ON c.oid=i.indexrelid+ WHERE i.indrelid='public.items'::regclass AND i.indisvalid AND c.relname=ANY($1)`,+ [ITEM_INDEXES.map((i) => i.name)],+ );+ if (indexes.rows[0].n !== ITEM_INDEXES.length) throw new Error("Items promotion requires all indexes ready");+ const triggers = await pool.query(`SELECT count(*)::int AS n FROM pg_trigger+ WHERE tgrelid IN ('public.session_entries'::regclass,'public.participants'::regclass,+ 'public.directory_members'::regclass,'public.legacy_tape_steps'::regclass)+ AND tgname IN ('items_copy_capture','items_copy_no_truncate') AND tgenabled='O'`);+ if (triggers.rows[0].n !== 8) throw new Error("Items promotion requires live change capture");+ }+ function release(): void {+ ready = true;+ for (const resolve of waiters) resolve();+ waiters.clear();+ }+ try {+ await validate();+ if (await completed()) release();+ else if (options.analyze) await pool.query("ANALYZE public.items");+ } catch (error) {+ await pool.end();+ throw error;+ }+ return {+ ready: () => ready,+ async preflight(): Promise<void> {+ if (!ready) await validate();+ },+ async refresh(): Promise<boolean> {+ if (ready) return true;+ return (refreshing ??= (async () => {+ if (await completed()) release();+ return ready;+ })().finally(() => {+ refreshing = undefined;+ }));+ },+ async promote(writersStopped: boolean): Promise<void> {+ if (!writersStopped) throw new Error("Items promotion requires confirmed stopped writers");+ if (ready) return;+ await (running ??= (async () => {+ const guard = await pool.connect();+ try {+ await guard.query("SELECT pg_advisory_lock(hashtext('qm:items-promotion'))");+ await validate();+ if (!(await completed())) {+ const client = new pg.Client({+ connectionString,+ ...pgCaOptions(),+ application_name: "qm-items-promotion",+ });+ await client.connect();+ try {+ await new ItemsCopyOperator(client).finish(true, { retireLeases: true });+ } finally {+ await client.end();+ }+ await applyPgMigrations(pool, [definePgMigration(ITEMS_CUTOVER_ID, ITEMS_CUTOVER_SQL)]);+ }+ release();+ } finally {+ try {+ await guard.query("SELECT pg_advisory_unlock(hashtext('qm:items-promotion'))");+ } finally {+ guard.release();+ }+ }+ })().finally(() => {+ running = undefined;+ }));+ },+ wait(signal?: AbortSignal): Promise<void> {+ if (ready) return Promise.resolve();+ return new Promise((resolve, reject) => {+ const finish = () => {+ waiters.delete(finish);+ signal?.removeEventListener("abort", abort);+ resolve();+ };+ const abort = () => {+ waiters.delete(finish);+ reject(signal?.reason);+ };+ if (signal?.aborted) return abort();+ waiters.add(finish);+ signal?.addEventListener("abort", abort, { once: true });+ });+ },+ close: () => pool.end(),+ };+}++export type ItemsPromotion = Awaited<ReturnType<typeof createItemsPromotion>>;diff --git a/src/sessions/postgres-session-store.ts b/src/sessions/postgres-session-store.tsindex c8868c8b0..62b59a14d 100644--- a/src/sessions/postgres-session-store.ts+++ b/src/sessions/postgres-session-store.ts@@ -227,7 +227,10 @@ const LAST_PREVIEW_LEN = 100;const storedPreview = (payload: unknown, maxLen: number): string => pgTextSafe(userMessagePreview(payload, maxLen));-export function createPostgresSessionStore(connectionString: string, opts: StoreOptions = {}): SessionStore {+export function createPostgresSessionStore(+ connectionString: string,+ opts: StoreOptions & { stageItemsCutover?: boolean } = {},+): SessionStore {const now = opts.now ?? (() => Date.now());const leaseTtlMs = opts.leaseTtlMs ?? 5 * 60_000;@@ -894,11 +897,15 @@ export function createPostgresSessionStore(connectionString: string, opts: Storeid: "sessions/store/0033-client-tools",statements: [`ALTER TABLE sessions ADD COLUMN IF NOT EXISTS client_tools JSONB`],},- itemsMigration,- {- id: "sessions/store/0036-items-cutover",- statements: ITEMS_CUTOVER_SQL,- },+ ...(opts.stageItemsCutover+ ? []+ : [+ itemsMigration,+ {+ id: "sessions/store/0036-items-cutover",+ statements: ITEMS_CUTOVER_SQL,+ },+ ]),],[{diff --git a/src/wiring.ts b/src/wiring.tsindex ef42f23fa..0178da3d8 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -1361,7 +1361,7 @@ export function buildApp(};const sessions: SessionStore =config.sessionStore === "postgres"- ? createPostgresSessionStore(requireDbUrl("SESSION_STORE"))+ ? createPostgresSessionStore(requireDbUrl("SESSION_STORE"), { stageItemsCutover: config.stageItemsCutover }): createMemorySessionStore();memorySessions.store = sessions;const toolOutput =diff --git a/test/items-promotion-routes.test.ts b/test/items-promotion-routes.test.tsnew file mode 100644index 000000000..9595da28d--- /dev/null+++ b/test/items-promotion-routes.test.ts@@ -0,0 +1,114 @@+import "./support/auto-fake-sprites.ts";+import assert from "node:assert/strict";+import type { AddressInfo } from "node:net";+import test from "node:test";+import { createServer } from "../src/api/server.ts";+import { signRequest } from "../src/auth/source-auth.ts";+import { buildApp } from "../src/wiring.ts";+import { createMemoryMap } from "../src/persistence/durable-map.ts";+import { createBackgroundOwnershipStore, type BackgroundOwnership } from "../src/runs/background-ownership.ts";+import { testConfig } from "./support/test-config.ts";++const source = "source-only-secret".repeat(3);+const control = "deployment-only-secret".repeat(3);++async function fixture() {+ let ready = false;+ let promoted = 0;+ const gate = Promise.withResolvers<void>();+ const built = buildApp(testConfig({ signingSecret: source }));+ const store = createBackgroundOwnershipStore(createMemoryMap<BackgroundOwnership>());+ const server = createServer(built.app, {+ signingSecret: source,+ portalIdentitySecret: "portal-identity-secret".repeat(3),+ capabilitySecret: "capability-only-secret".repeat(3),+ requireSignedPortalIdentity: true,+ backgroundOwnership: { store, instanceId: "instance-new", deploymentId: "core:new", active: () => false },+ deploymentControlSecret: control,+ itemsPromotion: {+ ready: () => ready,+ preflight: async () => {},+ refresh: async () => ready,+ promote: async () => {+ promoted++;+ ready = true;+ gate.resolve();+ },+ wait: () => gate.promise,+ close: async () => {},+ },+ });+ server.listen(0);+ const base = `http://localhost:${(server.address() as AddressInfo).port}`;+ let nonce = 0;+ return {+ promoted: () => promoted,+ release: () => {+ ready = true;+ gate.resolve();+ },+ get: (path: string) => fetch(base + path),+ request: async (method: string, body?: unknown, headers: Record<string, string> = {}) => {+ const raw = body === undefined ? "" : JSON.stringify(body);+ const ts = Math.floor(Date.now() / 1000) + nonce++;+ const path = "/v1/items-promotion";+ return fetch(base + path, {+ method,+ headers: {+ "content-type": "application/json",+ "x-timestamp": String(ts),+ "x-signature": signRequest(source, ts, `${method}\n${path}\n${raw}`),+ authorization: `Bearer ${control}`,+ ...headers,+ },+ ...(raw ? { body: raw } : {}),+ });+ },+ close: () => {+ gate.resolve();+ return new Promise<void>((resolve) => server.close(() => resolve()));+ },+ };+}++test("staged HTTP holds business requests but remains live and not ready", async () => {+ const f = await fixture();+ try {+ assert.equal((await f.get("/healthz")).status, 200);+ assert.equal((await f.get("/readyz")).status, 503);+ let responded = false;+ const held = f.get("/v1/sessions").then((r) => {+ responded = true;+ return r;+ });+ await new Promise((resolve) => setTimeout(resolve, 30));+ assert.equal(responded, false);+ f.release();+ assert.equal((await held).status, 401);+ assert.equal((await f.get("/readyz")).status, 200);+ } finally {+ await f.close();+ }+});++test("promotion requires source plus deployment credentials and an exact target", async () => {+ const f = await fixture();+ const body = { writersStopped: true, deploymentId: "core:new" };+ try {+ assert.equal((await f.request("POST", body, { authorization: "" })).status, 401);+ assert.equal((await f.request("POST", body, { "x-signature": "wrong" })).status, 401);+ for (const bad of [+ {},+ { ...body, writersStopped: false },+ { ...body, deploymentId: "core:old" },+ { ...body, extra: true },+ ])+ assert.equal((await f.request("POST", bad)).status, 400);+ assert.equal(f.promoted(), 0);+ assert.deepEqual(await (await f.request("GET")).json(), { deploymentId: "core:new", ready: false });+ assert.equal((await f.request("POST", body)).status, 200);+ assert.equal(f.promoted(), 1);+ } finally {+ await f.close();+ }+});diff --git a/test/postgres-items-promotion.test.ts b/test/postgres-items-promotion.test.tsnew file mode 100644index 000000000..a2d047bdc--- /dev/null+++ b/test/postgres-items-promotion.test.ts@@ -0,0 +1,102 @@+import assert from "node:assert/strict";+import test from "node:test";+import pg from "pg";+import { randomUUID } from "node:crypto";+import { applyPgMigrations, migrateRegisteredPgSchemas, registeredPgMigrations } from "../src/persistence/pg-pool.ts";+import { createPostgresSessionStore } from "../src/sessions/postgres-session-store.ts";+import { createItemsPromotion, ITEMS_CUTOVER_ID } from "../src/sessions/items-promotion.ts";+import { ItemsCopyOperator } from "../scripts/ops/items-copy.ts";+import { scopeId } from "../src/types.ts";++const url = process.env.DATABASE_URL;++test("staged core stays on the verified copy until authenticated promotion completes", { skip: !url }, async () => {+ const pool = new pg.Pool({ connectionString: url });+ const client = new pg.Client({ connectionString: url });+ await client.connect();+ createPostgresSessionStore(url!);+ const migrations = registeredPgMigrations(url!);+ let a: Awaited<ReturnType<typeof createItemsPromotion>> | undefined;+ let b: typeof a;+ let staged: ReturnType<typeof createPostgresSessionStore> | undefined;+ try {+ await applyPgMigrations(+ pool,+ migrations.filter((m) => m.id < "sessions/store/0035-items"),+ );+ const id = randomUUID();+ await pool.query(+ "INSERT INTO sessions(id,type,scope_id,thread_ref,created_at,messages) VALUES($1,'dm','personal:alice','web:promotion',1,1)",+ [id],+ );+ await pool.query(+ "INSERT INTO session_entries(session_id,seq,type,payload,scope_label,created_at) VALUES($1,0,'user','{\"text\":\"before\"}','personal:alice',1)",+ [id],+ );+ const op = new ItemsCopyOperator(client);+ await op.prepare();+ await assert.rejects(createItemsPromotion(url!), /verified staged copy/);+ while (await op.copyBatch()) continue;+ await op.completeCopy();+ await op.verify();+ await assert.rejects(createItemsPromotion(url!), /indexes ready/);+ await op.indexes();+ const stageUrl = new URL(url!);+ stageUrl.searchParams.set("application_name", "staged-promotion-test");+ staged = createPostgresSessionStore(stageUrl.toString(), { stageItemsCutover: true });+ assert.ok(+ !registeredPgMigrations(stageUrl.toString()).some((m) =>+ ["sessions/store/0035-items", ITEMS_CUTOVER_ID].includes(m.id),+ ),+ );+ await migrateRegisteredPgSchemas(stageUrl.toString());+ assert.equal((await pool.query("SELECT to_regclass('session_entries') AS name")).rows[0].name, "session_entries");+ a = await createItemsPromotion(url!);+ b = await createItemsPromotion(url!);+ assert.equal(a.ready(), false);+ const abort = new AbortController();+ const cancelled = a.wait(abort.signal);+ abort.abort(new Error("cancelled waiter"));+ await assert.rejects(cancelled, /cancelled waiter/);+ await pool.query("INSERT INTO session_leases(session_id,token,expires_at) VALUES($1,'old-writer',$2)", [+ id,+ Date.now() + 300_000,+ ]);+ await assert.rejects(a.promote(false), /confirmed stopped writers/);+ await assert.rejects(op.finish(true), /active session leases/);+ const fingerprint = (await pool.query("SELECT fingerprint FROM items_copy_ops.state")).rows[0].fingerprint;+ await pool.query("UPDATE items_copy_ops.state SET fingerprint='invalid'");+ await assert.rejects(a.promote(true), /verified staged copy/);+ assert.equal(a.ready(), false);+ assert.equal((await pool.query("SELECT count(*)::int AS n FROM session_leases")).rows[0].n, 1);+ await pool.query("UPDATE items_copy_ops.state SET fingerprint=$1", [fingerprint]);+ const waiting = a.wait();+ await Promise.all([a.promote(true), a.promote(true), b.promote(true)]);+ await waiting;+ assert.equal(a.ready(), true);+ assert.equal(await b.refresh(), true);+ await a.promote(true);+ assert.equal((await pool.query("SELECT to_regclass('session_entries') AS name")).rows[0].name, null);+ assert.equal((await pool.query("SELECT count(*)::int AS n FROM session_leases")).rows[0].n, 0);+ const { lease } = await staged.acquireLease(id);+ assert.ok(lease);+ await staged.append(lease, {+ type: "assistant",+ payload: { text: "after" },+ scopeLabel: scopeId("personal", "alice"),+ });+ await staged.releaseLease(lease);+ assert.deepEqual(+ (await pool.query("SELECT text FROM items ORDER BY seq")).rows.map((r) => r.text),+ ["before", "after"],+ );+ await applyPgMigrations(pool, migrations);+ await pool.query("UPDATE qm_schema_migrations SET checksum='invalid' WHERE id=$1", [ITEMS_CUTOVER_ID]);+ await assert.rejects(createItemsPromotion(url!), /checksum mismatch/);+ } finally {+ await a?.close();+ await b?.close();+ await client.end();+ await pool.end();+ }+});