Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Stage items cutover until deployment promotion

Regan Bell4ee45b57a8e5
diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.ts
index 4ce355a93..10a9bbe51 100644
--- a/cli/src/backends/aws.ts
+++ b/cli/src/backends/aws.ts
@@ -3126,6 +3126,26 @@ export async function awsBackgroundWorkStatus(
return { status, deploymentId: cohort.deploymentId, taskArns: cohort.taskArns, manifestId: cohort.manifest.id };
}
+export async function awsItemsPromotion(
+ config: QmConfig,
+ configDir: string,
+ options: { writersStopped?: boolean; candidate?: string } = {},
+): Promise<{ deploymentId: string; ready: boolean }> {
+ assertAwsCallerAccount(requireAws(config));
+ const cohort = await awsBackgroundCohort(config, configDir, options.candidate);
+ const transport = awsBackgroundWorkTransport(config, "/v1/items-promotion", 120_000);
+ const response = await transport(
+ options.writersStopped ? "POST" : "GET",
+ options.writersStopped ? JSON.stringify({ deploymentId: cohort.deploymentId, writersStopped: true }) : "",
+ );
+ if (response.status !== 200) throw new CliError(`items promotion failed with HTTP ${response.status}`);
+ const result = JSON.parse(response.body) as { deploymentId?: string; ready?: boolean };
+ if (result.deploymentId !== cohort.deploymentId || typeof result.ready !== "boolean")
+ throw new CliError("items promotion returned an unexpected deployment");
+ if (options.writersStopped && !result.ready) throw new CliError("items promotion has not completed");
+ return { deploymentId: cohort.deploymentId, ready: result.ready };
+}
+
export interface AwsBackgroundWorkCapacity {
manifestId: string;
deploymentId: string;
diff --git a/deploy/core/Dockerfile b/deploy/core/Dockerfile
index d35660561..b8ca1ad75 100644
--- a/deploy/core/Dockerfile
+++ b/deploy/core/Dockerfile
@@ -24,6 +24,7 @@ RUN npm ci --omit=dev && npm run audit:prod \
COPY --from=connector-sdk-build /app/.generated/connector-sdk ./.generated/connector-sdk
COPY src ./src
+COPY scripts/ops/items-copy.ts ./scripts/ops/items-copy.ts
COPY cli/src/canonical-json.ts cli/src/credential-paths.ts cli/src/scope-storage-key.ts cli/src/tool-descriptor.ts ./cli/src/
COPY cli/templates/slack-manifest.json ./cli/templates/slack-manifest.json
COPY skills-seed ./skills-seed
diff --git a/package.json b/package.json
index c946db626..967552409 100644
--- a/package.json
+++ b/package.json
@@ -48,7 +48,7 @@
"sandbox:local:build": "bash scripts/local-sandbox-build.sh",
"sandbox:porter:build": "docker build --platform linux/amd64 -t \"${PORTER_SANDBOX_IMAGE:?Set PORTER_SANDBOX_IMAGE to the registry tag to publish}\" porter && docker push \"$PORTER_SANDBOX_IMAGE\"",
"deploy:porter-runner-image": "docker build --platform linux/amd64 -t \"${PORTER_DEPLOY_RUNNER_IMAGE:?Set PORTER_DEPLOY_RUNNER_IMAGE to the registry tag to publish}\" -f deploy/porter-runner/Dockerfile . && docker push \"$PORTER_DEPLOY_RUNNER_IMAGE\"",
- "test:pg": "node scripts/run-pg-tests.mjs test/batch-paths.test.ts test/session-append-many.test.ts test/cron-queue.test.ts test/postgres-migration-startup.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-tapeless-store.test.ts test/postgres-legacy-tape-steps.test.ts test/postgres-items-copy.test.ts test/semantic-memory-pg.test.ts test/semantic-memory-http.test.ts test/semantic-memory-large-pg.test.ts test/postgres-instance-registry.test.ts test/readiness.test.ts test/postgres-grant-store.test.ts test/postgres-acl-file-id-migration.test.ts test/postgres-admin-grants.test.ts test/postgres-teams.test.ts test/postgres-decisions.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-linked-identity-membership.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-cron-run-as-migration.test.ts test/postgres-trigger-recipient-consent-migration.test.ts test/postgres-legacy-harness-migration.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-app-page-views.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/run-retry-delay.test.ts test/run-priority.test.ts test/postgres-run-stream.test.ts test/postgres-items-migration.test.ts test/postgres-items-contract.test.ts test/pi-harness-replay-fidelity.test.ts test/migrate-main.test.ts",
+ "test:pg": "node scripts/run-pg-tests.mjs test/batch-paths.test.ts test/session-append-many.test.ts test/cron-queue.test.ts test/postgres-migration-startup.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-tapeless-store.test.ts test/postgres-legacy-tape-steps.test.ts test/postgres-items-copy.test.ts test/semantic-memory-pg.test.ts test/semantic-memory-http.test.ts test/semantic-memory-large-pg.test.ts test/postgres-instance-registry.test.ts test/readiness.test.ts test/postgres-grant-store.test.ts test/postgres-acl-file-id-migration.test.ts test/postgres-admin-grants.test.ts test/postgres-teams.test.ts test/postgres-decisions.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-linked-identity-membership.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-cron-run-as-migration.test.ts test/postgres-trigger-recipient-consent-migration.test.ts test/postgres-legacy-harness-migration.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-app-page-views.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/run-retry-delay.test.ts test/run-priority.test.ts test/postgres-run-stream.test.ts test/postgres-items-migration.test.ts test/postgres-items-promotion.test.ts test/postgres-items-contract.test.ts test/pi-harness-replay-fidelity.test.ts test/migrate-main.test.ts",
"livetest": "node scripts/api-livetest.ts",
"bench:memory": "node --env-file-if-exists=.env scripts/memory-bench.ts",
"prepare": "node .husky/install.mjs",
diff --git a/scripts/ops/items-copy.ts b/scripts/ops/items-copy.ts
index 8df114067..e3dc06f48 100644
--- a/scripts/ops/items-copy.ts
+++ b/scripts/ops/items-copy.ts
@@ -7,7 +7,7 @@ import { sleep } from "../../src/util/async.ts";
import { ITEMS_TABLES_SQL, ITEM_INDEXES, itemIndexSql, itemsMigration } from "../../src/sessions/items-schema.ts";
import { itemsCopySelectedSql, itemsSelectedRowsSql, ITEMS_COPY_REPAIR_SQL } from "../../src/sessions/items-copy.ts";
-const fingerprint = createHash("sha256")
+export const fingerprint = createHash("sha256")
.update(JSON.stringify([ITEMS_TABLES_SQL, ITEM_INDEXES, ITEMS_COPY_REPAIR_SQL, itemsCopySelectedSql()]))
.digest("hex");
const state = "items_copy_ops.state";
@@ -312,7 +312,7 @@ export class ItemsCopyOperator {
}
}
- async finish(writersStopped: boolean) {
+ async finish(writersStopped: boolean, options: { retireLeases?: boolean } = {}) {
if (!writersStopped)
throw new Error("finish requires --writers-stopped; keep old writers stopped through the items cutover");
await this.transaction(async () => {
@@ -321,6 +321,7 @@ export class ItemsCopyOperator {
"LOCK TABLE public.session_entries,public.participants,public.directory_members,public.legacy_tape_steps IN SHARE ROW EXCLUSIVE MODE",
);
await this.assertState(["verified", "ready"]);
+ if (options.retireLeases) await this.client.query("DELETE FROM public.session_leases");
const {
rows: [leases],
} = await this.client.query(
diff --git a/src/api/deps.ts b/src/api/deps.ts
index 499019f69..c8160d046 100644
--- a/src/api/deps.ts
+++ b/src/api/deps.ts
@@ -1,4 +1,5 @@
import type { SlackWebClientFactory } from "../slack/web-client.ts";
+import type { ItemsPromotion } from "../sessions/items-promotion.ts";
import type { TeamStore } from "../teams/teams.ts";
import type { DecisionStore } from "../decisions/store.ts";
import type { ExternalSlackPolicies } from "../resolution/external-slack.ts";
@@ -85,6 +86,7 @@ import type { SlackInstallationStore, SlackSocketAppIdReader } from "../surfaces
import type { SlackAccountLink, ComposioReturn } from "./routes/composio.ts";
export interface ServerDeps {
+ itemsPromotion?: ItemsPromotion;
externalSlackPolicies?: ExternalSlackPolicies;
checkReadiness?: (signal: AbortSignal) => Promise<void>;
browserModelGateway?: ModelGatewayTransportConfig;
diff --git a/src/api/routes/index.ts b/src/api/routes/index.ts
index e9dd0d3b9..a79a14ed0 100644
--- a/src/api/routes/index.ts
+++ b/src/api/routes/index.ts
@@ -1,3 +1,4 @@
+import { itemsPromotionRoutes } from "./items-promotion.ts";
import { browserModelRoutes } from "./browser-model.ts";
import { deploymentLiveSmokeRoutes } from "./deployment-live-smoke.ts";
import { backgroundWorkRoutes } from "./background-work.ts";
@@ -87,6 +88,7 @@ export const apiRoutes: ReadonlyArray<Route<ApiCtx>> = [
...searchRoutes,
...deploymentLayerRoutes,
...backgroundWorkRoutes,
+ ...itemsPromotionRoutes,
...deploymentLiveSmokeRoutes,
...turnRoutes,
...runEventRoutes,
diff --git a/src/api/routes/items-promotion.ts b/src/api/routes/items-promotion.ts
new file mode 100644
index 000000000..3fcf8912c
--- /dev/null
+++ b/src/api/routes/items-promotion.ts
@@ -0,0 +1,30 @@
+import { requireDeploymentControl } from "./background-work.ts";
+import { sendJson } from "../http.ts";
+import { isObj } from "./shared.ts";
+import type { ApiCtx, Route } from "./route.ts";
+
+async function itemsPromotion(ctx: ApiCtx): Promise<void> {
+ if (!requireDeploymentControl(ctx)) return;
+ const promotion = ctx.deps.itemsPromotion;
+ if (!promotion) return sendJson(ctx.res, 409, { error: "items_promotion_not_staged" });
+ if (ctx.method === "GET") await promotion.preflight();
+ if (ctx.method === "POST") {
+ if (
+ !isObj(ctx.body) ||
+ ctx.body.writersStopped !== true ||
+ ctx.body.deploymentId !== ctx.deps.backgroundOwnership!.deploymentId ||
+ Object.keys(ctx.body).some((key) => !["writersStopped", "deploymentId"].includes(key))
+ )
+ return sendJson(ctx.res, 400, { error: "invalid_items_promotion" });
+ await promotion.promote(true);
+ }
+ sendJson(ctx.res, 200, {
+ deploymentId: ctx.deps.backgroundOwnership!.deploymentId,
+ ready: await promotion.refresh(),
+ });
+}
+
+export const itemsPromotionRoutes: ReadonlyArray<Route<ApiCtx>> = [
+ { method: "GET", path: "/v1/items-promotion", auth: "source", handle: itemsPromotion },
+ { method: "POST", path: "/v1/items-promotion", actor: "system", auth: "source", handle: itemsPromotion },
+];
diff --git a/src/api/server.ts b/src/api/server.ts
index e396962ad..6d0eba043 100644
--- a/src/api/server.ts
+++ b/src/api/server.ts
@@ -566,6 +566,30 @@ function buildServer(app: App, deps: ServerOptions, allowUnsignedSourceAuth: boo
server.maxConnections = 1024;
async function front(req: IncomingMessage, res: ServerResponse): Promise<void> {
+ const promotion = deps.itemsPromotion;
+ const path = new URL(req.url ?? "/", "http://core").pathname;
+ if (promotion && !promotion.ready()) {
+ if (req.method === "GET" && path === "/healthz") return sendJson(res, 200, { status: "staged" });
+ if (req.method === "GET" && path === "/readyz") return sendJson(res, 503, { ok: false });
+ const controlRequest =
+ path === "/v1/items-promotion" ||
+ (path === "/v1/background-work" && req.method === "GET") ||
+ path === "/v1/deployment-layer";
+ if (!controlRequest) {
+ const disconnected = new AbortController();
+ const abort = () => disconnected.abort();
+ res.once("close", abort);
+ try {
+ await promotion.wait(AbortSignal.any([disconnected.signal, AbortSignal.timeout(30_000)]));
+ } catch {
+ if (res.destroyed) return;
+ res.setHeader("Retry-After", "1");
+ return sendJson(res, 503, { error: "items_promotion_pending" });
+ } finally {
+ res.removeListener("close", abort);
+ }
+ }
+ }
armBodyDeadline(req, 30_000);
const base = baseCtx(req, res, wiring);
if (await proxyDeploymentSubdomain(base)) {
diff --git a/src/config.ts b/src/config.ts
index 0b97d112c..f428417c2 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -128,6 +128,7 @@ export interface Config {
backgroundJobTtlMaxMs: number;
backgroundWorkEnabled: boolean;
backgroundDeploymentId?: string;
+ stageItemsCutover?: boolean;
deploymentControlSecret?: string;
spendExportToken?: string;
buildSha?: string;
@@ -1486,6 +1487,7 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {
: {}),
sessionStore: env.SESSION_STORE === "postgres" ? "postgres" : "memory",
...(env.DATABASE_URL ? { databaseUrl: env.DATABASE_URL } : {}),
+ stageItemsCutover: env.QM_STAGE_ITEMS_CUTOVER === "1",
...(env.DATABASE_POOL_URL ? { databasePoolUrl: env.DATABASE_POOL_URL } : {}),
...(env.DATABASE_POOL_CA_CERT ? { databasePoolCaCert: env.DATABASE_POOL_CA_CERT } : {}),
...(env.DATABASE_POOL_MAX ? { databasePoolMax: numEnvStrict("DATABASE_POOL_MAX", env.DATABASE_POOL_MAX) } : {}),
diff --git a/src/index.ts b/src/index.ts
index 92e586639..c5f7c6512 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,4 +1,5 @@
import "./instrument.ts";
+import { createItemsPromotion } from "./sessions/items-promotion.ts";
import { createBackgroundController } from "./runs/background-controller.ts";
import { createManagedSlack } from "./surfaces/slack-managed.ts";
import { randomBytes } from "node:crypto";
@@ -15,7 +16,13 @@ import { migrateRegisteredPgSchemas } from "./persistence/pg-pool.ts";
const config = loadConfig();
+if (
+ config.stageItemsCutover &&
+ (config.sessionStore !== "postgres" || !config.databaseUrl || !config.backgroundDeploymentId)
+)
+ throw new Error("Staged items promotion requires Postgres and controlled background ownership");
const built = buildApp(config);
+const itemsPromotion = config.stageItemsCutover ? await createItemsPromotion(config.databaseUrl!) : undefined;
await migrateRegisteredPgSchemas(config.databaseUrl);
await built.sandboxResources.initialize();
const backfilledFires = await built.crons.backfillFires();
@@ -39,6 +46,7 @@ const managedSlack = process.env.QM_SLACK_SERVICE_URL
const server = createServer(built.app, {
...serverDeps(config, built, slackEnvironmentState, envSlackConfig?.botToken),
managedSlack,
+ itemsPromotion,
});
await built.config.hydrate?.();
@@ -46,7 +54,7 @@ await built.refreshCustomProviders();
await built.identity.hydrate();
await built.deploymentLayerReady;
built.deploymentLayerRefresh.start();
-built.runtime.start();
+if (!itemsPromotion) built.runtime.start();
server.listen(config.port, () => {
console.log(
@@ -55,6 +63,20 @@ server.listen(config.port, () => {
);
});
+if (itemsPromotion) {
+ const poll = setInterval(() => {
+ void itemsPromotion
+ .refresh()
+ .catch((error) => console.error("[qm] items promotion check failed:", errMessage(error)));
+ }, 500);
+ try {
+ await itemsPromotion.wait();
+ } finally {
+ clearInterval(poll);
+ }
+ built.runtime.start();
+}
+
if (config.deployAppsDomain) {
const domain = config.deployAppsDomain;
const probe = `qm-probe-${randomBytes(4).toString("hex")}.${domain}`;
@@ -192,6 +214,7 @@ function shutdown(signal: string): void {
async stop() {
await backgroundController?.stop();
await built.runtime.stop();
+ await itemsPromotion?.close();
},
releaseInFlightRuns: () => built.runtime.releaseInFlightRuns(),
},
diff --git a/src/migrate-main.ts b/src/migrate-main.ts
index 88b1cdaf5..8c6a8ef2c 100644
--- a/src/migrate-main.ts
+++ b/src/migrate-main.ts
@@ -1,3 +1,4 @@
+import { createItemsPromotion } from "./sessions/items-promotion.ts";
import { loadConfig } from "./config.ts";
import { migrateRegisteredPgSchemas } from "./persistence/pg-pool.ts";
import { buildApp } from "./wiring.ts";
@@ -5,6 +6,12 @@ import { buildApp } from "./wiring.ts";
const config = loadConfig();
const { runtime } = buildApp(config);
try {
+ if (config.stageItemsCutover) {
+ if (!config.databaseUrl || config.sessionStore !== "postgres")
+ throw new Error("Staged items promotion requires Postgres");
+ const promotion = await createItemsPromotion(config.databaseUrl, { analyze: true });
+ await promotion.close();
+ }
await migrateRegisteredPgSchemas(config.databaseUrl);
console.log("[qm:migrate] database migrations applied");
} finally {
diff --git a/src/sessions/items-cutover.ts b/src/sessions/items-cutover.ts
index aec1b32df..6fdc51f94 100644
--- a/src/sessions/items-cutover.ts
+++ b/src/sessions/items-cutover.ts
@@ -61,5 +61,4 @@ export const ITEMS_CUTOVER_SQL: readonly string[] = [
`DROP FUNCTION IF EXISTS participant_activity_on_insert_v3(), participant_activity_on_reopen_v3(),
participant_activity_on_entry_v3(), session_entry_boundary_v1()`,
`ALTER TABLE session_entries RENAME TO legacy_session_entries`,
- `ANALYZE items`,
];
diff --git a/src/sessions/items-promotion.ts b/src/sessions/items-promotion.ts
new file mode 100644
index 000000000..45fea0a3c
--- /dev/null
+++ b/src/sessions/items-promotion.ts
@@ -0,0 +1,138 @@
+import pg from "pg";
+import { applyPgMigrations, definePgMigration, pgMigrationChecksum, pgCaOptions } from "../persistence/pg-pool.ts";
+import { ItemsCopyOperator, fingerprint as itemsCopyFingerprint } from "../../scripts/ops/items-copy.ts";
+import { ITEM_INDEXES, itemsMigration } from "./items-schema.ts";
+import { ITEMS_CUTOVER_SQL } from "./items-cutover.ts";
+
+export const ITEMS_CUTOVER_ID = "sessions/store/0036-items-cutover";
+
+export async function createItemsPromotion(connectionString: string, options: { analyze?: boolean } = {}) {
+ const pool = new pg.Pool({ connectionString, ...pgCaOptions(), max: 2 });
+ let ready = false;
+ let running: Promise<void> | undefined;
+ let refreshing: Promise<boolean> | undefined;
+ const waiters = new Set<() => void>();
+ async function completed(): Promise<boolean> {
+ const { rows } = await pool.query("SELECT id,checksum FROM qm_schema_migrations WHERE id=ANY($1)", [
+ [itemsMigration.id, ITEMS_CUTOVER_ID],
+ ]);
+ const expected = new Map([
+ [itemsMigration.id, pgMigrationChecksum(itemsMigration.statements)],
+ [ITEMS_CUTOVER_ID, pgMigrationChecksum(ITEMS_CUTOVER_SQL)],
+ ]);
+ for (const row of rows)
+ if (expected.get(row.id) !== row.checksum) throw new Error("Items promotion migration checksum mismatch");
+ return rows.length === expected.size;
+ }
+ async function validate(): Promise<void> {
+ if (await completed()) return;
+ const { rows } = await pool.query(
+ `SELECT phase,source_oid='public.session_entries'::regclass AS source_valid,
+ target_oid='public.items'::regclass AS target_valid, fingerprint=$1 AS mapping_valid,
+ tenant_id=(SELECT id FROM public.qm_tenant) AS tenant_valid FROM items_copy_ops.state`,
+ [itemsCopyFingerprint],
+ );
+ if (
+ rows.length !== 1 ||
+ !["verified", "ready"].includes(rows[0].phase) ||
+ !rows[0].source_valid ||
+ !rows[0].target_valid ||
+ !rows[0].mapping_valid ||
+ !rows[0].tenant_valid
+ )
+ throw new Error("Items promotion requires a verified staged copy");
+ const indexes = await pool.query(
+ `SELECT count(*)::int AS n FROM pg_index i JOIN pg_class c ON c.oid=i.indexrelid
+ WHERE i.indrelid='public.items'::regclass AND i.indisvalid AND c.relname=ANY($1)`,
+ [ITEM_INDEXES.map((i) => i.name)],
+ );
+ if (indexes.rows[0].n !== ITEM_INDEXES.length) throw new Error("Items promotion requires all indexes ready");
+ const triggers = await pool.query(`SELECT count(*)::int AS n FROM pg_trigger
+ WHERE tgrelid IN ('public.session_entries'::regclass,'public.participants'::regclass,
+ 'public.directory_members'::regclass,'public.legacy_tape_steps'::regclass)
+ AND tgname IN ('items_copy_capture','items_copy_no_truncate') AND tgenabled='O'`);
+ if (triggers.rows[0].n !== 8) throw new Error("Items promotion requires live change capture");
+ }
+ function release(): void {
+ ready = true;
+ for (const resolve of waiters) resolve();
+ waiters.clear();
+ }
+ try {
+ await validate();
+ if (await completed()) release();
+ else if (options.analyze) await pool.query("ANALYZE public.items");
+ } catch (error) {
+ await pool.end();
+ throw error;
+ }
+ return {
+ ready: () => ready,
+ async preflight(): Promise<void> {
+ if (!ready) await validate();
+ },
+ async refresh(): Promise<boolean> {
+ if (ready) return true;
+ return (refreshing ??= (async () => {
+ if (await completed()) release();
+ return ready;
+ })().finally(() => {
+ refreshing = undefined;
+ }));
+ },
+ async promote(writersStopped: boolean): Promise<void> {
+ if (!writersStopped) throw new Error("Items promotion requires confirmed stopped writers");
+ if (ready) return;
+ await (running ??= (async () => {
+ const guard = await pool.connect();
+ try {
+ await guard.query("SELECT pg_advisory_lock(hashtext('qm:items-promotion'))");
+ await validate();
+ if (!(await completed())) {
+ const client = new pg.Client({
+ connectionString,
+ ...pgCaOptions(),
+ application_name: "qm-items-promotion",
+ });
+ await client.connect();
+ try {
+ await new ItemsCopyOperator(client).finish(true, { retireLeases: true });
+ } finally {
+ await client.end();
+ }
+ await applyPgMigrations(pool, [definePgMigration(ITEMS_CUTOVER_ID, ITEMS_CUTOVER_SQL)]);
+ }
+ release();
+ } finally {
+ try {
+ await guard.query("SELECT pg_advisory_unlock(hashtext('qm:items-promotion'))");
+ } finally {
+ guard.release();
+ }
+ }
+ })().finally(() => {
+ running = undefined;
+ }));
+ },
+ wait(signal?: AbortSignal): Promise<void> {
+ if (ready) return Promise.resolve();
+ return new Promise((resolve, reject) => {
+ const finish = () => {
+ waiters.delete(finish);
+ signal?.removeEventListener("abort", abort);
+ resolve();
+ };
+ const abort = () => {
+ waiters.delete(finish);
+ reject(signal?.reason);
+ };
+ if (signal?.aborted) return abort();
+ waiters.add(finish);
+ signal?.addEventListener("abort", abort, { once: true });
+ });
+ },
+ close: () => pool.end(),
+ };
+}
+
+export type ItemsPromotion = Awaited<ReturnType<typeof createItemsPromotion>>;
diff --git a/src/sessions/postgres-session-store.ts b/src/sessions/postgres-session-store.ts
index c8868c8b0..62b59a14d 100644
--- a/src/sessions/postgres-session-store.ts
+++ b/src/sessions/postgres-session-store.ts
@@ -227,7 +227,10 @@ const LAST_PREVIEW_LEN = 100;
const storedPreview = (payload: unknown, maxLen: number): string => pgTextSafe(userMessagePreview(payload, maxLen));
-export function createPostgresSessionStore(connectionString: string, opts: StoreOptions = {}): SessionStore {
+export function createPostgresSessionStore(
+ connectionString: string,
+ opts: StoreOptions & { stageItemsCutover?: boolean } = {},
+): SessionStore {
const now = opts.now ?? (() => Date.now());
const leaseTtlMs = opts.leaseTtlMs ?? 5 * 60_000;
@@ -894,11 +897,15 @@ export function createPostgresSessionStore(connectionString: string, opts: Store
id: "sessions/store/0033-client-tools",
statements: [`ALTER TABLE sessions ADD COLUMN IF NOT EXISTS client_tools JSONB`],
},
- itemsMigration,
- {
- id: "sessions/store/0036-items-cutover",
- statements: ITEMS_CUTOVER_SQL,
- },
+ ...(opts.stageItemsCutover
+ ? []
+ : [
+ itemsMigration,
+ {
+ id: "sessions/store/0036-items-cutover",
+ statements: ITEMS_CUTOVER_SQL,
+ },
+ ]),
],
[
{
diff --git a/src/wiring.ts b/src/wiring.ts
index ef42f23fa..0178da3d8 100644
--- a/src/wiring.ts
+++ b/src/wiring.ts
@@ -1361,7 +1361,7 @@ export function buildApp(
};
const sessions: SessionStore =
config.sessionStore === "postgres"
- ? createPostgresSessionStore(requireDbUrl("SESSION_STORE"))
+ ? createPostgresSessionStore(requireDbUrl("SESSION_STORE"), { stageItemsCutover: config.stageItemsCutover })
: createMemorySessionStore();
memorySessions.store = sessions;
const toolOutput =
diff --git a/test/items-promotion-routes.test.ts b/test/items-promotion-routes.test.ts
new file mode 100644
index 000000000..9595da28d
--- /dev/null
+++ b/test/items-promotion-routes.test.ts
@@ -0,0 +1,114 @@
+import "./support/auto-fake-sprites.ts";
+import assert from "node:assert/strict";
+import type { AddressInfo } from "node:net";
+import test from "node:test";
+import { createServer } from "../src/api/server.ts";
+import { signRequest } from "../src/auth/source-auth.ts";
+import { buildApp } from "../src/wiring.ts";
+import { createMemoryMap } from "../src/persistence/durable-map.ts";
+import { createBackgroundOwnershipStore, type BackgroundOwnership } from "../src/runs/background-ownership.ts";
+import { testConfig } from "./support/test-config.ts";
+
+const source = "source-only-secret".repeat(3);
+const control = "deployment-only-secret".repeat(3);
+
+async function fixture() {
+ let ready = false;
+ let promoted = 0;
+ const gate = Promise.withResolvers<void>();
+ const built = buildApp(testConfig({ signingSecret: source }));
+ const store = createBackgroundOwnershipStore(createMemoryMap<BackgroundOwnership>());
+ const server = createServer(built.app, {
+ signingSecret: source,
+ portalIdentitySecret: "portal-identity-secret".repeat(3),
+ capabilitySecret: "capability-only-secret".repeat(3),
+ requireSignedPortalIdentity: true,
+ backgroundOwnership: { store, instanceId: "instance-new", deploymentId: "core:new", active: () => false },
+ deploymentControlSecret: control,
+ itemsPromotion: {
+ ready: () => ready,
+ preflight: async () => {},
+ refresh: async () => ready,
+ promote: async () => {
+ promoted++;
+ ready = true;
+ gate.resolve();
+ },
+ wait: () => gate.promise,
+ close: async () => {},
+ },
+ });
+ server.listen(0);
+ const base = `http://localhost:${(server.address() as AddressInfo).port}`;
+ let nonce = 0;
+ return {
+ promoted: () => promoted,
+ release: () => {
+ ready = true;
+ gate.resolve();
+ },
+ get: (path: string) => fetch(base + path),
+ request: async (method: string, body?: unknown, headers: Record<string, string> = {}) => {
+ const raw = body === undefined ? "" : JSON.stringify(body);
+ const ts = Math.floor(Date.now() / 1000) + nonce++;
+ const path = "/v1/items-promotion";
+ return fetch(base + path, {
+ method,
+ headers: {
+ "content-type": "application/json",
+ "x-timestamp": String(ts),
+ "x-signature": signRequest(source, ts, `${method}\n${path}\n${raw}`),
+ authorization: `Bearer ${control}`,
+ ...headers,
+ },
+ ...(raw ? { body: raw } : {}),
+ });
+ },
+ close: () => {
+ gate.resolve();
+ return new Promise<void>((resolve) => server.close(() => resolve()));
+ },
+ };
+}
+
+test("staged HTTP holds business requests but remains live and not ready", async () => {
+ const f = await fixture();
+ try {
+ assert.equal((await f.get("/healthz")).status, 200);
+ assert.equal((await f.get("/readyz")).status, 503);
+ let responded = false;
+ const held = f.get("/v1/sessions").then((r) => {
+ responded = true;
+ return r;
+ });
+ await new Promise((resolve) => setTimeout(resolve, 30));
+ assert.equal(responded, false);
+ f.release();
+ assert.equal((await held).status, 401);
+ assert.equal((await f.get("/readyz")).status, 200);
+ } finally {
+ await f.close();
+ }
+});
+
+test("promotion requires source plus deployment credentials and an exact target", async () => {
+ const f = await fixture();
+ const body = { writersStopped: true, deploymentId: "core:new" };
+ try {
+ assert.equal((await f.request("POST", body, { authorization: "" })).status, 401);
+ assert.equal((await f.request("POST", body, { "x-signature": "wrong" })).status, 401);
+ for (const bad of [
+ {},
+ { ...body, writersStopped: false },
+ { ...body, deploymentId: "core:old" },
+ { ...body, extra: true },
+ ])
+ assert.equal((await f.request("POST", bad)).status, 400);
+ assert.equal(f.promoted(), 0);
+ assert.deepEqual(await (await f.request("GET")).json(), { deploymentId: "core:new", ready: false });
+ assert.equal((await f.request("POST", body)).status, 200);
+ assert.equal(f.promoted(), 1);
+ } finally {
+ await f.close();
+ }
+});
diff --git a/test/postgres-items-promotion.test.ts b/test/postgres-items-promotion.test.ts
new file mode 100644
index 000000000..a2d047bdc
--- /dev/null
+++ b/test/postgres-items-promotion.test.ts
@@ -0,0 +1,102 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import pg from "pg";
+import { randomUUID } from "node:crypto";
+import { applyPgMigrations, migrateRegisteredPgSchemas, registeredPgMigrations } from "../src/persistence/pg-pool.ts";
+import { createPostgresSessionStore } from "../src/sessions/postgres-session-store.ts";
+import { createItemsPromotion, ITEMS_CUTOVER_ID } from "../src/sessions/items-promotion.ts";
+import { ItemsCopyOperator } from "../scripts/ops/items-copy.ts";
+import { scopeId } from "../src/types.ts";
+
+const url = process.env.DATABASE_URL;
+
+test("staged core stays on the verified copy until authenticated promotion completes", { skip: !url }, async () => {
+ const pool = new pg.Pool({ connectionString: url });
+ const client = new pg.Client({ connectionString: url });
+ await client.connect();
+ createPostgresSessionStore(url!);
+ const migrations = registeredPgMigrations(url!);
+ let a: Awaited<ReturnType<typeof createItemsPromotion>> | undefined;
+ let b: typeof a;
+ let staged: ReturnType<typeof createPostgresSessionStore> | undefined;
+ try {
+ await applyPgMigrations(
+ pool,
+ migrations.filter((m) => m.id < "sessions/store/0035-items"),
+ );
+ const id = randomUUID();
+ await pool.query(
+ "INSERT INTO sessions(id,type,scope_id,thread_ref,created_at,messages) VALUES($1,'dm','personal:alice','web:promotion',1,1)",
+ [id],
+ );
+ await pool.query(
+ "INSERT INTO session_entries(session_id,seq,type,payload,scope_label,created_at) VALUES($1,0,'user','{\"text\":\"before\"}','personal:alice',1)",
+ [id],
+ );
+ const op = new ItemsCopyOperator(client);
+ await op.prepare();
+ await assert.rejects(createItemsPromotion(url!), /verified staged copy/);
+ while (await op.copyBatch()) continue;
+ await op.completeCopy();
+ await op.verify();
+ await assert.rejects(createItemsPromotion(url!), /indexes ready/);
+ await op.indexes();
+ const stageUrl = new URL(url!);
+ stageUrl.searchParams.set("application_name", "staged-promotion-test");
+ staged = createPostgresSessionStore(stageUrl.toString(), { stageItemsCutover: true });
+ assert.ok(
+ !registeredPgMigrations(stageUrl.toString()).some((m) =>
+ ["sessions/store/0035-items", ITEMS_CUTOVER_ID].includes(m.id),
+ ),
+ );
+ await migrateRegisteredPgSchemas(stageUrl.toString());
+ assert.equal((await pool.query("SELECT to_regclass('session_entries') AS name")).rows[0].name, "session_entries");
+ a = await createItemsPromotion(url!);
+ b = await createItemsPromotion(url!);
+ assert.equal(a.ready(), false);
+ const abort = new AbortController();
+ const cancelled = a.wait(abort.signal);
+ abort.abort(new Error("cancelled waiter"));
+ await assert.rejects(cancelled, /cancelled waiter/);
+ await pool.query("INSERT INTO session_leases(session_id,token,expires_at) VALUES($1,'old-writer',$2)", [
+ id,
+ Date.now() + 300_000,
+ ]);
+ await assert.rejects(a.promote(false), /confirmed stopped writers/);
+ await assert.rejects(op.finish(true), /active session leases/);
+ const fingerprint = (await pool.query("SELECT fingerprint FROM items_copy_ops.state")).rows[0].fingerprint;
+ await pool.query("UPDATE items_copy_ops.state SET fingerprint='invalid'");
+ await assert.rejects(a.promote(true), /verified staged copy/);
+ assert.equal(a.ready(), false);
+ assert.equal((await pool.query("SELECT count(*)::int AS n FROM session_leases")).rows[0].n, 1);
+ await pool.query("UPDATE items_copy_ops.state SET fingerprint=$1", [fingerprint]);
+ const waiting = a.wait();
+ await Promise.all([a.promote(true), a.promote(true), b.promote(true)]);
+ await waiting;
+ assert.equal(a.ready(), true);
+ assert.equal(await b.refresh(), true);
+ await a.promote(true);
+ assert.equal((await pool.query("SELECT to_regclass('session_entries') AS name")).rows[0].name, null);
+ assert.equal((await pool.query("SELECT count(*)::int AS n FROM session_leases")).rows[0].n, 0);
+ const { lease } = await staged.acquireLease(id);
+ assert.ok(lease);
+ await staged.append(lease, {
+ type: "assistant",
+ payload: { text: "after" },
+ scopeLabel: scopeId("personal", "alice"),
+ });
+ await staged.releaseLease(lease);
+ assert.deepEqual(
+ (await pool.query("SELECT text FROM items ORDER BY seq")).rows.map((r) => r.text),
+ ["before", "after"],
+ );
+ await applyPgMigrations(pool, migrations);
+ await pool.query("UPDATE qm_schema_migrations SET checksum='invalid' WHERE id=$1", [ITEMS_CUTOVER_ID]);
+ await assert.rejects(createItemsPromotion(url!), /checksum mismatch/);
+ } finally {
+ await a?.close();
+ await b?.close();
+ await client.end();
+ await pool.end();
+ }
+});