Multiplayer agent harness for work.
AWS apps: suspend instead of delete, rotate in the background, faster cold start (#2261)
Regan Bell453a46835f37
diff --git a/src/api/app-deployments.ts b/src/api/app-deployments.tsindex 9d0652b13..9d2cb2507 100644--- a/src/api/app-deployments.ts+++ b/src/api/app-deployments.ts@@ -34,7 +34,7 @@ export function createDeploymentMethods(| "setDeploymentAlwaysOn"| "setDeploymentEmbedAncestors"| "setDeploymentPublic"- | "keepAlwaysOnWarm"+ | "maintainDeployments"| "reachDeployment"| "deploymentLogsFor"| "deploymentGitRepoPath"@@ -160,8 +160,8 @@ export function createDeploymentMethods(setDeploymentPublic(idOrName, isPublic, actor) {return deps.deploy.setDeploymentPublic(idOrName, isPublic, actor);},- keepAlwaysOnWarm() {- return deps.deploy.keepAlwaysOnWarm();+ maintainDeployments() {+ return deps.deploy.maintainDeployments();},async reachDeployment(id, principalId, opts): Promise<Reach> {if (opts?.bypassAcl) return deps.deploy.reachDeployment(id, principalId, opts);diff --git a/src/api/app-types.ts b/src/api/app-types.tsindex d01e6e5a9..0945e2008 100644--- a/src/api/app-types.ts+++ b/src/api/app-types.ts@@ -582,7 +582,7 @@ export interface App {setDeploymentAlwaysOn(id: string, alwaysOn: boolean): Promise<Deployment>;setDeploymentEmbedAncestors(id: string, embedAncestors: string[]): Promise<Deployment>;setDeploymentPublic(idOrName: string, isPublic: boolean, actor: { createdBy: string }): Promise<Deployment>;- keepAlwaysOnWarm(): Promise<number>;+ maintainDeployments(): Promise<number>;reachDeployment(id: string, principalId: string, opts?: ReachOptions): Promise<Reach>;deploymentLogsFor(id: string,diff --git a/src/deploy/aws-deploy-provider.ts b/src/deploy/aws-deploy-provider.tsindex dbc12c829..76896df24 100644--- a/src/deploy/aws-deploy-provider.ts+++ b/src/deploy/aws-deploy-provider.ts@@ -46,6 +46,7 @@ const EXIT_APP_PORT_HELD = 97;const LIVE_REPLICATOR_PIDS = `ls_pids() { for p in /proc/[0-9]*; do c=$(tr '\\0' ' ' < "$p/cmdline" 2>/dev/null); case "\${c%% *}" in litestream|*/litestream) case "$c" in *${LITESTREAM_CONFIG}*) echo "\${p#/proc/}";; esac;; esac; done; }`;const ENDPOINT_PORT = 443;const RESOLVE_CACHE_MS = 15_000;+const ROTATE_AHEAD_MS = 30 * 60_000;const DEFAULT_INGRESS = (region: string): string =>`arn:aws:lambda:${region}:aws:network-connector:aws-network-connector:ALL_INGRESS`;@@ -60,6 +61,7 @@ export interface StoredDeployBody {lastSnapshotMs?: number;dataCredsAtMs?: number;rotateNotBeforeMs?: number;+ replacementMicrovmId?: string;orgId?: string;}@@ -104,7 +106,7 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployPconst store = opts.store ?? createMemoryMap<StoredDeployBody>();const launchQueue = createKeyedQueue<string>();const advisoryLock = opts.advisoryLock ?? createNoopAdvisoryLock();- const resolveCache = new Map<string, { endpoint: DeployEndpoint; untilMs: number }>();+ const resolveCache = new Map<string, { endpoint: DeployEndpoint; checkedMs: number; usedMs: number }>();const inflightResolves = new Map<string, Promise<DeployEndpoint | null>>();const resolveGens = new Map<string, number>();const bumpResolveGen = (id: string): void => {@@ -127,8 +129,13 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployPconst appPort = opts.appPort ?? APP_PORT_DEFAULT;const maximumDurationInSeconds = opts.maximumDurationInSeconds ?? 28_800;const rotateAfterMs = (opts.rotateAfterSeconds ?? 27_000) * 1000;+ const rotateDueMs = Math.max(0, rotateAfterMs - ROTATE_AHEAD_MS);+ const rotationDue = (stored: StoredDeployBody): boolean =>+ Date.now() - stored.createdAtMs > rotateDueMs &&+ !(stored.rotateNotBeforeMs && Date.now() < stored.rotateNotBeforeMs);const maxIdleDurationSeconds = opts.maxIdleDurationSeconds ?? 900;- const suspendedDurationSeconds = opts.suspendedDurationSeconds ?? 3600;+ const awakeForMs = Math.max(0, maxIdleDurationSeconds * 1000 - 60_000);+ const suspendedDurationSeconds = opts.suspendedDurationSeconds ?? maximumDurationInSeconds;const appsDomain = opts.appsDomain;const tokenTtlMinutes = opts.tokenTtlMinutes ?? 30;const dataBucket = opts.dataBucket;@@ -349,23 +356,35 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployPasync function ensureBody(d: Deployment): Promise<{ id: string; endpoint: string; fresh: boolean }> {return launchQueue(d.id, async () => {- const stored = await store.get(d.id);+ let stored = await store.get(d.id);+ if (stored?.replacementMicrovmId) {+ await api+ .terminate(stored.replacementMicrovmId)+ .catch((e) => swallow("aws-deploy: terminate orphaned replacement", e));+ const { replacementMicrovmId: _, ...rest } = stored;+ stored = rest;+ await store.put(d.id, stored);+ }+ let next: { id: string; endpoint: string } | undefined;if (stored) {const desc = await api.tryGetMicrovm(stored.microvmId);const alive = desc && desc.state !== "TERMINATED" && desc.state !== "TERMINATING";- const stale = Date.now() - stored.createdAtMs > rotateAfterMs;- if (alive && !stale) {- await ensureRunning(stored.microvmId, stored.endpoint);+ if (alive && !rotationDue(stored)) {+ await ensureRunning(stored.microvmId, stored.endpoint, desc);return { id: stored.microvmId, endpoint: stored.endpoint, fresh: false };}- if (alive && stale) {- await ensureRunning(stored.microvmId, stored.endpoint).catch(() => {});+ if (alive) {+ next = await launchBody(d.id, d.alwaysOn);+ const replacement = next;+ await store.put(d.id, { ...stored, replacementMicrovmId: replacement.id });+ await ensureRunning(stored.microvmId, stored.endpoint, desc).catch(() => {});const defer = async (label: string,e: unknown,extra: Partial<StoredDeployBody> = {},): Promise<{ id: string; endpoint: string; fresh: boolean }> => {swallow(label, e);+ await api.terminate(replacement.id).catch((e2) => swallow("aws-deploy: abandon replacement", e2));await store.put(d.id, { ...stored, rotateNotBeforeMs: Date.now() + 5 * 60_000, ...extra });return { id: stored.microvmId, endpoint: stored.endpoint, fresh: false };};@@ -386,7 +405,7 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployP}}}- const body = await launchBody(d.id, d.alwaysOn);+ const body = next ?? (await launchBody(d.id, d.alwaysOn));try {await hydrateData(d.id, body.id, body.endpoint);} catch (e) {@@ -555,7 +574,7 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployP!(stored.rotateNotBeforeMs && Date.now() < stored.rotateNotBeforeMs))return null;- await ensureRunning(stored.microvmId, stored.endpoint);+ await ensureRunning(stored.microvmId, stored.endpoint, desc);const exec = (script: string, timeoutSec: number) =>execRaw(stored.microvmId, stored.endpoint, script, timeoutSec);const report = appReporter(opts.onError, d, version.env);@@ -625,30 +644,55 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployPapply: (d, version) => place(d, version),+ async rotationDue(d): Promise<boolean> {+ const stored = await store.get(d.id);+ return !!stored && rotationDue(stored);+ },+reconcile: (d, version, input) => place(d, version, input),async resolveEndpoint(d, version): Promise<DeployEndpoint | null> {- const cached = resolveCache.get(d.id);- if (cached && Date.now() < cached.untilMs) return cached.endpoint;- const inflight = inflightResolves.get(d.id);- if (inflight) return inflight;const gen = resolveGens.get(d.id) ?? 0;const genCurrent = (): boolean => (resolveGens.get(d.id) ?? 0) === gen;- const resolve = resolveLive(d, version, genCurrent)- .then((endpoint) => {- if (endpoint && genCurrent()) {+ const revalidate = (): Promise<DeployEndpoint | null> => {+ const inflight = inflightResolves.get(d.id);+ if (inflight) return inflight;+ const resolve = resolveLive(d, version, genCurrent)+ .then((endpoint) => {+ if (!genCurrent()) return endpoint;+ if (!endpoint) {+ resolveCache.delete(d.id);+ return endpoint;+ }if (resolveCache.size > 500) {- for (const [k, v] of resolveCache) if (Date.now() >= v.untilMs) resolveCache.delete(k);+ for (const [k, v] of resolveCache) if (Date.now() - v.usedMs >= awakeForMs) resolveCache.delete(k);}- resolveCache.set(d.id, { endpoint, untilMs: Date.now() + RESOLVE_CACHE_MS });- }- return endpoint;- })- .finally(() => {- if (inflightResolves.get(d.id) === resolve) inflightResolves.delete(d.id);- });- inflightResolves.set(d.id, resolve);- return resolve;+ resolveCache.set(d.id, { endpoint, checkedMs: Date.now(), usedMs: Date.now() });+ return endpoint;+ })+ .finally(() => {+ if (inflightResolves.get(d.id) === resolve) inflightResolves.delete(d.id);+ });+ inflightResolves.set(d.id, resolve);+ return resolve;+ };+ const cached = resolveCache.get(d.id);+ const samePlacement = !d.endpoint || d.endpoint.host === cached?.endpoint.host;+ if (cached && samePlacement && Date.now() - cached.usedMs < awakeForMs) {+ cached.usedMs = Date.now();+ if (Date.now() - cached.checkedMs >= RESOLVE_CACHE_MS)+ void revalidate().catch((e) => swallow("aws-deploy: background revalidate", e));+ return cached.endpoint;+ }+ return revalidate();+ },++ async park(d): Promise<void> {+ if (d.alwaysOn) return;+ const stored = await store.get(d.id);+ if (!stored) return;+ resolveCache.delete(d.id);+ await api.suspend(stored.microvmId);},async logs(d, opts): Promise<string | null> {diff --git a/src/deploy/deploy-provider.ts b/src/deploy/deploy-provider.tsindex bd0563bc5..5a19bf1cc 100644--- a/src/deploy/deploy-provider.ts+++ b/src/deploy/deploy-provider.ts@@ -22,6 +22,8 @@ export interface DeployProvider {destroy(d: Deployment): Promise<void>;setAlwaysOn?(d: Deployment, alwaysOn: boolean): Promise<void>;resolveEndpoint?(d: Deployment, version: DeploymentVersion): Promise<DeployEndpoint | null>;+ rotationDue?(d: Deployment): Promise<boolean>;+ park?(d: Deployment): Promise<void>;/** Recent output from the running app (entrypoint stdout+stderr), newest last. */logs?(d: Deployment, opts: { tailLines: number }): Promise<string | null>;}diff --git a/src/deploy/deploy-service.ts b/src/deploy/deploy-service.tsindex cce5b2a60..937e2a648 100644--- a/src/deploy/deploy-service.ts+++ b/src/deploy/deploy-service.ts@@ -86,7 +86,7 @@ export interface DeployService {setDeploymentEmbedAncestors(id: string, embedAncestors: string[]): Promise<Deployment>;setDeploymentPublic(idOrName: string, isPublic: boolean, actor: { createdBy: string }): Promise<Deployment>;- keepAlwaysOnWarm(): Promise<number>;+ maintainDeployments(): Promise<number>;reachDeployment(idOrName: string, principalId: string, opts?: ReachOptions): Promise<Reach>;/** Recent app output (entrypoint stdout+stderr) for a deployment, newest last; null when the provider keeps none. */deploymentLogs(idOrName: string, opts: { tailLines: number }): Promise<string | null>;@@ -246,6 +246,17 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {});};+ const rotateDeployment = (id: string): Promise<void> =>+ withDeployLock(id, async () => {+ const cur = await deps.deployStore.get(id);+ if (!cur || cur.status !== "running" || !(await deps.provider.rotationDue?.(cur))) return;+ const v = currentVersionOf(cur);+ if (!v) return;+ const endpoint = await applyVersion(cur.id, v, cur.appliedVersion ?? cur.currentVersion);+ await markVersionRunning(cur.id, v.version, endpoint);+ await deps.provider.park?.(cur).catch((e) => swallow("deploy: park rotated app", e));+ });+const deploymentRef = (id: string): string => encodeRef(deployRef(id));const grantsOn = (d: Deployment): Promise<Grant[]> => deps.acl.grantsFor(d.ownerScopeId, deploymentRef(d.id));@@ -533,21 +544,26 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {return (await deps.deployStore.get(d.id))!;},- async keepAlwaysOnWarm() {+ async maintainDeployments() {const result = await leaderLease.hold("deployments:keep-warm", async () => {- let warmed = 0;+ let maintained = 0;for (const d of await deps.deployStore.list()) {- if (!d.alwaysOn || d.status !== "running") continue;+ if (d.status !== "running") continue;try {const cur = await deps.deployStore.get(d.id);- if (!cur?.alwaysOn || cur.status !== "running") continue;- await liveEndpoint(cur);- warmed++;+ if (!cur || cur.status !== "running") continue;+ if (await deps.provider.rotationDue?.(cur)) {+ await rotateDeployment(cur.id);+ maintained++;+ } else if (cur.alwaysOn) {+ await liveEndpoint(cur);+ maintained++;+ }} catch (e) {- console.error("%s", `[deploy] keep-warm failed for ${d.name ?? d.id}:`, errMessage(e));+ console.error("%s", `[deploy] maintenance failed for ${d.name ?? d.id}:`, errMessage(e));}}- return warmed;+ return maintained;});return result ?? 0;},diff --git a/src/deploy/shared-deploy-provider.ts b/src/deploy/shared-deploy-provider.tsindex df65369f5..e3edb1105 100644--- a/src/deploy/shared-deploy-provider.ts+++ b/src/deploy/shared-deploy-provider.ts@@ -140,7 +140,7 @@ export async function waitAppReady(exec: Exec, opts: AppReadyOptions): Promise<v`while [ "$(date +%s)" -lt "$end" ]; do pid=$(cat ${shq(APP_PID_PATH)} 2>/dev/null || true); ` +`code=; listening && { code=$(status); case "$code" in ""|[1-4]??) exit 0 ;; esac; }; ` +`[ -z "$pid" ] || alive "$pid" && [ ! -f ${EXITED} ] || died=1; ` +- `sleep 0.5; done; [ "$died" = 1 ] && exit ${EXIT_APP_EXITED}; ` ++ `sleep 0.1; done; [ "$died" = 1 ] && exit ${EXIT_APP_EXITED}; ` +`[ "$code" -ge 500 ] 2>/dev/null && { echo "$code"; exit ${EXIT_APP_SERVER_ERROR}; }; exit 1`;let left = opts.windowSec;let r: Awaited<ReturnType<Exec>>;diff --git a/src/sandbox/aws-microvm-api.ts b/src/sandbox/aws-microvm-api.tsindex a6b61b931..8b6f83d42 100644--- a/src/sandbox/aws-microvm-api.ts+++ b/src/sandbox/aws-microvm-api.ts@@ -219,8 +219,8 @@ export function createMicrovmApi(opts: AwsMicrovmApiOptions): AwsMicrovmApi {},async waitForState(id, target, waitOpts) {const timeoutMs = waitOpts?.timeoutMs ?? 120_000;- const intervalMs = waitOpts?.intervalMs ?? 1500;- const deadline = Date.now() + timeoutMs;+ const started = Date.now();+ const deadline = started + timeoutMs;let last: MicrovmDescription | null = null;while (Date.now() < deadline) {last = await api.getMicrovm(id);@@ -228,7 +228,7 @@ export function createMicrovmApi(opts: AwsMicrovmApiOptions): AwsMicrovmApi {if (last.state === "TERMINATED" && target !== "TERMINATED") {throw new AwsApiError(`microVM ${id} terminated while waiting for ${target}: ${last.stateReason ?? ""}`, 409);}- await sleep(intervalMs);+ await sleep(waitOpts?.intervalMs ?? pollInterval(started, 1500));}throw new AwsApiError(`timed out waiting for microVM ${id} to reach ${target} (last=${last?.state})`, 504);},@@ -236,6 +236,11 @@ export function createMicrovmApi(opts: AwsMicrovmApiOptions): AwsMicrovmApi {return api;}+const FAST_POLL_MS = 200;+const FAST_POLL_WINDOW_MS = 15_000;+const pollInterval = (startedMs: number, slowMs: number): number =>+ Date.now() - startedMs < FAST_POLL_WINDOW_MS ? FAST_POLL_MS : slowMs;+export interface VmFetchOptions {method?: string;body?: unknown;@@ -344,7 +349,8 @@ export function createMicrovmClient(api: AwsMicrovmApi, opts: MicrovmClientOptio}async function waitDaemon(id: string, endpoint: string): Promise<void> {- const deadline = Date.now() + 60_000;+ const started = Date.now();+ const deadline = started + 60_000;let lastErr = "";let throttled = 0;while (Date.now() < deadline) {@@ -366,7 +372,7 @@ export function createMicrovmClient(api: AwsMicrovmApi, opts: MicrovmClientOptiothrottled = 0;lastErr = errMessage(e);}- await sleep(2000);+ await sleep(pollInterval(started, 2000));}throw new Error(`microVM ${id} exec daemon never became reachable: ${lastErr}`);}diff --git a/src/wiring.ts b/src/wiring.tsindex bc6017f0d..2816d3365 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -2722,8 +2722,8 @@ export function buildApp(deployIdleTtlMs && deployIdleTtlMs > 0? createSweeper(() => app.reapIdleDeployments(deployIdleTtlMs), Math.max(5_000, Math.floor(deployIdleTtlMs / 4))): null;- const KEEP_WARM_INTERVAL_MS = 5 * 60_000;- const keepWarmSweeper = createSweeper(() => app.keepAlwaysOnWarm(), KEEP_WARM_INTERVAL_MS);+ const DEPLOY_MAINTENANCE_INTERVAL_MS = 5 * 60_000;+ const deployMaintenanceSweeper = createSweeper(() => app.maintainDeployments(), DEPLOY_MAINTENANCE_INTERVAL_MS);const deepIdleMachineMs = config.deepIdleMachineMs;const devIdleMachineMs = config.devIdleMachineMs;const SANDBOX_MAINTENANCE_INTERVAL_MS = 5 * 60_000;@@ -2775,7 +2775,7 @@ export function buildApp(composioReturnSweeper.start();fileUploads?.start();idleSweeper?.start();- keepWarmSweeper.start();+ deployMaintenanceSweeper.start();deepIdleSweeper?.start();wakeSweep.start();swarms?.start();@@ -2802,7 +2802,7 @@ export function buildApp(spendSweeper?.stop(),skillSyncEngine.stop(),idleSweeper?.stop(),- keepWarmSweeper.stop(),+ deployMaintenanceSweeper.stop(),deepIdleSweeper?.stop(),blobSweeper.stop(),composioReturnSweeper.stop(),diff --git a/test/aws-deploy-provider.test.ts b/test/aws-deploy-provider.test.tsindex 211cad25e..dbf875870 100644--- a/test/aws-deploy-provider.test.ts+++ b/test/aws-deploy-provider.test.ts@@ -25,6 +25,7 @@ interface Vm {function fakeApi() {const vms = new Map<string, Vm>();const counts = { run: 0, resume: 0, terminate: 0, token: 0, describe: 0 };+ const launches: Array<Parameters<AwsMicrovmApi["runMicrovm"]>[0]> = [];let n = 0;const api: AwsMicrovmApi = {async listImages() {@@ -39,8 +40,9 @@ function fakeApi() {async updateImage() {throw new Error("updateImage unused in deploy provider");},- async runMicrovm() {+ async runMicrovm(input) {counts.run++;+ launches.push(input);const id = `mvm-${++n}`;const endpoint = `${id}.lambda-microvm.${REGION}.on.aws`;vms.set(id, { state: "RUNNING", endpoint });@@ -80,7 +82,7 @@ function fakeApi() {return { microvmId: id, state: v?.state ?? target, ...(v?.endpoint ? { endpoint: v.endpoint } : {}) };},};- return { api, vms, counts };+ return { api, vms, counts, launches };}function fakeDaemon(@@ -337,6 +339,79 @@ test("resolveEndpoint (suspended): resumes the body and serves it warm", async (assert.equal(counts.run, 1, "no relaunch — same body");});+test("apply: an idle app stays suspended for its whole lifetime instead of being terminated after an hour", async () => {+ const { api, launches } = fakeApi();+ const { fetchImpl } = fakeDaemon();+ const p = provider(api, fetchImpl);+ await p.apply(deployment(ID), version(localSnapshot("a", "1")));+ assert.equal(launches.length, 1);+ assert.equal(launches[0]!.idlePolicy?.maxIdleDurationSeconds, 900, "still suspends after 15 idle minutes");+ assert.equal(+ launches[0]!.idlePolicy?.suspendedDurationSeconds,+ launches[0]!.maximumDurationInSeconds,+ "a suspended body survives until its maximum lifetime",+ );+});++test("apply: an explicit suspended-duration setting still wins", async () => {+ const { api, launches } = fakeApi();+ const { fetchImpl } = fakeDaemon();+ const p = provider(api, fetchImpl, createMemoryMap<StoredDeployBody>(), { suspendedDurationSeconds: 600 });+ await p.apply(deployment(ID), version(localSnapshot("a", "1")));+ assert.equal(launches[0]!.idlePolicy?.suspendedDurationSeconds, 600);+});++test("rotation: the replacement body is launched before the old one is terminated, and the old one stays resumable until then", async () => {+ const { api, vms } = fakeApi();+ const order: string[] = [];+ const realRun = api.runMicrovm.bind(api);+ const realTerminate = api.terminate.bind(api);+ api.runMicrovm = async (input) => {+ const r = await realRun(input);+ order.push(`run ${r.microvmId}`);+ return r;+ };+ api.terminate = async (id) => {+ order.push(`terminate ${id}`);+ return realTerminate(id);+ };+ const { fetchImpl } = fakeDaemon();+ const store = createMemoryMap<StoredDeployBody>();+ const p = provider(api, fetchImpl, store);+ const d = deployment(ID);+ await p.apply(d, version(localSnapshot("a", "1")));+ const first = (await store.get(ID))!;+ assert.equal(await p.rotationDue!(d), false);+ await store.put(ID, { ...first, createdAtMs: Date.now() - 7 * 3600_000 - 60_000 });+ vms.get(first.microvmId)!.state = "SUSPENDED";+ assert.equal(await p.rotationDue!(d), true, "due 30 minutes before the visit-path rotation");+ assert.ok(await p.resolveEndpoint!(d, version("/unused")), "visits still reach the old body meanwhile");++ await p.apply(d, version(localSnapshot("a", "1")));+ const second = (await store.get(ID))!;+ assert.notEqual(second.microvmId, first.microvmId);+ assert.deepEqual(order.slice(-2), [`run ${second.microvmId}`, `terminate ${first.microvmId}`]);+ assert.equal(await p.rotationDue!(d), false);+});++test("rotation: a replacement left behind by a crash mid-rotation is terminated on the next attempt", async () => {+ const { api, vms } = fakeApi();+ const { fetchImpl } = fakeDaemon();+ const store = createMemoryMap<StoredDeployBody>();+ const p = provider(api, fetchImpl, store);+ const d = deployment(ID);+ await p.apply(d, version(localSnapshot("a", "1")));+ const first = (await store.get(ID))!;+ vms.set("mvm-orphan", { state: "RUNNING", endpoint: "mvm-orphan.lambda-microvm.us-west-2.on.aws" });+ await store.put(ID, { ...first, replacementMicrovmId: "mvm-orphan" });++ await p.apply(d, version(localSnapshot("a", "1")));+ assert.equal(vms.get("mvm-orphan")!.state, "TERMINATED");+ const after = (await store.get(ID))!;+ assert.equal(after.microvmId, first.microvmId);+ assert.equal(after.replacementMicrovmId, undefined);+});+test("resolveEndpoint: reaches within the cache window skip the control plane", async () => {const { api, counts } = fakeApi();const { fetchImpl } = fakeDaemon();@@ -354,6 +429,60 @@ test("resolveEndpoint: reaches within the cache window skip the control plane",assert.equal(counts.describe, describesAfterFirst, "a fresh verdict is reused without re-asking the control plane");});+test("resolveEndpoint: an app in use is served from cache and re-checked in the background, not on the visit", async (t) => {+ t.mock.timers.enable({ apis: ["Date"], now: 1_000_000 });+ const { api, counts } = fakeApi();+ const { fetchImpl } = fakeDaemon();+ const store = createMemoryMap<StoredDeployBody>();+ const p = provider(api, fetchImpl, store);+ const d = deployment(ID);+ await p.apply(d, version(localSnapshot("a", "1")));+ const first = await p.resolveEndpoint!(d, version("/unused"));+ const describes = counts.describe;++ t.mock.timers.tick(60_000);+ const hot = await p.resolveEndpoint!(d, version("/unused"));+ assert.equal(hot!.host, first!.host, "the visit is answered from cache");+ await new Promise((r) => setImmediate(r));+ assert.equal(counts.describe, describes + 1, "a background re-check refreshed the verdict");++ const moved = { ...d, endpoint: { host: "elsewhere.on.aws", port: 443 } };+ await p.resolveEndpoint!(moved, version("/unused"));+ assert.equal(counts.describe, describes + 2, "a placement changed by another core is re-checked before serving");+});++test("resolveEndpoint: after the idle window the visit waits for a real check, since the machine may be suspended", async (t) => {+ t.mock.timers.enable({ apis: ["Date"], now: 1_000_000 });+ const { api, counts, vms } = fakeApi();+ const { fetchImpl } = fakeDaemon();+ const store = createMemoryMap<StoredDeployBody>();+ const p = provider(api, fetchImpl, store);+ const d = deployment(ID);+ await p.apply(d, version(localSnapshot("a", "1")));+ await p.resolveEndpoint!(d, version("/unused"));+ const body = (await store.get(ID))!;+ vms.get(body.microvmId)!.state = "SUSPENDED";++ t.mock.timers.tick(15 * 60_000);+ const resumesBefore = counts.resume;+ assert.ok(await p.resolveEndpoint!(d, version("/unused")));+ assert.equal(counts.resume, resumesBefore + 1, "the suspended machine is resumed before the visit is served");+});++test("park: suspends the app's machine unless it is always-on", async () => {+ const { api, vms } = fakeApi();+ const { fetchImpl } = fakeDaemon();+ const store = createMemoryMap<StoredDeployBody>();+ const p = provider(api, fetchImpl, store);+ const d = deployment(ID);+ await p.apply(d, version(localSnapshot("a", "1")));+ const body = (await store.get(ID))!;+ await p.park!({ ...d, alwaysOn: true });+ assert.equal(vms.get(body.microvmId)!.state, "RUNNING");+ await p.park!(d);+ assert.equal(vms.get(body.microvmId)!.state, "SUSPENDED");+});+test("resolveEndpoint: concurrent misses coalesce into one control-plane pass", async () => {const { api, counts } = fakeApi();const { fetchImpl } = fakeDaemon();@@ -371,8 +500,8 @@ test("resolveEndpoint: concurrent misses coalesce into one control-plane pass",assert.ok(a && b && c);assert.equal(counts.describe,- describesAfterApply + 2,- "three simultaneous misses share one tryGet + one ensureRunning describe",+ describesAfterApply + 1,+ "three simultaneous misses share one describe, reused by ensureRunning",);});@@ -719,8 +848,9 @@ test("app data: a failed rotate snapshot defers the rotation — the stale body});const e = await p.apply(deployment(ID), version(localSnapshot("a", "1")));- assert.equal(counts.terminate, 0, "the stale body is not terminated when its data can't be saved");- assert.equal(counts.run, 0, "no replacement launched");+ assert.equal(vms.get("mvm-old")!.state, "RUNNING", "the stale body is not terminated when its data can't be saved");+ assert.equal(counts.run, 1, "a replacement was launched ahead of the snapshot");+ assert.equal(vms.get("mvm-1")!.state, "TERMINATED", "and discarded instead of leaked");assert.equal(e.host, "mvm-old.lambda-microvm.us-west-2.on.aws", "the stale body keeps serving");});@@ -768,9 +898,10 @@ test("app data: a deferred rotation serves the stale body from resolveEndpoint iawait p.apply(d, version(localSnapshot("a", "1")));assert.ok((await store.get(ID))!.rotateNotBeforeMs! > Date.now(), "deferral stamped");+ const runsAfterDeferral = counts.run;const resolved = await p.resolveEndpoint!(d, version("/unused"));assert.ok(resolved, "stale-but-deferred body keeps serving");- assert.equal(counts.run, 0, "no replacement launched during the deferral");+ assert.equal(counts.run, runsAfterDeferral, "no replacement launched during the deferral");});test("litestream: shadow-WAL dir is excluded from both snapshot and hydrate", async () => {diff --git a/test/deploy-always-on.test.ts b/test/deploy-always-on.test.tsindex 171e2bb61..93f150079 100644--- a/test/deploy-always-on.test.ts+++ b/test/deploy-always-on.test.ts@@ -16,6 +16,8 @@ function svc(resolve?: () => Promise<DeployEndpoint | null>;setAlwaysOn?: DeployProvider["setAlwaysOn"];apply?: DeployProvider["apply"];+ rotationDue?: DeployProvider["rotationDue"];+ park?: DeployProvider["park"];} = {},) {const deployStore = createDeployStore();@@ -30,6 +32,8 @@ function svc(},destroy: async () => {},...(opts.setAlwaysOn ? { setAlwaysOn: opts.setAlwaysOn } : {}),+ ...(opts.rotationDue ? { rotationDue: opts.rotationDue } : {}),+ ...(opts.park ? { park: opts.park } : {}),...(opts.resolve? {resolveEndpoint: async () => {@@ -189,7 +193,7 @@ test("reaper re-checks alwaysOn under the deploy lock (no stale-snapshot destroyassert.equal((await deployStore.get(d.id))!.status, "running");});-test("keepAlwaysOnWarm re-ensures a live endpoint only for running always-on deployments", async () => {+test("maintainDeployments re-ensures a live endpoint only for running always-on deployments", async () => {const ctx = svc({ resolve: async () => ({ host: "127.0.0.1", port: 5001 }) });const { deploy } = ctx;const cold = await deploy.deploy({ ...owner, entrypoint: "x", files: [] });@@ -197,13 +201,13 @@ test("keepAlwaysOnWarm re-ensures a live endpoint only for running always-on depawait deploy.setDeploymentAlwaysOn(warm.id, true);const resolvesBefore = ctx.resolves;- const warmed = await deploy.keepAlwaysOnWarm();+ const warmed = await deploy.maintainDeployments();assert.equal(warmed, 1);assert.equal(ctx.resolves, resolvesBefore + 1);assert.ok(cold.id !== warm.id);});-test("keepAlwaysOnWarm skips a deployment whose flag was turned off after the list snapshot", async () => {+test("maintainDeployments skips a deployment whose flag was turned off after the list snapshot", async () => {const ctx = svc({ resolve: async () => null });const { deploy, deployStore } = ctx;const d = await deploy.deploy({ ...owner, entrypoint: "x", files: [] });@@ -215,24 +219,47 @@ test("keepAlwaysOnWarm skips a deployment whose flag was turned off after the lireturn snapshot;};const appliesBefore = ctx.applies;- const warmed = await deploy.keepAlwaysOnWarm();+ const warmed = await deploy.maintainDeployments();assert.equal(warmed, 0);assert.equal(ctx.applies, appliesBefore);});-test("keepAlwaysOnWarm relaunches when the platform has torn the app down", async () => {+test("maintainDeployments relaunches when the platform has torn the app down", async () => {const ctx = svc({ resolve: async () => null });const { deploy, deployStore } = ctx;const d = await deploy.deploy({ ...owner, entrypoint: "x", files: [] });await deploy.setDeploymentAlwaysOn(d.id, true);const appliesBefore = ctx.applies;- const warmed = await deploy.keepAlwaysOnWarm();+ const warmed = await deploy.maintainDeployments();assert.equal(warmed, 1);assert.ok(ctx.applies > appliesBefore, "expected a re-apply when no live endpoint exists");assert.equal((await deployStore.get(d.id))!.status, "running");});+test("maintainDeployments replaces a due app's machine in the background and parks the replacement", async () => {+ const due = new Set<string>();+ const parked: string[] = [];+ const ctx = svc({+ rotationDue: async (d) => due.has(d.id),+ park: async (d) => {+ parked.push(d.id);+ due.delete(d.id);+ },+ });+ const { deploy, deployStore } = ctx;+ const stale = await deploy.deploy({ ...owner, entrypoint: "x", files: [] });+ await deploy.deploy({ ...owner, entrypoint: "x", files: [] });+ due.add(stale.id);+ const appliesBefore = ctx.applies;++ const maintained = await deploy.maintainDeployments();+ assert.equal(maintained, 1, "only the due app is rotated");+ assert.equal(ctx.applies, appliesBefore + 1);+ assert.equal((await deployStore.get(stale.id))!.status, "running");+ assert.deepEqual(parked, [stale.id], "the replacement is suspended so the next visit is a fast resume");+});+test("always-on reaches the provider before persisting and retains the flag on failure", async () => {const seen: boolean[] = [];let reject = false;