Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

AWS apps: suspend instead of delete, rotate in the background, faster cold start (#2261)

Regan Bell453a46835f37
diff --git a/src/api/app-deployments.ts b/src/api/app-deployments.ts
index 9d0652b13..9d2cb2507 100644
--- a/src/api/app-deployments.ts
+++ b/src/api/app-deployments.ts
@@ -34,7 +34,7 @@ export function createDeploymentMethods(
| "setDeploymentAlwaysOn"
| "setDeploymentEmbedAncestors"
| "setDeploymentPublic"
- | "keepAlwaysOnWarm"
+ | "maintainDeployments"
| "reachDeployment"
| "deploymentLogsFor"
| "deploymentGitRepoPath"
@@ -160,8 +160,8 @@ export function createDeploymentMethods(
setDeploymentPublic(idOrName, isPublic, actor) {
return deps.deploy.setDeploymentPublic(idOrName, isPublic, actor);
},
- keepAlwaysOnWarm() {
- return deps.deploy.keepAlwaysOnWarm();
+ maintainDeployments() {
+ return deps.deploy.maintainDeployments();
},
async reachDeployment(id, principalId, opts): Promise<Reach> {
if (opts?.bypassAcl) return deps.deploy.reachDeployment(id, principalId, opts);
diff --git a/src/api/app-types.ts b/src/api/app-types.ts
index d01e6e5a9..0945e2008 100644
--- a/src/api/app-types.ts
+++ b/src/api/app-types.ts
@@ -582,7 +582,7 @@ export interface App {
setDeploymentAlwaysOn(id: string, alwaysOn: boolean): Promise<Deployment>;
setDeploymentEmbedAncestors(id: string, embedAncestors: string[]): Promise<Deployment>;
setDeploymentPublic(idOrName: string, isPublic: boolean, actor: { createdBy: string }): Promise<Deployment>;
- keepAlwaysOnWarm(): Promise<number>;
+ maintainDeployments(): Promise<number>;
reachDeployment(id: string, principalId: string, opts?: ReachOptions): Promise<Reach>;
deploymentLogsFor(
id: string,
diff --git a/src/deploy/aws-deploy-provider.ts b/src/deploy/aws-deploy-provider.ts
index dbc12c829..76896df24 100644
--- a/src/deploy/aws-deploy-provider.ts
+++ b/src/deploy/aws-deploy-provider.ts
@@ -46,6 +46,7 @@ const EXIT_APP_PORT_HELD = 97;
const LIVE_REPLICATOR_PIDS = `ls_pids() { for p in /proc/[0-9]*; do c=$(tr '\\0' ' ' < "$p/cmdline" 2>/dev/null); case "\${c%% *}" in litestream|*/litestream) case "$c" in *${LITESTREAM_CONFIG}*) echo "\${p#/proc/}";; esac;; esac; done; }`;
const ENDPOINT_PORT = 443;
const RESOLVE_CACHE_MS = 15_000;
+const ROTATE_AHEAD_MS = 30 * 60_000;
const DEFAULT_INGRESS = (region: string): string =>
`arn:aws:lambda:${region}:aws:network-connector:aws-network-connector:ALL_INGRESS`;
@@ -60,6 +61,7 @@ export interface StoredDeployBody {
lastSnapshotMs?: number;
dataCredsAtMs?: number;
rotateNotBeforeMs?: number;
+ replacementMicrovmId?: string;
orgId?: string;
}
@@ -104,7 +106,7 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployP
const store = opts.store ?? createMemoryMap<StoredDeployBody>();
const launchQueue = createKeyedQueue<string>();
const advisoryLock = opts.advisoryLock ?? createNoopAdvisoryLock();
- const resolveCache = new Map<string, { endpoint: DeployEndpoint; untilMs: number }>();
+ const resolveCache = new Map<string, { endpoint: DeployEndpoint; checkedMs: number; usedMs: number }>();
const inflightResolves = new Map<string, Promise<DeployEndpoint | null>>();
const resolveGens = new Map<string, number>();
const bumpResolveGen = (id: string): void => {
@@ -127,8 +129,13 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployP
const appPort = opts.appPort ?? APP_PORT_DEFAULT;
const maximumDurationInSeconds = opts.maximumDurationInSeconds ?? 28_800;
const rotateAfterMs = (opts.rotateAfterSeconds ?? 27_000) * 1000;
+ const rotateDueMs = Math.max(0, rotateAfterMs - ROTATE_AHEAD_MS);
+ const rotationDue = (stored: StoredDeployBody): boolean =>
+ Date.now() - stored.createdAtMs > rotateDueMs &&
+ !(stored.rotateNotBeforeMs && Date.now() < stored.rotateNotBeforeMs);
const maxIdleDurationSeconds = opts.maxIdleDurationSeconds ?? 900;
- const suspendedDurationSeconds = opts.suspendedDurationSeconds ?? 3600;
+ const awakeForMs = Math.max(0, maxIdleDurationSeconds * 1000 - 60_000);
+ const suspendedDurationSeconds = opts.suspendedDurationSeconds ?? maximumDurationInSeconds;
const appsDomain = opts.appsDomain;
const tokenTtlMinutes = opts.tokenTtlMinutes ?? 30;
const dataBucket = opts.dataBucket;
@@ -349,23 +356,35 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployP
async function ensureBody(d: Deployment): Promise<{ id: string; endpoint: string; fresh: boolean }> {
return launchQueue(d.id, async () => {
- const stored = await store.get(d.id);
+ let stored = await store.get(d.id);
+ if (stored?.replacementMicrovmId) {
+ await api
+ .terminate(stored.replacementMicrovmId)
+ .catch((e) => swallow("aws-deploy: terminate orphaned replacement", e));
+ const { replacementMicrovmId: _, ...rest } = stored;
+ stored = rest;
+ await store.put(d.id, stored);
+ }
+ let next: { id: string; endpoint: string } | undefined;
if (stored) {
const desc = await api.tryGetMicrovm(stored.microvmId);
const alive = desc && desc.state !== "TERMINATED" && desc.state !== "TERMINATING";
- const stale = Date.now() - stored.createdAtMs > rotateAfterMs;
- if (alive && !stale) {
- await ensureRunning(stored.microvmId, stored.endpoint);
+ if (alive && !rotationDue(stored)) {
+ await ensureRunning(stored.microvmId, stored.endpoint, desc);
return { id: stored.microvmId, endpoint: stored.endpoint, fresh: false };
}
- if (alive && stale) {
- await ensureRunning(stored.microvmId, stored.endpoint).catch(() => {});
+ if (alive) {
+ next = await launchBody(d.id, d.alwaysOn);
+ const replacement = next;
+ await store.put(d.id, { ...stored, replacementMicrovmId: replacement.id });
+ await ensureRunning(stored.microvmId, stored.endpoint, desc).catch(() => {});
const defer = async (
label: string,
e: unknown,
extra: Partial<StoredDeployBody> = {},
): Promise<{ id: string; endpoint: string; fresh: boolean }> => {
swallow(label, e);
+ await api.terminate(replacement.id).catch((e2) => swallow("aws-deploy: abandon replacement", e2));
await store.put(d.id, { ...stored, rotateNotBeforeMs: Date.now() + 5 * 60_000, ...extra });
return { id: stored.microvmId, endpoint: stored.endpoint, fresh: false };
};
@@ -386,7 +405,7 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployP
}
}
}
- const body = await launchBody(d.id, d.alwaysOn);
+ const body = next ?? (await launchBody(d.id, d.alwaysOn));
try {
await hydrateData(d.id, body.id, body.endpoint);
} catch (e) {
@@ -555,7 +574,7 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployP
!(stored.rotateNotBeforeMs && Date.now() < stored.rotateNotBeforeMs)
)
return null;
- await ensureRunning(stored.microvmId, stored.endpoint);
+ await ensureRunning(stored.microvmId, stored.endpoint, desc);
const exec = (script: string, timeoutSec: number) =>
execRaw(stored.microvmId, stored.endpoint, script, timeoutSec);
const report = appReporter(opts.onError, d, version.env);
@@ -625,30 +644,55 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployP
apply: (d, version) => place(d, version),
+ async rotationDue(d): Promise<boolean> {
+ const stored = await store.get(d.id);
+ return !!stored && rotationDue(stored);
+ },
+
reconcile: (d, version, input) => place(d, version, input),
async resolveEndpoint(d, version): Promise<DeployEndpoint | null> {
- const cached = resolveCache.get(d.id);
- if (cached && Date.now() < cached.untilMs) return cached.endpoint;
- const inflight = inflightResolves.get(d.id);
- if (inflight) return inflight;
const gen = resolveGens.get(d.id) ?? 0;
const genCurrent = (): boolean => (resolveGens.get(d.id) ?? 0) === gen;
- const resolve = resolveLive(d, version, genCurrent)
- .then((endpoint) => {
- if (endpoint && genCurrent()) {
+ const revalidate = (): Promise<DeployEndpoint | null> => {
+ const inflight = inflightResolves.get(d.id);
+ if (inflight) return inflight;
+ const resolve = resolveLive(d, version, genCurrent)
+ .then((endpoint) => {
+ if (!genCurrent()) return endpoint;
+ if (!endpoint) {
+ resolveCache.delete(d.id);
+ return endpoint;
+ }
if (resolveCache.size > 500) {
- for (const [k, v] of resolveCache) if (Date.now() >= v.untilMs) resolveCache.delete(k);
+ for (const [k, v] of resolveCache) if (Date.now() - v.usedMs >= awakeForMs) resolveCache.delete(k);
}
- resolveCache.set(d.id, { endpoint, untilMs: Date.now() + RESOLVE_CACHE_MS });
- }
- return endpoint;
- })
- .finally(() => {
- if (inflightResolves.get(d.id) === resolve) inflightResolves.delete(d.id);
- });
- inflightResolves.set(d.id, resolve);
- return resolve;
+ resolveCache.set(d.id, { endpoint, checkedMs: Date.now(), usedMs: Date.now() });
+ return endpoint;
+ })
+ .finally(() => {
+ if (inflightResolves.get(d.id) === resolve) inflightResolves.delete(d.id);
+ });
+ inflightResolves.set(d.id, resolve);
+ return resolve;
+ };
+ const cached = resolveCache.get(d.id);
+ const samePlacement = !d.endpoint || d.endpoint.host === cached?.endpoint.host;
+ if (cached && samePlacement && Date.now() - cached.usedMs < awakeForMs) {
+ cached.usedMs = Date.now();
+ if (Date.now() - cached.checkedMs >= RESOLVE_CACHE_MS)
+ void revalidate().catch((e) => swallow("aws-deploy: background revalidate", e));
+ return cached.endpoint;
+ }
+ return revalidate();
+ },
+
+ async park(d): Promise<void> {
+ if (d.alwaysOn) return;
+ const stored = await store.get(d.id);
+ if (!stored) return;
+ resolveCache.delete(d.id);
+ await api.suspend(stored.microvmId);
},
async logs(d, opts): Promise<string | null> {
diff --git a/src/deploy/deploy-provider.ts b/src/deploy/deploy-provider.ts
index bd0563bc5..5a19bf1cc 100644
--- a/src/deploy/deploy-provider.ts
+++ b/src/deploy/deploy-provider.ts
@@ -22,6 +22,8 @@ export interface DeployProvider {
destroy(d: Deployment): Promise<void>;
setAlwaysOn?(d: Deployment, alwaysOn: boolean): Promise<void>;
resolveEndpoint?(d: Deployment, version: DeploymentVersion): Promise<DeployEndpoint | null>;
+ rotationDue?(d: Deployment): Promise<boolean>;
+ park?(d: Deployment): Promise<void>;
/** Recent output from the running app (entrypoint stdout+stderr), newest last. */
logs?(d: Deployment, opts: { tailLines: number }): Promise<string | null>;
}
diff --git a/src/deploy/deploy-service.ts b/src/deploy/deploy-service.ts
index cce5b2a60..937e2a648 100644
--- a/src/deploy/deploy-service.ts
+++ b/src/deploy/deploy-service.ts
@@ -86,7 +86,7 @@ export interface DeployService {
setDeploymentEmbedAncestors(id: string, embedAncestors: string[]): Promise<Deployment>;
setDeploymentPublic(idOrName: string, isPublic: boolean, actor: { createdBy: string }): Promise<Deployment>;
- keepAlwaysOnWarm(): Promise<number>;
+ maintainDeployments(): Promise<number>;
reachDeployment(idOrName: string, principalId: string, opts?: ReachOptions): Promise<Reach>;
/** Recent app output (entrypoint stdout+stderr) for a deployment, newest last; null when the provider keeps none. */
deploymentLogs(idOrName: string, opts: { tailLines: number }): Promise<string | null>;
@@ -246,6 +246,17 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {
});
};
+ const rotateDeployment = (id: string): Promise<void> =>
+ withDeployLock(id, async () => {
+ const cur = await deps.deployStore.get(id);
+ if (!cur || cur.status !== "running" || !(await deps.provider.rotationDue?.(cur))) return;
+ const v = currentVersionOf(cur);
+ if (!v) return;
+ const endpoint = await applyVersion(cur.id, v, cur.appliedVersion ?? cur.currentVersion);
+ await markVersionRunning(cur.id, v.version, endpoint);
+ await deps.provider.park?.(cur).catch((e) => swallow("deploy: park rotated app", e));
+ });
+
const deploymentRef = (id: string): string => encodeRef(deployRef(id));
const grantsOn = (d: Deployment): Promise<Grant[]> => deps.acl.grantsFor(d.ownerScopeId, deploymentRef(d.id));
@@ -533,21 +544,26 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {
return (await deps.deployStore.get(d.id))!;
},
- async keepAlwaysOnWarm() {
+ async maintainDeployments() {
const result = await leaderLease.hold("deployments:keep-warm", async () => {
- let warmed = 0;
+ let maintained = 0;
for (const d of await deps.deployStore.list()) {
- if (!d.alwaysOn || d.status !== "running") continue;
+ if (d.status !== "running") continue;
try {
const cur = await deps.deployStore.get(d.id);
- if (!cur?.alwaysOn || cur.status !== "running") continue;
- await liveEndpoint(cur);
- warmed++;
+ if (!cur || cur.status !== "running") continue;
+ if (await deps.provider.rotationDue?.(cur)) {
+ await rotateDeployment(cur.id);
+ maintained++;
+ } else if (cur.alwaysOn) {
+ await liveEndpoint(cur);
+ maintained++;
+ }
} catch (e) {
- console.error("%s", `[deploy] keep-warm failed for ${d.name ?? d.id}:`, errMessage(e));
+ console.error("%s", `[deploy] maintenance failed for ${d.name ?? d.id}:`, errMessage(e));
}
}
- return warmed;
+ return maintained;
});
return result ?? 0;
},
diff --git a/src/deploy/shared-deploy-provider.ts b/src/deploy/shared-deploy-provider.ts
index df65369f5..e3edb1105 100644
--- a/src/deploy/shared-deploy-provider.ts
+++ b/src/deploy/shared-deploy-provider.ts
@@ -140,7 +140,7 @@ export async function waitAppReady(exec: Exec, opts: AppReadyOptions): Promise<v
`while [ "$(date +%s)" -lt "$end" ]; do pid=$(cat ${shq(APP_PID_PATH)} 2>/dev/null || true); ` +
`code=; listening && { code=$(status); case "$code" in ""|[1-4]??) exit 0 ;; esac; }; ` +
`[ -z "$pid" ] || alive "$pid" && [ ! -f ${EXITED} ] || died=1; ` +
- `sleep 0.5; done; [ "$died" = 1 ] && exit ${EXIT_APP_EXITED}; ` +
+ `sleep 0.1; done; [ "$died" = 1 ] && exit ${EXIT_APP_EXITED}; ` +
`[ "$code" -ge 500 ] 2>/dev/null && { echo "$code"; exit ${EXIT_APP_SERVER_ERROR}; }; exit 1`;
let left = opts.windowSec;
let r: Awaited<ReturnType<Exec>>;
diff --git a/src/sandbox/aws-microvm-api.ts b/src/sandbox/aws-microvm-api.ts
index a6b61b931..8b6f83d42 100644
--- a/src/sandbox/aws-microvm-api.ts
+++ b/src/sandbox/aws-microvm-api.ts
@@ -219,8 +219,8 @@ export function createMicrovmApi(opts: AwsMicrovmApiOptions): AwsMicrovmApi {
},
async waitForState(id, target, waitOpts) {
const timeoutMs = waitOpts?.timeoutMs ?? 120_000;
- const intervalMs = waitOpts?.intervalMs ?? 1500;
- const deadline = Date.now() + timeoutMs;
+ const started = Date.now();
+ const deadline = started + timeoutMs;
let last: MicrovmDescription | null = null;
while (Date.now() < deadline) {
last = await api.getMicrovm(id);
@@ -228,7 +228,7 @@ export function createMicrovmApi(opts: AwsMicrovmApiOptions): AwsMicrovmApi {
if (last.state === "TERMINATED" && target !== "TERMINATED") {
throw new AwsApiError(`microVM ${id} terminated while waiting for ${target}: ${last.stateReason ?? ""}`, 409);
}
- await sleep(intervalMs);
+ await sleep(waitOpts?.intervalMs ?? pollInterval(started, 1500));
}
throw new AwsApiError(`timed out waiting for microVM ${id} to reach ${target} (last=${last?.state})`, 504);
},
@@ -236,6 +236,11 @@ export function createMicrovmApi(opts: AwsMicrovmApiOptions): AwsMicrovmApi {
return api;
}
+const FAST_POLL_MS = 200;
+const FAST_POLL_WINDOW_MS = 15_000;
+const pollInterval = (startedMs: number, slowMs: number): number =>
+ Date.now() - startedMs < FAST_POLL_WINDOW_MS ? FAST_POLL_MS : slowMs;
+
export interface VmFetchOptions {
method?: string;
body?: unknown;
@@ -344,7 +349,8 @@ export function createMicrovmClient(api: AwsMicrovmApi, opts: MicrovmClientOptio
}
async function waitDaemon(id: string, endpoint: string): Promise<void> {
- const deadline = Date.now() + 60_000;
+ const started = Date.now();
+ const deadline = started + 60_000;
let lastErr = "";
let throttled = 0;
while (Date.now() < deadline) {
@@ -366,7 +372,7 @@ export function createMicrovmClient(api: AwsMicrovmApi, opts: MicrovmClientOptio
throttled = 0;
lastErr = errMessage(e);
}
- await sleep(2000);
+ await sleep(pollInterval(started, 2000));
}
throw new Error(`microVM ${id} exec daemon never became reachable: ${lastErr}`);
}
diff --git a/src/wiring.ts b/src/wiring.ts
index bc6017f0d..2816d3365 100644
--- a/src/wiring.ts
+++ b/src/wiring.ts
@@ -2722,8 +2722,8 @@ export function buildApp(
deployIdleTtlMs && deployIdleTtlMs > 0
? createSweeper(() => app.reapIdleDeployments(deployIdleTtlMs), Math.max(5_000, Math.floor(deployIdleTtlMs / 4)))
: null;
- const KEEP_WARM_INTERVAL_MS = 5 * 60_000;
- const keepWarmSweeper = createSweeper(() => app.keepAlwaysOnWarm(), KEEP_WARM_INTERVAL_MS);
+ const DEPLOY_MAINTENANCE_INTERVAL_MS = 5 * 60_000;
+ const deployMaintenanceSweeper = createSweeper(() => app.maintainDeployments(), DEPLOY_MAINTENANCE_INTERVAL_MS);
const deepIdleMachineMs = config.deepIdleMachineMs;
const devIdleMachineMs = config.devIdleMachineMs;
const SANDBOX_MAINTENANCE_INTERVAL_MS = 5 * 60_000;
@@ -2775,7 +2775,7 @@ export function buildApp(
composioReturnSweeper.start();
fileUploads?.start();
idleSweeper?.start();
- keepWarmSweeper.start();
+ deployMaintenanceSweeper.start();
deepIdleSweeper?.start();
wakeSweep.start();
swarms?.start();
@@ -2802,7 +2802,7 @@ export function buildApp(
spendSweeper?.stop(),
skillSyncEngine.stop(),
idleSweeper?.stop(),
- keepWarmSweeper.stop(),
+ deployMaintenanceSweeper.stop(),
deepIdleSweeper?.stop(),
blobSweeper.stop(),
composioReturnSweeper.stop(),
diff --git a/test/aws-deploy-provider.test.ts b/test/aws-deploy-provider.test.ts
index 211cad25e..dbf875870 100644
--- a/test/aws-deploy-provider.test.ts
+++ b/test/aws-deploy-provider.test.ts
@@ -25,6 +25,7 @@ interface Vm {
function fakeApi() {
const vms = new Map<string, Vm>();
const counts = { run: 0, resume: 0, terminate: 0, token: 0, describe: 0 };
+ const launches: Array<Parameters<AwsMicrovmApi["runMicrovm"]>[0]> = [];
let n = 0;
const api: AwsMicrovmApi = {
async listImages() {
@@ -39,8 +40,9 @@ function fakeApi() {
async updateImage() {
throw new Error("updateImage unused in deploy provider");
},
- async runMicrovm() {
+ async runMicrovm(input) {
counts.run++;
+ launches.push(input);
const id = `mvm-${++n}`;
const endpoint = `${id}.lambda-microvm.${REGION}.on.aws`;
vms.set(id, { state: "RUNNING", endpoint });
@@ -80,7 +82,7 @@ function fakeApi() {
return { microvmId: id, state: v?.state ?? target, ...(v?.endpoint ? { endpoint: v.endpoint } : {}) };
},
};
- return { api, vms, counts };
+ return { api, vms, counts, launches };
}
function fakeDaemon(
@@ -337,6 +339,79 @@ test("resolveEndpoint (suspended): resumes the body and serves it warm", async (
assert.equal(counts.run, 1, "no relaunch — same body");
});
+test("apply: an idle app stays suspended for its whole lifetime instead of being terminated after an hour", async () => {
+ const { api, launches } = fakeApi();
+ const { fetchImpl } = fakeDaemon();
+ const p = provider(api, fetchImpl);
+ await p.apply(deployment(ID), version(localSnapshot("a", "1")));
+ assert.equal(launches.length, 1);
+ assert.equal(launches[0]!.idlePolicy?.maxIdleDurationSeconds, 900, "still suspends after 15 idle minutes");
+ assert.equal(
+ launches[0]!.idlePolicy?.suspendedDurationSeconds,
+ launches[0]!.maximumDurationInSeconds,
+ "a suspended body survives until its maximum lifetime",
+ );
+});
+
+test("apply: an explicit suspended-duration setting still wins", async () => {
+ const { api, launches } = fakeApi();
+ const { fetchImpl } = fakeDaemon();
+ const p = provider(api, fetchImpl, createMemoryMap<StoredDeployBody>(), { suspendedDurationSeconds: 600 });
+ await p.apply(deployment(ID), version(localSnapshot("a", "1")));
+ assert.equal(launches[0]!.idlePolicy?.suspendedDurationSeconds, 600);
+});
+
+test("rotation: the replacement body is launched before the old one is terminated, and the old one stays resumable until then", async () => {
+ const { api, vms } = fakeApi();
+ const order: string[] = [];
+ const realRun = api.runMicrovm.bind(api);
+ const realTerminate = api.terminate.bind(api);
+ api.runMicrovm = async (input) => {
+ const r = await realRun(input);
+ order.push(`run ${r.microvmId}`);
+ return r;
+ };
+ api.terminate = async (id) => {
+ order.push(`terminate ${id}`);
+ return realTerminate(id);
+ };
+ const { fetchImpl } = fakeDaemon();
+ const store = createMemoryMap<StoredDeployBody>();
+ const p = provider(api, fetchImpl, store);
+ const d = deployment(ID);
+ await p.apply(d, version(localSnapshot("a", "1")));
+ const first = (await store.get(ID))!;
+ assert.equal(await p.rotationDue!(d), false);
+ await store.put(ID, { ...first, createdAtMs: Date.now() - 7 * 3600_000 - 60_000 });
+ vms.get(first.microvmId)!.state = "SUSPENDED";
+ assert.equal(await p.rotationDue!(d), true, "due 30 minutes before the visit-path rotation");
+ assert.ok(await p.resolveEndpoint!(d, version("/unused")), "visits still reach the old body meanwhile");
+
+ await p.apply(d, version(localSnapshot("a", "1")));
+ const second = (await store.get(ID))!;
+ assert.notEqual(second.microvmId, first.microvmId);
+ assert.deepEqual(order.slice(-2), [`run ${second.microvmId}`, `terminate ${first.microvmId}`]);
+ assert.equal(await p.rotationDue!(d), false);
+});
+
+test("rotation: a replacement left behind by a crash mid-rotation is terminated on the next attempt", async () => {
+ const { api, vms } = fakeApi();
+ const { fetchImpl } = fakeDaemon();
+ const store = createMemoryMap<StoredDeployBody>();
+ const p = provider(api, fetchImpl, store);
+ const d = deployment(ID);
+ await p.apply(d, version(localSnapshot("a", "1")));
+ const first = (await store.get(ID))!;
+ vms.set("mvm-orphan", { state: "RUNNING", endpoint: "mvm-orphan.lambda-microvm.us-west-2.on.aws" });
+ await store.put(ID, { ...first, replacementMicrovmId: "mvm-orphan" });
+
+ await p.apply(d, version(localSnapshot("a", "1")));
+ assert.equal(vms.get("mvm-orphan")!.state, "TERMINATED");
+ const after = (await store.get(ID))!;
+ assert.equal(after.microvmId, first.microvmId);
+ assert.equal(after.replacementMicrovmId, undefined);
+});
+
test("resolveEndpoint: reaches within the cache window skip the control plane", async () => {
const { api, counts } = fakeApi();
const { fetchImpl } = fakeDaemon();
@@ -354,6 +429,60 @@ test("resolveEndpoint: reaches within the cache window skip the control plane",
assert.equal(counts.describe, describesAfterFirst, "a fresh verdict is reused without re-asking the control plane");
});
+test("resolveEndpoint: an app in use is served from cache and re-checked in the background, not on the visit", async (t) => {
+ t.mock.timers.enable({ apis: ["Date"], now: 1_000_000 });
+ const { api, counts } = fakeApi();
+ const { fetchImpl } = fakeDaemon();
+ const store = createMemoryMap<StoredDeployBody>();
+ const p = provider(api, fetchImpl, store);
+ const d = deployment(ID);
+ await p.apply(d, version(localSnapshot("a", "1")));
+ const first = await p.resolveEndpoint!(d, version("/unused"));
+ const describes = counts.describe;
+
+ t.mock.timers.tick(60_000);
+ const hot = await p.resolveEndpoint!(d, version("/unused"));
+ assert.equal(hot!.host, first!.host, "the visit is answered from cache");
+ await new Promise((r) => setImmediate(r));
+ assert.equal(counts.describe, describes + 1, "a background re-check refreshed the verdict");
+
+ const moved = { ...d, endpoint: { host: "elsewhere.on.aws", port: 443 } };
+ await p.resolveEndpoint!(moved, version("/unused"));
+ assert.equal(counts.describe, describes + 2, "a placement changed by another core is re-checked before serving");
+});
+
+test("resolveEndpoint: after the idle window the visit waits for a real check, since the machine may be suspended", async (t) => {
+ t.mock.timers.enable({ apis: ["Date"], now: 1_000_000 });
+ const { api, counts, vms } = fakeApi();
+ const { fetchImpl } = fakeDaemon();
+ const store = createMemoryMap<StoredDeployBody>();
+ const p = provider(api, fetchImpl, store);
+ const d = deployment(ID);
+ await p.apply(d, version(localSnapshot("a", "1")));
+ await p.resolveEndpoint!(d, version("/unused"));
+ const body = (await store.get(ID))!;
+ vms.get(body.microvmId)!.state = "SUSPENDED";
+
+ t.mock.timers.tick(15 * 60_000);
+ const resumesBefore = counts.resume;
+ assert.ok(await p.resolveEndpoint!(d, version("/unused")));
+ assert.equal(counts.resume, resumesBefore + 1, "the suspended machine is resumed before the visit is served");
+});
+
+test("park: suspends the app's machine unless it is always-on", async () => {
+ const { api, vms } = fakeApi();
+ const { fetchImpl } = fakeDaemon();
+ const store = createMemoryMap<StoredDeployBody>();
+ const p = provider(api, fetchImpl, store);
+ const d = deployment(ID);
+ await p.apply(d, version(localSnapshot("a", "1")));
+ const body = (await store.get(ID))!;
+ await p.park!({ ...d, alwaysOn: true });
+ assert.equal(vms.get(body.microvmId)!.state, "RUNNING");
+ await p.park!(d);
+ assert.equal(vms.get(body.microvmId)!.state, "SUSPENDED");
+});
+
test("resolveEndpoint: concurrent misses coalesce into one control-plane pass", async () => {
const { api, counts } = fakeApi();
const { fetchImpl } = fakeDaemon();
@@ -371,8 +500,8 @@ test("resolveEndpoint: concurrent misses coalesce into one control-plane pass",
assert.ok(a && b && c);
assert.equal(
counts.describe,
- describesAfterApply + 2,
- "three simultaneous misses share one tryGet + one ensureRunning describe",
+ describesAfterApply + 1,
+ "three simultaneous misses share one describe, reused by ensureRunning",
);
});
@@ -719,8 +848,9 @@ test("app data: a failed rotate snapshot defers the rotation — the stale body
});
const e = await p.apply(deployment(ID), version(localSnapshot("a", "1")));
- assert.equal(counts.terminate, 0, "the stale body is not terminated when its data can't be saved");
- assert.equal(counts.run, 0, "no replacement launched");
+ assert.equal(vms.get("mvm-old")!.state, "RUNNING", "the stale body is not terminated when its data can't be saved");
+ assert.equal(counts.run, 1, "a replacement was launched ahead of the snapshot");
+ assert.equal(vms.get("mvm-1")!.state, "TERMINATED", "and discarded instead of leaked");
assert.equal(e.host, "mvm-old.lambda-microvm.us-west-2.on.aws", "the stale body keeps serving");
});
@@ -768,9 +898,10 @@ test("app data: a deferred rotation serves the stale body from resolveEndpoint i
await p.apply(d, version(localSnapshot("a", "1")));
assert.ok((await store.get(ID))!.rotateNotBeforeMs! > Date.now(), "deferral stamped");
+ const runsAfterDeferral = counts.run;
const resolved = await p.resolveEndpoint!(d, version("/unused"));
assert.ok(resolved, "stale-but-deferred body keeps serving");
- assert.equal(counts.run, 0, "no replacement launched during the deferral");
+ assert.equal(counts.run, runsAfterDeferral, "no replacement launched during the deferral");
});
test("litestream: shadow-WAL dir is excluded from both snapshot and hydrate", async () => {
diff --git a/test/deploy-always-on.test.ts b/test/deploy-always-on.test.ts
index 171e2bb61..93f150079 100644
--- a/test/deploy-always-on.test.ts
+++ b/test/deploy-always-on.test.ts
@@ -16,6 +16,8 @@ function svc(
resolve?: () => Promise<DeployEndpoint | null>;
setAlwaysOn?: DeployProvider["setAlwaysOn"];
apply?: DeployProvider["apply"];
+ rotationDue?: DeployProvider["rotationDue"];
+ park?: DeployProvider["park"];
} = {},
) {
const deployStore = createDeployStore();
@@ -30,6 +32,8 @@ function svc(
},
destroy: async () => {},
...(opts.setAlwaysOn ? { setAlwaysOn: opts.setAlwaysOn } : {}),
+ ...(opts.rotationDue ? { rotationDue: opts.rotationDue } : {}),
+ ...(opts.park ? { park: opts.park } : {}),
...(opts.resolve
? {
resolveEndpoint: async () => {
@@ -189,7 +193,7 @@ test("reaper re-checks alwaysOn under the deploy lock (no stale-snapshot destroy
assert.equal((await deployStore.get(d.id))!.status, "running");
});
-test("keepAlwaysOnWarm re-ensures a live endpoint only for running always-on deployments", async () => {
+test("maintainDeployments re-ensures a live endpoint only for running always-on deployments", async () => {
const ctx = svc({ resolve: async () => ({ host: "127.0.0.1", port: 5001 }) });
const { deploy } = ctx;
const cold = await deploy.deploy({ ...owner, entrypoint: "x", files: [] });
@@ -197,13 +201,13 @@ test("keepAlwaysOnWarm re-ensures a live endpoint only for running always-on dep
await deploy.setDeploymentAlwaysOn(warm.id, true);
const resolvesBefore = ctx.resolves;
- const warmed = await deploy.keepAlwaysOnWarm();
+ const warmed = await deploy.maintainDeployments();
assert.equal(warmed, 1);
assert.equal(ctx.resolves, resolvesBefore + 1);
assert.ok(cold.id !== warm.id);
});
-test("keepAlwaysOnWarm skips a deployment whose flag was turned off after the list snapshot", async () => {
+test("maintainDeployments skips a deployment whose flag was turned off after the list snapshot", async () => {
const ctx = svc({ resolve: async () => null });
const { deploy, deployStore } = ctx;
const d = await deploy.deploy({ ...owner, entrypoint: "x", files: [] });
@@ -215,24 +219,47 @@ test("keepAlwaysOnWarm skips a deployment whose flag was turned off after the li
return snapshot;
};
const appliesBefore = ctx.applies;
- const warmed = await deploy.keepAlwaysOnWarm();
+ const warmed = await deploy.maintainDeployments();
assert.equal(warmed, 0);
assert.equal(ctx.applies, appliesBefore);
});
-test("keepAlwaysOnWarm relaunches when the platform has torn the app down", async () => {
+test("maintainDeployments relaunches when the platform has torn the app down", async () => {
const ctx = svc({ resolve: async () => null });
const { deploy, deployStore } = ctx;
const d = await deploy.deploy({ ...owner, entrypoint: "x", files: [] });
await deploy.setDeploymentAlwaysOn(d.id, true);
const appliesBefore = ctx.applies;
- const warmed = await deploy.keepAlwaysOnWarm();
+ const warmed = await deploy.maintainDeployments();
assert.equal(warmed, 1);
assert.ok(ctx.applies > appliesBefore, "expected a re-apply when no live endpoint exists");
assert.equal((await deployStore.get(d.id))!.status, "running");
});
+test("maintainDeployments replaces a due app's machine in the background and parks the replacement", async () => {
+ const due = new Set<string>();
+ const parked: string[] = [];
+ const ctx = svc({
+ rotationDue: async (d) => due.has(d.id),
+ park: async (d) => {
+ parked.push(d.id);
+ due.delete(d.id);
+ },
+ });
+ const { deploy, deployStore } = ctx;
+ const stale = await deploy.deploy({ ...owner, entrypoint: "x", files: [] });
+ await deploy.deploy({ ...owner, entrypoint: "x", files: [] });
+ due.add(stale.id);
+ const appliesBefore = ctx.applies;
+
+ const maintained = await deploy.maintainDeployments();
+ assert.equal(maintained, 1, "only the due app is rotated");
+ assert.equal(ctx.applies, appliesBefore + 1);
+ assert.equal((await deployStore.get(stale.id))!.status, "running");
+ assert.deepEqual(parked, [stale.id], "the replacement is suspended so the next visit is a fast resume");
+});
+
test("always-on reaches the provider before persisting and retains the flag on failure", async () => {
const seen: boolean[] = [];
let reject = false;