Skip to content

The open-source app everyone uses to manage agents at work

Add database-free process liveness probes (#16101)

Devin Foleya0651c96f8b4
diff --git a/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md
index 8177461f43..0ae5e2a499 100644
--- a/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md
+++ b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md
@@ -399,6 +399,13 @@ flowchart LR
explicit deep readiness still tests DB health, and signed claim/bootstrap
mutations retain their existing validation and tracking.
+ Implementation progress: `/api/liveness` reports startup state and the public
+ commit without auth/session resolution, SQL, or provider I/O. It is mounted
+ after hostname and signed-bootstrap validation, before standby/auth, and its
+ exact GET/HEAD route stays reachable without adding a sleep hold. `/api/health`
+ retains database readiness and signed bootstrap. Callers must opt into the
+ liveness endpoint; this does not claim all external probes are migrated.
+
- [ ] **DB-05 — Audit external probes and configured endpoints.** Inspect the
fleet gateway, uptime monitors, deployment checks, DB pool settings, and Neon
suspend configuration. The [ECS example](../../docker/ecs-task-definition.json)
diff --git a/maps/agent-access-map/instance-operations.md b/maps/agent-access-map/instance-operations.md
index d51cccf323..9728d1085b 100644
--- a/maps/agent-access-map/instance-operations.md
+++ b/maps/agent-access-map/instance-operations.md
@@ -88,6 +88,14 @@ Review: `baseline`.
API: `present` — representative operation IDs:
- `GET /api/health`.
+- `GET /api/liveness` and `HEAD /api/liveness` — public process startup status
+ and build commit only; no database readiness or workspace identity.
+
+Liveness is anonymous and runs before actor/session resolution. Hostname and
+signed-bootstrap endpoint validation still apply. It grants no instance control
+authority. [Liveness tests](../../server/src/__tests__/liveness.test.ts) cover idle
+drain, invalid ambient credentials, unavailable SQL, and startup recovery with
+local Express fixtures; no live provider or agent access is claimed.
Agent access: `partial` — REST health plus local CLI doctor.
diff --git a/maps/feature-map/instance-operations.md b/maps/feature-map/instance-operations.md
index 0e02f78a36..e83ef87450 100644
--- a/maps/feature-map/instance-operations.md
+++ b/maps/feature-map/instance-operations.md
@@ -14,6 +14,12 @@ Agent access: [companion](../agent-access-map/instance-operations.md).
- `diagnostics`: use doctor/health output to distinguish readiness from a listening port.
- `backup`: create and inspect backups and verify restore only into a separate disposable target.
+Routine HTTP monitoring may use `GET /api/liveness` (or `HEAD`). It returns only
+process startup status and the public build commit without SQL or authentication.
+It remains reachable during an idle drain. Use `/api/health` for explicit database
+readiness, signed bootstrap, and authorized diagnostics; liveness does not
+establish database health or claim a workspace.
+
## How to get to it (user POV)
### `operator-cli`
diff --git a/server/src/__tests__/liveness.test.ts b/server/src/__tests__/liveness.test.ts
new file mode 100644
index 0000000000..d8ce096df5
--- /dev/null
+++ b/server/src/__tests__/liveness.test.ts
@@ -0,0 +1,109 @@
+import { readFileSync } from "node:fs";
+import express from "express";
+import request from "supertest";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import type { Db } from "@paperclipai/db";
+import { livenessRoutes } from "../routes/liveness.js";
+import { healthRoutes } from "../routes/health.js";
+import { privateHostnameGuard } from "../middleware/private-hostname-guard.js";
+import { cloudRuntimeIdentityMiddleware } from "../middleware/cloud-runtime-identity.js";
+import { cloudWarmStandbyMiddleware } from "../middleware/cloud-warm-standby.js";
+import { idleAdmissionMiddleware, trackIdleRequestHandlers } from "../middleware/idle-admission.js";
+import { idleWorkSnapshot, startTaskDrain, stopTaskDrain } from "../services/task-admission.js";
+import { resetStartupRecoveryStateForTests, setStartupRecoveryPhase } from "../startup-recovery-state.js";
+import { getServerInfoSnapshot } from "../server-info.js";
+
+afterEach(() => { stopTaskDrain(); resetStartupRecoveryStateForTests(); });
+
+function fixture(standby = false) {
+ const db = { execute: vi.fn().mockRejectedValue(new Error("database offline")) } as unknown as Db;
+ const auth = vi.fn((_req, _res, next) => next());
+ const health = healthRoutes(db, {
+ deploymentMode: "authenticated", deploymentExposure: "public", authReady: true,
+ companyDeletionEnabled: false, runtimeEnv: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fixture-token" },
+ isWarmStandby: () => standby,
+ });
+ const app = express();
+ app.use(idleAdmissionMiddleware);
+ app.use(privateHostnameGuard({ enabled: true, allowedHostnames: ["allowed.example"], bindHost: "0.0.0.0" }));
+ app.use(cloudRuntimeIdentityMiddleware(db));
+ app.use("/api/liveness", livenessRoutes());
+ app.use(cloudWarmStandbyMiddleware(() => standby, health));
+ app.use(auth);
+ app.use("/api/health", health);
+ app.get("/api/private", (_req, res) => res.sendStatus(401));
+ trackIdleRequestHandlers(app);
+ const http = request.agent(app).set("host", "allowed.example");
+ return { app, http, db, auth };
+}
+
+describe("database-free process liveness", () => {
+ it.each([false, true])("does not resolve auth or query SQL with an unavailable database (standby: %s)", async standby => {
+ const f = fixture(standby);
+ for (let i = 0; i < 3; i++) {
+ const response = await f.http.get("/api/liveness")
+ .set("authorization", "Bearer invalid-fixture")
+ .set("cookie", "session_token=invalid-fixture")
+ .set("x-paperclip-cloud-tenant-token", "invalid-fixture");
+ expect(response.status).toBe(200);
+ expect(response.headers["cache-control"]).toBe("no-store");
+ const serverInfo = getServerInfoSnapshot();
+ expect(response.body).toEqual({ status: "ok", commit: serverInfo.git.available ? serverInfo.git.fullSha : null });
+ }
+ const head = await f.http.head("/api/liveness").expect(200);
+ expect(head.text).toBeUndefined();
+ expect(f.auth).not.toHaveBeenCalled();
+ expect(f.db.execute).not.toHaveBeenCalled();
+ expect(idleWorkSnapshot().active).toBe(0);
+ if (!standby) {
+ await f.http.get("/api/health").expect(503);
+ expect(f.db.execute).toHaveBeenCalledOnce();
+ await f.http.get("/api/private").expect(401);
+ expect(f.auth).toHaveBeenCalledTimes(2);
+ }
+ });
+
+ it.each(["starting", "recovering"] as const)("reports %s until startup recovery is complete", async phase => {
+ const f = fixture();
+ setStartupRecoveryPhase(phase);
+ await f.http.get("/api/liveness").expect(503).expect(response => expect(response.body.status).toBe("starting"));
+ setStartupRecoveryPhase("ready");
+ await f.http.get("/api/liveness").expect(200);
+ expect(f.auth).not.toHaveBeenCalled();
+ expect(f.db.execute).not.toHaveBeenCalled();
+ });
+
+ it("retains hostname and signed bootstrap endpoint validation", async () => {
+ const f = fixture();
+ await f.http.get("/api/liveness").set("host", "denied.example").expect(403);
+ await f.http.get("/api/liveness").set("x-paperclip-cloud-runtime-identity", "invalid-fixture")
+ .expect(400).expect({ error: "cloud_runtime_identity_wrong_endpoint" });
+ expect(f.auth).not.toHaveBeenCalled();
+ expect(f.db.execute).not.toHaveBeenCalled();
+ });
+
+ it("stays reachable during drain without exempting writes or other paths", async () => {
+ const f = fixture();
+ const before = idleWorkSnapshot();
+ startTaskDrain({ purpose: "idle", ttlMs: 60_000 });
+ for (const path of ["/api/liveness", "/api/liveness/"]) {
+ await f.http.get(path).expect(200);
+ await f.http.head(path).expect(200);
+ await f.http.post(path).expect(503);
+ }
+ await f.http.get("/api/liveness/details").expect(503);
+ await f.http.get("/api/private").expect(503);
+ expect(idleWorkSnapshot()).toEqual(before);
+ expect(f.auth).not.toHaveBeenCalled();
+ expect(f.db.execute).not.toHaveBeenCalled();
+ });
+
+ it("mounts the production route after host and bootstrap validation and before standby/auth", () => {
+ const source = readFileSync(new URL("../app.ts", import.meta.url), "utf8");
+ const route = source.indexOf('app.use("/api/liveness", livenessRoutes())');
+ expect(route).toBeGreaterThan(source.indexOf("privateHostnameGuard({"));
+ expect(route).toBeGreaterThan(source.indexOf("app.use(cloudRuntimeIdentityMiddleware(db))"));
+ expect(route).toBeLessThan(source.indexOf("app.use(cloudWarmStandbyMiddleware("));
+ expect(route).toBeLessThan(source.indexOf("actorMiddleware(db,"));
+ });
+});
diff --git a/server/src/__tests__/openapi-routes.test.ts b/server/src/__tests__/openapi-routes.test.ts
index a4093e52e2..3ab8e3d3de 100644
--- a/server/src/__tests__/openapi-routes.test.ts
+++ b/server/src/__tests__/openapi-routes.test.ts
@@ -55,6 +55,7 @@ const apiPrefixes: Record<string, string> = {
"folders.ts": "/api",
"goals.ts": "/api",
"health.ts": "/api/health",
+ "liveness.ts": "/api/liveness",
"inbox-agent-policy.ts": "/api",
"inbox-dismissals.ts": "/api",
"instance-database-backups.ts": "/api",
@@ -87,9 +88,9 @@ const apiPrefixes: Record<string, string> = {
};
const ROUTE_LITERAL_PATTERN =
- /router\.(get|post|put|patch|delete)\(\s*["'`]([^"'`]+)["'`]/g;
-const ROUTE_ARRAY_PATTERN = /router\.(get|post|put|patch|delete)\(\s*\[([^\]]+)\]/g;
-const ROUTER_METHOD_PATTERN = /router\.(get|post|put|patch|delete)\(/;
+ /router\.(get|head|post|put|patch|delete)\(\s*["'`]([^"'`]+)["'`]/g;
+const ROUTE_ARRAY_PATTERN = /router\.(get|head|post|put|patch|delete)\(\s*\[([^\]]+)\]/g;
+const ROUTER_METHOD_PATTERN = /router\.(get|head|post|put|patch|delete)\(/;
const HTTP_METHODS = new Set([
"get",
"put",
@@ -183,7 +184,7 @@ function resolveMountedPath(file: string, prefix: string, routePath: string) {
) {
return routePath;
}
- if ((file === "companies.ts" || file === "health.ts") && routePath === "/") {
+ if ((file === "companies.ts" || file === "health.ts" || file === "liveness.ts") && routePath === "/") {
return prefix;
}
if (file === "companies.ts" || file === "health.ts") {
@@ -296,6 +297,17 @@ function loadSpecRoutes() {
}
describe("openapi routes", () => {
+ it("documents anonymous liveness separately from database readiness", () => {
+ const spec = buildOpenApiSpec() as any;
+ for (const method of ["get", "head"]) {
+ const operation = spec.paths["/api/liveness"][method];
+ expect(operation.security).toEqual([]);
+ expect(operation.responses[503]).toBeDefined();
+ }
+ expect(Object.keys(spec.paths["/api/liveness"].get.responses[200].content["application/json"].schema.properties))
+ .toEqual(["status", "commit"]);
+ expect(spec.paths["/api/health"].get.responses[503]).toBeDefined();
+ });
it("documents only writable agent update fields and lifecycle status requests", () => {
const document = buildOpenApiSpec() as any;
const properties = document.paths["/api/agents/{id}"].patch.requestBody.content["application/json"].schema.properties;
diff --git a/server/src/app.ts b/server/src/app.ts
index 5aa7c79b1d..b81bb16b9d 100644
--- a/server/src/app.ts
+++ b/server/src/app.ts
@@ -69,6 +69,7 @@ import {
} from "./services/company-import-transfers.js";
import { companyTransferRunService } from "./services/company-transfer-runs.js";
import { healthRoutes } from "./routes/health.js";
+import { livenessRoutes } from "./routes/liveness.js";
import { cloudRuntimeIdentityMiddleware } from "./middleware/cloud-runtime-identity.js";
import { cloudControlMiddleware } from "./middleware/cloud-control.js";
import { cloudRoutes } from "./routes/cloud.js";
@@ -606,6 +607,9 @@ export async function createApp(
const publicMcpIngress = Router();
app.use(cloudRuntimeIdentityMiddleware(db));
+ // Routine probes stop before session/actor resolution, even after claim.
+ // Keep hostname validation and the signed-bootstrap endpoint guard above.
+ app.use("/api/liveness", livenessRoutes());
// A signed claim above commits identity before any normal request can seed
// company data. Unclaimed probes bypass session resolution as well as SQL.
app.use(cloudWarmStandbyMiddleware(isWarmStandby, health, staticUi));
diff --git a/server/src/middleware/idle-admission.ts b/server/src/middleware/idle-admission.ts
index 9540d704dd..9695883733 100644
--- a/server/src/middleware/idle-admission.ts
+++ b/server/src/middleware/idle-admission.ts
@@ -25,6 +25,12 @@ function isControlRequest(req: Request) {
/** Install before body parsers, auth, webhooks and tool ingress. */
export const idleAdmissionMiddleware: RequestHandler = (req, res, next) => {
+ // This exact route is synchronous and has no auth, SQL, or accepted work.
+ // Probing it must not prevent an otherwise quiescent instance from draining.
+ if (req.path.replace(/\/$/, "") === "/api/liveness" && ["GET", "HEAD"].includes(req.method)) {
+ next();
+ return;
+ }
const controlRequest = isControlRequest(req);
if (!controlRequest && isIdleTaskDrainActive()) {
res.set("Retry-After", "1").status(503).json({ error: "instance_preparing_to_sleep" });
diff --git a/server/src/routes/liveness.ts b/server/src/routes/liveness.ts
new file mode 100644
index 0000000000..d3b638ddca
--- /dev/null
+++ b/server/src/routes/liveness.ts
@@ -0,0 +1,20 @@
+import { Router, type RequestHandler } from "express";
+import { getServerInfoSnapshot, type ServerInfoSnapshot } from "../server-info.js";
+import { getStartupRecoveryState } from "../startup-recovery-state.js";
+
+/** Routine probes report process readiness without auth, SQL, or provider I/O.
+ * Database readiness and signed bootstrap remain on /api/health. */
+export function livenessRoutes(serverInfo: ServerInfoSnapshot = getServerInfoSnapshot()) {
+ const router = Router();
+ const commit = serverInfo.git.available ? serverInfo.git.fullSha : null;
+ const respond: RequestHandler = (_req, res) => {
+ const ready = getStartupRecoveryState().phase === "ready";
+ res.set("Cache-Control", "no-store").status(ready ? 200 : 503).json({
+ status: ready ? "ok" : "starting",
+ commit,
+ });
+ };
+ router.head("/", respond);
+ router.get("/", respond);
+ return router;
+}
diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts
index 25b6c7278f..532e3a66a2 100644
--- a/server/src/routes/openapi.ts
+++ b/server/src/routes/openapi.ts
@@ -1331,6 +1331,8 @@ const RUNTIME_TOOLS_OPERATIONS = new Set([
const PUBLIC_OPERATIONS = new Set([
"GET /api/agent-avatars/{version}/{palette}/{file}",
"GET /api/health",
+ "GET /api/liveness",
+ "HEAD /api/liveness",
"GET /api/openapi.json",
"GET /api/board-claim/{token}",
"POST /api/cli-auth/challenges",
@@ -1871,6 +1873,28 @@ function applyDocumentFixups(document: any): any {
// ─── Health ──────────────────────────────────────────────────────────────────
+registry.registerPath({
+ method: "get",
+ path: "/api/liveness",
+ tags: ["health"],
+ summary: "Process startup status without database access",
+ description: "Runs before actor/session resolution. Reports only in-memory startup recovery and the public build commit. Use /api/health for explicit database readiness or signed bootstrap. No workspace identity or control authority is granted.",
+ responses: {
+ 200: r.ok(z.object({ status: z.literal("ok"), commit: z.string().nullable() }).strict()),
+ 503: { description: "Startup recovery is not complete", content: {
+ "application/json": { schema: z.object({ status: z.literal("starting"), commit: z.string().nullable() }).strict() },
+ } },
+ },
+});
+
+registry.registerPath({
+ method: "head",
+ path: "/api/liveness",
+ tags: ["health"],
+ summary: "Process startup status without a response body",
+ responses: { 200: { description: "Process startup recovery is complete" }, 503: { description: "Startup recovery is not complete" } },
+});
+
// Shared by the healthy and database-unreachable responses: full details
// (including serverInfo) ride only on board/agent-actor responses.
const healthServerInfoSchema = z