The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
Add database-free process liveness probes (#16101)
Devin Foleya0651c96f8b4
diff --git a/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.mdindex 8177461f43..0ae5e2a499 100644--- a/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md+++ b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md@@ -399,6 +399,13 @@ flowchart LRexplicit deep readiness still tests DB health, and signed claim/bootstrapmutations retain their existing validation and tracking.+ Implementation progress: `/api/liveness` reports startup state and the public+ commit without auth/session resolution, SQL, or provider I/O. It is mounted+ after hostname and signed-bootstrap validation, before standby/auth, and its+ exact GET/HEAD route stays reachable without adding a sleep hold. `/api/health`+ retains database readiness and signed bootstrap. Callers must opt into the+ liveness endpoint; this does not claim all external probes are migrated.+- [ ] **DB-05 — Audit external probes and configured endpoints.** Inspect thefleet gateway, uptime monitors, deployment checks, DB pool settings, and Neonsuspend configuration. The [ECS example](../../docker/ecs-task-definition.json)diff --git a/maps/agent-access-map/instance-operations.md b/maps/agent-access-map/instance-operations.mdindex d51cccf323..9728d1085b 100644--- a/maps/agent-access-map/instance-operations.md+++ b/maps/agent-access-map/instance-operations.md@@ -88,6 +88,14 @@ Review: `baseline`.API: `present` — representative operation IDs:- `GET /api/health`.+- `GET /api/liveness` and `HEAD /api/liveness` — public process startup status+ and build commit only; no database readiness or workspace identity.++Liveness is anonymous and runs before actor/session resolution. Hostname and+signed-bootstrap endpoint validation still apply. It grants no instance control+authority. [Liveness tests](../../server/src/__tests__/liveness.test.ts) cover idle+drain, invalid ambient credentials, unavailable SQL, and startup recovery with+local Express fixtures; no live provider or agent access is claimed.Agent access: `partial` — REST health plus local CLI doctor.diff --git a/maps/feature-map/instance-operations.md b/maps/feature-map/instance-operations.mdindex 0e02f78a36..e83ef87450 100644--- a/maps/feature-map/instance-operations.md+++ b/maps/feature-map/instance-operations.md@@ -14,6 +14,12 @@ Agent access: [companion](../agent-access-map/instance-operations.md).- `diagnostics`: use doctor/health output to distinguish readiness from a listening port.- `backup`: create and inspect backups and verify restore only into a separate disposable target.+Routine HTTP monitoring may use `GET /api/liveness` (or `HEAD`). It returns only+process startup status and the public build commit without SQL or authentication.+It remains reachable during an idle drain. Use `/api/health` for explicit database+readiness, signed bootstrap, and authorized diagnostics; liveness does not+establish database health or claim a workspace.+## How to get to it (user POV)### `operator-cli`diff --git a/server/src/__tests__/liveness.test.ts b/server/src/__tests__/liveness.test.tsnew file mode 100644index 0000000000..d8ce096df5--- /dev/null+++ b/server/src/__tests__/liveness.test.ts@@ -0,0 +1,109 @@+import { readFileSync } from "node:fs";+import express from "express";+import request from "supertest";+import { afterEach, describe, expect, it, vi } from "vitest";+import type { Db } from "@paperclipai/db";+import { livenessRoutes } from "../routes/liveness.js";+import { healthRoutes } from "../routes/health.js";+import { privateHostnameGuard } from "../middleware/private-hostname-guard.js";+import { cloudRuntimeIdentityMiddleware } from "../middleware/cloud-runtime-identity.js";+import { cloudWarmStandbyMiddleware } from "../middleware/cloud-warm-standby.js";+import { idleAdmissionMiddleware, trackIdleRequestHandlers } from "../middleware/idle-admission.js";+import { idleWorkSnapshot, startTaskDrain, stopTaskDrain } from "../services/task-admission.js";+import { resetStartupRecoveryStateForTests, setStartupRecoveryPhase } from "../startup-recovery-state.js";+import { getServerInfoSnapshot } from "../server-info.js";++afterEach(() => { stopTaskDrain(); resetStartupRecoveryStateForTests(); });++function fixture(standby = false) {+ const db = { execute: vi.fn().mockRejectedValue(new Error("database offline")) } as unknown as Db;+ const auth = vi.fn((_req, _res, next) => next());+ const health = healthRoutes(db, {+ deploymentMode: "authenticated", deploymentExposure: "public", authReady: true,+ companyDeletionEnabled: false, runtimeEnv: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fixture-token" },+ isWarmStandby: () => standby,+ });+ const app = express();+ app.use(idleAdmissionMiddleware);+ app.use(privateHostnameGuard({ enabled: true, allowedHostnames: ["allowed.example"], bindHost: "0.0.0.0" }));+ app.use(cloudRuntimeIdentityMiddleware(db));+ app.use("/api/liveness", livenessRoutes());+ app.use(cloudWarmStandbyMiddleware(() => standby, health));+ app.use(auth);+ app.use("/api/health", health);+ app.get("/api/private", (_req, res) => res.sendStatus(401));+ trackIdleRequestHandlers(app);+ const http = request.agent(app).set("host", "allowed.example");+ return { app, http, db, auth };+}++describe("database-free process liveness", () => {+ it.each([false, true])("does not resolve auth or query SQL with an unavailable database (standby: %s)", async standby => {+ const f = fixture(standby);+ for (let i = 0; i < 3; i++) {+ const response = await f.http.get("/api/liveness")+ .set("authorization", "Bearer invalid-fixture")+ .set("cookie", "session_token=invalid-fixture")+ .set("x-paperclip-cloud-tenant-token", "invalid-fixture");+ expect(response.status).toBe(200);+ expect(response.headers["cache-control"]).toBe("no-store");+ const serverInfo = getServerInfoSnapshot();+ expect(response.body).toEqual({ status: "ok", commit: serverInfo.git.available ? serverInfo.git.fullSha : null });+ }+ const head = await f.http.head("/api/liveness").expect(200);+ expect(head.text).toBeUndefined();+ expect(f.auth).not.toHaveBeenCalled();+ expect(f.db.execute).not.toHaveBeenCalled();+ expect(idleWorkSnapshot().active).toBe(0);+ if (!standby) {+ await f.http.get("/api/health").expect(503);+ expect(f.db.execute).toHaveBeenCalledOnce();+ await f.http.get("/api/private").expect(401);+ expect(f.auth).toHaveBeenCalledTimes(2);+ }+ });++ it.each(["starting", "recovering"] as const)("reports %s until startup recovery is complete", async phase => {+ const f = fixture();+ setStartupRecoveryPhase(phase);+ await f.http.get("/api/liveness").expect(503).expect(response => expect(response.body.status).toBe("starting"));+ setStartupRecoveryPhase("ready");+ await f.http.get("/api/liveness").expect(200);+ expect(f.auth).not.toHaveBeenCalled();+ expect(f.db.execute).not.toHaveBeenCalled();+ });++ it("retains hostname and signed bootstrap endpoint validation", async () => {+ const f = fixture();+ await f.http.get("/api/liveness").set("host", "denied.example").expect(403);+ await f.http.get("/api/liveness").set("x-paperclip-cloud-runtime-identity", "invalid-fixture")+ .expect(400).expect({ error: "cloud_runtime_identity_wrong_endpoint" });+ expect(f.auth).not.toHaveBeenCalled();+ expect(f.db.execute).not.toHaveBeenCalled();+ });++ it("stays reachable during drain without exempting writes or other paths", async () => {+ const f = fixture();+ const before = idleWorkSnapshot();+ startTaskDrain({ purpose: "idle", ttlMs: 60_000 });+ for (const path of ["/api/liveness", "/api/liveness/"]) {+ await f.http.get(path).expect(200);+ await f.http.head(path).expect(200);+ await f.http.post(path).expect(503);+ }+ await f.http.get("/api/liveness/details").expect(503);+ await f.http.get("/api/private").expect(503);+ expect(idleWorkSnapshot()).toEqual(before);+ expect(f.auth).not.toHaveBeenCalled();+ expect(f.db.execute).not.toHaveBeenCalled();+ });++ it("mounts the production route after host and bootstrap validation and before standby/auth", () => {+ const source = readFileSync(new URL("../app.ts", import.meta.url), "utf8");+ const route = source.indexOf('app.use("/api/liveness", livenessRoutes())');+ expect(route).toBeGreaterThan(source.indexOf("privateHostnameGuard({"));+ expect(route).toBeGreaterThan(source.indexOf("app.use(cloudRuntimeIdentityMiddleware(db))"));+ expect(route).toBeLessThan(source.indexOf("app.use(cloudWarmStandbyMiddleware("));+ expect(route).toBeLessThan(source.indexOf("actorMiddleware(db,"));+ });+});diff --git a/server/src/__tests__/openapi-routes.test.ts b/server/src/__tests__/openapi-routes.test.tsindex a4093e52e2..3ab8e3d3de 100644--- a/server/src/__tests__/openapi-routes.test.ts+++ b/server/src/__tests__/openapi-routes.test.ts@@ -55,6 +55,7 @@ const apiPrefixes: Record<string, string> = {"folders.ts": "/api","goals.ts": "/api","health.ts": "/api/health",+ "liveness.ts": "/api/liveness","inbox-agent-policy.ts": "/api","inbox-dismissals.ts": "/api","instance-database-backups.ts": "/api",@@ -87,9 +88,9 @@ const apiPrefixes: Record<string, string> = {};const ROUTE_LITERAL_PATTERN =- /router\.(get|post|put|patch|delete)\(\s*["'`]([^"'`]+)["'`]/g;-const ROUTE_ARRAY_PATTERN = /router\.(get|post|put|patch|delete)\(\s*\[([^\]]+)\]/g;-const ROUTER_METHOD_PATTERN = /router\.(get|post|put|patch|delete)\(/;+ /router\.(get|head|post|put|patch|delete)\(\s*["'`]([^"'`]+)["'`]/g;+const ROUTE_ARRAY_PATTERN = /router\.(get|head|post|put|patch|delete)\(\s*\[([^\]]+)\]/g;+const ROUTER_METHOD_PATTERN = /router\.(get|head|post|put|patch|delete)\(/;const HTTP_METHODS = new Set(["get","put",@@ -183,7 +184,7 @@ function resolveMountedPath(file: string, prefix: string, routePath: string) {) {return routePath;}- if ((file === "companies.ts" || file === "health.ts") && routePath === "/") {+ if ((file === "companies.ts" || file === "health.ts" || file === "liveness.ts") && routePath === "/") {return prefix;}if (file === "companies.ts" || file === "health.ts") {@@ -296,6 +297,17 @@ function loadSpecRoutes() {}describe("openapi routes", () => {+ it("documents anonymous liveness separately from database readiness", () => {+ const spec = buildOpenApiSpec() as any;+ for (const method of ["get", "head"]) {+ const operation = spec.paths["/api/liveness"][method];+ expect(operation.security).toEqual([]);+ expect(operation.responses[503]).toBeDefined();+ }+ expect(Object.keys(spec.paths["/api/liveness"].get.responses[200].content["application/json"].schema.properties))+ .toEqual(["status", "commit"]);+ expect(spec.paths["/api/health"].get.responses[503]).toBeDefined();+ });it("documents only writable agent update fields and lifecycle status requests", () => {const document = buildOpenApiSpec() as any;const properties = document.paths["/api/agents/{id}"].patch.requestBody.content["application/json"].schema.properties;diff --git a/server/src/app.ts b/server/src/app.tsindex 5aa7c79b1d..b81bb16b9d 100644--- a/server/src/app.ts+++ b/server/src/app.ts@@ -69,6 +69,7 @@ import {} from "./services/company-import-transfers.js";import { companyTransferRunService } from "./services/company-transfer-runs.js";import { healthRoutes } from "./routes/health.js";+import { livenessRoutes } from "./routes/liveness.js";import { cloudRuntimeIdentityMiddleware } from "./middleware/cloud-runtime-identity.js";import { cloudControlMiddleware } from "./middleware/cloud-control.js";import { cloudRoutes } from "./routes/cloud.js";@@ -606,6 +607,9 @@ export async function createApp(const publicMcpIngress = Router();app.use(cloudRuntimeIdentityMiddleware(db));+ // Routine probes stop before session/actor resolution, even after claim.+ // Keep hostname validation and the signed-bootstrap endpoint guard above.+ app.use("/api/liveness", livenessRoutes());// A signed claim above commits identity before any normal request can seed// company data. Unclaimed probes bypass session resolution as well as SQL.app.use(cloudWarmStandbyMiddleware(isWarmStandby, health, staticUi));diff --git a/server/src/middleware/idle-admission.ts b/server/src/middleware/idle-admission.tsindex 9540d704dd..9695883733 100644--- a/server/src/middleware/idle-admission.ts+++ b/server/src/middleware/idle-admission.ts@@ -25,6 +25,12 @@ function isControlRequest(req: Request) {/** Install before body parsers, auth, webhooks and tool ingress. */export const idleAdmissionMiddleware: RequestHandler = (req, res, next) => {+ // This exact route is synchronous and has no auth, SQL, or accepted work.+ // Probing it must not prevent an otherwise quiescent instance from draining.+ if (req.path.replace(/\/$/, "") === "/api/liveness" && ["GET", "HEAD"].includes(req.method)) {+ next();+ return;+ }const controlRequest = isControlRequest(req);if (!controlRequest && isIdleTaskDrainActive()) {res.set("Retry-After", "1").status(503).json({ error: "instance_preparing_to_sleep" });diff --git a/server/src/routes/liveness.ts b/server/src/routes/liveness.tsnew file mode 100644index 0000000000..d3b638ddca--- /dev/null+++ b/server/src/routes/liveness.ts@@ -0,0 +1,20 @@+import { Router, type RequestHandler } from "express";+import { getServerInfoSnapshot, type ServerInfoSnapshot } from "../server-info.js";+import { getStartupRecoveryState } from "../startup-recovery-state.js";++/** Routine probes report process readiness without auth, SQL, or provider I/O.+ * Database readiness and signed bootstrap remain on /api/health. */+export function livenessRoutes(serverInfo: ServerInfoSnapshot = getServerInfoSnapshot()) {+ const router = Router();+ const commit = serverInfo.git.available ? serverInfo.git.fullSha : null;+ const respond: RequestHandler = (_req, res) => {+ const ready = getStartupRecoveryState().phase === "ready";+ res.set("Cache-Control", "no-store").status(ready ? 200 : 503).json({+ status: ready ? "ok" : "starting",+ commit,+ });+ };+ router.head("/", respond);+ router.get("/", respond);+ return router;+}diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.tsindex 25b6c7278f..532e3a66a2 100644--- a/server/src/routes/openapi.ts+++ b/server/src/routes/openapi.ts@@ -1331,6 +1331,8 @@ const RUNTIME_TOOLS_OPERATIONS = new Set([const PUBLIC_OPERATIONS = new Set(["GET /api/agent-avatars/{version}/{palette}/{file}","GET /api/health",+ "GET /api/liveness",+ "HEAD /api/liveness","GET /api/openapi.json","GET /api/board-claim/{token}","POST /api/cli-auth/challenges",@@ -1871,6 +1873,28 @@ function applyDocumentFixups(document: any): any {// ─── Health ──────────────────────────────────────────────────────────────────+registry.registerPath({+ method: "get",+ path: "/api/liveness",+ tags: ["health"],+ summary: "Process startup status without database access",+ description: "Runs before actor/session resolution. Reports only in-memory startup recovery and the public build commit. Use /api/health for explicit database readiness or signed bootstrap. No workspace identity or control authority is granted.",+ responses: {+ 200: r.ok(z.object({ status: z.literal("ok"), commit: z.string().nullable() }).strict()),+ 503: { description: "Startup recovery is not complete", content: {+ "application/json": { schema: z.object({ status: z.literal("starting"), commit: z.string().nullable() }).strict() },+ } },+ },+});++registry.registerPath({+ method: "head",+ path: "/api/liveness",+ tags: ["health"],+ summary: "Process startup status without a response body",+ responses: { 200: { description: "Process startup recovery is complete" }, 503: { description: "Startup recovery is not complete" } },+});+// Shared by the healthy and database-unreachable responses: full details// (including serverInfo) ride only on board/agent-actor responses.const healthServerInfoSchema = z