Skip to content

Open-source manufacturing ERP, MES and QMS. Quoting, MRP, inventory, shop floor, quality and lot/serial traceability on one Postgres schema, with a REST API and MCP server. Self-host or use Carbon Cloud.

fix(health): the status code answers whether the pod can serve

Sidwebworks93692c2b3fae
diff --git a/apps/erp/app/routes/health.test.ts b/apps/erp/app/routes/health.test.ts
index 6d10785400..8de402c78d 100644
--- a/apps/erp/app/routes/health.test.ts
+++ b/apps/erp/app/routes/health.test.ts
@@ -46,13 +46,13 @@ describe("health loader", () => {
expect(body.checks.redis).toBe("down");
});
- it("reports the database down (never throws) on a query error", async () => {
+ it("returns 503 when the database is down — the code is the readiness gate", async () => {
select.mockResolvedValue({ error: { message: "boom" } });
const response = await loader();
const body = await response.json();
- expect(response.status).toBe(200);
+ expect(response.status).toBe(503);
expect(body.status).toBe("degraded");
expect(body.checks.database).toBe("down");
});
diff --git a/apps/erp/app/routes/health.ts b/apps/erp/app/routes/health.ts
index ff409d74b0..0a820f5fa0 100644
--- a/apps/erp/app/routes/health.ts
+++ b/apps/erp/app/routes/health.ts
@@ -1,10 +1,20 @@
import { getCarbonServiceRole } from "@carbon/auth/client.server";
import { redis } from "@carbon/kv";
-// Public health check: no auth, and it ALWAYS returns HTTP 200. This endpoint
-// backs the load balancer's target-group health check, so a transient dependency
-// blip must not mark every task unhealthy and cycle the whole service. The body
-// reports per-dependency status for monitoring (NIST 800-171 3.14.6) instead.
+// Public health check: no auth. The status CODE answers the one question a
+// probe or load balancer can act on — can this pod serve — so it turns on
+// the database alone: 503 without it, because nothing works without it.
+// Redis is reported in the body but does not gate the code: a cache blip
+// degrades the app, and pulling every pod out of the Service over it would
+// turn a degradation into the very outage it reported. The body carries the
+// full per-dependency truth for monitoring (NIST 800-171 3.14.6).
+//
+// Kubernetes reads only the code (any 2xx passes), which is why the old
+// always-200 contract made every probe a hollow gate: a rollout would cut
+// over to a pod the moment its HTTP server bound, whether or not it could
+// serve a single request. The chart's liveness probe does NOT hit this
+// endpoint (it is a TCP check), so a dead database makes pods unready —
+// never killed.
const CHECK_TIMEOUT_MS = 2_000;
// Resolve a dependency probe to a boolean, treating a throw or a slow response as
@@ -40,10 +50,8 @@ export async function loader() {
database: databaseUp ? "up" : "down"
};
- const healthy = redisUp && databaseUp;
-
return Response.json(
- { status: healthy ? "healthy" : "degraded", checks },
- { status: 200 }
+ { status: redisUp && databaseUp ? "healthy" : "degraded", checks },
+ { status: databaseUp ? 200 : 503 }
);
}
diff --git a/apps/mes/app/routes/_public+/health.tsx b/apps/mes/app/routes/_public+/health.tsx
index b3d4d2aaeb..0a820f5fa0 100644
--- a/apps/mes/app/routes/_public+/health.tsx
+++ b/apps/mes/app/routes/_public+/health.tsx
@@ -1,15 +1,57 @@
+import { getCarbonServiceRole } from "@carbon/auth/client.server";
+import { redis } from "@carbon/kv";
+
+// Public health check: no auth. The status CODE answers the one question a
+// probe or load balancer can act on — can this pod serve — so it turns on
+// the database alone: 503 without it, because nothing works without it.
+// Redis is reported in the body but does not gate the code: a cache blip
+// degrades the app, and pulling every pod out of the Service over it would
+// turn a degradation into the very outage it reported. The body carries the
+// full per-dependency truth for monitoring (NIST 800-171 3.14.6).
+//
+// Kubernetes reads only the code (any 2xx passes), which is why the old
+// always-200 contract made every probe a hollow gate: a rollout would cut
+// over to a pod the moment its HTTP server bound, whether or not it could
+// serve a single request. The chart's liveness probe does NOT hit this
+// endpoint (it is a TCP check), so a dead database makes pods unready —
+// never killed.
+const CHECK_TIMEOUT_MS = 2_000;
+
+// Resolve a dependency probe to a boolean, treating a throw or a slow response as
+// "down" so one hung dependency can't hang the endpoint.
+function probe(check: () => Promise<boolean>): Promise<boolean> {
+ return Promise.race([
+ check().catch(() => false),
+ new Promise<boolean>((resolve) =>
+ setTimeout(() => resolve(false), CHECK_TIMEOUT_MS)
+ )
+ ]);
+}
+
export async function loader() {
- return new Response(
- JSON.stringify({
- status: "ok",
- timestamp: new Date().toISOString()
- }),
- {
- status: 200,
- headers: {
- "Content-Type": "application/json",
- "Cache-Control": "no-cache, no-store, must-revalidate"
- }
- }
+ const client = getCarbonServiceRole();
+
+ const [redisUp, databaseUp] = await Promise.all([
+ // redis is resilience-wrapped: ping() resolves null instead of throwing when
+ // Redis is unreachable.
+ probe(async () => !!(await redis.ping())),
+ // Cheapest possible round-trip to Postgres via PostgREST: HEAD with an
+ // estimated count, no rows returned.
+ probe(async () => {
+ const { error } = await client
+ .from("company")
+ .select("id", { head: true, count: "estimated" });
+ return !error;
+ })
+ ]);
+
+ const checks = {
+ redis: redisUp ? "up" : "down",
+ database: databaseUp ? "up" : "down"
+ };
+
+ return Response.json(
+ { status: redisUp && databaseUp ? "healthy" : "degraded", checks },
+ { status: databaseUp ? 200 : 503 }
);
}