Skip to content

Build your personal knowledge base with Trilium Notes

fix(healthcheck): probe over TLS instead of assuming the server is up

Elian Doran3f698fcca188
diff --git a/apps/server/spec/support/self_signed_tls.ts b/apps/server/spec/support/self_signed_tls.ts
new file mode 100644
index 0000000000..af2cfa19c2
--- /dev/null
+++ b/apps/server/spec/support/self_signed_tls.ts
@@ -0,0 +1,30 @@
+/**
+ * A throwaway self-signed certificate for tests that need a real TLS listener, generated for
+ * CN=localhost and valid until 2126. It secures nothing: the key is public in this repository,
+ * and it exists only so a test can start an https server without shelling out to openssl.
+ */
+export const SELF_SIGNED_KEY = `-----BEGIN PRIVATE KEY-----
+MIIBeQIBADCCAQMGByqGSM49AgEwgfcCAQEwLAYHKoZIzj0BAQIhAP////8AAAAB
+AAAAAAAAAAAAAAAA////////////////MFsEIP////8AAAABAAAAAAAAAAAAAAAA
+///////////////8BCBaxjXYqjqT57PrvVV2mIa8ZR0GsMxTsPY7zjw+J9JgSwMV
+AMSdNgiG5wSTamZ44ROdJreBn36QBEEEaxfR8uEsQkf4vOblY6RA8ncDfYEt6zOg
+9KE5RdiYwpZP40Li/hp/m47n60p8D54WK84zV2sxXs7LtkBoN79R9QIhAP////8A
+AAAA//////////+85vqtpxeehPO5ysL8YyVRAgEBBG0wawIBAQQg8eQb0bVq1ju5
+wxOdYjheoL+FBX3aicSmNQX04su4MbahRANCAARCiHeo9UeRkTq3QwhTyKXZm8NG
+7k4pIcmyuJXLQTzpYysWHpRR09P9JcnyWmaU5S64k5aE7FBjQX242H3ZkAyB
+-----END PRIVATE KEY-----`;
+
+export const SELF_SIGNED_CERT = `-----BEGIN CERTIFICATE-----
+MIICMzCCAdmgAwIBAgIJANd2MheI5A34MAoGCCqGSM49BAMCMBQxEjAQBgNVBAMM
+CWxvY2FsaG9zdDAgFw0yNjA4MjMxNTA0MzJaGA8yMTI2MDczMDE1MDQzMlowFDES
+MBAGA1UEAwwJbG9jYWxob3N0MIIBSzCCAQMGByqGSM49AgEwgfcCAQEwLAYHKoZI
+zj0BAQIhAP////8AAAABAAAAAAAAAAAAAAAA////////////////MFsEIP////8A
+AAABAAAAAAAAAAAAAAAA///////////////8BCBaxjXYqjqT57PrvVV2mIa8ZR0G
+sMxTsPY7zjw+J9JgSwMVAMSdNgiG5wSTamZ44ROdJreBn36QBEEEaxfR8uEsQkf4
+vOblY6RA8ncDfYEt6zOg9KE5RdiYwpZP40Li/hp/m47n60p8D54WK84zV2sxXs7L
+tkBoN79R9QIhAP////8AAAAA//////////+85vqtpxeehPO5ysL8YyVRAgEBA0IA
+BEKId6j1R5GROrdDCFPIpdmbw0buTikhybK4lctBPOljKxYelFHT0/0lyfJaZpTl
+LriTloTsUGNBfbjYfdmQDIGjHjAcMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAA
+ATAKBggqhkjOPQQDAgNIADBFAiBLTh7kJfkGlb435qNAXAy3g9JxQ3Q2DBQJa10T
+Nd+riQIhAMVAviJGOgeezLlI7HR9vS7eUaFFAKWGMKMyhoFTh79s
+-----END CERTIFICATE-----`;
diff --git a/apps/server/src/services/healthcheck.spec.ts b/apps/server/src/services/healthcheck.spec.ts
index 9cc1bcd186..b0b16f6471 100644
--- a/apps/server/src/services/healthcheck.spec.ts
+++ b/apps/server/src/services/healthcheck.spec.ts
@@ -1,10 +1,12 @@
import fs from "fs";
import http from "http";
+import https from "https";
import net from "net";
import os from "os";
import path from "path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { SELF_SIGNED_CERT, SELF_SIGNED_KEY } from "../../spec/support/self_signed_tls.js";
import { probeHealth, resolveHealthcheckTarget } from "./healthcheck.js";
//#region Probing the server
@@ -90,16 +92,36 @@ describe("the docker healthcheck probe", () => {
expect(logged).toEqual([ "ERROR" ]);
});
- it("reports healthy without contacting the server when trilium terminates TLS", async () => {
- let reached = false;
- const port = await startServer((_req, res) => {
- reached = true;
+ it("speaks TLS when trilium terminates it, rather than assuming the server is up", async () => {
+ const seen: (string | undefined)[] = [];
+ const port = await startTlsServer((req, res) => {
+ seen.push(req.url);
+ res.statusCode = 200;
res.end();
});
expect(await probeHealth({ https: true, port, host: LOCALHOST })).toBe(0);
- expect(reached).toBe(false);
- expect(logged).toEqual([]);
+ expect(seen).toEqual([ "/api/health-check" ]);
+ expect(logged).toEqual([ "STATUS: 200" ]);
+ });
+
+ it("reports unhealthy over TLS when the server is gone, not passing blindly", async () => {
+ const port = await reserveFreePort();
+
+ expect(await probeHealth({ https: true, port, host: LOCALHOST })).toBe(1);
+ expect(logged).toEqual([ "ERROR" ]);
+ });
+
+ it("accepts the self-signed certificate a loopback probe cannot verify", async () => {
+ // The certificate secures nothing here: the probe is checking that the server answers,
+ // not who it is, and a self-signed or expired cert must not read as unhealthy.
+ const port = await startTlsServer((_req, res) => {
+ res.statusCode = 503;
+ res.end();
+ });
+
+ expect(await probeHealth({ https: true, port, host: LOCALHOST })).toBe(1);
+ expect(logged).toEqual([ "STATUS: 503" ]);
});
it("accepts a 200 whatever the body says", async () => {
@@ -150,6 +172,19 @@ async function startServer(handler: http.RequestListener) {
return address.port;
}
+async function startTlsServer(handler: http.RequestListener) {
+ const tls = { key: SELF_SIGNED_KEY, cert: SELF_SIGNED_CERT };
+ const server = track(https.createServer(tls, handler));
+ await new Promise<void>((resolve) => server.listen(0, LOCALHOST, resolve));
+
+ const address = server.address();
+ if (address === null || typeof address === "string") {
+ throw new Error("expected the server to be listening on a TCP port");
+ }
+
+ return address.port;
+}
+
async function startUnixServer(handler: http.RequestListener) {
const socketPath = path.join(makeTempDir(), "trilium.sock");
const server = track(http.createServer(handler));
diff --git a/apps/server/src/services/healthcheck.ts b/apps/server/src/services/healthcheck.ts
index 3738e71ad4..44510248c4 100644
--- a/apps/server/src/services/healthcheck.ts
+++ b/apps/server/src/services/healthcheck.ts
@@ -1,12 +1,13 @@
import fs from "fs";
import http from "http";
+import https from "https";
import ini from "ini";
import os from "os";
import path from "path";
/** Where the probe looks for the server, as the running instance resolved it. */
export interface HealthcheckTarget {
- /** True when Trilium terminates TLS itself, which the probe cannot speak. */
+ /** True when Trilium terminates TLS itself, so the probe has to speak it too. */
https: boolean;
/** TCP port to probe, or 0 when `host` names a Unix socket instead. */
port: number;
@@ -23,23 +24,25 @@ const TIMEOUT_MS = 2000;
* Requests `/api/health-check` and resolves the exit code Docker reads from the probe:
* 0 for healthy, 1 for unhealthy.
*/
-export function probeHealth({ https, port, host }: HealthcheckTarget): Promise<number> {
- if (https) {
- // Trilium-terminated TLS is not supported yet, PRs are welcome. A reverse proxy that
- // terminates TLS leaves `https` false, so the probe still runs there.
- return Promise.resolve(0);
- }
-
+export function probeHealth({ https: overTls, port, host }: HealthcheckTarget): Promise<number> {
return new Promise((resolve) => {
- const options: http.RequestOptions = { timeout: TIMEOUT_MS };
+ const transport = overTls ? https : http;
+ const scheme = overTls ? "https" : "http";
+ const options: https.RequestOptions = {
+ timeout: TIMEOUT_MS,
+ // The probe asks whether the server answers, not who it is. A loopback certificate is
+ // usually self-signed and issued for a name this request does not use, so verifying it
+ // would report a healthy server as dead.
+ rejectUnauthorized: false
+ };
const onResponse = (res: http.IncomingMessage) => {
console.log(`STATUS: ${res.statusCode}`);
resolve(res.statusCode === 200 ? 0 : 1);
};
const request = port !== 0
- ? http.request(`http://${host}:${port}${PROBE_PATH}`, options, onResponse)
- : http.request({ ...options, socketPath: host, path: PROBE_PATH }, onResponse);
+ ? transport.request(`${scheme}://${host}:${port}${PROBE_PATH}`, options, onResponse)
+ : transport.request({ ...options, socketPath: host, path: PROBE_PATH }, onResponse);
// `timeout` only arms the socket's idle timer, so the request has to be torn down here.
// Without this a wedged server that still completes the handshake is waited on forever.