Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

feat(logging): capture all process output through pino (logging-capture-plan)

Carl-Gerhard Lindesvärd5bceac94625c
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 6a3815cbc..fd47a327c 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -1,6 +1,9 @@
/** biome-ignore-all lint/suspicious/useAwait: fastify need async or done callbacks */
process.env.TZ = 'UTC';
+import './utils/intercept-output';
+
+import { rawStderrWrite } from '@openpanel/logger';
import sourceMapSupport from 'source-map-support';
import { buildApp } from './app';
import { shutdown } from './utils/graceful-shutdown';
@@ -29,14 +32,25 @@ const startServer = async () => {
// exit immediately so Docker can respawn the container. Anything
// slower than this lets bad requests keep hitting a poisoned process
// and risks blocking past Docker's stop_grace_period (→ SIGKILL).
+ // Mirror fatals to the real stderr (bypassing the output interceptor,
+ // so nothing ships twice) — the OTLP flush window can be lost on the
+ // way down, and `docker logs` must always show why we died.
process.on('uncaughtException', (error) => {
logger.fatal({ err: error }, 'Uncaught exception — exiting');
+ rawStderrWrite(
+ `Uncaught exception — exiting: ${error?.stack ?? error}\n`,
+ );
// Flush pino, then hard-exit. unref() so the safety net doesn't
// keep the loop alive on its own.
setTimeout(() => process.exit(1), 1000).unref();
});
process.on('unhandledRejection', (reason, promise) => {
logger.fatal({ reason, promise }, 'Unhandled rejection — exiting');
+ rawStderrWrite(
+ `Unhandled rejection — exiting: ${
+ reason instanceof Error ? reason.stack : String(reason)
+ }\n`,
+ );
setTimeout(() => process.exit(1), 1000).unref();
});
}
diff --git a/apps/api/src/utils/intercept-output.ts b/apps/api/src/utils/intercept-output.ts
new file mode 100644
index 000000000..f6b225429
--- /dev/null
+++ b/apps/api/src/utils/intercept-output.ts
@@ -0,0 +1,6 @@
+import { interceptProcessOutput } from '@openpanel/logger';
+import { logger } from './logger';
+
+// Side-effect module — must stay the first import in index.ts so output from
+// the rest of the boot sequence is captured too.
+interceptProcessOutput(logger);
diff --git a/apps/worker/src/boot-workers.ts b/apps/worker/src/boot-workers.ts
index d3451bb50..e2a6e1968 100644
--- a/apps/worker/src/boot-workers.ts
+++ b/apps/worker/src/boot-workers.ts
@@ -1,5 +1,6 @@
import { performance } from 'node:perf_hooks';
import { setTimeout as sleep } from 'node:timers/promises';
+import { rawStderrWrite } from '@openpanel/logger';
import {
cohortComputeQueue,
cronQueue,
@@ -403,13 +404,22 @@ export function bootWorkers() {
// uncaughtException / unhandledRejection: process state is corrupt.
// Don't try to drain — log and exit fast so Docker respawns us.
+ // Mirror fatals to the real stderr (bypassing the output interceptor, so
+ // nothing ships twice) — the OTLP flush window can be lost on the way
+ // down, and `docker logs` must always show why we died.
process.on('uncaughtException', (error) => {
logger.fatal({ err: error }, 'Uncaught exception — exiting');
+ rawStderrWrite(`Uncaught exception — exiting: ${error?.stack ?? error}\n`);
setShuttingDown(true);
setTimeout(() => process.exit(1), 1000).unref();
});
process.on('unhandledRejection', (reason, promise) => {
logger.fatal({ reason, promise }, 'Unhandled rejection — exiting');
+ rawStderrWrite(
+ `Unhandled rejection — exiting: ${
+ reason instanceof Error ? reason.stack : String(reason)
+ }\n`,
+ );
setShuttingDown(true);
setTimeout(() => process.exit(1), 1000).unref();
});
diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts
index 0bfbc4856..c22d891c8 100644
--- a/apps/worker/src/index.ts
+++ b/apps/worker/src/index.ts
@@ -1,3 +1,5 @@
+import './utils/intercept-output';
+
import { createBullBoard } from '@bull-board/api';
import { BullMQAdapter } from '@bull-board/api/bullMQAdapter';
import { ExpressAdapter } from '@bull-board/express';
@@ -160,7 +162,7 @@ async function start() {
});
app.listen(PORT, () => {
- console.log(`For the UI, open http://localhost:${PORT}/`);
+ logger.info(`For the UI, open http://localhost:${PORT}/`);
});
if (process.env.DISABLE_WORKERS === undefined) {
diff --git a/apps/worker/src/utils/intercept-output.ts b/apps/worker/src/utils/intercept-output.ts
new file mode 100644
index 000000000..f6b225429
--- /dev/null
+++ b/apps/worker/src/utils/intercept-output.ts
@@ -0,0 +1,6 @@
+import { interceptProcessOutput } from '@openpanel/logger';
+import { logger } from './logger';
+
+// Side-effect module — must stay the first import in index.ts so output from
+// the rest of the boot sequence is captured too.
+interceptProcessOutput(logger);
diff --git a/packages/db/src/prisma-client.ts b/packages/db/src/prisma-client.ts
index 534b3bb67..3edb87d0d 100644
--- a/packages/db/src/prisma-client.ts
+++ b/packages/db/src/prisma-client.ts
@@ -1,5 +1,6 @@
import { getSubscriptionState } from '@openpanel/payments/subscription-state';
import { PrismaClient } from './generated/prisma/client';
+import { logger } from './logger';
export * from './generated/prisma/client';
@@ -10,9 +11,24 @@ const subscriptionStateNeeds = {
} as const;
const getPrismaClient = () => {
- const prisma = new PrismaClient({
- log: ['error'],
- }).$extends({
+ // emit: 'event' keeps the engine from writing prisma:error lines straight
+ // to stderr, so they flow through pino to the OTLP pipeline instead.
+ const client = new PrismaClient({
+ log: [
+ { emit: 'event', level: 'error' },
+ { emit: 'event', level: 'warn' },
+ ],
+ });
+
+ // $on only exists on the base client — keep it before $extends.
+ client.$on('error', (event) => {
+ logger.error({ target: event.target }, `prisma: ${event.message}`);
+ });
+ client.$on('warn', (event) => {
+ logger.warn({ target: event.target }, `prisma: ${event.message}`);
+ });
+
+ const prisma = client.$extends({
result: {
organization: {
subscriptionState: {
diff --git a/packages/logger/index.ts b/packages/logger/index.ts
index 588f07b1a..6e7df04a4 100644
--- a/packages/logger/index.ts
+++ b/packages/logger/index.ts
@@ -6,6 +6,19 @@ export type ILogger = Logger;
const logLevel = process.env.LOG_LEVEL ?? 'info';
const silent = process.env.LOG_SILENT === 'true';
+// Exactly one shipping path at a time (see logging-capture-plan.md):
+// - 'otlp': pino ships via the HyperDX transport (requires HYPERDX_API_KEY).
+// - 'stdout': pino writes JSON to stdout; an external collector ships it.
+const logExporter =
+ process.env.LOG_EXPORTER ??
+ (process.env.HYPERDX_API_KEY ? 'otlp' : 'stdout');
+
+// Originals captured before interceptProcessOutput wraps the streams. Code
+// that must bypass capture (e.g. crash handlers mirroring fatals to stderr)
+// uses these so the line isn't re-ingested and shipped twice.
+export const rawStdoutWrite = process.stdout.write.bind(process.stdout);
+export const rawStderrWrite = process.stderr.write.bind(process.stderr);
+
// Substring match (lowercased). Catches camelCase, snake_case, prefixed and
// suffixed variants in one entry — e.g. 'token' covers accessToken,
// refresh_token, jwtToken, etc.
@@ -73,7 +86,7 @@ export function createLogger({ name }: { name: string }): ILogger {
.filter(Boolean)
.join('-');
- const useHyperDX = !!process.env.HYPERDX_API_KEY;
+ const useHyperDX = logExporter === 'otlp' && !!process.env.HYPERDX_API_KEY;
const usePretty = !useHyperDX && process.env.NODE_ENV !== 'production';
return pino({
@@ -85,7 +98,11 @@ export function createLogger({ name }: { name: string }): ILogger {
return redactSensitive(obj) as Record<string, unknown>;
},
},
- mixin: useHyperDX ? HyperDX.getPinoMixinFunction : undefined,
+ // Keep trace_id/span_id on every line even in stdout mode so trace↔log
+ // correlation survives when a collector does the shipping.
+ mixin: process.env.HYPERDX_API_KEY
+ ? HyperDX.getPinoMixinFunction
+ : undefined,
transport: useHyperDX
? HyperDX.getPinoTransport(logLevel, {
detectResources: true,
@@ -103,3 +120,113 @@ export function createLogger({ name }: { name: string }): ILogger {
: undefined,
});
}
+
+const MAX_INTERCEPTED_LINE_LENGTH = 8192;
+
+let intercepted = false;
+
+/**
+ * Route everything written to process.stdout/stderr through the given pino
+ * logger, so output that bypasses our loggers (Prisma engine lines, dependency
+ * console.*, Node warnings) still reaches the OTLP pipeline. Call it before
+ * anything else runs in the app entry.
+ *
+ * No feedback loop: pino writes via sonic-boom straight to the file
+ * descriptor (and transports write from a worker thread), so pino's own
+ * output never passes through these wrappers. A reentrancy guard covers any
+ * exotic transport that does.
+ *
+ * In otlp mode raw lines are also passed through to the original stream so
+ * `docker logs` stays useful. In stdout mode pino's JSON line on stdout IS
+ * the container output — teeing would print everything twice.
+ */
+export function interceptProcessOutput(logger: ILogger): void {
+ if (intercepted) {
+ return;
+ }
+ // Keep local dev output untouched.
+ if (
+ process.env.NODE_ENV !== 'production' &&
+ !process.env.HYPERDX_API_KEY &&
+ process.env.LOG_EXPORTER === undefined
+ ) {
+ return;
+ }
+ intercepted = true;
+
+ const passthrough = logExporter !== 'stdout';
+ let logging = false;
+
+ const wrap = (
+ stream: NodeJS.WriteStream,
+ original: typeof rawStdoutWrite,
+ emit: (line: string) => void
+ ) => {
+ let buffer = '';
+
+ const emitLines = (chunk: string) => {
+ if (logging) {
+ return;
+ }
+ buffer += chunk;
+ let newlineIndex = buffer.indexOf('\n');
+ while (newlineIndex !== -1) {
+ const line = buffer.slice(0, newlineIndex).trimEnd();
+ buffer = buffer.slice(newlineIndex + 1);
+ if (line) {
+ logging = true;
+ try {
+ emit(line.slice(0, MAX_INTERCEPTED_LINE_LENGTH));
+ } finally {
+ logging = false;
+ }
+ }
+ newlineIndex = buffer.indexOf('\n');
+ }
+ if (buffer.length > MAX_INTERCEPTED_LINE_LENGTH) {
+ const line = buffer;
+ buffer = '';
+ logging = true;
+ try {
+ emit(line.slice(0, MAX_INTERCEPTED_LINE_LENGTH));
+ } finally {
+ logging = false;
+ }
+ }
+ };
+
+ const write = (
+ chunk: Uint8Array | string,
+ encodingOrCallback?: BufferEncoding | ((error?: Error | null) => void),
+ callback?: (error?: Error | null) => void
+ ): boolean => {
+ try {
+ emitLines(
+ typeof chunk === 'string'
+ ? chunk
+ : Buffer.from(chunk).toString('utf8')
+ );
+ } catch {
+ // Interception must never break the stream.
+ }
+ if (passthrough) {
+ return original(chunk as never, encodingOrCallback as never, callback);
+ }
+ const cb =
+ typeof encodingOrCallback === 'function' ? encodingOrCallback : callback;
+ if (typeof cb === 'function') {
+ process.nextTick(cb);
+ }
+ return true;
+ };
+
+ stream.write = write as typeof stream.write;
+ };
+
+ wrap(process.stdout, rawStdoutWrite, (line) =>
+ logger.info({ source: 'stdout' }, line)
+ );
+ wrap(process.stderr, rawStderrWrite, (line) =>
+ logger.error({ source: 'stderr' }, line)
+ );
+}