OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
feat(logging): capture all process output through pino (logging-capture-plan)
Carl-Gerhard Lindesvärd5bceac94625c
diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 6a3815cbc..fd47a327c 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -1,6 +1,9 @@/** biome-ignore-all lint/suspicious/useAwait: fastify need async or done callbacks */process.env.TZ = 'UTC';+import './utils/intercept-output';++import { rawStderrWrite } from '@openpanel/logger';import sourceMapSupport from 'source-map-support';import { buildApp } from './app';import { shutdown } from './utils/graceful-shutdown';@@ -29,14 +32,25 @@ const startServer = async () => {// exit immediately so Docker can respawn the container. Anything// slower than this lets bad requests keep hitting a poisoned process// and risks blocking past Docker's stop_grace_period (→ SIGKILL).+ // Mirror fatals to the real stderr (bypassing the output interceptor,+ // so nothing ships twice) — the OTLP flush window can be lost on the+ // way down, and `docker logs` must always show why we died.process.on('uncaughtException', (error) => {logger.fatal({ err: error }, 'Uncaught exception — exiting');+ rawStderrWrite(+ `Uncaught exception — exiting: ${error?.stack ?? error}\n`,+ );// Flush pino, then hard-exit. unref() so the safety net doesn't// keep the loop alive on its own.setTimeout(() => process.exit(1), 1000).unref();});process.on('unhandledRejection', (reason, promise) => {logger.fatal({ reason, promise }, 'Unhandled rejection — exiting');+ rawStderrWrite(+ `Unhandled rejection — exiting: ${+ reason instanceof Error ? reason.stack : String(reason)+ }\n`,+ );setTimeout(() => process.exit(1), 1000).unref();});}diff --git a/apps/api/src/utils/intercept-output.ts b/apps/api/src/utils/intercept-output.tsnew file mode 100644index 000000000..f6b225429--- /dev/null+++ b/apps/api/src/utils/intercept-output.ts@@ -0,0 +1,6 @@+import { interceptProcessOutput } from '@openpanel/logger';+import { logger } from './logger';++// Side-effect module — must stay the first import in index.ts so output from+// the rest of the boot sequence is captured too.+interceptProcessOutput(logger);diff --git a/apps/worker/src/boot-workers.ts b/apps/worker/src/boot-workers.tsindex d3451bb50..e2a6e1968 100644--- a/apps/worker/src/boot-workers.ts+++ b/apps/worker/src/boot-workers.ts@@ -1,5 +1,6 @@import { performance } from 'node:perf_hooks';import { setTimeout as sleep } from 'node:timers/promises';+import { rawStderrWrite } from '@openpanel/logger';import {cohortComputeQueue,cronQueue,@@ -403,13 +404,22 @@ export function bootWorkers() {// uncaughtException / unhandledRejection: process state is corrupt.// Don't try to drain — log and exit fast so Docker respawns us.+ // Mirror fatals to the real stderr (bypassing the output interceptor, so+ // nothing ships twice) — the OTLP flush window can be lost on the way+ // down, and `docker logs` must always show why we died.process.on('uncaughtException', (error) => {logger.fatal({ err: error }, 'Uncaught exception — exiting');+ rawStderrWrite(`Uncaught exception — exiting: ${error?.stack ?? error}\n`);setShuttingDown(true);setTimeout(() => process.exit(1), 1000).unref();});process.on('unhandledRejection', (reason, promise) => {logger.fatal({ reason, promise }, 'Unhandled rejection — exiting');+ rawStderrWrite(+ `Unhandled rejection — exiting: ${+ reason instanceof Error ? reason.stack : String(reason)+ }\n`,+ );setShuttingDown(true);setTimeout(() => process.exit(1), 1000).unref();});diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.tsindex 0bfbc4856..c22d891c8 100644--- a/apps/worker/src/index.ts+++ b/apps/worker/src/index.ts@@ -1,3 +1,5 @@+import './utils/intercept-output';+import { createBullBoard } from '@bull-board/api';import { BullMQAdapter } from '@bull-board/api/bullMQAdapter';import { ExpressAdapter } from '@bull-board/express';@@ -160,7 +162,7 @@ async function start() {});app.listen(PORT, () => {- console.log(`For the UI, open http://localhost:${PORT}/`);+ logger.info(`For the UI, open http://localhost:${PORT}/`);});if (process.env.DISABLE_WORKERS === undefined) {diff --git a/apps/worker/src/utils/intercept-output.ts b/apps/worker/src/utils/intercept-output.tsnew file mode 100644index 000000000..f6b225429--- /dev/null+++ b/apps/worker/src/utils/intercept-output.ts@@ -0,0 +1,6 @@+import { interceptProcessOutput } from '@openpanel/logger';+import { logger } from './logger';++// Side-effect module — must stay the first import in index.ts so output from+// the rest of the boot sequence is captured too.+interceptProcessOutput(logger);diff --git a/packages/db/src/prisma-client.ts b/packages/db/src/prisma-client.tsindex 534b3bb67..3edb87d0d 100644--- a/packages/db/src/prisma-client.ts+++ b/packages/db/src/prisma-client.ts@@ -1,5 +1,6 @@import { getSubscriptionState } from '@openpanel/payments/subscription-state';import { PrismaClient } from './generated/prisma/client';+import { logger } from './logger';export * from './generated/prisma/client';@@ -10,9 +11,24 @@ const subscriptionStateNeeds = {} as const;const getPrismaClient = () => {- const prisma = new PrismaClient({- log: ['error'],- }).$extends({+ // emit: 'event' keeps the engine from writing prisma:error lines straight+ // to stderr, so they flow through pino to the OTLP pipeline instead.+ const client = new PrismaClient({+ log: [+ { emit: 'event', level: 'error' },+ { emit: 'event', level: 'warn' },+ ],+ });++ // $on only exists on the base client — keep it before $extends.+ client.$on('error', (event) => {+ logger.error({ target: event.target }, `prisma: ${event.message}`);+ });+ client.$on('warn', (event) => {+ logger.warn({ target: event.target }, `prisma: ${event.message}`);+ });++ const prisma = client.$extends({result: {organization: {subscriptionState: {diff --git a/packages/logger/index.ts b/packages/logger/index.tsindex 588f07b1a..6e7df04a4 100644--- a/packages/logger/index.ts+++ b/packages/logger/index.ts@@ -6,6 +6,19 @@ export type ILogger = Logger;const logLevel = process.env.LOG_LEVEL ?? 'info';const silent = process.env.LOG_SILENT === 'true';+// Exactly one shipping path at a time (see logging-capture-plan.md):+// - 'otlp': pino ships via the HyperDX transport (requires HYPERDX_API_KEY).+// - 'stdout': pino writes JSON to stdout; an external collector ships it.+const logExporter =+ process.env.LOG_EXPORTER ??+ (process.env.HYPERDX_API_KEY ? 'otlp' : 'stdout');++// Originals captured before interceptProcessOutput wraps the streams. Code+// that must bypass capture (e.g. crash handlers mirroring fatals to stderr)+// uses these so the line isn't re-ingested and shipped twice.+export const rawStdoutWrite = process.stdout.write.bind(process.stdout);+export const rawStderrWrite = process.stderr.write.bind(process.stderr);+// Substring match (lowercased). Catches camelCase, snake_case, prefixed and// suffixed variants in one entry — e.g. 'token' covers accessToken,// refresh_token, jwtToken, etc.@@ -73,7 +86,7 @@ export function createLogger({ name }: { name: string }): ILogger {.filter(Boolean).join('-');- const useHyperDX = !!process.env.HYPERDX_API_KEY;+ const useHyperDX = logExporter === 'otlp' && !!process.env.HYPERDX_API_KEY;const usePretty = !useHyperDX && process.env.NODE_ENV !== 'production';return pino({@@ -85,7 +98,11 @@ export function createLogger({ name }: { name: string }): ILogger {return redactSensitive(obj) as Record<string, unknown>;},},- mixin: useHyperDX ? HyperDX.getPinoMixinFunction : undefined,+ // Keep trace_id/span_id on every line even in stdout mode so trace↔log+ // correlation survives when a collector does the shipping.+ mixin: process.env.HYPERDX_API_KEY+ ? HyperDX.getPinoMixinFunction+ : undefined,transport: useHyperDX? HyperDX.getPinoTransport(logLevel, {detectResources: true,@@ -103,3 +120,113 @@ export function createLogger({ name }: { name: string }): ILogger {: undefined,});}++const MAX_INTERCEPTED_LINE_LENGTH = 8192;++let intercepted = false;++/**+ * Route everything written to process.stdout/stderr through the given pino+ * logger, so output that bypasses our loggers (Prisma engine lines, dependency+ * console.*, Node warnings) still reaches the OTLP pipeline. Call it before+ * anything else runs in the app entry.+ *+ * No feedback loop: pino writes via sonic-boom straight to the file+ * descriptor (and transports write from a worker thread), so pino's own+ * output never passes through these wrappers. A reentrancy guard covers any+ * exotic transport that does.+ *+ * In otlp mode raw lines are also passed through to the original stream so+ * `docker logs` stays useful. In stdout mode pino's JSON line on stdout IS+ * the container output — teeing would print everything twice.+ */+export function interceptProcessOutput(logger: ILogger): void {+ if (intercepted) {+ return;+ }+ // Keep local dev output untouched.+ if (+ process.env.NODE_ENV !== 'production' &&+ !process.env.HYPERDX_API_KEY &&+ process.env.LOG_EXPORTER === undefined+ ) {+ return;+ }+ intercepted = true;++ const passthrough = logExporter !== 'stdout';+ let logging = false;++ const wrap = (+ stream: NodeJS.WriteStream,+ original: typeof rawStdoutWrite,+ emit: (line: string) => void+ ) => {+ let buffer = '';++ const emitLines = (chunk: string) => {+ if (logging) {+ return;+ }+ buffer += chunk;+ let newlineIndex = buffer.indexOf('\n');+ while (newlineIndex !== -1) {+ const line = buffer.slice(0, newlineIndex).trimEnd();+ buffer = buffer.slice(newlineIndex + 1);+ if (line) {+ logging = true;+ try {+ emit(line.slice(0, MAX_INTERCEPTED_LINE_LENGTH));+ } finally {+ logging = false;+ }+ }+ newlineIndex = buffer.indexOf('\n');+ }+ if (buffer.length > MAX_INTERCEPTED_LINE_LENGTH) {+ const line = buffer;+ buffer = '';+ logging = true;+ try {+ emit(line.slice(0, MAX_INTERCEPTED_LINE_LENGTH));+ } finally {+ logging = false;+ }+ }+ };++ const write = (+ chunk: Uint8Array | string,+ encodingOrCallback?: BufferEncoding | ((error?: Error | null) => void),+ callback?: (error?: Error | null) => void+ ): boolean => {+ try {+ emitLines(+ typeof chunk === 'string'+ ? chunk+ : Buffer.from(chunk).toString('utf8')+ );+ } catch {+ // Interception must never break the stream.+ }+ if (passthrough) {+ return original(chunk as never, encodingOrCallback as never, callback);+ }+ const cb =+ typeof encodingOrCallback === 'function' ? encodingOrCallback : callback;+ if (typeof cb === 'function') {+ process.nextTick(cb);+ }+ return true;+ };++ stream.write = write as typeof stream.write;+ };++ wrap(process.stdout, rawStdoutWrite, (line) =>+ logger.info({ source: 'stdout' }, line)+ );+ wrap(process.stderr, rawStderrWrite, (line) =>+ logger.error({ source: 'stderr' }, line)+ );+}