Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(open-knowledge): drain telemetry file sink before provider shutdown (#1616)

Andrew Mikofalvyb747ad62a007
diff --git a/.changeset/telemetry-shutdown-drain-file-sink-first.md b/.changeset/telemetry-shutdown-drain-file-sink-first.md
new file mode 100644
index 000000000..54dc7e426
--- /dev/null
+++ b/.changeset/telemetry-shutdown-drain-file-sink-first.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge-server": patch
+---
+
+Fix telemetry shutdown losing local spans when the OTLP collector is unreachable. `shutdownTelemetry()` now drains the local file-sink exporter on its own awaited promise before the provider-wide teardown, instead of relying on `tracerProvider.shutdown()`. The provider fan-out is `Promise.all(...).then(resolve, reject)`, which short-circuits the moment the OTLP push processor rejects (a fast ECONNREFUSED when no collector is listening) — pre-empting the file pipeline's still-in-flight disk write. That dropped the very spans `ok diagnose bundle` exists to harvest and caused the intermittent `Telemetry > Gate combination 4` failure. Draining the file sink first makes on-disk span capture independent of the push pipeline's fate; the whole sequence still rides the existing shutdown-timeout race, so a pathological filesystem stall cannot deadlock teardown.
diff --git a/packages/server/src/telemetry.ts b/packages/server/src/telemetry.ts
index f4115e1eb..2a80de2d2 100644
--- a/packages/server/src/telemetry.ts
+++ b/packages/server/src/telemetry.ts
@@ -21,6 +21,7 @@ const TRACER_NAME = 'open-knowledge-server';
let tracerProvider: BasicTracerProvider | null = null;
let meterProvider: MeterProvider | null = null;
+let fileSpanExporter: FileSpanExporter | null = null;
function noopResult(): { tracer: Tracer; meter: Meter } {
return {
@@ -65,14 +66,11 @@ export function initTelemetry(opts: InitTelemetryOptions = {}): { tracer: Tracer
opts.localSink?.attributeDenylist ?? DEFAULT_TELEMETRY_ATTRIBUTE_DENYLIST;
spanProcessors.push(new ScrubbingSpanProcessor({ attributeDenylist }));
if (fileSinkEnabled && opts.localSink !== undefined) {
- spanProcessors.push(
- new SimpleSpanProcessor(
- new FileSpanExporter({
- contentDir: opts.localSink.contentDir,
- maxBytes: opts.localSink.spansMaxBytes,
- }),
- ),
- );
+ fileSpanExporter = new FileSpanExporter({
+ contentDir: opts.localSink.contentDir,
+ maxBytes: opts.localSink.spansMaxBytes,
+ });
+ spanProcessors.push(new SimpleSpanProcessor(fileSpanExporter));
}
if (pushEnabled) {
spanProcessors.push(new BatchSpanProcessor(new OTLPTraceExporter()));
@@ -126,6 +124,7 @@ export function initTelemetry(opts: InitTelemetryOptions = {}): { tracer: Tracer
} else {
log.error({ err: e }, 'failed to initialize OpenTelemetry — falling back to no-op');
meterProvider = null;
+ fileSpanExporter = null;
}
}
@@ -150,14 +149,21 @@ export async function shutdownTelemetry(): Promise<void> {
}
}
if (!tracerProvider && !meterProvider) return;
- const shutdownPromise = Promise.all([
- tracerProvider?.shutdown().catch((e: unknown) => {
- log.warn({ err: e }, 'tracer provider shutdown failed');
- }),
- meterProvider?.shutdown().catch((e: unknown) => {
- log.warn({ err: e }, 'meter provider shutdown failed');
- }),
- ]);
+ const fileFlush = fileSpanExporter
+ ? fileSpanExporter.forceFlush().catch((e: unknown) => {
+ log.warn({ err: e }, 'telemetry file-sink flush failed during shutdown');
+ })
+ : Promise.resolve();
+ const shutdownPromise = fileFlush.then(() =>
+ Promise.all([
+ tracerProvider?.shutdown().catch((e: unknown) => {
+ log.warn({ err: e }, 'tracer provider shutdown failed');
+ }),
+ meterProvider?.shutdown().catch((e: unknown) => {
+ log.warn({ err: e }, 'meter provider shutdown failed');
+ }),
+ ]),
+ );
let timer: ReturnType<typeof setTimeout> | undefined;
const timedOut = await Promise.race([
shutdownPromise.then(() => {
@@ -174,6 +180,7 @@ export async function shutdownTelemetry(): Promise<void> {
}
tracerProvider = null;
meterProvider = null;
+ fileSpanExporter = null;
trace.disable();
metrics.disable();
context.disable();