Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

feat(server): sampler wiring + runtime and queue-depth gauges (#2730)

Andrew Mikofalvye4c11f7c3037
diff --git a/.changeset/server-observability-quick-wins.md b/.changeset/server-observability-quick-wins.md
new file mode 100644
index 000000000..137ed8a59
--- /dev/null
+++ b/.changeset/server-observability-quick-wins.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+New operational metrics for cloud/server operators (all zero-overhead when telemetry is disabled): event-loop delay percentiles (`ok.server.event_loop.delay_ms`), CPU utilization (`ok.server.cpu.utilization`), extended memory sections (`external`, `array_buffers`) on `ok.server.memory.usage_megabytes`, a loaded-documents gauge (`ok.server.docs.loaded`), and write-spine queue depths (`ok.persistence.queue.depth`, `ok.bridge.drain_backlog`). Error responses now stamp their correlation UUID on the active trace span (`ok.error.instance`) so a client-reported error can be joined against its trace. The standard `OTEL_TRACES_SAMPLER` / `OTEL_TRACES_SAMPLER_ARG` sampling contract is now pinned by tests and documented, including its interaction with the local bug-bundle span sink.
diff --git a/packages/server/src/boot.ts b/packages/server/src/boot.ts
index 8865647d5..28114d2da 100644
--- a/packages/server/src/boot.ts
+++ b/packages/server/src/boot.ts
@@ -58,7 +58,7 @@ import { createMcpHttpHandler } from './mcp-http.ts';
import { mountMcpAndApi } from './mcp-mount.ts';
import { MissingOkConfigError } from './missing-ok-config-error.ts';
import { createServer, type ServerInstance, type ServerOptions } from './server-factory.ts';
-import { installServerMemoryGauge } from './server-memory-telemetry.ts';
+import { installServerMemoryGauge, installServerRuntimeGauges } from './server-memory-telemetry.ts';
import { reconcileSkillInstalls } from './skill-reconcile.ts';
import { initTelemetry, shutdownTelemetry, withSpan } from './telemetry.ts';
import {
@@ -436,6 +436,7 @@ export async function bootServer(opts: BootServerOptions): Promise<BootedServer>
// `.ok/local/` runtime-state root as the span/log sinks.
initToleranceTelemetryWriter(sinkProjectDir);
installServerMemoryGauge();
+ installServerRuntimeGauges();
// Wrap the orchestration in an `ok.boot` span so telemetry can slice
// boot-failure rates by worktree kind. Computed here (not inside
diff --git a/packages/server/src/http/error-response.test.ts b/packages/server/src/http/error-response.test.ts
index b52c04227..3b6f91978 100644
--- a/packages/server/src/http/error-response.test.ts
+++ b/packages/server/src/http/error-response.test.ts
@@ -518,3 +518,49 @@ describe('createStreamingErrorWriter — writableEnded guard', () => {
expect(writeCalls.length).toBe(1);
});
});
+
+describe('errorResponse — trace correlation (ok.error.instance span attribute)', () => {
+ test('stamps the minted instance UUID on the active span', async () => {
+ // Requires a real context manager: setActiveSpanAttributes reads
+ // trace.getSpan(context.active()), which the default no-op context
+ // manager never populates.
+ const { context: apiContext, trace } = await import('@opentelemetry/api');
+ const { AsyncLocalStorageContextManager } = await import('@opentelemetry/context-async-hooks');
+ const { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } = await import(
+ '@opentelemetry/sdk-trace-base'
+ );
+ const exporter = new InMemorySpanExporter();
+ const provider = new BasicTracerProvider({
+ spanProcessors: [new SimpleSpanProcessor(exporter)],
+ });
+ trace.setGlobalTracerProvider(provider);
+ apiContext.setGlobalContextManager(new AsyncLocalStorageContextManager());
+ try {
+ const { res, endCalls } = makeMockRes();
+ trace.getTracer('test').startActiveSpan('request-span', (span) => {
+ errorResponse(res, 500, 'urn:ok:error:internal-server-error', 'Boom.');
+ span.end();
+ });
+ const spans = exporter.getFinishedSpans();
+ expect(spans).toHaveLength(1);
+ const stamped = spans[0].attributes['ok.error.instance'];
+ expect(typeof stamped).toBe('string');
+ // The stamped value must be the SAME correlation ID the client
+ // received in the problem+json body — that equality is what makes
+ // trace ↔ client-report joins possible.
+ const body = JSON.parse(endCalls[0]);
+ expect(stamped).toBe(body.instance);
+ } finally {
+ await provider.shutdown();
+ trace.disable();
+ apiContext.disable();
+ }
+ });
+
+ test('is a no-op without an active span (no throw, response unchanged)', () => {
+ const { res, writeHeadCalls, endCalls } = makeMockRes();
+ expect(() => errorResponse(res, 404, 'urn:ok:error:not-found', 'Missing.')).not.toThrow();
+ expect(writeHeadCalls.length).toBe(1);
+ expect(JSON.parse(endCalls[0]).title).toBe('Missing.');
+ });
+});
diff --git a/packages/server/src/http/error-response.ts b/packages/server/src/http/error-response.ts
index 0c37c8f9f..b3e11020e 100644
--- a/packages/server/src/http/error-response.ts
+++ b/packages/server/src/http/error-response.ts
@@ -47,7 +47,7 @@ import {
} from '@inkeep/open-knowledge-core';
import type { Counter } from '@opentelemetry/api';
import { getLogger } from '../logger.ts';
-import { getMeter } from '../telemetry.ts';
+import { getMeter, setActiveSpanAttributes } from '../telemetry.ts';
// Lazy logger accessor — `loggerFactory.configure()` (used by capture-logger
// tests in `server-factory.test.ts` and `logger.test.ts`) clears the
@@ -166,6 +166,12 @@ export function errorResponse(
// log-grep workflows still pattern-match a flat string.
// `ProblemDetailsSchema.instance` in core validates the prefix shape.
const instance = options.instance ?? `urn:uuid:${randomUUID()}`;
+ // Mirror the correlation ID onto the active request span so a trace found
+ // in Tempo can be joined against the `instance` a client reported (and
+ // vice versa). No-op when no span is active or OTel is disabled. A UUID is
+ // a sanctioned span attribute — the cardinality STOP rule governs metric
+ // labels, and `ok.api.error.count` below stays UUID-free.
+ setActiveSpanAttributes({ 'ok.error.instance': instance });
// Defense-in-depth: if a handler has already started writing (async race,
// programming error at any of the ~286 call sites), `res.writeHead()`
diff --git a/packages/server/src/persistence.ts b/packages/server/src/persistence.ts
index cf7650195..444f9912f 100644
--- a/packages/server/src/persistence.ts
+++ b/packages/server/src/persistence.ts
@@ -577,6 +577,19 @@ function toStoreFailure(err: unknown): StoreFailure {
return { code, message };
}
+/**
+ * Read-only snapshot of a persistence instance's pending-store depths, sampled
+ * by the workload observable gauges at metric-export time. Owned here (the
+ * domain module that owns those queues) rather than by the telemetry module
+ * that consumes it, so telemetry depends on persistence and not the reverse.
+ */
+export interface PersistenceQueueDepths {
+ /** Stores parked during a branch operation, awaiting replay. */
+ readonly branchDeferred: number;
+ /** Docs whose store cycle is currently deferring on the quiescence gate. */
+ readonly quiescenceDeferred: number;
+}
+
export interface PersistenceHandle {
extension: Extension;
flushDeferredStores: (mode?: 'within-branch' | 'discard-stale') => Promise<void>;
@@ -617,6 +630,12 @@ export interface PersistenceHandle {
*/
flushContributors: () => Promise<void>;
waitForPendingCommits: () => Promise<void>;
+ /**
+ * Read-only snapshot of this instance's pending-store depths, for the
+ * workload observable gauges. Sampled at metric-export time only; never
+ * mutates state.
+ */
+ getQueueDepths: () => PersistenceQueueDepths;
/**
* Config-doc persistence context. Exposed so the file-watcher
* orchestration in `server-factory.ts` can call `applyExternalConfigChange`
@@ -2355,6 +2374,13 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis
agentWriteStores.add(documentName);
}
+ function getQueueDepths(): PersistenceQueueDepths {
+ return {
+ branchDeferred: deferredStores.size,
+ quiescenceDeferred: persistenceDeferCounts.size,
+ };
+ }
+
return {
extension,
flushDeferredStores,
@@ -2364,6 +2390,7 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis
takeStoreFailure,
takeStoreDivergence,
markAgentWriteStore,
+ getQueueDepths,
configPersistenceCtx,
managedArtifactCtx,
};
diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.ts
index 7413d3a22..816010342 100644
--- a/packages/server/src/server-factory.ts
+++ b/packages/server/src/server-factory.ts
@@ -155,6 +155,11 @@ import {
import { acquireServerLock, markServerLockDraining, releaseServerLock } from './server-lock.ts';
import { createServerObserverExtension } from './server-observer-extension.ts';
import type { PairedWriteOrigin } from './server-observers.ts';
+import {
+ installServerWorkloadGauges,
+ registerLoadedDocsProvider,
+ registerPersistenceQueueDepthProvider,
+} from './server-workload-telemetry.ts';
import { shadowOpGateFor } from './shadow-op-gate.ts';
import {
commitUpstreamImport,
@@ -883,6 +888,10 @@ export function createServer(options: ServerOptions): ServerInstance {
let loadedPrincipal: Principal | null = null;
const forceUnloadSet = new Set<Document>();
let shutdownAllowsUnload = false;
+ // Unregister hooks for the pull-based workload gauge providers registered
+ // in the init `try` below; called from destroy() so a torn-down instance
+ // stops contributing to (and being retained by) the process-wide gauges.
+ const unregisterWorkloadProviders: Array<() => void> = [];
// Assigned synchronously in the init `try` immediately after `new Hocuspocus` (before the try
// completes or awaits). Call sites (disk reconcile, API extension) only run after boot returns.
let forceUnloadDocument!: (document: Document) => Promise<void>;
@@ -1205,6 +1214,18 @@ export function createServer(options: ServerOptions): ServerInstance {
extensions: [persistence.extension],
});
+ // Workload observable gauges (loaded docs, persistence queue depths,
+ // bridge drain backlog). Providers are sampled only at metric-export
+ // time; registering them is a Set add and safe when OTel is disabled.
+ // Installed here (not just boot.ts) because the dev-plugin path enters
+ // createServer directly; install is idempotent.
+ const hp = hocuspocus;
+ unregisterWorkloadProviders.push(
+ registerLoadedDocsProvider(() => hp.documents.size),
+ registerPersistenceQueueDepthProvider(() => persistence.getQueueDepths()),
+ );
+ installServerWorkloadGauges();
+
// Hocuspocus unloads documents as soon as the last WebSocket disconnects.
// That is unsafe with client-side y-indexeddb: a browser refresh leaves a
// durable client copy of the same Yjs items, while the server rebuilds a
@@ -1667,6 +1688,15 @@ export function createServer(options: ServerOptions): ServerInstance {
}),
);
} catch (err) {
+ // Init failed after the workload providers may have registered. The caller
+ // never receives an instance to call destroy() on, so drain the providers
+ // here too — otherwise their closures (capturing `hocuspocus`/`persistence`)
+ // leak into the process-global registries and keep summing off a
+ // half-torn-down object. Idempotent via splice(0), so destroy()'s later
+ // drain stays a no-op.
+ for (const unregister of unregisterWorkloadProviders.splice(0)) {
+ unregister();
+ }
releaseServerLock(lockDir);
throw err;
}
@@ -2399,6 +2429,13 @@ export function createServer(options: ServerOptions): ServerInstance {
const phaseErrors: Array<{ phase: string; error: string }> = [];
shutdownAllowsUnload = true;
+ // Stop contributing to the process-wide workload gauges before any
+ // teardown phase runs — a mid-destroy sample would otherwise read
+ // half-dismantled structures.
+ for (const unregister of unregisterWorkloadProviders.splice(0)) {
+ unregister();
+ }
+
// Advertise teardown FIRST — before any flush work. Readers (MCP
// discovery, desktop attach, spawners) see `draining: true` and stop
// dialing our port or treating lock-presence as "serving"; supervisors
diff --git a/packages/server/src/server-memory-telemetry.test.ts b/packages/server/src/server-memory-telemetry.test.ts
index 8872e4d1e..a5c83826e 100644
--- a/packages/server/src/server-memory-telemetry.test.ts
+++ b/packages/server/src/server-memory-telemetry.test.ts
@@ -8,21 +8,26 @@ import {
PeriodicExportingMetricReader,
} from '@opentelemetry/sdk-metrics';
import {
- __resetServerMemoryGaugeForTests,
+ __resetServerRuntimeTelemetryForTests,
installServerMemoryGauge,
+ installServerRuntimeGauges,
} from './server-memory-telemetry.ts';
const METRIC = 'ok.server.memory.usage_megabytes';
+const EVENT_LOOP_METRIC = 'ok.server.event_loop.delay_ms';
+const CPU_METRIC = 'ok.server.cpu.utilization';
describe('installServerMemoryGauge — no-op meter (OTel disabled)', () => {
test('does not throw and is idempotent with the default no-op meter', () => {
metrics.disable();
- __resetServerMemoryGaugeForTests();
+ __resetServerRuntimeTelemetryForTests();
expect(() => {
installServerMemoryGauge();
installServerMemoryGauge();
+ installServerRuntimeGauges();
+ installServerRuntimeGauges();
}).not.toThrow();
- __resetServerMemoryGaugeForTests();
+ __resetServerRuntimeTelemetryForTests();
});
});
@@ -36,25 +41,28 @@ describe('installServerMemoryGauge — registered meter', () => {
reader = new PeriodicExportingMetricReader({ exporter, exportIntervalMillis: 60_000 });
provider = new MeterProvider({ readers: [reader] });
metrics.setGlobalMeterProvider(provider);
- __resetServerMemoryGaugeForTests();
+ __resetServerRuntimeTelemetryForTests();
});
afterAll(async () => {
await provider.shutdown();
metrics.disable();
- __resetServerMemoryGaugeForTests();
+ __resetServerRuntimeTelemetryForTests();
});
- async function collectSections(): Promise<Array<{ value: number; section: unknown }>> {
+ async function collectPoints(
+ metricName: string,
+ label: string,
+ ): Promise<Array<{ value: number; label: unknown }>> {
exporter.reset();
await reader.forceFlush();
- const out: Array<{ value: number; section: unknown }> = [];
+ const out: Array<{ value: number; label: unknown }> = [];
for (const rm of exporter.getMetrics()) {
for (const sm of rm.scopeMetrics) {
for (const metric of sm.metrics) {
- if (metric.descriptor.name !== METRIC) continue;
+ if (metric.descriptor.name !== metricName) continue;
for (const dp of metric.dataPoints as Array<DataPoint<number>>) {
- out.push({ value: dp.value, section: dp.attributes.section });
+ out.push({ value: dp.value, label: dp.attributes[label] });
}
}
}
@@ -62,18 +70,66 @@ describe('installServerMemoryGauge — registered meter', () => {
return out;
}
- test('records exactly the three bounded sections with positive values', async () => {
+ test('records exactly the five bounded sections', async () => {
installServerMemoryGauge();
- const points = await collectSections();
- expect(points.map((p) => p.section).sort()).toEqual(['heap_total', 'heap_used', 'rss']);
+ const points = await collectPoints(METRIC, 'section');
+ expect(points.map((p) => p.label).sort()).toEqual([
+ 'array_buffers',
+ 'external',
+ 'heap_total',
+ 'heap_used',
+ 'rss',
+ ]);
for (const p of points) {
- expect(p.value).toBeGreaterThan(0);
+ expect(p.value).toBeGreaterThanOrEqual(0);
}
+ // The heap sections are always non-zero in a live process; external and
+ // array_buffers can legitimately be near-zero, hence the >= 0 floor above.
+ const heapUsed = points.find((p) => p.label === 'heap_used');
+ expect(heapUsed?.value).toBeGreaterThan(0);
});
test('is idempotent — a second install does not duplicate the series', async () => {
installServerMemoryGauge();
- const points = await collectSections();
- expect(points).toHaveLength(3);
+ const points = await collectPoints(METRIC, 'section');
+ expect(points).toHaveLength(5);
+ });
+
+ test('event-loop gauge arms on first export and reports p50/p99 on the next', async () => {
+ installServerRuntimeGauges();
+ // First collection only enables the lazily-started histogram — the
+ // export window has no data yet, so no points are observed.
+ const first = await collectPoints(EVENT_LOOP_METRIC, 'stat');
+ expect(first).toHaveLength(0);
+ // Poll instead of sleeping a fixed interval: a heavily-loaded CI runner may
+ // not iterate the event loop enough within a fixed wall-clock window. Each
+ // forceFlush past the arming one observes p50/p99, so this converges to two
+ // points as soon as the loop ticks the histogram once.
+ let second: Awaited<ReturnType<typeof collectPoints>> = [];
+ const deadline = Date.now() + 2000;
+ while (second.length < 2 && Date.now() < deadline) {
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ second = await collectPoints(EVENT_LOOP_METRIC, 'stat');
+ }
+ expect(second.map((p) => p.label).sort()).toEqual(['p50', 'p99']);
+ for (const p of second) {
+ expect(p.value).toBeGreaterThanOrEqual(0);
+ expect(Number.isFinite(p.value)).toBe(true);
+ }
+ });
+
+ test('cpu gauge records the baseline on first export and utilization on the next', async () => {
+ installServerRuntimeGauges();
+ // Two callbacks are required: the first records the baseline sample and
+ // observes nothing; the second computes the delta and reports both modes.
+ // Collect twice explicitly so the test does not depend on a prior test
+ // having already driven the first callback via shared module-level state.
+ await collectPoints(CPU_METRIC, 'mode');
+ const second = await collectPoints(CPU_METRIC, 'mode');
+ expect(second.map((p) => p.label).sort()).toEqual(['system', 'user']);
+ for (const p of second) {
+ expect(p.value).toBeGreaterThanOrEqual(0);
+ expect(Number.isFinite(p.value)).toBe(true);
+ }
});
});
diff --git a/packages/server/src/server-memory-telemetry.ts b/packages/server/src/server-memory-telemetry.ts
index 650228e28..e8fddf827 100644
--- a/packages/server/src/server-memory-telemetry.ts
+++ b/packages/server/src/server-memory-telemetry.ts
@@ -1,28 +1,54 @@
/**
- * Server process memory exposed as a bounded OpenTelemetry observable gauge.
+ * Server process runtime health exposed as bounded OpenTelemetry observable
+ * gauges: memory by section, event-loop delay percentiles, and CPU
+ * utilization.
*
* Server-side memory growth (e.g. a large `?showAll=true` walk) is invisible
- * from the renderer. This gauge samples `process.memoryUsage()` — via the
- * shared `captureServerMemorySnapshot()` helper — at each metric-export
- * interval. Cardinality is fixed at three series through the `section` enum.
+ * from the renderer. The memory gauge samples `process.memoryUsage()` — via
+ * the shared `captureServerMemorySnapshot()` helper — at each metric-export
+ * interval. Cardinality is fixed through the `section` enum.
*
* Zero overhead when OTel is disabled: `getMeter()` returns the API no-op
- * meter, whose observable gauge never invokes the callback.
+ * meter, whose observable gauge never invokes the callback. The event-loop
+ * delay histogram is likewise only enabled from inside the gauge callback,
+ * so its constant sampling cost is paid only when a real meter is live.
*/
+import { monitorEventLoopDelay } from 'node:perf_hooks';
import type { ObservableGauge, ObservableResult } from '@opentelemetry/api';
import { captureServerMemorySnapshot } from './perf-measurement.ts';
import { getMeter, onTelemetryShutdown } from './telemetry.ts';
let cachedGauge: ObservableGauge | null = null;
+let cachedEventLoopGauge: ObservableGauge | null = null;
+let cachedCpuGauge: ObservableGauge | null = null;
-// Drop the cached gauge whenever telemetry shuts down. The gauge is bound to
-// the meter provider torn down by shutdownTelemetry; keeping the cache would
-// make the next installServerMemoryGauge() a no-op (idempotency guard below) so
-// the callback would never rebind to the freshly-initialized meter. Registered
-// once at import — onTelemetryShutdown dedups, and a reset hook is cheaper than
-// the alternative (telemetry.ts importing this module → circular).
+const NANOS_PER_MS = 1e6;
+
+/**
+ * Lazily-enabled event-loop delay histogram. `monitorEventLoopDelay` costs a
+ * periodic timer while enabled, so it is only started from inside the gauge
+ * callback — which the no-op meter never invokes — preserving the
+ * zero-overhead-when-disabled contract. Reset after each observation so each
+ * export interval reports its own window rather than a since-boot aggregate.
+ */
+let eventLoopHistogram: ReturnType<typeof monitorEventLoopDelay> | null = null;
+
+/** Previous CPU/wall sample for utilization deltas. Null until first callback. */
+let lastCpuSample: { cpu: NodeJS.CpuUsage; hrtimeNs: bigint } | null = null;
+
+// Drop the cached gauges whenever telemetry shuts down. The gauges are bound
+// to the meter provider torn down by shutdownTelemetry; keeping the cache
+// would make the next install call a no-op (idempotency guards below) so the
+// callbacks would never rebind to the freshly-initialized meter. Registered
+// once at import — onTelemetryShutdown dedups, and a reset hook is cheaper
+// than the alternative (telemetry.ts importing this module → circular).
onTelemetryShutdown(() => {
cachedGauge = null;
+ cachedEventLoopGauge = null;
+ cachedCpuGauge = null;
+ eventLoopHistogram?.disable();
+ eventLoopHistogram = null;
+ lastCpuSample = null;
});
/**
@@ -34,7 +60,7 @@ export function installServerMemoryGauge(): void {
if (cachedGauge) return;
const gauge = getMeter().createObservableGauge('ok.server.memory.usage_megabytes', {
description:
- 'Server process memory by section. Bounded labels: section ∈ {heap_used, heap_total, rss}.',
+ 'Server process memory by section. Bounded labels: section ∈ {heap_used, heap_total, rss, external, array_buffers}.',
unit: 'MB',
});
gauge.addCallback((result: ObservableResult) => {
@@ -42,11 +68,74 @@ export function installServerMemoryGauge(): void {
result.observe(snapshot.heapUsedMb, { section: 'heap_used' });
result.observe(snapshot.heapTotalMb, { section: 'heap_total' });
result.observe(snapshot.rssMb, { section: 'rss' });
+ result.observe(snapshot.externalMb, { section: 'external' });
+ result.observe(snapshot.arrayBuffersMb, { section: 'array_buffers' });
});
cachedGauge = gauge;
}
-/** Drop the cached gauge so a test can rebind against a fresh meter. Test-only. */
-export function __resetServerMemoryGaugeForTests(): void {
+/**
+ * Register the event-loop delay and CPU utilization gauges. Idempotent per
+ * instrument, same lifecycle contract as {@link installServerMemoryGauge}.
+ *
+ * Event-loop delay reports p50/p99 in milliseconds over the window since the
+ * previous export (the histogram resets after each read); the first callback
+ * only enables the histogram, so the first export interval observes nothing.
+ * CPU utilization reports user/system CPU time as a fraction of wall time
+ * since the previous callback (may exceed 1 on multi-core parallelism); the
+ * first callback only records the baseline sample.
+ */
+export function installServerRuntimeGauges(): void {
+ if (!cachedEventLoopGauge) {
+ const gauge = getMeter().createObservableGauge('ok.server.event_loop.delay_ms', {
+ description:
+ 'Event-loop delay percentiles over the last export window. Bounded labels: stat ∈ {p50, p99}.',
+ unit: 'ms',
+ });
+ gauge.addCallback((result: ObservableResult) => {
+ if (!eventLoopHistogram) {
+ eventLoopHistogram = monitorEventLoopDelay({ resolution: 20 });
+ eventLoopHistogram.enable();
+ return;
+ }
+ result.observe(eventLoopHistogram.percentile(50) / NANOS_PER_MS, { stat: 'p50' });
+ result.observe(eventLoopHistogram.percentile(99) / NANOS_PER_MS, { stat: 'p99' });
+ eventLoopHistogram.reset();
+ });
+ cachedEventLoopGauge = gauge;
+ }
+
+ if (!cachedCpuGauge) {
+ const gauge = getMeter().createObservableGauge('ok.server.cpu.utilization', {
+ description:
+ 'Process CPU time as a fraction of wall time since the previous export. Bounded labels: mode ∈ {user, system}.',
+ unit: '1',
+ });
+ gauge.addCallback((result: ObservableResult) => {
+ const cpu = process.cpuUsage();
+ const hrtimeNs = process.hrtime.bigint();
+ const prev = lastCpuSample;
+ lastCpuSample = { cpu, hrtimeNs };
+ if (!prev) return;
+ const wallMicros = Number(hrtimeNs - prev.hrtimeNs) / 1_000;
+ if (wallMicros <= 0) return;
+ result.observe((cpu.user - prev.cpu.user) / wallMicros, { mode: 'user' });
+ result.observe((cpu.system - prev.cpu.system) / wallMicros, { mode: 'system' });
+ });
+ cachedCpuGauge = gauge;
+ }
+}
+
+/**
+ * Drop every cached runtime instrument (memory, event-loop, CPU) plus the
+ * event-loop histogram and CPU baseline so a test can rebind against a fresh
+ * meter. Test-only.
+ */
+export function __resetServerRuntimeTelemetryForTests(): void {
cachedGauge = null;
+ cachedEventLoopGauge = null;
+ cachedCpuGauge = null;
+ eventLoopHistogram?.disable();
+ eventLoopHistogram = null;
+ lastCpuSample = null;
}
diff --git a/packages/server/src/server-observers.ts b/packages/server/src/server-observers.ts
index d7081b90c..688a4a0f4 100644
--- a/packages/server/src/server-observers.ts
+++ b/packages/server/src/server-observers.ts
@@ -79,6 +79,7 @@ import {
incrementServerObserverError,
incrementServerObserverFire,
} from './metrics.ts';
+import { registerBridgeDirtyProbe } from './server-workload-telemetry.ts';
import { type ShadowHandle, saveInMemoryCheckpoint } from './shadow-repo.ts';
import { setActiveSpanAttributes, withSpanSync } from './telemetry.ts';
@@ -2039,6 +2040,9 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void
xmlFragment.observeDeep(observerA);
ytext.observe(observerB);
doc.on('afterAllTransactions', afterAll);
+ // Pull-based backlog probe for the workload gauges: sampled only at
+ // metric-export time, so the observer hot path stays untouched.
+ const unregisterDirtyProbe = registerBridgeDirtyProbe(() => xmlDirty || textDirty);
// Quiescence tracking lives in its own module to avoid `Date.now()` /
// `setTimeout` here (precedent #13(b) — bridge-no-wallclock guard).
// `attachQuiescenceTracker` hooks `afterTransaction` + `afterAllTransactions`
@@ -2047,6 +2051,7 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void
// ─── Cleanup ───────────────────────────────────────────────
return () => {
+ unregisterDirtyProbe();
detachQuiescence();
doc.off('afterAllTransactions', afterAll);
xmlFragment.unobserveDeep(observerA);
diff --git a/packages/server/src/server-workload-telemetry.test.ts b/packages/server/src/server-workload-telemetry.test.ts
new file mode 100644
index 000000000..d33c65992
--- /dev/null
+++ b/packages/server/src/server-workload-telemetry.test.ts
@@ -0,0 +1,129 @@
+import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
+import { metrics } from '@opentelemetry/api';
+import {
+ AggregationTemporality,
+ type DataPoint,
+ InMemoryMetricExporter,
+ MeterProvider,
+ PeriodicExportingMetricReader,
+} from '@opentelemetry/sdk-metrics';
+import {
+ __resetServerWorkloadTelemetryForTests,
+ installServerWorkloadGauges,
+ registerBridgeDirtyProbe,
+ registerLoadedDocsProvider,
+ registerPersistenceQueueDepthProvider,
+} from './server-workload-telemetry.ts';
+
+const DOCS_METRIC = 'ok.server.docs.loaded';
+const QUEUE_METRIC = 'ok.persistence.queue.depth';
+const BACKLOG_METRIC = 'ok.bridge.drain_backlog';
+
+describe('server workload telemetry — no-op meter (OTel disabled)', () => {
+ test('install and register/unregister are safe with the default no-op meter', () => {
+ metrics.disable();
+ __resetServerWorkloadTelemetryForTests();
+ expect(() => {
+ installServerWorkloadGauges();
+ installServerWorkloadGauges();
+ const un1 = registerLoadedDocsProvider(() => 3);
+ const un2 = registerPersistenceQueueDepthProvider(() => ({
+ branchDeferred: 0,
+ quiescenceDeferred: 0,
+ }));
+ const un3 = registerBridgeDirtyProbe(() => false);
+ un1();
+ un2();
+ un3();
+ }).not.toThrow();
+ __resetServerWorkloadTelemetryForTests();
+ });
+});
+
+describe('server workload telemetry — registered meter', () => {
+ let exporter: InMemoryMetricExporter;
+ let reader: PeriodicExportingMetricReader;
+ let provider: MeterProvider;
+
+ beforeAll(() => {
+ exporter = new InMemoryMetricExporter(AggregationTemporality.CUMULATIVE);
+ reader = new PeriodicExportingMetricReader({ exporter, exportIntervalMillis: 60_000 });
+ provider = new MeterProvider({ readers: [reader] });
+ metrics.setGlobalMeterProvider(provider);
+ });
+
+ beforeEach(() => {
+ __resetServerWorkloadTelemetryForTests();
+ installServerWorkloadGauges();
+ });
+
+ afterAll(async () => {
+ await provider.shutdown();
+ metrics.disable();
+ __resetServerWorkloadTelemetryForTests();
+ });
+
+ async function collect(
+ metricName: string,
+ ): Promise<Array<{ value: number; attributes: Record<string, unknown> }>> {
+ exporter.reset();
+ await reader.forceFlush();
+ const out: Array<{ value: number; attributes: Record<string, unknown> }> = [];
+ for (const rm of exporter.getMetrics()) {
+ for (const sm of rm.scopeMetrics) {
+ for (const metric of sm.metrics) {
+ if (metric.descriptor.name !== metricName) continue;
+ for (const dp of metric.dataPoints as Array<DataPoint<number>>) {
+ out.push({ value: dp.value, attributes: dp.attributes });
+ }
+ }
+ }
+ }
+ return out;
+ }
+
+ test('ok.server.docs.loaded sums registered providers and drops unregistered ones', async () => {
+ const un1 = registerLoadedDocsProvider(() => 4);
+ const un2 = registerLoadedDocsProvider(() => 2);
+ let points = await collect(DOCS_METRIC);
+ expect(points).toHaveLength(1);
+ expect(points[0].value).toBe(6);
+
+ un2();
+ points = await collect(DOCS_METRIC);
+ expect(points[0].value).toBe(4);
+ un1();
+ points = await collect(DOCS_METRIC);
+ expect(points[0].value).toBe(0);
+ });
+
+ test('ok.persistence.queue.depth reports both bounded queue labels', async () => {
+ registerPersistenceQueueDepthProvider(() => ({ branchDeferred: 3, quiescenceDeferred: 1 }));
+ const points = await collect(QUEUE_METRIC);
+ const byQueue = new Map(points.map((p) => [p.attributes.queue, p.value]));
+ expect([...byQueue.keys()].sort()).toEqual(['branch_deferred', 'quiescence_deferred']);
+ expect(byQueue.get('branch_deferred')).toBe(3);
+ expect(byQueue.get('quiescence_deferred')).toBe(1);
+ });
+
+ test('ok.bridge.drain_backlog counts only probes reporting dirty', async () => {
+ let dirtyA = false;
+ registerBridgeDirtyProbe(() => dirtyA);
+ registerBridgeDirtyProbe(() => true);
+ let points = await collect(BACKLOG_METRIC);
+ expect(points[0].value).toBe(1);
+
+ dirtyA = true;
+ points = await collect(BACKLOG_METRIC);
+ expect(points[0].value).toBe(2);
+ });
+
+ test('a throwing provider is skipped without poisoning the remaining sum', async () => {
+ registerLoadedDocsProvider(() => {
+ throw new Error('torn down');
+ });
+ registerLoadedDocsProvider(() => 5);
+ const points = await collect(DOCS_METRIC);
+ expect(points[0].value).toBe(5);
+ });
+});
diff --git a/packages/server/src/server-workload-telemetry.ts b/packages/server/src/server-workload-telemetry.ts
new file mode 100644
index 000000000..e12f5c247
--- /dev/null
+++ b/packages/server/src/server-workload-telemetry.ts
@@ -0,0 +1,152 @@
+/**
+ * Write-spine workload exposed as bounded OpenTelemetry observable gauges:
+ * loaded Y.Doc count, persistence queue depths, and bridge drain backlog.
+ *
+ * Pull-based by design: producers (server-factory, persistence, the server
+ * observer bridge) register read-only provider closures here at construction
+ * and unregister at teardown; the gauge callbacks sample the providers only
+ * at metric-export time. That keeps the write spine's hot paths untouched —
+ * no counter increments inside observers or store hooks — and keeps the
+ * registries safe to populate when OTel is disabled (a Set add/delete; the
+ * no-op meter never invokes the callbacks).
+ */
+import type { ObservableGauge, ObservableResult } from '@opentelemetry/api';
+import type { PersistenceQueueDepths } from './persistence.ts';
+import { getMeter, onTelemetryShutdown } from './telemetry.ts';
+
+const loadedDocsProviders = new Set<() => number>();
+const persistenceQueueProviders = new Set<() => PersistenceQueueDepths>();
+const bridgeDirtyProbes = new Set<() => boolean>();
+
+let cachedLoadedDocsGauge: ObservableGauge | null = null;
+let cachedQueueDepthGauge: ObservableGauge | null = null;
+let cachedDrainBacklogGauge: ObservableGauge | null = null;
+
+// Cached instruments drop on telemetry shutdown so the next install rebinds
+// against the fresh meter (same lifecycle contract as the server-memory
+// gauge). Provider registries survive — they mirror live server objects
+// whose lifecycle is owned by their register/unregister call sites.
+onTelemetryShutdown(() => {
+ cachedLoadedDocsGauge = null;
+ cachedQueueDepthGauge = null;
+ cachedDrainBacklogGauge = null;
+});
+
+/**
+ * Register a provider for the currently-loaded server-side Y.Doc count
+ * (typically `() => hocuspocus.documents.size`, which includes synthetic
+ * system/config docs). Returns an unregister function; call it at server
+ * teardown so the registry doesn't retain the closure.
+ */
+export function registerLoadedDocsProvider(provider: () => number): () => void {
+ loadedDocsProviders.add(provider);
+ return () => {
+ loadedDocsProviders.delete(provider);
+ };
+}
+
+/** Register a persistence queue-depth provider. Returns an unregister function. */
+export function registerPersistenceQueueDepthProvider(
+ provider: () => PersistenceQueueDepths,
+): () => void {
+ persistenceQueueProviders.add(provider);
+ return () => {
+ persistenceQueueProviders.delete(provider);
+ };
+}
+
+/**
+ * Register a per-doc probe reporting whether the doc's bridge observers hold
+ * an un-settled dirty flag (an `afterAllTransactions` settlement is owed).
+ * Returns an unregister function; the observer cleanup path must call it.
+ */
+export function registerBridgeDirtyProbe(probe: () => boolean): () => void {
+ bridgeDirtyProbes.add(probe);
+ return () => {
+ bridgeDirtyProbes.delete(probe);
+ };
+}
+
+/**
+ * Register the workload gauges against the currently-registered global meter.
+ * Idempotent — a second call is a no-op so a double boot can't
+ * double-register the callbacks. A throwing provider is skipped rather than
+ * propagated: instrumentation must never feed back into the write spine.
+ */
+export function installServerWorkloadGauges(): void {
+ if (cachedLoadedDocsGauge && cachedQueueDepthGauge && cachedDrainBacklogGauge) return;
+
+ if (!cachedLoadedDocsGauge) {
+ const gauge = getMeter().createObservableGauge('ok.server.docs.loaded', {
+ description:
+ 'Server-side Y.Docs currently resident in memory (includes synthetic system/config docs).',
+ unit: '{documents}',
+ });
+ gauge.addCallback((result: ObservableResult) => {
+ let total = 0;
+ for (const provider of loadedDocsProviders) {
+ try {
+ total += provider();
+ } catch {
+ // Skip a torn-down provider; the remaining sum stays meaningful.
+ }
+ }
+ result.observe(total);
+ });
+ cachedLoadedDocsGauge = gauge;
+ }
+
+ if (!cachedQueueDepthGauge) {
+ const gauge = getMeter().createObservableGauge('ok.persistence.queue.depth', {
+ description:
+ 'Pending persistence stores by queue. Bounded labels: queue ∈ {branch_deferred, quiescence_deferred}.',
+ unit: '{documents}',
+ });
+ gauge.addCallback((result: ObservableResult) => {
+ let branchDeferred = 0;
+ let quiescenceDeferred = 0;
+ for (const provider of persistenceQueueProviders) {
+ try {
+ const depths = provider();
+ branchDeferred += depths.branchDeferred;
+ quiescenceDeferred += depths.quiescenceDeferred;
+ } catch {
+ // Skip a torn-down provider; the remaining sum stays meaningful.
+ }
+ }
+ result.observe(branchDeferred, { queue: 'branch_deferred' });
+ result.observe(quiescenceDeferred, { queue: 'quiescence_deferred' });
+ });
+ cachedQueueDepthGauge = gauge;
+ }
+
+ if (!cachedDrainBacklogGauge) {
+ const gauge = getMeter().createObservableGauge('ok.bridge.drain_backlog', {
+ description:
+ 'Docs whose bridge observers hold an un-settled dirty flag at sample time (settlement is synchronous per drain, so a sustained non-zero value indicates a stuck drain).',
+ unit: '{documents}',
+ });
+ gauge.addCallback((result: ObservableResult) => {
+ let dirty = 0;
+ for (const probe of bridgeDirtyProbes) {
+ try {
+ if (probe()) dirty++;
+ } catch {
+ // Skip a torn-down probe; the remaining count stays meaningful.
+ }
+ }
+ result.observe(dirty);
+ });
+ cachedDrainBacklogGauge = gauge;
+ }
+}
+
+/** Drop cached instruments and clear registries. Test-only. */
+export function __resetServerWorkloadTelemetryForTests(): void {
+ cachedLoadedDocsGauge = null;
+ cachedQueueDepthGauge = null;
+ cachedDrainBacklogGauge = null;
+ loadedDocsProviders.clear();
+ persistenceQueueProviders.clear();
+ bridgeDirtyProbes.clear();
+}
diff --git a/packages/server/src/telemetry.test.ts b/packages/server/src/telemetry.test.ts
index 5325b01bf..46f4b9dc2 100644
--- a/packages/server/src/telemetry.test.ts
+++ b/packages/server/src/telemetry.test.ts
@@ -514,4 +514,81 @@ describe('Telemetry', () => {
}
});
});
+
+ describe('Sampler env contract (OTEL_TRACES_SAMPLER)', () => {
+ // BasicTracerProvider builds its sampler from the standard env vars when
+ // initTelemetry passes none (unset → parentbased_always_on, sample-all).
+ // These tests pin that contract so an SDK bump that drops env-config
+ // sampling — the 2.x line moved it into a shim layer — fails loudly
+ // instead of silently changing sampling behavior.
+ const SAMPLER_ENV_KEYS = ['OTEL_TRACES_SAMPLER', 'OTEL_TRACES_SAMPLER_ARG'] as const;
+
+ function withSamplerEnv(
+ env: Partial<Record<(typeof SAMPLER_ENV_KEYS)[number], string>>,
+ fn: () => void,
+ ): void {
+ const saved = SAMPLER_ENV_KEYS.map((k) => [k, process.env[k]] as const);
+ for (const k of SAMPLER_ENV_KEYS) delete process.env[k];
+ for (const [k, v] of Object.entries(env)) process.env[k] = v;
+ try {
+ fn();
+ } finally {
+ for (const [k, v] of saved) {
+ if (v === undefined) delete process.env[k];
+ else process.env[k] = v;
+ }
+ }
+ }
+
+ it('samples everything by default (env unset) — spans record and reach the file sink', () => {
+ withSamplerEnv({}, () => {
+ initTelemetry({ localSink: makeLocalSinkOpts(tmp) });
+ const span = getTracer().startSpan('sampler-default-span');
+ expect(span.isRecording()).toBe(true);
+ span.end();
+ });
+ });
+
+ it('honors OTEL_TRACES_SAMPLER=always_off — spans become non-recording and skip the file sink', async () => {
+ withSamplerEnv({ OTEL_TRACES_SAMPLER: 'always_off' }, () => {
+ initTelemetry({ localSink: makeLocalSinkOpts(tmp) });
+ const span = getTracer().startSpan('sampler-off-span');
+ expect(span.isRecording()).toBe(false);
+ span.end();
+ });
+ // Sampling runs at span creation, before any processor — a dropped
+ // span never reaches the ScrubbingSpanProcessor → file-sink chain, so
+ // nothing lands on disk. This is the bug-bundle implication the
+ // initTelemetry docstring documents.
+ await shutdownTelemetry();
+ const spansFile = spansCurrentPath(tmp);
+ if (existsSync(spansFile)) {
+ expect(readFileSync(spansFile, 'utf-8').trim()).toBe('');
+ }
+ });
+
+ it('honors OTEL_TRACES_SAMPLER=parentbased_traceidratio with ARG=0 (root spans dropped)', () => {
+ withSamplerEnv(
+ { OTEL_TRACES_SAMPLER: 'parentbased_traceidratio', OTEL_TRACES_SAMPLER_ARG: '0' },
+ () => {
+ initTelemetry({ localSink: makeLocalSinkOpts(tmp) });
+ const span = getTracer().startSpan('sampler-ratio-zero-span');
+ expect(span.isRecording()).toBe(false);
+ span.end();
+ },
+ );
+ });
+
+ it('honors OTEL_TRACES_SAMPLER=parentbased_traceidratio with ARG=1 (all root spans kept)', () => {
+ withSamplerEnv(
+ { OTEL_TRACES_SAMPLER: 'parentbased_traceidratio', OTEL_TRACES_SAMPLER_ARG: '1' },
+ () => {
+ initTelemetry({ localSink: makeLocalSinkOpts(tmp) });
+ const span = getTracer().startSpan('sampler-ratio-one-span');
+ expect(span.isRecording()).toBe(true);
+ span.end();
+ },
+ );
+ });
+ });
});
diff --git a/packages/server/src/telemetry.ts b/packages/server/src/telemetry.ts
index 9531aeee6..80e9cd918 100644
--- a/packages/server/src/telemetry.ts
+++ b/packages/server/src/telemetry.ts
@@ -89,6 +89,15 @@ export interface InitTelemetryOptions {
* JSONL form of aggregated time-series has low marginal signal for
* bug-report debugging).
*
+ * Sampling: the standard `OTEL_TRACES_SAMPLER` / `OTEL_TRACES_SAMPLER_ARG`
+ * env vars are honored by `BasicTracerProvider` itself (no sampler is passed
+ * here, so its constructor builds one from env; unset defaults to
+ * `parentbased_always_on`, i.e. sample-all). Sampling happens at span
+ * creation, BEFORE any processor runs — a ratio sampler therefore also thins
+ * the local file sink that `ok diagnose bundle` harvests. Leave the vars
+ * unset (the default) when complete bug-bundle span capture matters; the
+ * sampler-env contract is pinned by telemetry.test.ts.
+ *
* Idempotent — calling twice returns the same providers.
*
* Failure modes branch on which subsystem threw: