Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
feat(server): sampler wiring + runtime and queue-depth gauges (#2730)
Andrew Mikofalvye4c11f7c3037
diff --git a/.changeset/server-observability-quick-wins.md b/.changeset/server-observability-quick-wins.mdnew file mode 100644index 000000000..137ed8a59--- /dev/null+++ b/.changeset/server-observability-quick-wins.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++New operational metrics for cloud/server operators (all zero-overhead when telemetry is disabled): event-loop delay percentiles (`ok.server.event_loop.delay_ms`), CPU utilization (`ok.server.cpu.utilization`), extended memory sections (`external`, `array_buffers`) on `ok.server.memory.usage_megabytes`, a loaded-documents gauge (`ok.server.docs.loaded`), and write-spine queue depths (`ok.persistence.queue.depth`, `ok.bridge.drain_backlog`). Error responses now stamp their correlation UUID on the active trace span (`ok.error.instance`) so a client-reported error can be joined against its trace. The standard `OTEL_TRACES_SAMPLER` / `OTEL_TRACES_SAMPLER_ARG` sampling contract is now pinned by tests and documented, including its interaction with the local bug-bundle span sink.diff --git a/packages/server/src/boot.ts b/packages/server/src/boot.tsindex 8865647d5..28114d2da 100644--- a/packages/server/src/boot.ts+++ b/packages/server/src/boot.ts@@ -58,7 +58,7 @@ import { createMcpHttpHandler } from './mcp-http.ts';import { mountMcpAndApi } from './mcp-mount.ts';import { MissingOkConfigError } from './missing-ok-config-error.ts';import { createServer, type ServerInstance, type ServerOptions } from './server-factory.ts';-import { installServerMemoryGauge } from './server-memory-telemetry.ts';+import { installServerMemoryGauge, installServerRuntimeGauges } from './server-memory-telemetry.ts';import { reconcileSkillInstalls } from './skill-reconcile.ts';import { initTelemetry, shutdownTelemetry, withSpan } from './telemetry.ts';import {@@ -436,6 +436,7 @@ export async function bootServer(opts: BootServerOptions): Promise<BootedServer>// `.ok/local/` runtime-state root as the span/log sinks.initToleranceTelemetryWriter(sinkProjectDir);installServerMemoryGauge();+ installServerRuntimeGauges();// Wrap the orchestration in an `ok.boot` span so telemetry can slice// boot-failure rates by worktree kind. Computed here (not insidediff --git a/packages/server/src/http/error-response.test.ts b/packages/server/src/http/error-response.test.tsindex b52c04227..3b6f91978 100644--- a/packages/server/src/http/error-response.test.ts+++ b/packages/server/src/http/error-response.test.ts@@ -518,3 +518,49 @@ describe('createStreamingErrorWriter — writableEnded guard', () => {expect(writeCalls.length).toBe(1);});});++describe('errorResponse — trace correlation (ok.error.instance span attribute)', () => {+ test('stamps the minted instance UUID on the active span', async () => {+ // Requires a real context manager: setActiveSpanAttributes reads+ // trace.getSpan(context.active()), which the default no-op context+ // manager never populates.+ const { context: apiContext, trace } = await import('@opentelemetry/api');+ const { AsyncLocalStorageContextManager } = await import('@opentelemetry/context-async-hooks');+ const { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } = await import(+ '@opentelemetry/sdk-trace-base'+ );+ const exporter = new InMemorySpanExporter();+ const provider = new BasicTracerProvider({+ spanProcessors: [new SimpleSpanProcessor(exporter)],+ });+ trace.setGlobalTracerProvider(provider);+ apiContext.setGlobalContextManager(new AsyncLocalStorageContextManager());+ try {+ const { res, endCalls } = makeMockRes();+ trace.getTracer('test').startActiveSpan('request-span', (span) => {+ errorResponse(res, 500, 'urn:ok:error:internal-server-error', 'Boom.');+ span.end();+ });+ const spans = exporter.getFinishedSpans();+ expect(spans).toHaveLength(1);+ const stamped = spans[0].attributes['ok.error.instance'];+ expect(typeof stamped).toBe('string');+ // The stamped value must be the SAME correlation ID the client+ // received in the problem+json body — that equality is what makes+ // trace ↔ client-report joins possible.+ const body = JSON.parse(endCalls[0]);+ expect(stamped).toBe(body.instance);+ } finally {+ await provider.shutdown();+ trace.disable();+ apiContext.disable();+ }+ });++ test('is a no-op without an active span (no throw, response unchanged)', () => {+ const { res, writeHeadCalls, endCalls } = makeMockRes();+ expect(() => errorResponse(res, 404, 'urn:ok:error:not-found', 'Missing.')).not.toThrow();+ expect(writeHeadCalls.length).toBe(1);+ expect(JSON.parse(endCalls[0]).title).toBe('Missing.');+ });+});diff --git a/packages/server/src/http/error-response.ts b/packages/server/src/http/error-response.tsindex 0c37c8f9f..b3e11020e 100644--- a/packages/server/src/http/error-response.ts+++ b/packages/server/src/http/error-response.ts@@ -47,7 +47,7 @@ import {} from '@inkeep/open-knowledge-core';import type { Counter } from '@opentelemetry/api';import { getLogger } from '../logger.ts';-import { getMeter } from '../telemetry.ts';+import { getMeter, setActiveSpanAttributes } from '../telemetry.ts';// Lazy logger accessor — `loggerFactory.configure()` (used by capture-logger// tests in `server-factory.test.ts` and `logger.test.ts`) clears the@@ -166,6 +166,12 @@ export function errorResponse(// log-grep workflows still pattern-match a flat string.// `ProblemDetailsSchema.instance` in core validates the prefix shape.const instance = options.instance ?? `urn:uuid:${randomUUID()}`;+ // Mirror the correlation ID onto the active request span so a trace found+ // in Tempo can be joined against the `instance` a client reported (and+ // vice versa). No-op when no span is active or OTel is disabled. A UUID is+ // a sanctioned span attribute — the cardinality STOP rule governs metric+ // labels, and `ok.api.error.count` below stays UUID-free.+ setActiveSpanAttributes({ 'ok.error.instance': instance });// Defense-in-depth: if a handler has already started writing (async race,// programming error at any of the ~286 call sites), `res.writeHead()`diff --git a/packages/server/src/persistence.ts b/packages/server/src/persistence.tsindex cf7650195..444f9912f 100644--- a/packages/server/src/persistence.ts+++ b/packages/server/src/persistence.ts@@ -577,6 +577,19 @@ function toStoreFailure(err: unknown): StoreFailure {return { code, message };}+/**+ * Read-only snapshot of a persistence instance's pending-store depths, sampled+ * by the workload observable gauges at metric-export time. Owned here (the+ * domain module that owns those queues) rather than by the telemetry module+ * that consumes it, so telemetry depends on persistence and not the reverse.+ */+export interface PersistenceQueueDepths {+ /** Stores parked during a branch operation, awaiting replay. */+ readonly branchDeferred: number;+ /** Docs whose store cycle is currently deferring on the quiescence gate. */+ readonly quiescenceDeferred: number;+}+export interface PersistenceHandle {extension: Extension;flushDeferredStores: (mode?: 'within-branch' | 'discard-stale') => Promise<void>;@@ -617,6 +630,12 @@ export interface PersistenceHandle {*/flushContributors: () => Promise<void>;waitForPendingCommits: () => Promise<void>;+ /**+ * Read-only snapshot of this instance's pending-store depths, for the+ * workload observable gauges. Sampled at metric-export time only; never+ * mutates state.+ */+ getQueueDepths: () => PersistenceQueueDepths;/*** Config-doc persistence context. Exposed so the file-watcher* orchestration in `server-factory.ts` can call `applyExternalConfigChange`@@ -2355,6 +2374,13 @@ export function createPersistenceExtension(options?: PersistenceOptions): PersisagentWriteStores.add(documentName);}+ function getQueueDepths(): PersistenceQueueDepths {+ return {+ branchDeferred: deferredStores.size,+ quiescenceDeferred: persistenceDeferCounts.size,+ };+ }+return {extension,flushDeferredStores,@@ -2364,6 +2390,7 @@ export function createPersistenceExtension(options?: PersistenceOptions): PersistakeStoreFailure,takeStoreDivergence,markAgentWriteStore,+ getQueueDepths,configPersistenceCtx,managedArtifactCtx,};diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.tsindex 7413d3a22..816010342 100644--- a/packages/server/src/server-factory.ts+++ b/packages/server/src/server-factory.ts@@ -155,6 +155,11 @@ import {import { acquireServerLock, markServerLockDraining, releaseServerLock } from './server-lock.ts';import { createServerObserverExtension } from './server-observer-extension.ts';import type { PairedWriteOrigin } from './server-observers.ts';+import {+ installServerWorkloadGauges,+ registerLoadedDocsProvider,+ registerPersistenceQueueDepthProvider,+} from './server-workload-telemetry.ts';import { shadowOpGateFor } from './shadow-op-gate.ts';import {commitUpstreamImport,@@ -883,6 +888,10 @@ export function createServer(options: ServerOptions): ServerInstance {let loadedPrincipal: Principal | null = null;const forceUnloadSet = new Set<Document>();let shutdownAllowsUnload = false;+ // Unregister hooks for the pull-based workload gauge providers registered+ // in the init `try` below; called from destroy() so a torn-down instance+ // stops contributing to (and being retained by) the process-wide gauges.+ const unregisterWorkloadProviders: Array<() => void> = [];// Assigned synchronously in the init `try` immediately after `new Hocuspocus` (before the try// completes or awaits). Call sites (disk reconcile, API extension) only run after boot returns.let forceUnloadDocument!: (document: Document) => Promise<void>;@@ -1205,6 +1214,18 @@ export function createServer(options: ServerOptions): ServerInstance {extensions: [persistence.extension],});+ // Workload observable gauges (loaded docs, persistence queue depths,+ // bridge drain backlog). Providers are sampled only at metric-export+ // time; registering them is a Set add and safe when OTel is disabled.+ // Installed here (not just boot.ts) because the dev-plugin path enters+ // createServer directly; install is idempotent.+ const hp = hocuspocus;+ unregisterWorkloadProviders.push(+ registerLoadedDocsProvider(() => hp.documents.size),+ registerPersistenceQueueDepthProvider(() => persistence.getQueueDepths()),+ );+ installServerWorkloadGauges();+// Hocuspocus unloads documents as soon as the last WebSocket disconnects.// That is unsafe with client-side y-indexeddb: a browser refresh leaves a// durable client copy of the same Yjs items, while the server rebuilds a@@ -1667,6 +1688,15 @@ export function createServer(options: ServerOptions): ServerInstance {}),);} catch (err) {+ // Init failed after the workload providers may have registered. The caller+ // never receives an instance to call destroy() on, so drain the providers+ // here too — otherwise their closures (capturing `hocuspocus`/`persistence`)+ // leak into the process-global registries and keep summing off a+ // half-torn-down object. Idempotent via splice(0), so destroy()'s later+ // drain stays a no-op.+ for (const unregister of unregisterWorkloadProviders.splice(0)) {+ unregister();+ }releaseServerLock(lockDir);throw err;}@@ -2399,6 +2429,13 @@ export function createServer(options: ServerOptions): ServerInstance {const phaseErrors: Array<{ phase: string; error: string }> = [];shutdownAllowsUnload = true;+ // Stop contributing to the process-wide workload gauges before any+ // teardown phase runs — a mid-destroy sample would otherwise read+ // half-dismantled structures.+ for (const unregister of unregisterWorkloadProviders.splice(0)) {+ unregister();+ }+// Advertise teardown FIRST — before any flush work. Readers (MCP// discovery, desktop attach, spawners) see `draining: true` and stop// dialing our port or treating lock-presence as "serving"; supervisorsdiff --git a/packages/server/src/server-memory-telemetry.test.ts b/packages/server/src/server-memory-telemetry.test.tsindex 8872e4d1e..a5c83826e 100644--- a/packages/server/src/server-memory-telemetry.test.ts+++ b/packages/server/src/server-memory-telemetry.test.ts@@ -8,21 +8,26 @@ import {PeriodicExportingMetricReader,} from '@opentelemetry/sdk-metrics';import {- __resetServerMemoryGaugeForTests,+ __resetServerRuntimeTelemetryForTests,installServerMemoryGauge,+ installServerRuntimeGauges,} from './server-memory-telemetry.ts';const METRIC = 'ok.server.memory.usage_megabytes';+const EVENT_LOOP_METRIC = 'ok.server.event_loop.delay_ms';+const CPU_METRIC = 'ok.server.cpu.utilization';describe('installServerMemoryGauge — no-op meter (OTel disabled)', () => {test('does not throw and is idempotent with the default no-op meter', () => {metrics.disable();- __resetServerMemoryGaugeForTests();+ __resetServerRuntimeTelemetryForTests();expect(() => {installServerMemoryGauge();installServerMemoryGauge();+ installServerRuntimeGauges();+ installServerRuntimeGauges();}).not.toThrow();- __resetServerMemoryGaugeForTests();+ __resetServerRuntimeTelemetryForTests();});});@@ -36,25 +41,28 @@ describe('installServerMemoryGauge — registered meter', () => {reader = new PeriodicExportingMetricReader({ exporter, exportIntervalMillis: 60_000 });provider = new MeterProvider({ readers: [reader] });metrics.setGlobalMeterProvider(provider);- __resetServerMemoryGaugeForTests();+ __resetServerRuntimeTelemetryForTests();});afterAll(async () => {await provider.shutdown();metrics.disable();- __resetServerMemoryGaugeForTests();+ __resetServerRuntimeTelemetryForTests();});- async function collectSections(): Promise<Array<{ value: number; section: unknown }>> {+ async function collectPoints(+ metricName: string,+ label: string,+ ): Promise<Array<{ value: number; label: unknown }>> {exporter.reset();await reader.forceFlush();- const out: Array<{ value: number; section: unknown }> = [];+ const out: Array<{ value: number; label: unknown }> = [];for (const rm of exporter.getMetrics()) {for (const sm of rm.scopeMetrics) {for (const metric of sm.metrics) {- if (metric.descriptor.name !== METRIC) continue;+ if (metric.descriptor.name !== metricName) continue;for (const dp of metric.dataPoints as Array<DataPoint<number>>) {- out.push({ value: dp.value, section: dp.attributes.section });+ out.push({ value: dp.value, label: dp.attributes[label] });}}}@@ -62,18 +70,66 @@ describe('installServerMemoryGauge — registered meter', () => {return out;}- test('records exactly the three bounded sections with positive values', async () => {+ test('records exactly the five bounded sections', async () => {installServerMemoryGauge();- const points = await collectSections();- expect(points.map((p) => p.section).sort()).toEqual(['heap_total', 'heap_used', 'rss']);+ const points = await collectPoints(METRIC, 'section');+ expect(points.map((p) => p.label).sort()).toEqual([+ 'array_buffers',+ 'external',+ 'heap_total',+ 'heap_used',+ 'rss',+ ]);for (const p of points) {- expect(p.value).toBeGreaterThan(0);+ expect(p.value).toBeGreaterThanOrEqual(0);}+ // The heap sections are always non-zero in a live process; external and+ // array_buffers can legitimately be near-zero, hence the >= 0 floor above.+ const heapUsed = points.find((p) => p.label === 'heap_used');+ expect(heapUsed?.value).toBeGreaterThan(0);});test('is idempotent — a second install does not duplicate the series', async () => {installServerMemoryGauge();- const points = await collectSections();- expect(points).toHaveLength(3);+ const points = await collectPoints(METRIC, 'section');+ expect(points).toHaveLength(5);+ });++ test('event-loop gauge arms on first export and reports p50/p99 on the next', async () => {+ installServerRuntimeGauges();+ // First collection only enables the lazily-started histogram — the+ // export window has no data yet, so no points are observed.+ const first = await collectPoints(EVENT_LOOP_METRIC, 'stat');+ expect(first).toHaveLength(0);+ // Poll instead of sleeping a fixed interval: a heavily-loaded CI runner may+ // not iterate the event loop enough within a fixed wall-clock window. Each+ // forceFlush past the arming one observes p50/p99, so this converges to two+ // points as soon as the loop ticks the histogram once.+ let second: Awaited<ReturnType<typeof collectPoints>> = [];+ const deadline = Date.now() + 2000;+ while (second.length < 2 && Date.now() < deadline) {+ await new Promise((resolve) => setTimeout(resolve, 20));+ second = await collectPoints(EVENT_LOOP_METRIC, 'stat');+ }+ expect(second.map((p) => p.label).sort()).toEqual(['p50', 'p99']);+ for (const p of second) {+ expect(p.value).toBeGreaterThanOrEqual(0);+ expect(Number.isFinite(p.value)).toBe(true);+ }+ });++ test('cpu gauge records the baseline on first export and utilization on the next', async () => {+ installServerRuntimeGauges();+ // Two callbacks are required: the first records the baseline sample and+ // observes nothing; the second computes the delta and reports both modes.+ // Collect twice explicitly so the test does not depend on a prior test+ // having already driven the first callback via shared module-level state.+ await collectPoints(CPU_METRIC, 'mode');+ const second = await collectPoints(CPU_METRIC, 'mode');+ expect(second.map((p) => p.label).sort()).toEqual(['system', 'user']);+ for (const p of second) {+ expect(p.value).toBeGreaterThanOrEqual(0);+ expect(Number.isFinite(p.value)).toBe(true);+ }});});diff --git a/packages/server/src/server-memory-telemetry.ts b/packages/server/src/server-memory-telemetry.tsindex 650228e28..e8fddf827 100644--- a/packages/server/src/server-memory-telemetry.ts+++ b/packages/server/src/server-memory-telemetry.ts@@ -1,28 +1,54 @@/**- * Server process memory exposed as a bounded OpenTelemetry observable gauge.+ * Server process runtime health exposed as bounded OpenTelemetry observable+ * gauges: memory by section, event-loop delay percentiles, and CPU+ * utilization.** Server-side memory growth (e.g. a large `?showAll=true` walk) is invisible- * from the renderer. This gauge samples `process.memoryUsage()` — via the- * shared `captureServerMemorySnapshot()` helper — at each metric-export- * interval. Cardinality is fixed at three series through the `section` enum.+ * from the renderer. The memory gauge samples `process.memoryUsage()` — via+ * the shared `captureServerMemorySnapshot()` helper — at each metric-export+ * interval. Cardinality is fixed through the `section` enum.** Zero overhead when OTel is disabled: `getMeter()` returns the API no-op- * meter, whose observable gauge never invokes the callback.+ * meter, whose observable gauge never invokes the callback. The event-loop+ * delay histogram is likewise only enabled from inside the gauge callback,+ * so its constant sampling cost is paid only when a real meter is live.*/+import { monitorEventLoopDelay } from 'node:perf_hooks';import type { ObservableGauge, ObservableResult } from '@opentelemetry/api';import { captureServerMemorySnapshot } from './perf-measurement.ts';import { getMeter, onTelemetryShutdown } from './telemetry.ts';let cachedGauge: ObservableGauge | null = null;+let cachedEventLoopGauge: ObservableGauge | null = null;+let cachedCpuGauge: ObservableGauge | null = null;-// Drop the cached gauge whenever telemetry shuts down. The gauge is bound to-// the meter provider torn down by shutdownTelemetry; keeping the cache would-// make the next installServerMemoryGauge() a no-op (idempotency guard below) so-// the callback would never rebind to the freshly-initialized meter. Registered-// once at import — onTelemetryShutdown dedups, and a reset hook is cheaper than-// the alternative (telemetry.ts importing this module → circular).+const NANOS_PER_MS = 1e6;++/**+ * Lazily-enabled event-loop delay histogram. `monitorEventLoopDelay` costs a+ * periodic timer while enabled, so it is only started from inside the gauge+ * callback — which the no-op meter never invokes — preserving the+ * zero-overhead-when-disabled contract. Reset after each observation so each+ * export interval reports its own window rather than a since-boot aggregate.+ */+let eventLoopHistogram: ReturnType<typeof monitorEventLoopDelay> | null = null;++/** Previous CPU/wall sample for utilization deltas. Null until first callback. */+let lastCpuSample: { cpu: NodeJS.CpuUsage; hrtimeNs: bigint } | null = null;++// Drop the cached gauges whenever telemetry shuts down. The gauges are bound+// to the meter provider torn down by shutdownTelemetry; keeping the cache+// would make the next install call a no-op (idempotency guards below) so the+// callbacks would never rebind to the freshly-initialized meter. Registered+// once at import — onTelemetryShutdown dedups, and a reset hook is cheaper+// than the alternative (telemetry.ts importing this module → circular).onTelemetryShutdown(() => {cachedGauge = null;+ cachedEventLoopGauge = null;+ cachedCpuGauge = null;+ eventLoopHistogram?.disable();+ eventLoopHistogram = null;+ lastCpuSample = null;});/**@@ -34,7 +60,7 @@ export function installServerMemoryGauge(): void {if (cachedGauge) return;const gauge = getMeter().createObservableGauge('ok.server.memory.usage_megabytes', {description:- 'Server process memory by section. Bounded labels: section ∈ {heap_used, heap_total, rss}.',+ 'Server process memory by section. Bounded labels: section ∈ {heap_used, heap_total, rss, external, array_buffers}.',unit: 'MB',});gauge.addCallback((result: ObservableResult) => {@@ -42,11 +68,74 @@ export function installServerMemoryGauge(): void {result.observe(snapshot.heapUsedMb, { section: 'heap_used' });result.observe(snapshot.heapTotalMb, { section: 'heap_total' });result.observe(snapshot.rssMb, { section: 'rss' });+ result.observe(snapshot.externalMb, { section: 'external' });+ result.observe(snapshot.arrayBuffersMb, { section: 'array_buffers' });});cachedGauge = gauge;}-/** Drop the cached gauge so a test can rebind against a fresh meter. Test-only. */-export function __resetServerMemoryGaugeForTests(): void {+/**+ * Register the event-loop delay and CPU utilization gauges. Idempotent per+ * instrument, same lifecycle contract as {@link installServerMemoryGauge}.+ *+ * Event-loop delay reports p50/p99 in milliseconds over the window since the+ * previous export (the histogram resets after each read); the first callback+ * only enables the histogram, so the first export interval observes nothing.+ * CPU utilization reports user/system CPU time as a fraction of wall time+ * since the previous callback (may exceed 1 on multi-core parallelism); the+ * first callback only records the baseline sample.+ */+export function installServerRuntimeGauges(): void {+ if (!cachedEventLoopGauge) {+ const gauge = getMeter().createObservableGauge('ok.server.event_loop.delay_ms', {+ description:+ 'Event-loop delay percentiles over the last export window. Bounded labels: stat ∈ {p50, p99}.',+ unit: 'ms',+ });+ gauge.addCallback((result: ObservableResult) => {+ if (!eventLoopHistogram) {+ eventLoopHistogram = monitorEventLoopDelay({ resolution: 20 });+ eventLoopHistogram.enable();+ return;+ }+ result.observe(eventLoopHistogram.percentile(50) / NANOS_PER_MS, { stat: 'p50' });+ result.observe(eventLoopHistogram.percentile(99) / NANOS_PER_MS, { stat: 'p99' });+ eventLoopHistogram.reset();+ });+ cachedEventLoopGauge = gauge;+ }++ if (!cachedCpuGauge) {+ const gauge = getMeter().createObservableGauge('ok.server.cpu.utilization', {+ description:+ 'Process CPU time as a fraction of wall time since the previous export. Bounded labels: mode ∈ {user, system}.',+ unit: '1',+ });+ gauge.addCallback((result: ObservableResult) => {+ const cpu = process.cpuUsage();+ const hrtimeNs = process.hrtime.bigint();+ const prev = lastCpuSample;+ lastCpuSample = { cpu, hrtimeNs };+ if (!prev) return;+ const wallMicros = Number(hrtimeNs - prev.hrtimeNs) / 1_000;+ if (wallMicros <= 0) return;+ result.observe((cpu.user - prev.cpu.user) / wallMicros, { mode: 'user' });+ result.observe((cpu.system - prev.cpu.system) / wallMicros, { mode: 'system' });+ });+ cachedCpuGauge = gauge;+ }+}++/**+ * Drop every cached runtime instrument (memory, event-loop, CPU) plus the+ * event-loop histogram and CPU baseline so a test can rebind against a fresh+ * meter. Test-only.+ */+export function __resetServerRuntimeTelemetryForTests(): void {cachedGauge = null;+ cachedEventLoopGauge = null;+ cachedCpuGauge = null;+ eventLoopHistogram?.disable();+ eventLoopHistogram = null;+ lastCpuSample = null;}diff --git a/packages/server/src/server-observers.ts b/packages/server/src/server-observers.tsindex d7081b90c..688a4a0f4 100644--- a/packages/server/src/server-observers.ts+++ b/packages/server/src/server-observers.ts@@ -79,6 +79,7 @@ import {incrementServerObserverError,incrementServerObserverFire,} from './metrics.ts';+import { registerBridgeDirtyProbe } from './server-workload-telemetry.ts';import { type ShadowHandle, saveInMemoryCheckpoint } from './shadow-repo.ts';import { setActiveSpanAttributes, withSpanSync } from './telemetry.ts';@@ -2039,6 +2040,9 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => voidxmlFragment.observeDeep(observerA);ytext.observe(observerB);doc.on('afterAllTransactions', afterAll);+ // Pull-based backlog probe for the workload gauges: sampled only at+ // metric-export time, so the observer hot path stays untouched.+ const unregisterDirtyProbe = registerBridgeDirtyProbe(() => xmlDirty || textDirty);// Quiescence tracking lives in its own module to avoid `Date.now()` /// `setTimeout` here (precedent #13(b) — bridge-no-wallclock guard).// `attachQuiescenceTracker` hooks `afterTransaction` + `afterAllTransactions`@@ -2047,6 +2051,7 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void// ─── Cleanup ───────────────────────────────────────────────return () => {+ unregisterDirtyProbe();detachQuiescence();doc.off('afterAllTransactions', afterAll);xmlFragment.unobserveDeep(observerA);diff --git a/packages/server/src/server-workload-telemetry.test.ts b/packages/server/src/server-workload-telemetry.test.tsnew file mode 100644index 000000000..d33c65992--- /dev/null+++ b/packages/server/src/server-workload-telemetry.test.ts@@ -0,0 +1,129 @@+import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';+import { metrics } from '@opentelemetry/api';+import {+ AggregationTemporality,+ type DataPoint,+ InMemoryMetricExporter,+ MeterProvider,+ PeriodicExportingMetricReader,+} from '@opentelemetry/sdk-metrics';+import {+ __resetServerWorkloadTelemetryForTests,+ installServerWorkloadGauges,+ registerBridgeDirtyProbe,+ registerLoadedDocsProvider,+ registerPersistenceQueueDepthProvider,+} from './server-workload-telemetry.ts';++const DOCS_METRIC = 'ok.server.docs.loaded';+const QUEUE_METRIC = 'ok.persistence.queue.depth';+const BACKLOG_METRIC = 'ok.bridge.drain_backlog';++describe('server workload telemetry — no-op meter (OTel disabled)', () => {+ test('install and register/unregister are safe with the default no-op meter', () => {+ metrics.disable();+ __resetServerWorkloadTelemetryForTests();+ expect(() => {+ installServerWorkloadGauges();+ installServerWorkloadGauges();+ const un1 = registerLoadedDocsProvider(() => 3);+ const un2 = registerPersistenceQueueDepthProvider(() => ({+ branchDeferred: 0,+ quiescenceDeferred: 0,+ }));+ const un3 = registerBridgeDirtyProbe(() => false);+ un1();+ un2();+ un3();+ }).not.toThrow();+ __resetServerWorkloadTelemetryForTests();+ });+});++describe('server workload telemetry — registered meter', () => {+ let exporter: InMemoryMetricExporter;+ let reader: PeriodicExportingMetricReader;+ let provider: MeterProvider;++ beforeAll(() => {+ exporter = new InMemoryMetricExporter(AggregationTemporality.CUMULATIVE);+ reader = new PeriodicExportingMetricReader({ exporter, exportIntervalMillis: 60_000 });+ provider = new MeterProvider({ readers: [reader] });+ metrics.setGlobalMeterProvider(provider);+ });++ beforeEach(() => {+ __resetServerWorkloadTelemetryForTests();+ installServerWorkloadGauges();+ });++ afterAll(async () => {+ await provider.shutdown();+ metrics.disable();+ __resetServerWorkloadTelemetryForTests();+ });++ async function collect(+ metricName: string,+ ): Promise<Array<{ value: number; attributes: Record<string, unknown> }>> {+ exporter.reset();+ await reader.forceFlush();+ const out: Array<{ value: number; attributes: Record<string, unknown> }> = [];+ for (const rm of exporter.getMetrics()) {+ for (const sm of rm.scopeMetrics) {+ for (const metric of sm.metrics) {+ if (metric.descriptor.name !== metricName) continue;+ for (const dp of metric.dataPoints as Array<DataPoint<number>>) {+ out.push({ value: dp.value, attributes: dp.attributes });+ }+ }+ }+ }+ return out;+ }++ test('ok.server.docs.loaded sums registered providers and drops unregistered ones', async () => {+ const un1 = registerLoadedDocsProvider(() => 4);+ const un2 = registerLoadedDocsProvider(() => 2);+ let points = await collect(DOCS_METRIC);+ expect(points).toHaveLength(1);+ expect(points[0].value).toBe(6);++ un2();+ points = await collect(DOCS_METRIC);+ expect(points[0].value).toBe(4);+ un1();+ points = await collect(DOCS_METRIC);+ expect(points[0].value).toBe(0);+ });++ test('ok.persistence.queue.depth reports both bounded queue labels', async () => {+ registerPersistenceQueueDepthProvider(() => ({ branchDeferred: 3, quiescenceDeferred: 1 }));+ const points = await collect(QUEUE_METRIC);+ const byQueue = new Map(points.map((p) => [p.attributes.queue, p.value]));+ expect([...byQueue.keys()].sort()).toEqual(['branch_deferred', 'quiescence_deferred']);+ expect(byQueue.get('branch_deferred')).toBe(3);+ expect(byQueue.get('quiescence_deferred')).toBe(1);+ });++ test('ok.bridge.drain_backlog counts only probes reporting dirty', async () => {+ let dirtyA = false;+ registerBridgeDirtyProbe(() => dirtyA);+ registerBridgeDirtyProbe(() => true);+ let points = await collect(BACKLOG_METRIC);+ expect(points[0].value).toBe(1);++ dirtyA = true;+ points = await collect(BACKLOG_METRIC);+ expect(points[0].value).toBe(2);+ });++ test('a throwing provider is skipped without poisoning the remaining sum', async () => {+ registerLoadedDocsProvider(() => {+ throw new Error('torn down');+ });+ registerLoadedDocsProvider(() => 5);+ const points = await collect(DOCS_METRIC);+ expect(points[0].value).toBe(5);+ });+});diff --git a/packages/server/src/server-workload-telemetry.ts b/packages/server/src/server-workload-telemetry.tsnew file mode 100644index 000000000..e12f5c247--- /dev/null+++ b/packages/server/src/server-workload-telemetry.ts@@ -0,0 +1,152 @@+/**+ * Write-spine workload exposed as bounded OpenTelemetry observable gauges:+ * loaded Y.Doc count, persistence queue depths, and bridge drain backlog.+ *+ * Pull-based by design: producers (server-factory, persistence, the server+ * observer bridge) register read-only provider closures here at construction+ * and unregister at teardown; the gauge callbacks sample the providers only+ * at metric-export time. That keeps the write spine's hot paths untouched —+ * no counter increments inside observers or store hooks — and keeps the+ * registries safe to populate when OTel is disabled (a Set add/delete; the+ * no-op meter never invokes the callbacks).+ */+import type { ObservableGauge, ObservableResult } from '@opentelemetry/api';+import type { PersistenceQueueDepths } from './persistence.ts';+import { getMeter, onTelemetryShutdown } from './telemetry.ts';++const loadedDocsProviders = new Set<() => number>();+const persistenceQueueProviders = new Set<() => PersistenceQueueDepths>();+const bridgeDirtyProbes = new Set<() => boolean>();++let cachedLoadedDocsGauge: ObservableGauge | null = null;+let cachedQueueDepthGauge: ObservableGauge | null = null;+let cachedDrainBacklogGauge: ObservableGauge | null = null;++// Cached instruments drop on telemetry shutdown so the next install rebinds+// against the fresh meter (same lifecycle contract as the server-memory+// gauge). Provider registries survive — they mirror live server objects+// whose lifecycle is owned by their register/unregister call sites.+onTelemetryShutdown(() => {+ cachedLoadedDocsGauge = null;+ cachedQueueDepthGauge = null;+ cachedDrainBacklogGauge = null;+});++/**+ * Register a provider for the currently-loaded server-side Y.Doc count+ * (typically `() => hocuspocus.documents.size`, which includes synthetic+ * system/config docs). Returns an unregister function; call it at server+ * teardown so the registry doesn't retain the closure.+ */+export function registerLoadedDocsProvider(provider: () => number): () => void {+ loadedDocsProviders.add(provider);+ return () => {+ loadedDocsProviders.delete(provider);+ };+}++/** Register a persistence queue-depth provider. Returns an unregister function. */+export function registerPersistenceQueueDepthProvider(+ provider: () => PersistenceQueueDepths,+): () => void {+ persistenceQueueProviders.add(provider);+ return () => {+ persistenceQueueProviders.delete(provider);+ };+}++/**+ * Register a per-doc probe reporting whether the doc's bridge observers hold+ * an un-settled dirty flag (an `afterAllTransactions` settlement is owed).+ * Returns an unregister function; the observer cleanup path must call it.+ */+export function registerBridgeDirtyProbe(probe: () => boolean): () => void {+ bridgeDirtyProbes.add(probe);+ return () => {+ bridgeDirtyProbes.delete(probe);+ };+}++/**+ * Register the workload gauges against the currently-registered global meter.+ * Idempotent — a second call is a no-op so a double boot can't+ * double-register the callbacks. A throwing provider is skipped rather than+ * propagated: instrumentation must never feed back into the write spine.+ */+export function installServerWorkloadGauges(): void {+ if (cachedLoadedDocsGauge && cachedQueueDepthGauge && cachedDrainBacklogGauge) return;++ if (!cachedLoadedDocsGauge) {+ const gauge = getMeter().createObservableGauge('ok.server.docs.loaded', {+ description:+ 'Server-side Y.Docs currently resident in memory (includes synthetic system/config docs).',+ unit: '{documents}',+ });+ gauge.addCallback((result: ObservableResult) => {+ let total = 0;+ for (const provider of loadedDocsProviders) {+ try {+ total += provider();+ } catch {+ // Skip a torn-down provider; the remaining sum stays meaningful.+ }+ }+ result.observe(total);+ });+ cachedLoadedDocsGauge = gauge;+ }++ if (!cachedQueueDepthGauge) {+ const gauge = getMeter().createObservableGauge('ok.persistence.queue.depth', {+ description:+ 'Pending persistence stores by queue. Bounded labels: queue ∈ {branch_deferred, quiescence_deferred}.',+ unit: '{documents}',+ });+ gauge.addCallback((result: ObservableResult) => {+ let branchDeferred = 0;+ let quiescenceDeferred = 0;+ for (const provider of persistenceQueueProviders) {+ try {+ const depths = provider();+ branchDeferred += depths.branchDeferred;+ quiescenceDeferred += depths.quiescenceDeferred;+ } catch {+ // Skip a torn-down provider; the remaining sum stays meaningful.+ }+ }+ result.observe(branchDeferred, { queue: 'branch_deferred' });+ result.observe(quiescenceDeferred, { queue: 'quiescence_deferred' });+ });+ cachedQueueDepthGauge = gauge;+ }++ if (!cachedDrainBacklogGauge) {+ const gauge = getMeter().createObservableGauge('ok.bridge.drain_backlog', {+ description:+ 'Docs whose bridge observers hold an un-settled dirty flag at sample time (settlement is synchronous per drain, so a sustained non-zero value indicates a stuck drain).',+ unit: '{documents}',+ });+ gauge.addCallback((result: ObservableResult) => {+ let dirty = 0;+ for (const probe of bridgeDirtyProbes) {+ try {+ if (probe()) dirty++;+ } catch {+ // Skip a torn-down probe; the remaining count stays meaningful.+ }+ }+ result.observe(dirty);+ });+ cachedDrainBacklogGauge = gauge;+ }+}++/** Drop cached instruments and clear registries. Test-only. */+export function __resetServerWorkloadTelemetryForTests(): void {+ cachedLoadedDocsGauge = null;+ cachedQueueDepthGauge = null;+ cachedDrainBacklogGauge = null;+ loadedDocsProviders.clear();+ persistenceQueueProviders.clear();+ bridgeDirtyProbes.clear();+}diff --git a/packages/server/src/telemetry.test.ts b/packages/server/src/telemetry.test.tsindex 5325b01bf..46f4b9dc2 100644--- a/packages/server/src/telemetry.test.ts+++ b/packages/server/src/telemetry.test.ts@@ -514,4 +514,81 @@ describe('Telemetry', () => {}});});++ describe('Sampler env contract (OTEL_TRACES_SAMPLER)', () => {+ // BasicTracerProvider builds its sampler from the standard env vars when+ // initTelemetry passes none (unset → parentbased_always_on, sample-all).+ // These tests pin that contract so an SDK bump that drops env-config+ // sampling — the 2.x line moved it into a shim layer — fails loudly+ // instead of silently changing sampling behavior.+ const SAMPLER_ENV_KEYS = ['OTEL_TRACES_SAMPLER', 'OTEL_TRACES_SAMPLER_ARG'] as const;++ function withSamplerEnv(+ env: Partial<Record<(typeof SAMPLER_ENV_KEYS)[number], string>>,+ fn: () => void,+ ): void {+ const saved = SAMPLER_ENV_KEYS.map((k) => [k, process.env[k]] as const);+ for (const k of SAMPLER_ENV_KEYS) delete process.env[k];+ for (const [k, v] of Object.entries(env)) process.env[k] = v;+ try {+ fn();+ } finally {+ for (const [k, v] of saved) {+ if (v === undefined) delete process.env[k];+ else process.env[k] = v;+ }+ }+ }++ it('samples everything by default (env unset) — spans record and reach the file sink', () => {+ withSamplerEnv({}, () => {+ initTelemetry({ localSink: makeLocalSinkOpts(tmp) });+ const span = getTracer().startSpan('sampler-default-span');+ expect(span.isRecording()).toBe(true);+ span.end();+ });+ });++ it('honors OTEL_TRACES_SAMPLER=always_off — spans become non-recording and skip the file sink', async () => {+ withSamplerEnv({ OTEL_TRACES_SAMPLER: 'always_off' }, () => {+ initTelemetry({ localSink: makeLocalSinkOpts(tmp) });+ const span = getTracer().startSpan('sampler-off-span');+ expect(span.isRecording()).toBe(false);+ span.end();+ });+ // Sampling runs at span creation, before any processor — a dropped+ // span never reaches the ScrubbingSpanProcessor → file-sink chain, so+ // nothing lands on disk. This is the bug-bundle implication the+ // initTelemetry docstring documents.+ await shutdownTelemetry();+ const spansFile = spansCurrentPath(tmp);+ if (existsSync(spansFile)) {+ expect(readFileSync(spansFile, 'utf-8').trim()).toBe('');+ }+ });++ it('honors OTEL_TRACES_SAMPLER=parentbased_traceidratio with ARG=0 (root spans dropped)', () => {+ withSamplerEnv(+ { OTEL_TRACES_SAMPLER: 'parentbased_traceidratio', OTEL_TRACES_SAMPLER_ARG: '0' },+ () => {+ initTelemetry({ localSink: makeLocalSinkOpts(tmp) });+ const span = getTracer().startSpan('sampler-ratio-zero-span');+ expect(span.isRecording()).toBe(false);+ span.end();+ },+ );+ });++ it('honors OTEL_TRACES_SAMPLER=parentbased_traceidratio with ARG=1 (all root spans kept)', () => {+ withSamplerEnv(+ { OTEL_TRACES_SAMPLER: 'parentbased_traceidratio', OTEL_TRACES_SAMPLER_ARG: '1' },+ () => {+ initTelemetry({ localSink: makeLocalSinkOpts(tmp) });+ const span = getTracer().startSpan('sampler-ratio-one-span');+ expect(span.isRecording()).toBe(true);+ span.end();+ },+ );+ });+ });});diff --git a/packages/server/src/telemetry.ts b/packages/server/src/telemetry.tsindex 9531aeee6..80e9cd918 100644--- a/packages/server/src/telemetry.ts+++ b/packages/server/src/telemetry.ts@@ -89,6 +89,15 @@ export interface InitTelemetryOptions {* JSONL form of aggregated time-series has low marginal signal for* bug-report debugging).*+ * Sampling: the standard `OTEL_TRACES_SAMPLER` / `OTEL_TRACES_SAMPLER_ARG`+ * env vars are honored by `BasicTracerProvider` itself (no sampler is passed+ * here, so its constructor builds one from env; unset defaults to+ * `parentbased_always_on`, i.e. sample-all). Sampling happens at span+ * creation, BEFORE any processor runs — a ratio sampler therefore also thins+ * the local file sink that `ok diagnose bundle` harvests. Leave the vars+ * unset (the default) when complete bug-bundle span capture matters; the+ * sampler-env contract is pinned by telemetry.test.ts.+ ** Idempotent — calling twice returns the same providers.** Failure modes branch on which subsystem threw: