Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
feat(server): WS lifecycle gauges, trace-context extraction, MCP tool spans (#2797)
Andrew Mikofalvye1345c0a7aba
diff --git a/.changeset/ok-ws-mcp-telemetry.md b/.changeset/ok-ws-mcp-telemetry.mdnew file mode 100644index 000000000..28ed1bdc5--- /dev/null+++ b/.changeset/ok-ws-mcp-telemetry.md@@ -0,0 +1,9 @@+---+"@inkeep/open-knowledge": patch+---++Add WebSocket-lifecycle and MCP-tool observability to the server (all opt-in via OpenTelemetry, zero overhead when disabled):++- New observable gauges: `ok.ws.connections.active` (live collab connections, labeled `kind ∈ {websocket, direct}`), `ok.sessions.active`, and `ok.sessions.limit` — a session-cap stall now shows up as active pinned at the limit instead of undiagnosable 503s.+- The `traceparent` the browser already appends to the collab WebSocket URL is now extracted server-side, so the `sync.handshake` span parents to the user's browser trace instead of starting a disconnected root.+- Every MCP tool invocation now gets a `mcp.tool.<name>` span plus an `ok.mcp.tool.duration` histogram and `ok.mcp.tool.errors` counter (bounded tool-name labels), instrumented once at the dispatch spine — including the HTTP MCP endpoint, which previously had no per-tool instrumentation at all.diff --git a/packages/app/src/editor/collab-otel.ts b/packages/app/src/editor/collab-otel.tsindex 2a0d5d999..1b606e8a3 100644--- a/packages/app/src/editor/collab-otel.ts+++ b/packages/app/src/editor/collab-otel.ts@@ -2,11 +2,11 @@* Hocuspocus WebSocket ↔ OTel trace context bridge.** The browser's native WebSocket API cannot set headers (unlike fetch), so- * we inject the current trace context as a URL query parameter. The server's- * `onConnect` hook (packages/server/src/server-factory.ts, future wiring) extracts- * `traceparent` from `requestParameters` and attaches it to the session- * origin, so downstream spans (`persistence.onStoreDocument`,- * `shadow.commitWip`, etc.) can parent back to the original browser trace.+ * we inject the current trace context as a URL query parameter. The server+ * extracts `traceparent` / `tracestate` from `requestParameters` in its+ * sync-handshake extension (packages/server/src/sync-handshake-span-extension.ts),+ * so the server-side `sync.handshake` span parents back to the original+ * browser trace instead of starting a disconnected root.** This is additive — when OTel is disabled the helper returns the URL* unchanged.diff --git a/packages/server/src/agent-sessions.ts b/packages/server/src/agent-sessions.tsindex 7d3079030..6627a8b03 100644--- a/packages/server/src/agent-sessions.ts+++ b/packages/server/src/agent-sessions.ts@@ -655,6 +655,16 @@ export class AgentSessionManager {this.maxSessions = options.maxSessions ?? MAX_AGENT_SESSIONS;}+ /** Number of live sessions currently retained. Read-only occupancy probe. */+ public get liveSessionCount(): number {+ return this.sessions.size;+ }++ /** The hard cap `getSession` enforces (`MAX_AGENT_SESSIONS` unless overridden). */+ public get sessionLimit(): number {+ return this.maxSessions;+ }+private sessionKey(docName: string, agentId: string): string {return `${docName}\0${agentId}`;}diff --git a/packages/server/src/mcp/tool-logging.ts b/packages/server/src/mcp/tool-logging.tsindex fd7e6b88f..1555e3830 100644--- a/packages/server/src/mcp/tool-logging.ts+++ b/packages/server/src/mcp/tool-logging.ts@@ -1,5 +1,6 @@import type { AgentIdentity } from './agent-identity.ts';import { type McpLogger, runWithMcpLogger } from './logger.ts';+import { wrapToolHandlerForTelemetry } from './tool-telemetry.ts';import type { ServerInstance } from './tools/shared.ts';interface LoggedToolServerOptions {@@ -228,13 +229,19 @@ export function wrapToolHandlerForLogging(/*** Return a server instance whose `tool(...)` and `registerTool(...)`- * registration wraps handlers with request-aware structured logging.+ * registration wraps handlers with dispatch telemetry (span + duration/error+ * metrics; see tool-telemetry.ts) and, when a logger is supplied,+ * request-aware structured logging. Telemetry wrapping is unconditional —+ * the HTTP MCP endpoint registers without a logger but runs in the same+ * process as the OTel SDK, and skipping the wrap there would leave the one+ * telemetry-live registration path uninstrumented.*/export function createLoggedServer(server: ServerInstance,opts: LoggedToolServerOptions,): ServerInstance {- if (!opts.logger) return server;+ const instrument = (name: string, handler: AnyToolHandler): AnyToolHandler =>+ wrapToolHandlerForTelemetry(name, wrapToolHandlerForLogging(name, handler, opts));const originalTool = (server as unknown as { tool: AnyToolHandler }).tool.bind(server);// registerTool is the modern MCP SDK registration API; older mocks/test stubs may not@@ -252,11 +259,7 @@ export function createLoggedServer(return originalTool(...toolArgs);}const nextArgs = [...toolArgs];- nextArgs[nextArgs.length - 1] = wrapToolHandlerForLogging(- name,- handler as AnyToolHandler,- opts,- );+ nextArgs[nextArgs.length - 1] = instrument(name, handler as AnyToolHandler);return originalTool(...nextArgs);}) as AnyToolHandler;@@ -269,7 +272,7 @@ export function createLoggedServer(if (typeof cb !== 'function') {return originalRegisterTool(name, config, cb);}- const wrappedCb = wrapToolHandlerForLogging(name, cb, opts);+ const wrappedCb = instrument(name, cb);return originalRegisterTool(name, config, wrappedCb);}) as unknown as typeof server.registerTool;}diff --git a/packages/server/src/mcp/tool-telemetry.test.ts b/packages/server/src/mcp/tool-telemetry.test.tsnew file mode 100644index 000000000..94f819c60--- /dev/null+++ b/packages/server/src/mcp/tool-telemetry.test.ts@@ -0,0 +1,214 @@+/**+ * Tests for the MCP tool-dispatch telemetry wrapper.+ *+ * The wrapper sits at the registration spine (createLoggedServer) so every+ * tool gets a `mcp.tool.<name>` span, a duration histogram point, and error+ * counting — including the HTTP MCP endpoint, which registers without a+ * logger and would otherwise be the one uninstrumented telemetry-live path.+ *+ * Production wiring uses the OTLP exporter via `initTelemetry`; these+ * assertions use InMemory exporters purely as a unit-test capture surface,+ * mirroring the sync-handshake-span-extension.test.ts pattern.+ */+import { context, metrics, SpanStatusCode, trace } from '@opentelemetry/api';+import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';+import {+ AggregationTemporality,+ type DataPoint,+ InMemoryMetricExporter,+ MeterProvider,+ PeriodicExportingMetricReader,+} from '@opentelemetry/sdk-metrics';+import {+ BasicTracerProvider,+ InMemorySpanExporter,+ type ReadableSpan,+ SimpleSpanProcessor,+} from '@opentelemetry/sdk-trace-base';+import { afterEach, beforeEach, describe, expect, test } from 'vitest';+import { shutdownTelemetry } from '../telemetry.ts';+import { createLoggedServer } from './tool-logging.ts';+import { wrapToolHandlerForTelemetry } from './tool-telemetry.ts';++const DURATION_METRIC = 'ok.mcp.tool.duration';+const ERRORS_METRIC = 'ok.mcp.tool.errors';++let spanExporter: InMemorySpanExporter;+let tracerProvider: BasicTracerProvider;+let metricExporter: InMemoryMetricExporter;+let metricReader: PeriodicExportingMetricReader;+let meterProvider: MeterProvider;++beforeEach(async () => {+ // Drop instrument caches bound to any previous test's provider.+ await shutdownTelemetry();+ spanExporter = new InMemorySpanExporter();+ tracerProvider = new BasicTracerProvider({+ spanProcessors: [new SimpleSpanProcessor(spanExporter)],+ });+ trace.setGlobalTracerProvider(tracerProvider);+ context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());+ metricExporter = new InMemoryMetricExporter(AggregationTemporality.CUMULATIVE);+ metricReader = new PeriodicExportingMetricReader({+ exporter: metricExporter,+ exportIntervalMillis: 60_000,+ });+ meterProvider = new MeterProvider({ readers: [metricReader] });+ metrics.setGlobalMeterProvider(meterProvider);+});++afterEach(async () => {+ await tracerProvider.shutdown();+ await meterProvider.shutdown();+ trace.disable();+ metrics.disable();+ context.disable();+ await shutdownTelemetry();+});++function spansByName(name: string): ReadableSpan[] {+ return spanExporter.getFinishedSpans().filter((s) => s.name === name);+}++async function collectPoints(+ metricName: string,+): Promise<Array<{ value: unknown; attributes: Record<string, unknown> }>> {+ await metricReader.forceFlush();+ const out: Array<{ value: unknown; attributes: Record<string, unknown> }> = [];+ for (const rm of metricExporter.getMetrics()) {+ for (const sm of rm.scopeMetrics) {+ for (const metric of sm.metrics) {+ if (metric.descriptor.name !== metricName) continue;+ for (const dp of metric.dataPoints as Array<DataPoint<unknown>>) {+ out.push({ value: dp.value, attributes: dp.attributes });+ }+ }+ }+ }+ return out;+}++describe('wrapToolHandlerForTelemetry', () => {+ test('emits an mcp.tool.<name> span with a bounded tool-name attribute', async () => {+ const wrapped = wrapToolHandlerForTelemetry('write', async () => ({ content: [] }));+ await wrapped({ docName: 'notes/test' });+ const spans = spansByName('mcp.tool.write');+ expect(spans.length).toBe(1);+ expect(spans[0]?.attributes['mcp.tool.name']).toBe('write');+ expect(spans[0]?.status.code).not.toBe(SpanStatusCode.ERROR);+ });++ test('records a duration histogram point labeled by tool', async () => {+ const wrapped = wrapToolHandlerForTelemetry('search', async () => ({ content: [] }));+ await wrapped({ query: 'q' });+ const points = await collectPoints(DURATION_METRIC);+ expect(points.length).toBe(1);+ expect(points[0]?.attributes.tool).toBe('search');+ const histogramValue = points[0]?.value as { count: number };+ expect(histogramValue.count).toBe(1);+ });++ test('passes the result through unchanged and preserves invocation args', async () => {+ const seen: unknown[] = [];+ const result = { content: [{ type: 'text', text: 'ok' }] };+ const wrapped = wrapToolHandlerForTelemetry('exec', async (...args: unknown[]) => {+ seen.push(...args);+ return result;+ });+ const extra = { requestId: 'req-1' };+ await expect(wrapped({ command: 'ls' }, extra)).resolves.toBe(result);+ expect(seen).toEqual([{ command: 'ls' }, extra]);+ });++ test('counts an isError result as error_result and marks the span ERROR', async () => {+ const wrapped = wrapToolHandlerForTelemetry('edit', async () => ({+ isError: true,+ content: [{ type: 'text', text: 'no such doc' }],+ }));+ const result = (await wrapped({ docName: 'missing' })) as { isError: boolean };+ // The error result still flows back to the MCP client unchanged.+ expect(result.isError).toBe(true);++ const spans = spansByName('mcp.tool.edit');+ expect(spans[0]?.status.code).toBe(SpanStatusCode.ERROR);+ const errorPoints = await collectPoints(ERRORS_METRIC);+ expect(errorPoints.length).toBe(1);+ expect(errorPoints[0]?.attributes).toMatchObject({ tool: 'edit', kind: 'error_result' });+ expect(errorPoints[0]?.value).toBe(1);+ });++ test('counts a thrown error as exception, marks the span ERROR, and rethrows', async () => {+ const wrapped = wrapToolHandlerForTelemetry('move', async () => {+ throw new Error('boom');+ });+ await expect(wrapped({ from: 'a', to: 'b' })).rejects.toThrow('boom');++ const spans = spansByName('mcp.tool.move');+ expect(spans.length).toBe(1);+ expect(spans[0]?.status.code).toBe(SpanStatusCode.ERROR);+ const errorPoints = await collectPoints(ERRORS_METRIC);+ expect(errorPoints[0]?.attributes).toMatchObject({ tool: 'move', kind: 'exception' });+ // Duration still records on the throw path.+ const durationPoints = await collectPoints(DURATION_METRIC);+ expect(durationPoints[0]?.attributes.tool).toBe('move');+ });++ test('is a safe no-op passthrough when no OTel SDK is registered', async () => {+ trace.disable();+ metrics.disable();+ await shutdownTelemetry();+ const wrapped = wrapToolHandlerForTelemetry('write', async () => 'plain-result');+ await expect(wrapped({})).resolves.toBe('plain-result');+ });+});++describe('createLoggedServer telemetry wrapping', () => {+ test('wraps registerTool handlers even when no logger is supplied', async () => {+ // The HTTP MCP endpoint registers without a logger — the previous+ // logger-gated early return would have left it uninstrumented.+ let capturedHandler: ((...args: unknown[]) => unknown) | undefined;+ const fakeServer = {+ tool: () => 'legacy-registered',+ registerTool: (_name: string, _config: unknown, handler: (...args: unknown[]) => unknown) => {+ capturedHandler = handler;+ return 'registered-tool';+ },+ };++ const wrapped = createLoggedServer(fakeServer as never, {});+ const originalHandler = async () => ({ content: [] });+ (+ wrapped as unknown as {+ registerTool: (name: string, config: unknown, cb: unknown) => unknown;+ }+ ).registerTool('history', { description: 'desc' }, originalHandler);++ expect(capturedHandler).toBeDefined();+ expect(capturedHandler).not.toBe(originalHandler);+ await capturedHandler?.({ docName: 'notes/test' });+ expect(spansByName('mcp.tool.history').length).toBe(1);+ });++ test('wraps legacy tool() handlers without a logger', async () => {+ let capturedHandler: ((...args: unknown[]) => unknown) | undefined;+ const fakeServer = {+ tool: (...args: unknown[]) => {+ capturedHandler = args.at(-1) as (...args: unknown[]) => unknown;+ return 'registered';+ },+ };++ const wrapped = createLoggedServer(fakeServer as never, {});+ const originalHandler = async () => ({ content: [] });+ (wrapped as unknown as { tool: (...args: unknown[]) => unknown }).tool(+ 'palette',+ 'desc',+ {},+ originalHandler,+ );++ expect(capturedHandler).not.toBe(originalHandler);+ await capturedHandler?.({});+ expect(spansByName('mcp.tool.palette').length).toBe(1);+ });+});diff --git a/packages/server/src/mcp/tool-telemetry.ts b/packages/server/src/mcp/tool-telemetry.tsnew file mode 100644index 000000000..6bf0f5221--- /dev/null+++ b/packages/server/src/mcp/tool-telemetry.ts@@ -0,0 +1,90 @@+/**+ * OpenTelemetry instrumentation for MCP tool dispatch.+ *+ * One wrapper at the registration spine (`createLoggedServer` in+ * tool-logging.ts) instruments every tool without per-tool edits:+ *+ * - a `mcp.tool.<name>` span around each invocation+ * - `ok.mcp.tool.duration` histogram (ms)+ * - `ok.mcp.tool.errors` counter, split by `kind` (exception vs isError+ * result — MCP handlers report most failures as `isError: true` results+ * rather than throws, and a throw-only counter would miss them)+ *+ * The `tool` label is bounded: tool names are static string literals at the+ * registration sites in tools/index.ts (~21 names), never caller-supplied.+ * No doc names, paths, or argument content reach span attributes or metric+ * labels.+ *+ * Instruments live in the same process the tools run in. In the collab+ * server (`ok start` HTTP MCP endpoint) they bind to the provider+ * `initTelemetry` registered; in the stdio `ok mcp` proxy no SDK is+ * registered, so the no-op tracer/meter make every call here a no-op —+ * matching the zero-overhead-when-disabled contract.+ *+ * Cached instruments drop on telemetry shutdown so the next tool call+ * rebinds against the fresh meter (same lifecycle contract as+ * server-workload-telemetry.ts).+ */+import { type Counter, type Histogram, SpanStatusCode } from '@opentelemetry/api';+import { getMeter, onTelemetryShutdown, withSpan } from '../telemetry.ts';++type AnyToolHandler = (...args: unknown[]) => unknown;++let cachedDurationHistogram: Histogram | null = null;+let cachedErrorCounter: Counter | null = null;++onTelemetryShutdown(() => {+ cachedDurationHistogram = null;+ cachedErrorCounter = null;+});++function durationHistogram(): Histogram {+ cachedDurationHistogram ??= getMeter().createHistogram('ok.mcp.tool.duration', {+ description: 'MCP tool invocation duration. Bounded label: tool (static registry names).',+ unit: 'ms',+ });+ return cachedDurationHistogram;+}++function errorCounter(): Counter {+ cachedErrorCounter ??= getMeter().createCounter('ok.mcp.tool.errors', {+ description:+ 'MCP tool invocation failures. Bounded labels: tool (static registry names), kind ∈ {exception, error_result}.',+ unit: '{errors}',+ });+ return cachedErrorCounter;+}++function isErrorResult(result: unknown): boolean {+ return (+ typeof result === 'object' &&+ result !== null &&+ (result as { isError?: unknown }).isError === true+ );+}++/**+ * Wrap an MCP tool handler with a dispatch span + duration/error metrics.+ * Applied unconditionally at registration (unlike logging, which needs a+ * logger); with no OTel SDK registered every instrument is the API no-op.+ */+export function wrapToolHandlerForTelemetry(name: string, handler: AnyToolHandler): AnyToolHandler {+ return (...invocationArgs: unknown[]) =>+ withSpan(`mcp.tool.${name}`, { attributes: { 'mcp.tool.name': name } }, async (span) => {+ const startedAt = performance.now();+ try {+ const result = await handler(...invocationArgs);+ if (isErrorResult(result)) {+ span.setStatus({ code: SpanStatusCode.ERROR, message: 'tool returned isError result' });+ errorCounter().add(1, { tool: name, kind: 'error_result' });+ }+ return result;+ } catch (err) {+ // withSpan records the exception and sets ERROR status on rethrow.+ errorCounter().add(1, { tool: name, kind: 'exception' });+ throw err;+ } finally {+ durationHistogram().record(performance.now() - startedAt, { tool: name });+ }+ });+}diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.tsindex 0fce5b441..d7520d703 100644--- a/packages/server/src/server-factory.ts+++ b/packages/server/src/server-factory.ts@@ -172,6 +172,8 @@ import { createServerObserverExtension } from './server-observer-extension.ts';import type { PairedWriteOrigin } from './server-observers.ts';import {installServerWorkloadGauges,+ registerAgentSessionCountsProvider,+ registerConnectionCountsProvider,registerLoadedDocsProvider,registerPersistenceQueueDepthProvider,} from './server-workload-telemetry.ts';@@ -1270,6 +1272,16 @@ export function createServer(options: ServerOptions): ServerInstance {unregisterWorkloadProviders.push(registerLoadedDocsProvider(() => hp.documents.size),registerPersistenceQueueDepthProvider(() => persistence.getQueueDepths()),+ registerConnectionCountsProvider(() => {+ // Hocuspocus's own count is deduplicated sockets + DirectConnections;+ // subtracting the per-doc direct totals splits the two transports so+ // the gauge can label them (a socket attached to N docs counts once).+ let direct = 0;+ for (const doc of hp.documents.values()) {+ direct += doc.directConnectionsCount;+ }+ return { websocket: hp.getConnectionsCount() - direct, direct };+ }),);installServerWorkloadGauges();@@ -1421,6 +1433,16 @@ export function createServer(options: ServerOptions): ServerInstance {agentPresenceBroadcaster = new AgentPresenceBroadcaster(hocuspocus);sessionManager = new AgentSessionManager(hocuspocus);+ // Registered here (not with the connection provider above) because the+ // session manager doesn't exist yet at that point; gauges only sample at+ // metric-export time, so late registration is safe.+ const sm = sessionManager;+ unregisterWorkloadProviders.push(+ registerAgentSessionCountsProvider(() => ({+ active: sm.liveSessionCount,+ limit: sm.sessionLimit,+ })),+ );const liveDerivedIndexExtension = createLiveDerivedIndexExtension({backlinkIndex,tagIndex,diff --git a/packages/server/src/server-workload-telemetry.test.ts b/packages/server/src/server-workload-telemetry.test.tsindex d33c65992..d740928e7 100644--- a/packages/server/src/server-workload-telemetry.test.ts+++ b/packages/server/src/server-workload-telemetry.test.ts@@ -1,4 +1,3 @@-import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';import { metrics } from '@opentelemetry/api';import {AggregationTemporality,@@ -7,10 +6,13 @@ import {MeterProvider,PeriodicExportingMetricReader,} from '@opentelemetry/sdk-metrics';+import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'vitest';import {__resetServerWorkloadTelemetryForTests,installServerWorkloadGauges,+ registerAgentSessionCountsProvider,registerBridgeDirtyProbe,+ registerConnectionCountsProvider,registerLoadedDocsProvider,registerPersistenceQueueDepthProvider,} from './server-workload-telemetry.ts';@@ -18,6 +20,9 @@ import {const DOCS_METRIC = 'ok.server.docs.loaded';const QUEUE_METRIC = 'ok.persistence.queue.depth';const BACKLOG_METRIC = 'ok.bridge.drain_backlog';+const CONNECTIONS_METRIC = 'ok.ws.connections.active';+const SESSIONS_ACTIVE_METRIC = 'ok.sessions.active';+const SESSIONS_LIMIT_METRIC = 'ok.sessions.limit';describe('server workload telemetry — no-op meter (OTel disabled)', () => {test('install and register/unregister are safe with the default no-op meter', () => {@@ -32,9 +37,13 @@ describe('server workload telemetry — no-op meter (OTel disabled)', () => {quiescenceDeferred: 0,}));const un3 = registerBridgeDirtyProbe(() => false);+ const un4 = registerConnectionCountsProvider(() => ({ websocket: 1, direct: 2 }));+ const un5 = registerAgentSessionCountsProvider(() => ({ active: 0, limit: 256 }));un1();un2();un3();+ un4();+ un5();}).not.toThrow();__resetServerWorkloadTelemetryForTests();});@@ -126,4 +135,47 @@ describe('server workload telemetry — registered meter', () => {const points = await collect(DOCS_METRIC);expect(points[0].value).toBe(5);});++ test('ok.ws.connections.active reports both bounded kind labels and drops unregistered providers', async () => {+ const un = registerConnectionCountsProvider(() => ({ websocket: 3, direct: 5 }));+ registerConnectionCountsProvider(() => ({ websocket: 1, direct: 0 }));+ let points = await collect(CONNECTIONS_METRIC);+ let byKind = new Map(points.map((p) => [p.attributes.kind, p.value]));+ expect([...byKind.keys()].sort()).toEqual(['direct', 'websocket']);+ expect(byKind.get('websocket')).toBe(4);+ expect(byKind.get('direct')).toBe(5);++ un();+ points = await collect(CONNECTIONS_METRIC);+ byKind = new Map(points.map((p) => [p.attributes.kind, p.value]));+ expect(byKind.get('websocket')).toBe(1);+ expect(byKind.get('direct')).toBe(0);+ });++ test('ok.sessions.active and ok.sessions.limit track occupancy against the cap', async () => {+ let active = 2;+ registerAgentSessionCountsProvider(() => ({ active, limit: 256 }));+ let activePoints = await collect(SESSIONS_ACTIVE_METRIC);+ let limitPoints = await collect(SESSIONS_LIMIT_METRIC);+ expect(activePoints[0].value).toBe(2);+ expect(limitPoints[0].value).toBe(256);++ // A cap stall surfaces as active pinned at limit.+ active = 256;+ activePoints = await collect(SESSIONS_ACTIVE_METRIC);+ limitPoints = await collect(SESSIONS_LIMIT_METRIC);+ expect(activePoints[0].value).toBe(256);+ expect(activePoints[0].value).toBe(limitPoints[0].value);+ });++ test('a throwing session provider is skipped for both session gauges', async () => {+ registerAgentSessionCountsProvider(() => {+ throw new Error('torn down');+ });+ registerAgentSessionCountsProvider(() => ({ active: 7, limit: 256 }));+ const activePoints = await collect(SESSIONS_ACTIVE_METRIC);+ const limitPoints = await collect(SESSIONS_LIMIT_METRIC);+ expect(activePoints[0].value).toBe(7);+ expect(limitPoints[0].value).toBe(256);+ });});diff --git a/packages/server/src/server-workload-telemetry.ts b/packages/server/src/server-workload-telemetry.tsindex e12f5c247..df43bdd07 100644--- a/packages/server/src/server-workload-telemetry.ts+++ b/packages/server/src/server-workload-telemetry.ts@@ -1,6 +1,7 @@/*** Write-spine workload exposed as bounded OpenTelemetry observable gauges:- * loaded Y.Doc count, persistence queue depths, and bridge drain backlog.+ * loaded Y.Doc count, persistence queue depths, bridge drain backlog, live+ * collab connections, and agent-session occupancy vs the hard cap.** Pull-based by design: producers (server-factory, persistence, the server* observer bridge) register read-only provider closures here at construction@@ -14,13 +15,34 @@ import type { ObservableGauge, ObservableResult } from '@opentelemetry/api';import type { PersistenceQueueDepths } from './persistence.ts';import { getMeter, onTelemetryShutdown } from './telemetry.ts';+/** Live collab-connection counts by transport kind. */+export interface ConnectionCounts {+ /** Deduplicated client WebSocket sockets (one socket can attach many docs). */+ websocket: number;+ /** In-process DirectConnections (agent sessions + server-held system/config docs). */+ direct: number;+}++/** Agent-session occupancy against the server's hard cap. */+export interface AgentSessionCounts {+ /** Live (docName, agentId) sessions currently retained. */+ active: number;+ /** The session cap (`MAX_AGENT_SESSIONS` unless overridden). */+ limit: number;+}+const loadedDocsProviders = new Set<() => number>();const persistenceQueueProviders = new Set<() => PersistenceQueueDepths>();const bridgeDirtyProbes = new Set<() => boolean>();+const connectionCountsProviders = new Set<() => ConnectionCounts>();+const agentSessionCountsProviders = new Set<() => AgentSessionCounts>();let cachedLoadedDocsGauge: ObservableGauge | null = null;let cachedQueueDepthGauge: ObservableGauge | null = null;let cachedDrainBacklogGauge: ObservableGauge | null = null;+let cachedConnectionsGauge: ObservableGauge | null = null;+let cachedSessionsActiveGauge: ObservableGauge | null = null;+let cachedSessionsLimitGauge: ObservableGauge | null = null;// Cached instruments drop on telemetry shutdown so the next install rebinds// against the fresh meter (same lifecycle contract as the server-memory@@ -30,6 +52,9 @@ onTelemetryShutdown(() => {cachedLoadedDocsGauge = null;cachedQueueDepthGauge = null;cachedDrainBacklogGauge = null;+ cachedConnectionsGauge = null;+ cachedSessionsActiveGauge = null;+ cachedSessionsLimitGauge = null;});/**@@ -67,6 +92,32 @@ export function registerBridgeDirtyProbe(probe: () => boolean): () => void {};}+/**+ * Register a provider for live collab-connection counts (WebSocket sockets+ * + DirectConnections). Returns an unregister function; call it at server+ * teardown so the registry doesn't retain the closure.+ */+export function registerConnectionCountsProvider(provider: () => ConnectionCounts): () => void {+ connectionCountsProviders.add(provider);+ return () => {+ connectionCountsProviders.delete(provider);+ };+}++/**+ * Register a provider for agent-session occupancy (`AgentSessionManager`+ * live-session count + its hard cap). Returns an unregister function; call+ * it at server teardown. A session-cap stall (`getSession` throwing+ * `AgentSessionCapacityError` → 503s at the HTTP boundary) is invisible+ * without this — active pinned at the limit is the diagnostic signature.+ */+export function registerAgentSessionCountsProvider(provider: () => AgentSessionCounts): () => void {+ agentSessionCountsProviders.add(provider);+ return () => {+ agentSessionCountsProviders.delete(provider);+ };+}+/*** Register the workload gauges against the currently-registered global meter.* Idempotent — a second call is a no-op so a double boot can't@@ -74,7 +125,16 @@ export function registerBridgeDirtyProbe(probe: () => boolean): () => void {* propagated: instrumentation must never feed back into the write spine.*/export function installServerWorkloadGauges(): void {- if (cachedLoadedDocsGauge && cachedQueueDepthGauge && cachedDrainBacklogGauge) return;+ if (+ cachedLoadedDocsGauge &&+ cachedQueueDepthGauge &&+ cachedDrainBacklogGauge &&+ cachedConnectionsGauge &&+ cachedSessionsActiveGauge &&+ cachedSessionsLimitGauge+ ) {+ return;+ }if (!cachedLoadedDocsGauge) {const gauge = getMeter().createObservableGauge('ok.server.docs.loaded', {@@ -139,6 +199,67 @@ export function installServerWorkloadGauges(): void {});cachedDrainBacklogGauge = gauge;}++ if (!cachedConnectionsGauge) {+ const gauge = getMeter().createObservableGauge('ok.ws.connections.active', {+ description:+ 'Live collab connections. Bounded labels: kind ∈ {websocket, direct}. websocket = deduplicated client sockets; direct = in-process DirectConnections (agent sessions + server-held system/config docs).',+ unit: '{connections}',+ });+ gauge.addCallback((result: ObservableResult) => {+ let websocket = 0;+ let direct = 0;+ for (const provider of connectionCountsProviders) {+ try {+ const counts = provider();+ websocket += counts.websocket;+ direct += counts.direct;+ } catch {+ // Skip a torn-down provider; the remaining sum stays meaningful.+ }+ }+ result.observe(websocket, { kind: 'websocket' });+ result.observe(direct, { kind: 'direct' });+ });+ cachedConnectionsGauge = gauge;+ }++ if (!cachedSessionsActiveGauge || !cachedSessionsLimitGauge) {+ const activeGauge = getMeter().createObservableGauge('ok.sessions.active', {+ description:+ 'Live (docName, agentId) agent sessions. Pinned at ok.sessions.limit means new sessions are being refused with 503s.',+ unit: '{sessions}',+ });+ const limitGauge = getMeter().createObservableGauge('ok.sessions.limit', {+ description:+ 'Hard cap on live agent sessions (summed across server instances in this process).',+ unit: '{sessions}',+ });+ activeGauge.addCallback((result: ObservableResult) => {+ let active = 0;+ for (const provider of agentSessionCountsProviders) {+ try {+ active += provider().active;+ } catch {+ // Skip a torn-down provider; the remaining sum stays meaningful.+ }+ }+ result.observe(active);+ });+ limitGauge.addCallback((result: ObservableResult) => {+ let limit = 0;+ for (const provider of agentSessionCountsProviders) {+ try {+ limit += provider().limit;+ } catch {+ // Skip a torn-down provider; the remaining sum stays meaningful.+ }+ }+ result.observe(limit);+ });+ cachedSessionsActiveGauge = activeGauge;+ cachedSessionsLimitGauge = limitGauge;+ }}/** Drop cached instruments and clear registries. Test-only. */@@ -146,7 +267,12 @@ export function __resetServerWorkloadTelemetryForTests(): void {cachedLoadedDocsGauge = null;cachedQueueDepthGauge = null;cachedDrainBacklogGauge = null;+ cachedConnectionsGauge = null;+ cachedSessionsActiveGauge = null;+ cachedSessionsLimitGauge = null;loadedDocsProviders.clear();persistenceQueueProviders.clear();bridgeDirtyProbes.clear();+ connectionCountsProviders.clear();+ agentSessionCountsProviders.clear();}diff --git a/packages/server/src/sync-handshake-span-extension.test.ts b/packages/server/src/sync-handshake-span-extension.test.tsindex 839f6375a..ea97af5cb 100644--- a/packages/server/src/sync-handshake-span-extension.test.ts+++ b/packages/server/src/sync-handshake-span-extension.test.ts@@ -14,15 +14,16 @@** Skips system / config docs — those bypass the markdown bridge entirely.*/-import { afterEach, beforeEach, describe, expect, test } from 'bun:test';-import { context, metrics, trace } from '@opentelemetry/api';+import { context, metrics, propagation, trace } from '@opentelemetry/api';import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';+import { W3CTraceContextPropagator } from '@opentelemetry/core';import {BasicTracerProvider,InMemorySpanExporter,type ReadableSpan,SimpleSpanProcessor,} from '@opentelemetry/sdk-trace-base';+import { afterEach, beforeEach, describe, expect, test } from 'vitest';import * as Y from 'yjs';import { createSyncHandshakeSpanExtension } from './sync-handshake-span-extension.ts';@@ -36,6 +37,9 @@ function setupExporter(): void {});trace.setGlobalTracerProvider(provider);context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());+ // Mirrors initTelemetry's production registration — the extraction tests+ // exercise the same W3C propagator the server runs with.+ propagation.setGlobalPropagator(new W3CTraceContextPropagator());}async function teardownExporter(): Promise<void> {@@ -43,6 +47,7 @@ async function teardownExporter(): Promise<void> {trace.disable();metrics.disable();context.disable();+ propagation.disable();}function spansByName(name: string): ReadableSpan[] {@@ -59,9 +64,16 @@ function spansByName(name: string): ReadableSpan[] {// biome-ignore lint/suspicious/noExplicitAny: structural test payloadtype AfterLoadPayload = any;-function makePayload(opts: { documentName: string; mountId?: string }): AfterLoadPayload {+function makePayload(opts: {+ documentName: string;+ mountId?: string;+ traceparent?: string;+ tracestate?: string;+}): AfterLoadPayload {const params = new URLSearchParams();if (opts.mountId !== undefined) params.set('mountId', opts.mountId);+ if (opts.traceparent !== undefined) params.set('traceparent', opts.traceparent);+ if (opts.tracestate !== undefined) params.set('tracestate', opts.tracestate);return {documentName: opts.documentName,document: new Y.Doc(),@@ -177,6 +189,63 @@ describe('sync.handshake span — basic emission', () => {});});+describe('sync.handshake span — WS traceparent extraction', () => {+ // W3C trace-context values the browser's appendTraceContextToCollabUrl+ // would inject into the collab URL. Hardcoded so tests are deterministic.+ const TRACE_ID = '0af7651916cd43dd8448eb211c80319c';+ const PARENT_SPAN_ID = 'b7ad6b7169203331';+ const TRACEPARENT = `00-${TRACE_ID}-${PARENT_SPAN_ID}-01`;++ test('parents the span to the browser trace when traceparent is present', async () => {+ const extension = createSyncHandshakeSpanExtension();+ await extension.afterLoadDocument?.(+ makePayload({ documentName: 'README', mountId: UUID_A, traceparent: TRACEPARENT }),+ );+ const spans = spansByName('sync.handshake');+ expect(spans.length).toBe(1);+ const span = spans[0];+ expect(span?.spanContext().traceId).toBe(TRACE_ID);+ expect(span?.parentSpanContext?.spanId).toBe(PARENT_SPAN_ID);+ });++ test('carries tracestate through extraction alongside traceparent', async () => {+ const extension = createSyncHandshakeSpanExtension();+ await extension.afterLoadDocument?.(+ makePayload({+ documentName: 'README',+ traceparent: TRACEPARENT,+ tracestate: 'vendor=value',+ }),+ );+ const spans = spansByName('sync.handshake');+ expect(spans.length).toBe(1);+ // Same trace lineage; tracestate presence must not break extraction.+ expect(spans[0]?.spanContext().traceId).toBe(TRACE_ID);+ });++ test('starts a root span when no traceparent is supplied', async () => {+ const extension = createSyncHandshakeSpanExtension();+ await extension.afterLoadDocument?.(makePayload({ documentName: 'README' }));+ const spans = spansByName('sync.handshake');+ expect(spans.length).toBe(1);+ expect(spans[0]?.parentSpanContext).toBeUndefined();+ expect(spans[0]?.spanContext().traceId).not.toBe(TRACE_ID);+ });++ test('falls back to a root span on a malformed traceparent without throwing', async () => {+ // The W3C propagator ignores values it cannot parse — extract returns+ // the context unchanged, so the span becomes a root rather than failing+ // the handshake.+ const extension = createSyncHandshakeSpanExtension();+ await extension.afterLoadDocument?.(+ makePayload({ documentName: 'README', traceparent: 'garbage-not-a-traceparent' }),+ );+ const spans = spansByName('sync.handshake');+ expect(spans.length).toBe(1);+ expect(spans[0]?.parentSpanContext).toBeUndefined();+ });+});+describe('sync.handshake span — OTel SDK fault isolation', () => {test('OTel emission throw does not propagate out of afterLoadDocument', async () => {// The OTel API contract says start/end must not throw, but andiff --git a/packages/server/src/sync-handshake-span-extension.ts b/packages/server/src/sync-handshake-span-extension.tsindex a205c77a9..8fc611a74 100644--- a/packages/server/src/sync-handshake-span-extension.ts+++ b/packages/server/src/sync-handshake-span-extension.ts@@ -31,7 +31,7 @@* must not fail the sync handshake.*/import type { Extension } from '@hocuspocus/server';-import type { Attributes } from '@opentelemetry/api';+import { type Attributes, context as otelContext, propagation } from '@opentelemetry/api';import { isConfigDoc, isSystemDoc } from './cc1-broadcast.ts';import { getLogger } from './logger.ts';import { withSpanSync } from './telemetry.ts';@@ -64,8 +64,31 @@ export function createSyncHandshakeSpanExtension(): Extension {// `withSpanSync` (not `withSpan`) so afterLoadDocument doesn't pay an// awaited-Promise microtask hop per WebSocket connection even when// OTel is disabled and the span body is a no-op.+ //+ // The browser cannot set headers on a native WebSocket, so the+ // frontend appends `traceparent` / `tracestate` to the collab URL+ // (appendTraceContextToCollabUrl in the app's collab-otel.ts) and+ // they arrive here as query params. Extracting them parents this+ // span to the browser's trace instead of starting a disconnected+ // root. A malformed traceparent is ignored by the W3C propagator+ // (extract returns the context unchanged); with OTel disabled the+ // no-op propagator does the same, so the extra work is one small+ // object allocation on a once-per-doc-load path.try {- withSpanSync('sync.handshake', { attributes }, () => {});+ const emitSpan = (): void => {+ withSpanSync('sync.handshake', { attributes }, () => {});+ };+ const traceparent = requestParameters?.get('traceparent');+ if (traceparent !== null && traceparent !== undefined) {+ const carrier: Record<string, string> = { traceparent };+ const tracestate = requestParameters?.get('tracestate');+ if (tracestate !== null && tracestate !== undefined) {+ carrier.tracestate = tracestate;+ }+ otelContext.with(propagation.extract(otelContext.active(), carrier), emitSpan);+ } else {+ emitSpan();+ }} catch (err) {// Pass the Error through under `err` so the pino serializer keeps// its `.stack` — coercing to `.message` discards the call-frame