Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

feat(server): WS lifecycle gauges, trace-context extraction, MCP tool spans (#2797)

Andrew Mikofalvye1345c0a7aba
diff --git a/.changeset/ok-ws-mcp-telemetry.md b/.changeset/ok-ws-mcp-telemetry.md
new file mode 100644
index 000000000..28ed1bdc5
--- /dev/null
+++ b/.changeset/ok-ws-mcp-telemetry.md
@@ -0,0 +1,9 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Add WebSocket-lifecycle and MCP-tool observability to the server (all opt-in via OpenTelemetry, zero overhead when disabled):
+
+- New observable gauges: `ok.ws.connections.active` (live collab connections, labeled `kind ∈ {websocket, direct}`), `ok.sessions.active`, and `ok.sessions.limit` — a session-cap stall now shows up as active pinned at the limit instead of undiagnosable 503s.
+- The `traceparent` the browser already appends to the collab WebSocket URL is now extracted server-side, so the `sync.handshake` span parents to the user's browser trace instead of starting a disconnected root.
+- Every MCP tool invocation now gets a `mcp.tool.<name>` span plus an `ok.mcp.tool.duration` histogram and `ok.mcp.tool.errors` counter (bounded tool-name labels), instrumented once at the dispatch spine — including the HTTP MCP endpoint, which previously had no per-tool instrumentation at all.
diff --git a/packages/app/src/editor/collab-otel.ts b/packages/app/src/editor/collab-otel.ts
index 2a0d5d999..1b606e8a3 100644
--- a/packages/app/src/editor/collab-otel.ts
+++ b/packages/app/src/editor/collab-otel.ts
@@ -2,11 +2,11 @@
* Hocuspocus WebSocket ↔ OTel trace context bridge.
*
* The browser's native WebSocket API cannot set headers (unlike fetch), so
- * we inject the current trace context as a URL query parameter. The server's
- * `onConnect` hook (packages/server/src/server-factory.ts, future wiring) extracts
- * `traceparent` from `requestParameters` and attaches it to the session
- * origin, so downstream spans (`persistence.onStoreDocument`,
- * `shadow.commitWip`, etc.) can parent back to the original browser trace.
+ * we inject the current trace context as a URL query parameter. The server
+ * extracts `traceparent` / `tracestate` from `requestParameters` in its
+ * sync-handshake extension (packages/server/src/sync-handshake-span-extension.ts),
+ * so the server-side `sync.handshake` span parents back to the original
+ * browser trace instead of starting a disconnected root.
*
* This is additive — when OTel is disabled the helper returns the URL
* unchanged.
diff --git a/packages/server/src/agent-sessions.ts b/packages/server/src/agent-sessions.ts
index 7d3079030..6627a8b03 100644
--- a/packages/server/src/agent-sessions.ts
+++ b/packages/server/src/agent-sessions.ts
@@ -655,6 +655,16 @@ export class AgentSessionManager {
this.maxSessions = options.maxSessions ?? MAX_AGENT_SESSIONS;
}
+ /** Number of live sessions currently retained. Read-only occupancy probe. */
+ public get liveSessionCount(): number {
+ return this.sessions.size;
+ }
+
+ /** The hard cap `getSession` enforces (`MAX_AGENT_SESSIONS` unless overridden). */
+ public get sessionLimit(): number {
+ return this.maxSessions;
+ }
+
private sessionKey(docName: string, agentId: string): string {
return `${docName}\0${agentId}`;
}
diff --git a/packages/server/src/mcp/tool-logging.ts b/packages/server/src/mcp/tool-logging.ts
index fd7e6b88f..1555e3830 100644
--- a/packages/server/src/mcp/tool-logging.ts
+++ b/packages/server/src/mcp/tool-logging.ts
@@ -1,5 +1,6 @@
import type { AgentIdentity } from './agent-identity.ts';
import { type McpLogger, runWithMcpLogger } from './logger.ts';
+import { wrapToolHandlerForTelemetry } from './tool-telemetry.ts';
import type { ServerInstance } from './tools/shared.ts';
interface LoggedToolServerOptions {
@@ -228,13 +229,19 @@ export function wrapToolHandlerForLogging(
/**
* Return a server instance whose `tool(...)` and `registerTool(...)`
- * registration wraps handlers with request-aware structured logging.
+ * registration wraps handlers with dispatch telemetry (span + duration/error
+ * metrics; see tool-telemetry.ts) and, when a logger is supplied,
+ * request-aware structured logging. Telemetry wrapping is unconditional —
+ * the HTTP MCP endpoint registers without a logger but runs in the same
+ * process as the OTel SDK, and skipping the wrap there would leave the one
+ * telemetry-live registration path uninstrumented.
*/
export function createLoggedServer(
server: ServerInstance,
opts: LoggedToolServerOptions,
): ServerInstance {
- if (!opts.logger) return server;
+ const instrument = (name: string, handler: AnyToolHandler): AnyToolHandler =>
+ wrapToolHandlerForTelemetry(name, wrapToolHandlerForLogging(name, handler, opts));
const originalTool = (server as unknown as { tool: AnyToolHandler }).tool.bind(server);
// registerTool is the modern MCP SDK registration API; older mocks/test stubs may not
@@ -252,11 +259,7 @@ export function createLoggedServer(
return originalTool(...toolArgs);
}
const nextArgs = [...toolArgs];
- nextArgs[nextArgs.length - 1] = wrapToolHandlerForLogging(
- name,
- handler as AnyToolHandler,
- opts,
- );
+ nextArgs[nextArgs.length - 1] = instrument(name, handler as AnyToolHandler);
return originalTool(...nextArgs);
}) as AnyToolHandler;
@@ -269,7 +272,7 @@ export function createLoggedServer(
if (typeof cb !== 'function') {
return originalRegisterTool(name, config, cb);
}
- const wrappedCb = wrapToolHandlerForLogging(name, cb, opts);
+ const wrappedCb = instrument(name, cb);
return originalRegisterTool(name, config, wrappedCb);
}) as unknown as typeof server.registerTool;
}
diff --git a/packages/server/src/mcp/tool-telemetry.test.ts b/packages/server/src/mcp/tool-telemetry.test.ts
new file mode 100644
index 000000000..94f819c60
--- /dev/null
+++ b/packages/server/src/mcp/tool-telemetry.test.ts
@@ -0,0 +1,214 @@
+/**
+ * Tests for the MCP tool-dispatch telemetry wrapper.
+ *
+ * The wrapper sits at the registration spine (createLoggedServer) so every
+ * tool gets a `mcp.tool.<name>` span, a duration histogram point, and error
+ * counting — including the HTTP MCP endpoint, which registers without a
+ * logger and would otherwise be the one uninstrumented telemetry-live path.
+ *
+ * Production wiring uses the OTLP exporter via `initTelemetry`; these
+ * assertions use InMemory exporters purely as a unit-test capture surface,
+ * mirroring the sync-handshake-span-extension.test.ts pattern.
+ */
+import { context, metrics, SpanStatusCode, trace } from '@opentelemetry/api';
+import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
+import {
+ AggregationTemporality,
+ type DataPoint,
+ InMemoryMetricExporter,
+ MeterProvider,
+ PeriodicExportingMetricReader,
+} from '@opentelemetry/sdk-metrics';
+import {
+ BasicTracerProvider,
+ InMemorySpanExporter,
+ type ReadableSpan,
+ SimpleSpanProcessor,
+} from '@opentelemetry/sdk-trace-base';
+import { afterEach, beforeEach, describe, expect, test } from 'vitest';
+import { shutdownTelemetry } from '../telemetry.ts';
+import { createLoggedServer } from './tool-logging.ts';
+import { wrapToolHandlerForTelemetry } from './tool-telemetry.ts';
+
+const DURATION_METRIC = 'ok.mcp.tool.duration';
+const ERRORS_METRIC = 'ok.mcp.tool.errors';
+
+let spanExporter: InMemorySpanExporter;
+let tracerProvider: BasicTracerProvider;
+let metricExporter: InMemoryMetricExporter;
+let metricReader: PeriodicExportingMetricReader;
+let meterProvider: MeterProvider;
+
+beforeEach(async () => {
+ // Drop instrument caches bound to any previous test's provider.
+ await shutdownTelemetry();
+ spanExporter = new InMemorySpanExporter();
+ tracerProvider = new BasicTracerProvider({
+ spanProcessors: [new SimpleSpanProcessor(spanExporter)],
+ });
+ trace.setGlobalTracerProvider(tracerProvider);
+ context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());
+ metricExporter = new InMemoryMetricExporter(AggregationTemporality.CUMULATIVE);
+ metricReader = new PeriodicExportingMetricReader({
+ exporter: metricExporter,
+ exportIntervalMillis: 60_000,
+ });
+ meterProvider = new MeterProvider({ readers: [metricReader] });
+ metrics.setGlobalMeterProvider(meterProvider);
+});
+
+afterEach(async () => {
+ await tracerProvider.shutdown();
+ await meterProvider.shutdown();
+ trace.disable();
+ metrics.disable();
+ context.disable();
+ await shutdownTelemetry();
+});
+
+function spansByName(name: string): ReadableSpan[] {
+ return spanExporter.getFinishedSpans().filter((s) => s.name === name);
+}
+
+async function collectPoints(
+ metricName: string,
+): Promise<Array<{ value: unknown; attributes: Record<string, unknown> }>> {
+ await metricReader.forceFlush();
+ const out: Array<{ value: unknown; attributes: Record<string, unknown> }> = [];
+ for (const rm of metricExporter.getMetrics()) {
+ for (const sm of rm.scopeMetrics) {
+ for (const metric of sm.metrics) {
+ if (metric.descriptor.name !== metricName) continue;
+ for (const dp of metric.dataPoints as Array<DataPoint<unknown>>) {
+ out.push({ value: dp.value, attributes: dp.attributes });
+ }
+ }
+ }
+ }
+ return out;
+}
+
+describe('wrapToolHandlerForTelemetry', () => {
+ test('emits an mcp.tool.<name> span with a bounded tool-name attribute', async () => {
+ const wrapped = wrapToolHandlerForTelemetry('write', async () => ({ content: [] }));
+ await wrapped({ docName: 'notes/test' });
+ const spans = spansByName('mcp.tool.write');
+ expect(spans.length).toBe(1);
+ expect(spans[0]?.attributes['mcp.tool.name']).toBe('write');
+ expect(spans[0]?.status.code).not.toBe(SpanStatusCode.ERROR);
+ });
+
+ test('records a duration histogram point labeled by tool', async () => {
+ const wrapped = wrapToolHandlerForTelemetry('search', async () => ({ content: [] }));
+ await wrapped({ query: 'q' });
+ const points = await collectPoints(DURATION_METRIC);
+ expect(points.length).toBe(1);
+ expect(points[0]?.attributes.tool).toBe('search');
+ const histogramValue = points[0]?.value as { count: number };
+ expect(histogramValue.count).toBe(1);
+ });
+
+ test('passes the result through unchanged and preserves invocation args', async () => {
+ const seen: unknown[] = [];
+ const result = { content: [{ type: 'text', text: 'ok' }] };
+ const wrapped = wrapToolHandlerForTelemetry('exec', async (...args: unknown[]) => {
+ seen.push(...args);
+ return result;
+ });
+ const extra = { requestId: 'req-1' };
+ await expect(wrapped({ command: 'ls' }, extra)).resolves.toBe(result);
+ expect(seen).toEqual([{ command: 'ls' }, extra]);
+ });
+
+ test('counts an isError result as error_result and marks the span ERROR', async () => {
+ const wrapped = wrapToolHandlerForTelemetry('edit', async () => ({
+ isError: true,
+ content: [{ type: 'text', text: 'no such doc' }],
+ }));
+ const result = (await wrapped({ docName: 'missing' })) as { isError: boolean };
+ // The error result still flows back to the MCP client unchanged.
+ expect(result.isError).toBe(true);
+
+ const spans = spansByName('mcp.tool.edit');
+ expect(spans[0]?.status.code).toBe(SpanStatusCode.ERROR);
+ const errorPoints = await collectPoints(ERRORS_METRIC);
+ expect(errorPoints.length).toBe(1);
+ expect(errorPoints[0]?.attributes).toMatchObject({ tool: 'edit', kind: 'error_result' });
+ expect(errorPoints[0]?.value).toBe(1);
+ });
+
+ test('counts a thrown error as exception, marks the span ERROR, and rethrows', async () => {
+ const wrapped = wrapToolHandlerForTelemetry('move', async () => {
+ throw new Error('boom');
+ });
+ await expect(wrapped({ from: 'a', to: 'b' })).rejects.toThrow('boom');
+
+ const spans = spansByName('mcp.tool.move');
+ expect(spans.length).toBe(1);
+ expect(spans[0]?.status.code).toBe(SpanStatusCode.ERROR);
+ const errorPoints = await collectPoints(ERRORS_METRIC);
+ expect(errorPoints[0]?.attributes).toMatchObject({ tool: 'move', kind: 'exception' });
+ // Duration still records on the throw path.
+ const durationPoints = await collectPoints(DURATION_METRIC);
+ expect(durationPoints[0]?.attributes.tool).toBe('move');
+ });
+
+ test('is a safe no-op passthrough when no OTel SDK is registered', async () => {
+ trace.disable();
+ metrics.disable();
+ await shutdownTelemetry();
+ const wrapped = wrapToolHandlerForTelemetry('write', async () => 'plain-result');
+ await expect(wrapped({})).resolves.toBe('plain-result');
+ });
+});
+
+describe('createLoggedServer telemetry wrapping', () => {
+ test('wraps registerTool handlers even when no logger is supplied', async () => {
+ // The HTTP MCP endpoint registers without a logger — the previous
+ // logger-gated early return would have left it uninstrumented.
+ let capturedHandler: ((...args: unknown[]) => unknown) | undefined;
+ const fakeServer = {
+ tool: () => 'legacy-registered',
+ registerTool: (_name: string, _config: unknown, handler: (...args: unknown[]) => unknown) => {
+ capturedHandler = handler;
+ return 'registered-tool';
+ },
+ };
+
+ const wrapped = createLoggedServer(fakeServer as never, {});
+ const originalHandler = async () => ({ content: [] });
+ (
+ wrapped as unknown as {
+ registerTool: (name: string, config: unknown, cb: unknown) => unknown;
+ }
+ ).registerTool('history', { description: 'desc' }, originalHandler);
+
+ expect(capturedHandler).toBeDefined();
+ expect(capturedHandler).not.toBe(originalHandler);
+ await capturedHandler?.({ docName: 'notes/test' });
+ expect(spansByName('mcp.tool.history').length).toBe(1);
+ });
+
+ test('wraps legacy tool() handlers without a logger', async () => {
+ let capturedHandler: ((...args: unknown[]) => unknown) | undefined;
+ const fakeServer = {
+ tool: (...args: unknown[]) => {
+ capturedHandler = args.at(-1) as (...args: unknown[]) => unknown;
+ return 'registered';
+ },
+ };
+
+ const wrapped = createLoggedServer(fakeServer as never, {});
+ const originalHandler = async () => ({ content: [] });
+ (wrapped as unknown as { tool: (...args: unknown[]) => unknown }).tool(
+ 'palette',
+ 'desc',
+ {},
+ originalHandler,
+ );
+
+ expect(capturedHandler).not.toBe(originalHandler);
+ await capturedHandler?.({});
+ expect(spansByName('mcp.tool.palette').length).toBe(1);
+ });
+});
diff --git a/packages/server/src/mcp/tool-telemetry.ts b/packages/server/src/mcp/tool-telemetry.ts
new file mode 100644
index 000000000..6bf0f5221
--- /dev/null
+++ b/packages/server/src/mcp/tool-telemetry.ts
@@ -0,0 +1,90 @@
+/**
+ * OpenTelemetry instrumentation for MCP tool dispatch.
+ *
+ * One wrapper at the registration spine (`createLoggedServer` in
+ * tool-logging.ts) instruments every tool without per-tool edits:
+ *
+ * - a `mcp.tool.<name>` span around each invocation
+ * - `ok.mcp.tool.duration` histogram (ms)
+ * - `ok.mcp.tool.errors` counter, split by `kind` (exception vs isError
+ * result — MCP handlers report most failures as `isError: true` results
+ * rather than throws, and a throw-only counter would miss them)
+ *
+ * The `tool` label is bounded: tool names are static string literals at the
+ * registration sites in tools/index.ts (~21 names), never caller-supplied.
+ * No doc names, paths, or argument content reach span attributes or metric
+ * labels.
+ *
+ * Instruments live in the same process the tools run in. In the collab
+ * server (`ok start` HTTP MCP endpoint) they bind to the provider
+ * `initTelemetry` registered; in the stdio `ok mcp` proxy no SDK is
+ * registered, so the no-op tracer/meter make every call here a no-op —
+ * matching the zero-overhead-when-disabled contract.
+ *
+ * Cached instruments drop on telemetry shutdown so the next tool call
+ * rebinds against the fresh meter (same lifecycle contract as
+ * server-workload-telemetry.ts).
+ */
+import { type Counter, type Histogram, SpanStatusCode } from '@opentelemetry/api';
+import { getMeter, onTelemetryShutdown, withSpan } from '../telemetry.ts';
+
+type AnyToolHandler = (...args: unknown[]) => unknown;
+
+let cachedDurationHistogram: Histogram | null = null;
+let cachedErrorCounter: Counter | null = null;
+
+onTelemetryShutdown(() => {
+ cachedDurationHistogram = null;
+ cachedErrorCounter = null;
+});
+
+function durationHistogram(): Histogram {
+ cachedDurationHistogram ??= getMeter().createHistogram('ok.mcp.tool.duration', {
+ description: 'MCP tool invocation duration. Bounded label: tool (static registry names).',
+ unit: 'ms',
+ });
+ return cachedDurationHistogram;
+}
+
+function errorCounter(): Counter {
+ cachedErrorCounter ??= getMeter().createCounter('ok.mcp.tool.errors', {
+ description:
+ 'MCP tool invocation failures. Bounded labels: tool (static registry names), kind ∈ {exception, error_result}.',
+ unit: '{errors}',
+ });
+ return cachedErrorCounter;
+}
+
+function isErrorResult(result: unknown): boolean {
+ return (
+ typeof result === 'object' &&
+ result !== null &&
+ (result as { isError?: unknown }).isError === true
+ );
+}
+
+/**
+ * Wrap an MCP tool handler with a dispatch span + duration/error metrics.
+ * Applied unconditionally at registration (unlike logging, which needs a
+ * logger); with no OTel SDK registered every instrument is the API no-op.
+ */
+export function wrapToolHandlerForTelemetry(name: string, handler: AnyToolHandler): AnyToolHandler {
+ return (...invocationArgs: unknown[]) =>
+ withSpan(`mcp.tool.${name}`, { attributes: { 'mcp.tool.name': name } }, async (span) => {
+ const startedAt = performance.now();
+ try {
+ const result = await handler(...invocationArgs);
+ if (isErrorResult(result)) {
+ span.setStatus({ code: SpanStatusCode.ERROR, message: 'tool returned isError result' });
+ errorCounter().add(1, { tool: name, kind: 'error_result' });
+ }
+ return result;
+ } catch (err) {
+ // withSpan records the exception and sets ERROR status on rethrow.
+ errorCounter().add(1, { tool: name, kind: 'exception' });
+ throw err;
+ } finally {
+ durationHistogram().record(performance.now() - startedAt, { tool: name });
+ }
+ });
+}
diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.ts
index 0fce5b441..d7520d703 100644
--- a/packages/server/src/server-factory.ts
+++ b/packages/server/src/server-factory.ts
@@ -172,6 +172,8 @@ import { createServerObserverExtension } from './server-observer-extension.ts';
import type { PairedWriteOrigin } from './server-observers.ts';
import {
installServerWorkloadGauges,
+ registerAgentSessionCountsProvider,
+ registerConnectionCountsProvider,
registerLoadedDocsProvider,
registerPersistenceQueueDepthProvider,
} from './server-workload-telemetry.ts';
@@ -1270,6 +1272,16 @@ export function createServer(options: ServerOptions): ServerInstance {
unregisterWorkloadProviders.push(
registerLoadedDocsProvider(() => hp.documents.size),
registerPersistenceQueueDepthProvider(() => persistence.getQueueDepths()),
+ registerConnectionCountsProvider(() => {
+ // Hocuspocus's own count is deduplicated sockets + DirectConnections;
+ // subtracting the per-doc direct totals splits the two transports so
+ // the gauge can label them (a socket attached to N docs counts once).
+ let direct = 0;
+ for (const doc of hp.documents.values()) {
+ direct += doc.directConnectionsCount;
+ }
+ return { websocket: hp.getConnectionsCount() - direct, direct };
+ }),
);
installServerWorkloadGauges();
@@ -1421,6 +1433,16 @@ export function createServer(options: ServerOptions): ServerInstance {
agentPresenceBroadcaster = new AgentPresenceBroadcaster(hocuspocus);
sessionManager = new AgentSessionManager(hocuspocus);
+ // Registered here (not with the connection provider above) because the
+ // session manager doesn't exist yet at that point; gauges only sample at
+ // metric-export time, so late registration is safe.
+ const sm = sessionManager;
+ unregisterWorkloadProviders.push(
+ registerAgentSessionCountsProvider(() => ({
+ active: sm.liveSessionCount,
+ limit: sm.sessionLimit,
+ })),
+ );
const liveDerivedIndexExtension = createLiveDerivedIndexExtension({
backlinkIndex,
tagIndex,
diff --git a/packages/server/src/server-workload-telemetry.test.ts b/packages/server/src/server-workload-telemetry.test.ts
index d33c65992..d740928e7 100644
--- a/packages/server/src/server-workload-telemetry.test.ts
+++ b/packages/server/src/server-workload-telemetry.test.ts
@@ -1,4 +1,3 @@
-import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
import { metrics } from '@opentelemetry/api';
import {
AggregationTemporality,
@@ -7,10 +6,13 @@ import {
MeterProvider,
PeriodicExportingMetricReader,
} from '@opentelemetry/sdk-metrics';
+import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'vitest';
import {
__resetServerWorkloadTelemetryForTests,
installServerWorkloadGauges,
+ registerAgentSessionCountsProvider,
registerBridgeDirtyProbe,
+ registerConnectionCountsProvider,
registerLoadedDocsProvider,
registerPersistenceQueueDepthProvider,
} from './server-workload-telemetry.ts';
@@ -18,6 +20,9 @@ import {
const DOCS_METRIC = 'ok.server.docs.loaded';
const QUEUE_METRIC = 'ok.persistence.queue.depth';
const BACKLOG_METRIC = 'ok.bridge.drain_backlog';
+const CONNECTIONS_METRIC = 'ok.ws.connections.active';
+const SESSIONS_ACTIVE_METRIC = 'ok.sessions.active';
+const SESSIONS_LIMIT_METRIC = 'ok.sessions.limit';
describe('server workload telemetry — no-op meter (OTel disabled)', () => {
test('install and register/unregister are safe with the default no-op meter', () => {
@@ -32,9 +37,13 @@ describe('server workload telemetry — no-op meter (OTel disabled)', () => {
quiescenceDeferred: 0,
}));
const un3 = registerBridgeDirtyProbe(() => false);
+ const un4 = registerConnectionCountsProvider(() => ({ websocket: 1, direct: 2 }));
+ const un5 = registerAgentSessionCountsProvider(() => ({ active: 0, limit: 256 }));
un1();
un2();
un3();
+ un4();
+ un5();
}).not.toThrow();
__resetServerWorkloadTelemetryForTests();
});
@@ -126,4 +135,47 @@ describe('server workload telemetry — registered meter', () => {
const points = await collect(DOCS_METRIC);
expect(points[0].value).toBe(5);
});
+
+ test('ok.ws.connections.active reports both bounded kind labels and drops unregistered providers', async () => {
+ const un = registerConnectionCountsProvider(() => ({ websocket: 3, direct: 5 }));
+ registerConnectionCountsProvider(() => ({ websocket: 1, direct: 0 }));
+ let points = await collect(CONNECTIONS_METRIC);
+ let byKind = new Map(points.map((p) => [p.attributes.kind, p.value]));
+ expect([...byKind.keys()].sort()).toEqual(['direct', 'websocket']);
+ expect(byKind.get('websocket')).toBe(4);
+ expect(byKind.get('direct')).toBe(5);
+
+ un();
+ points = await collect(CONNECTIONS_METRIC);
+ byKind = new Map(points.map((p) => [p.attributes.kind, p.value]));
+ expect(byKind.get('websocket')).toBe(1);
+ expect(byKind.get('direct')).toBe(0);
+ });
+
+ test('ok.sessions.active and ok.sessions.limit track occupancy against the cap', async () => {
+ let active = 2;
+ registerAgentSessionCountsProvider(() => ({ active, limit: 256 }));
+ let activePoints = await collect(SESSIONS_ACTIVE_METRIC);
+ let limitPoints = await collect(SESSIONS_LIMIT_METRIC);
+ expect(activePoints[0].value).toBe(2);
+ expect(limitPoints[0].value).toBe(256);
+
+ // A cap stall surfaces as active pinned at limit.
+ active = 256;
+ activePoints = await collect(SESSIONS_ACTIVE_METRIC);
+ limitPoints = await collect(SESSIONS_LIMIT_METRIC);
+ expect(activePoints[0].value).toBe(256);
+ expect(activePoints[0].value).toBe(limitPoints[0].value);
+ });
+
+ test('a throwing session provider is skipped for both session gauges', async () => {
+ registerAgentSessionCountsProvider(() => {
+ throw new Error('torn down');
+ });
+ registerAgentSessionCountsProvider(() => ({ active: 7, limit: 256 }));
+ const activePoints = await collect(SESSIONS_ACTIVE_METRIC);
+ const limitPoints = await collect(SESSIONS_LIMIT_METRIC);
+ expect(activePoints[0].value).toBe(7);
+ expect(limitPoints[0].value).toBe(256);
+ });
});
diff --git a/packages/server/src/server-workload-telemetry.ts b/packages/server/src/server-workload-telemetry.ts
index e12f5c247..df43bdd07 100644
--- a/packages/server/src/server-workload-telemetry.ts
+++ b/packages/server/src/server-workload-telemetry.ts
@@ -1,6 +1,7 @@
/**
* Write-spine workload exposed as bounded OpenTelemetry observable gauges:
- * loaded Y.Doc count, persistence queue depths, and bridge drain backlog.
+ * loaded Y.Doc count, persistence queue depths, bridge drain backlog, live
+ * collab connections, and agent-session occupancy vs the hard cap.
*
* Pull-based by design: producers (server-factory, persistence, the server
* observer bridge) register read-only provider closures here at construction
@@ -14,13 +15,34 @@ import type { ObservableGauge, ObservableResult } from '@opentelemetry/api';
import type { PersistenceQueueDepths } from './persistence.ts';
import { getMeter, onTelemetryShutdown } from './telemetry.ts';
+/** Live collab-connection counts by transport kind. */
+export interface ConnectionCounts {
+ /** Deduplicated client WebSocket sockets (one socket can attach many docs). */
+ websocket: number;
+ /** In-process DirectConnections (agent sessions + server-held system/config docs). */
+ direct: number;
+}
+
+/** Agent-session occupancy against the server's hard cap. */
+export interface AgentSessionCounts {
+ /** Live (docName, agentId) sessions currently retained. */
+ active: number;
+ /** The session cap (`MAX_AGENT_SESSIONS` unless overridden). */
+ limit: number;
+}
+
const loadedDocsProviders = new Set<() => number>();
const persistenceQueueProviders = new Set<() => PersistenceQueueDepths>();
const bridgeDirtyProbes = new Set<() => boolean>();
+const connectionCountsProviders = new Set<() => ConnectionCounts>();
+const agentSessionCountsProviders = new Set<() => AgentSessionCounts>();
let cachedLoadedDocsGauge: ObservableGauge | null = null;
let cachedQueueDepthGauge: ObservableGauge | null = null;
let cachedDrainBacklogGauge: ObservableGauge | null = null;
+let cachedConnectionsGauge: ObservableGauge | null = null;
+let cachedSessionsActiveGauge: ObservableGauge | null = null;
+let cachedSessionsLimitGauge: ObservableGauge | null = null;
// Cached instruments drop on telemetry shutdown so the next install rebinds
// against the fresh meter (same lifecycle contract as the server-memory
@@ -30,6 +52,9 @@ onTelemetryShutdown(() => {
cachedLoadedDocsGauge = null;
cachedQueueDepthGauge = null;
cachedDrainBacklogGauge = null;
+ cachedConnectionsGauge = null;
+ cachedSessionsActiveGauge = null;
+ cachedSessionsLimitGauge = null;
});
/**
@@ -67,6 +92,32 @@ export function registerBridgeDirtyProbe(probe: () => boolean): () => void {
};
}
+/**
+ * Register a provider for live collab-connection counts (WebSocket sockets
+ * + DirectConnections). Returns an unregister function; call it at server
+ * teardown so the registry doesn't retain the closure.
+ */
+export function registerConnectionCountsProvider(provider: () => ConnectionCounts): () => void {
+ connectionCountsProviders.add(provider);
+ return () => {
+ connectionCountsProviders.delete(provider);
+ };
+}
+
+/**
+ * Register a provider for agent-session occupancy (`AgentSessionManager`
+ * live-session count + its hard cap). Returns an unregister function; call
+ * it at server teardown. A session-cap stall (`getSession` throwing
+ * `AgentSessionCapacityError` → 503s at the HTTP boundary) is invisible
+ * without this — active pinned at the limit is the diagnostic signature.
+ */
+export function registerAgentSessionCountsProvider(provider: () => AgentSessionCounts): () => void {
+ agentSessionCountsProviders.add(provider);
+ return () => {
+ agentSessionCountsProviders.delete(provider);
+ };
+}
+
/**
* Register the workload gauges against the currently-registered global meter.
* Idempotent — a second call is a no-op so a double boot can't
@@ -74,7 +125,16 @@ export function registerBridgeDirtyProbe(probe: () => boolean): () => void {
* propagated: instrumentation must never feed back into the write spine.
*/
export function installServerWorkloadGauges(): void {
- if (cachedLoadedDocsGauge && cachedQueueDepthGauge && cachedDrainBacklogGauge) return;
+ if (
+ cachedLoadedDocsGauge &&
+ cachedQueueDepthGauge &&
+ cachedDrainBacklogGauge &&
+ cachedConnectionsGauge &&
+ cachedSessionsActiveGauge &&
+ cachedSessionsLimitGauge
+ ) {
+ return;
+ }
if (!cachedLoadedDocsGauge) {
const gauge = getMeter().createObservableGauge('ok.server.docs.loaded', {
@@ -139,6 +199,67 @@ export function installServerWorkloadGauges(): void {
});
cachedDrainBacklogGauge = gauge;
}
+
+ if (!cachedConnectionsGauge) {
+ const gauge = getMeter().createObservableGauge('ok.ws.connections.active', {
+ description:
+ 'Live collab connections. Bounded labels: kind ∈ {websocket, direct}. websocket = deduplicated client sockets; direct = in-process DirectConnections (agent sessions + server-held system/config docs).',
+ unit: '{connections}',
+ });
+ gauge.addCallback((result: ObservableResult) => {
+ let websocket = 0;
+ let direct = 0;
+ for (const provider of connectionCountsProviders) {
+ try {
+ const counts = provider();
+ websocket += counts.websocket;
+ direct += counts.direct;
+ } catch {
+ // Skip a torn-down provider; the remaining sum stays meaningful.
+ }
+ }
+ result.observe(websocket, { kind: 'websocket' });
+ result.observe(direct, { kind: 'direct' });
+ });
+ cachedConnectionsGauge = gauge;
+ }
+
+ if (!cachedSessionsActiveGauge || !cachedSessionsLimitGauge) {
+ const activeGauge = getMeter().createObservableGauge('ok.sessions.active', {
+ description:
+ 'Live (docName, agentId) agent sessions. Pinned at ok.sessions.limit means new sessions are being refused with 503s.',
+ unit: '{sessions}',
+ });
+ const limitGauge = getMeter().createObservableGauge('ok.sessions.limit', {
+ description:
+ 'Hard cap on live agent sessions (summed across server instances in this process).',
+ unit: '{sessions}',
+ });
+ activeGauge.addCallback((result: ObservableResult) => {
+ let active = 0;
+ for (const provider of agentSessionCountsProviders) {
+ try {
+ active += provider().active;
+ } catch {
+ // Skip a torn-down provider; the remaining sum stays meaningful.
+ }
+ }
+ result.observe(active);
+ });
+ limitGauge.addCallback((result: ObservableResult) => {
+ let limit = 0;
+ for (const provider of agentSessionCountsProviders) {
+ try {
+ limit += provider().limit;
+ } catch {
+ // Skip a torn-down provider; the remaining sum stays meaningful.
+ }
+ }
+ result.observe(limit);
+ });
+ cachedSessionsActiveGauge = activeGauge;
+ cachedSessionsLimitGauge = limitGauge;
+ }
}
/** Drop cached instruments and clear registries. Test-only. */
@@ -146,7 +267,12 @@ export function __resetServerWorkloadTelemetryForTests(): void {
cachedLoadedDocsGauge = null;
cachedQueueDepthGauge = null;
cachedDrainBacklogGauge = null;
+ cachedConnectionsGauge = null;
+ cachedSessionsActiveGauge = null;
+ cachedSessionsLimitGauge = null;
loadedDocsProviders.clear();
persistenceQueueProviders.clear();
bridgeDirtyProbes.clear();
+ connectionCountsProviders.clear();
+ agentSessionCountsProviders.clear();
}
diff --git a/packages/server/src/sync-handshake-span-extension.test.ts b/packages/server/src/sync-handshake-span-extension.test.ts
index 839f6375a..ea97af5cb 100644
--- a/packages/server/src/sync-handshake-span-extension.test.ts
+++ b/packages/server/src/sync-handshake-span-extension.test.ts
@@ -14,15 +14,16 @@
*
* Skips system / config docs — those bypass the markdown bridge entirely.
*/
-import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
-import { context, metrics, trace } from '@opentelemetry/api';
+import { context, metrics, propagation, trace } from '@opentelemetry/api';
import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
+import { W3CTraceContextPropagator } from '@opentelemetry/core';
import {
BasicTracerProvider,
InMemorySpanExporter,
type ReadableSpan,
SimpleSpanProcessor,
} from '@opentelemetry/sdk-trace-base';
+import { afterEach, beforeEach, describe, expect, test } from 'vitest';
import * as Y from 'yjs';
import { createSyncHandshakeSpanExtension } from './sync-handshake-span-extension.ts';
@@ -36,6 +37,9 @@ function setupExporter(): void {
});
trace.setGlobalTracerProvider(provider);
context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());
+ // Mirrors initTelemetry's production registration — the extraction tests
+ // exercise the same W3C propagator the server runs with.
+ propagation.setGlobalPropagator(new W3CTraceContextPropagator());
}
async function teardownExporter(): Promise<void> {
@@ -43,6 +47,7 @@ async function teardownExporter(): Promise<void> {
trace.disable();
metrics.disable();
context.disable();
+ propagation.disable();
}
function spansByName(name: string): ReadableSpan[] {
@@ -59,9 +64,16 @@ function spansByName(name: string): ReadableSpan[] {
// biome-ignore lint/suspicious/noExplicitAny: structural test payload
type AfterLoadPayload = any;
-function makePayload(opts: { documentName: string; mountId?: string }): AfterLoadPayload {
+function makePayload(opts: {
+ documentName: string;
+ mountId?: string;
+ traceparent?: string;
+ tracestate?: string;
+}): AfterLoadPayload {
const params = new URLSearchParams();
if (opts.mountId !== undefined) params.set('mountId', opts.mountId);
+ if (opts.traceparent !== undefined) params.set('traceparent', opts.traceparent);
+ if (opts.tracestate !== undefined) params.set('tracestate', opts.tracestate);
return {
documentName: opts.documentName,
document: new Y.Doc(),
@@ -177,6 +189,63 @@ describe('sync.handshake span — basic emission', () => {
});
});
+describe('sync.handshake span — WS traceparent extraction', () => {
+ // W3C trace-context values the browser's appendTraceContextToCollabUrl
+ // would inject into the collab URL. Hardcoded so tests are deterministic.
+ const TRACE_ID = '0af7651916cd43dd8448eb211c80319c';
+ const PARENT_SPAN_ID = 'b7ad6b7169203331';
+ const TRACEPARENT = `00-${TRACE_ID}-${PARENT_SPAN_ID}-01`;
+
+ test('parents the span to the browser trace when traceparent is present', async () => {
+ const extension = createSyncHandshakeSpanExtension();
+ await extension.afterLoadDocument?.(
+ makePayload({ documentName: 'README', mountId: UUID_A, traceparent: TRACEPARENT }),
+ );
+ const spans = spansByName('sync.handshake');
+ expect(spans.length).toBe(1);
+ const span = spans[0];
+ expect(span?.spanContext().traceId).toBe(TRACE_ID);
+ expect(span?.parentSpanContext?.spanId).toBe(PARENT_SPAN_ID);
+ });
+
+ test('carries tracestate through extraction alongside traceparent', async () => {
+ const extension = createSyncHandshakeSpanExtension();
+ await extension.afterLoadDocument?.(
+ makePayload({
+ documentName: 'README',
+ traceparent: TRACEPARENT,
+ tracestate: 'vendor=value',
+ }),
+ );
+ const spans = spansByName('sync.handshake');
+ expect(spans.length).toBe(1);
+ // Same trace lineage; tracestate presence must not break extraction.
+ expect(spans[0]?.spanContext().traceId).toBe(TRACE_ID);
+ });
+
+ test('starts a root span when no traceparent is supplied', async () => {
+ const extension = createSyncHandshakeSpanExtension();
+ await extension.afterLoadDocument?.(makePayload({ documentName: 'README' }));
+ const spans = spansByName('sync.handshake');
+ expect(spans.length).toBe(1);
+ expect(spans[0]?.parentSpanContext).toBeUndefined();
+ expect(spans[0]?.spanContext().traceId).not.toBe(TRACE_ID);
+ });
+
+ test('falls back to a root span on a malformed traceparent without throwing', async () => {
+ // The W3C propagator ignores values it cannot parse — extract returns
+ // the context unchanged, so the span becomes a root rather than failing
+ // the handshake.
+ const extension = createSyncHandshakeSpanExtension();
+ await extension.afterLoadDocument?.(
+ makePayload({ documentName: 'README', traceparent: 'garbage-not-a-traceparent' }),
+ );
+ const spans = spansByName('sync.handshake');
+ expect(spans.length).toBe(1);
+ expect(spans[0]?.parentSpanContext).toBeUndefined();
+ });
+});
+
describe('sync.handshake span — OTel SDK fault isolation', () => {
test('OTel emission throw does not propagate out of afterLoadDocument', async () => {
// The OTel API contract says start/end must not throw, but an
diff --git a/packages/server/src/sync-handshake-span-extension.ts b/packages/server/src/sync-handshake-span-extension.ts
index a205c77a9..8fc611a74 100644
--- a/packages/server/src/sync-handshake-span-extension.ts
+++ b/packages/server/src/sync-handshake-span-extension.ts
@@ -31,7 +31,7 @@
* must not fail the sync handshake.
*/
import type { Extension } from '@hocuspocus/server';
-import type { Attributes } from '@opentelemetry/api';
+import { type Attributes, context as otelContext, propagation } from '@opentelemetry/api';
import { isConfigDoc, isSystemDoc } from './cc1-broadcast.ts';
import { getLogger } from './logger.ts';
import { withSpanSync } from './telemetry.ts';
@@ -64,8 +64,31 @@ export function createSyncHandshakeSpanExtension(): Extension {
// `withSpanSync` (not `withSpan`) so afterLoadDocument doesn't pay an
// awaited-Promise microtask hop per WebSocket connection even when
// OTel is disabled and the span body is a no-op.
+ //
+ // The browser cannot set headers on a native WebSocket, so the
+ // frontend appends `traceparent` / `tracestate` to the collab URL
+ // (appendTraceContextToCollabUrl in the app's collab-otel.ts) and
+ // they arrive here as query params. Extracting them parents this
+ // span to the browser's trace instead of starting a disconnected
+ // root. A malformed traceparent is ignored by the W3C propagator
+ // (extract returns the context unchanged); with OTel disabled the
+ // no-op propagator does the same, so the extra work is one small
+ // object allocation on a once-per-doc-load path.
try {
- withSpanSync('sync.handshake', { attributes }, () => {});
+ const emitSpan = (): void => {
+ withSpanSync('sync.handshake', { attributes }, () => {});
+ };
+ const traceparent = requestParameters?.get('traceparent');
+ if (traceparent !== null && traceparent !== undefined) {
+ const carrier: Record<string, string> = { traceparent };
+ const tracestate = requestParameters?.get('tracestate');
+ if (tracestate !== null && tracestate !== undefined) {
+ carrier.tracestate = tracestate;
+ }
+ otelContext.with(propagation.extract(otelContext.active(), carrier), emitSpan);
+ } else {
+ emitSpan();
+ }
} catch (err) {
// Pass the Error through under `err` so the pino serializer keeps
// its `.stack` — coercing to `.message` discards the call-frame