Skip to content

Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.

feat(telemetry): add zero-dependency OTLP span exporter and context propagation

Khuram Niaz5423e14826c7
diff --git a/__tests__/telemetry/disabled.spec.ts b/__tests__/telemetry/disabled.spec.ts
new file mode 100644
index 0000000..8847621
--- /dev/null
+++ b/__tests__/telemetry/disabled.spec.ts
@@ -0,0 +1,68 @@
+import { loadTelemetry, mockFetchOk } from "./helpers";
+
+describe("telemetry config", () => {
+ afterEach(() => {
+ vi.unstubAllEnvs();
+ vi.unstubAllGlobals();
+ });
+
+ it("is disabled when no endpoint is set", async () => {
+ const { config } = await loadTelemetry();
+ expect(config.TELEMETRY_ENABLED).toBe(false);
+ expect(config.CAPTURE_CONTENT).toBe(false);
+ expect(config.TRACES_URL).toBe("");
+ });
+
+ it("derives the two OTLP urls from the base endpoint, trailing slash and all", async () => {
+ const { config } = await loadTelemetry({
+ endpoint: "http://homelab:5080/api/default/",
+ });
+ expect(config.TELEMETRY_ENABLED).toBe(true);
+ expect(config.TRACES_URL).toBe("http://homelab:5080/api/default/v1/traces");
+ expect(config.LOGS_URL).toBe("http://homelab:5080/api/default/v1/logs");
+ });
+
+ it("keeps content capture off unless the endpoint is also set", async () => {
+ const off = await loadTelemetry({ captureContent: true });
+ expect(off.config.CAPTURE_CONTENT).toBe(false);
+
+ const on = await loadTelemetry({
+ endpoint: "http://homelab:5080/api/default",
+ captureContent: true,
+ });
+ expect(on.config.CAPTURE_CONTENT).toBe(true);
+ });
+
+ it("splits header pairs on the first '=' so base64 padding survives", async () => {
+ const { config } = await loadTelemetry({
+ endpoint: "http://homelab:5080/api/default",
+ headers: "Authorization=Basic dXNlckBleGFtcGxlLmNvbTp0b2tlbg==,X-Extra=1",
+ });
+ expect(config.OTLP_HEADERS).toEqual({
+ Authorization: "Basic dXNlckBleGFtcGxlLmNvbTp0b2tlbg==",
+ "X-Extra": "1",
+ });
+ });
+ it("allocates nothing and never touches fetch when disabled", async () => {
+ const fetchMock = mockFetchOk();
+ const { span, otlp } = await loadTelemetry();
+
+ const a = span.startSpan("agent.chat.turn", { "jobsync.surface": "agent.chat" });
+ const b = span.startSpan("agent.nested.review_resume");
+ a.setAttrs({ "gen_ai.usage.input_tokens": 10 });
+ a.setError(new Error("ignored"));
+ a.end();
+ b.end();
+
+ // The same frozen no-op, not two objects.
+ expect(a).toBe(b);
+ expect(otlp.getTelemetryStats().queuedSpans).toBe(0);
+ expect(fetchMock).not.toHaveBeenCalled();
+ });
+
+ it("runInSpan still runs the function when disabled", async () => {
+ const { span } = await loadTelemetry();
+ const noop = span.startSpan("x");
+ expect(span.runInSpan(noop, () => "ran")).toBe("ran");
+ });
+});
diff --git a/__tests__/telemetry/helpers.ts b/__tests__/telemetry/helpers.ts
new file mode 100644
index 0000000..248e35b
--- /dev/null
+++ b/__tests__/telemetry/helpers.ts
@@ -0,0 +1,39 @@
+import { vi } from "vitest";
+
+type TelemetryEnv = {
+ endpoint?: string;
+ captureContent?: boolean;
+ headers?: string;
+ serviceName?: string;
+};
+
+// config.ts reads process.env once at module load, so every scenario needs a
+// fresh module graph. The ALS and the queue live on globalThis and survive
+// resetModules, hence the explicit queue reset below.
+export async function loadTelemetry(env: TelemetryEnv = {}) {
+ vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", env.endpoint ?? "");
+ vi.stubEnv("OTEL_CAPTURE_CONTENT", env.captureContent ? "true" : "false");
+ vi.stubEnv("OTEL_EXPORTER_OTLP_HEADERS", env.headers ?? "");
+ vi.stubEnv("OTEL_SERVICE_NAME", env.serviceName ?? "jobsync-test");
+ vi.resetModules();
+
+ const otlp = await import("@/lib/telemetry/otlp");
+ otlp.resetTelemetryStateForTests();
+
+ return {
+ config: await import("@/lib/telemetry/config"),
+ span: await import("@/lib/telemetry/span"),
+ genai: await import("@/lib/telemetry/genai"),
+ logger: await import("@/lib/telemetry/logger"),
+ otlp,
+ };
+}
+
+export function mockFetchOk() {
+ // Typed like fetch so specs can read the url and body off mock.calls.
+ const fetchMock = vi.fn(
+ async (_url: string, _init?: RequestInit) => new Response("{}", { status: 200 }),
+ );
+ vi.stubGlobal("fetch", fetchMock);
+ return fetchMock;
+}
diff --git a/__tests__/telemetry/otlpPayload.spec.ts b/__tests__/telemetry/otlpPayload.spec.ts
new file mode 100644
index 0000000..b2d9fee
--- /dev/null
+++ b/__tests__/telemetry/otlpPayload.spec.ts
@@ -0,0 +1,121 @@
+import { loadTelemetry } from "./helpers";
+import { APP_CONSTANTS } from "@/lib/constants";
+
+const ENDPOINT = "http://homelab:5080/api/default";
+
+function span(overrides: Record<string, unknown> = {}) {
+ return {
+ name: "agent.chat.turn",
+ traceId: "5b8aa5a2d2c872e8321cf37308d69df2",
+ spanId: "051581bf3cb55c13",
+ startedAt: 1_756_500_000_000,
+ endedAt: 1_756_500_001_000,
+ attrs: {},
+ ...overrides,
+ } as any;
+}
+
+describe("otlp payload shape", () => {
+ afterEach(() => {
+ vi.unstubAllEnvs();
+ vi.unstubAllGlobals();
+ });
+
+ it("nests resourceSpans / scopeSpans and carries service.name", async () => {
+ const { otlp } = await loadTelemetry({
+ endpoint: ENDPOINT,
+ serviceName: "jobsync-test",
+ });
+ const payload = otlp.buildTracePayload([span()]) as any;
+
+ expect(payload.resourceSpans).toHaveLength(1);
+ expect(payload.resourceSpans[0].resource.attributes).toContainEqual({
+ key: "service.name",
+ value: { stringValue: "jobsync-test" },
+ });
+ const encoded = payload.resourceSpans[0].scopeSpans[0].spans[0];
+ expect(encoded.traceId).toBe("5b8aa5a2d2c872e8321cf37308d69df2");
+ expect(encoded.spanId).toBe("051581bf3cb55c13");
+ expect(encoded.parentSpanId).toBeUndefined();
+ expect(encoded.kind).toBe(1);
+ expect(encoded.startTimeUnixNano).toBe("1756500000000000000");
+ expect(encoded.endTimeUnixNano).toBe("1756500001000000000");
+ });
+
+ it("types attribute values: string, int, double, bool, and JSON for objects", async () => {
+ const { otlp } = await loadTelemetry({ endpoint: ENDPOINT });
+ const payload = otlp.buildTracePayload([
+ span({
+ attrs: {
+ "gen_ai.request.model": "qwen3.5:9b",
+ "gen_ai.usage.input_tokens": 1234,
+ "gen_ai.request.temperature": 0.1,
+ "jobsync.prefix_changed": true,
+ "gen_ai.prompt": { role: "user", content: "hi" },
+ },
+ }),
+ ]) as any;
+ const attrs = payload.resourceSpans[0].scopeSpans[0].spans[0].attributes;
+ const byKey = Object.fromEntries(attrs.map((a: any) => [a.key, a.value]));
+
+ expect(byKey["gen_ai.request.model"]).toEqual({ stringValue: "qwen3.5:9b" });
+ expect(byKey["gen_ai.usage.input_tokens"]).toEqual({ intValue: "1234" });
+ expect(byKey["gen_ai.request.temperature"]).toEqual({ doubleValue: 0.1 });
+ expect(byKey["jobsync.prefix_changed"]).toEqual({ boolValue: true });
+ expect(byKey["gen_ai.prompt"]).toEqual({
+ stringValue: '{"role":"user","content":"hi"}',
+ });
+ });
+
+ it("caps an oversized value and names the capped key", async () => {
+ const { otlp } = await loadTelemetry({ endpoint: ENDPOINT });
+ const huge = "x".repeat(APP_CONSTANTS.TELEMETRY_MAX_ATTR_CHARS + 500);
+ const payload = otlp.buildTracePayload([
+ span({ attrs: { "gen_ai.prompt": huge } }),
+ ]) as any;
+ const attrs = payload.resourceSpans[0].scopeSpans[0].spans[0].attributes;
+ const byKey = Object.fromEntries(attrs.map((a: any) => [a.key, a.value]));
+
+ expect(byKey["gen_ai.prompt"].stringValue).toHaveLength(
+ APP_CONSTANTS.TELEMETRY_MAX_ATTR_CHARS,
+ );
+ // The telemetry cap is a different fact from jobsync.input.truncated:
+ // the model still saw the whole thing.
+ expect(byKey["jobsync.telemetry.capped_attrs"]).toEqual({
+ stringValue: "gen_ai.prompt",
+ });
+ });
+
+ it("sets an error status when the span recorded one", async () => {
+ const { otlp } = await loadTelemetry({ endpoint: ENDPOINT });
+ const payload = otlp.buildTracePayload([
+ span({ error: true, errorMessage: "boom" }),
+ ]) as any;
+ expect(payload.resourceSpans[0].scopeSpans[0].spans[0].status).toEqual({
+ code: 2,
+ message: "boom",
+ });
+ });
+
+ it("builds a log payload with severity, body and trace correlation", async () => {
+ const { otlp } = await loadTelemetry({ endpoint: ENDPOINT });
+ const payload = otlp.buildLogPayload([
+ {
+ timestamp: 1_756_500_000_000,
+ severityNumber: 17,
+ severityText: "ERROR",
+ body: "scrape failed",
+ attrs: { "automation.id": "a1" },
+ traceId: "5b8aa5a2d2c872e8321cf37308d69df2",
+ spanId: "051581bf3cb55c13",
+ },
+ ]) as any;
+ const record = payload.resourceLogs[0].scopeLogs[0].logRecords[0];
+
+ expect(record.timeUnixNano).toBe("1756500000000000000");
+ expect(record.severityNumber).toBe(17);
+ expect(record.severityText).toBe("ERROR");
+ expect(record.body).toEqual({ stringValue: "scrape failed" });
+ expect(record.traceId).toBe("5b8aa5a2d2c872e8321cf37308d69df2");
+ });
+});
diff --git a/__tests__/telemetry/queue.spec.ts b/__tests__/telemetry/queue.spec.ts
new file mode 100644
index 0000000..f8aec50
--- /dev/null
+++ b/__tests__/telemetry/queue.spec.ts
@@ -0,0 +1,70 @@
+import { loadTelemetry, mockFetchOk } from "./helpers";
+import { APP_CONSTANTS } from "@/lib/constants";
+
+const ENDPOINT = "http://homelab:5080/api/default";
+
+function span(name: string) {
+ return {
+ name,
+ traceId: "a".repeat(32),
+ spanId: "b".repeat(16),
+ startedAt: 1_756_500_000_000,
+ endedAt: 1_756_500_000_100,
+ attrs: {},
+ } as any;
+}
+
+describe("otlp queue and flush timer", () => {
+ beforeEach(() => vi.useFakeTimers());
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.unstubAllEnvs();
+ vi.unstubAllGlobals();
+ });
+
+ it("enqueueing schedules a flush instead of performing one", async () => {
+ const fetchMock = mockFetchOk();
+ const { otlp } = await loadTelemetry({ endpoint: ENDPOINT });
+
+ for (let i = 0; i < APP_CONSTANTS.TELEMETRY_BATCH_SIZE + 10; i++) {
+ otlp.enqueueSpan(span(`s${i}`));
+ }
+ // The whole point: no serialization and no POST in the caller's stack.
+ expect(fetchMock).not.toHaveBeenCalled();
+
+ await vi.advanceTimersByTimeAsync(APP_CONSTANTS.TELEMETRY_FLUSH_INTERVAL_MS);
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ expect(fetchMock.mock.calls[0]![0]).toBe(`${ENDPOINT}/v1/traces`);
+ });
+
+ it("drops the oldest span past the queue cap", async () => {
+ mockFetchOk();
+ const { otlp } = await loadTelemetry({ endpoint: ENDPOINT });
+
+ for (let i = 0; i < APP_CONSTANTS.TELEMETRY_MAX_QUEUE + 5; i++) {
+ otlp.enqueueSpan(span(`s${i}`));
+ }
+ const stats = otlp.getTelemetryStats();
+ expect(stats.queuedSpans).toBe(APP_CONSTANTS.TELEMETRY_MAX_QUEUE);
+ expect(stats.dropped).toBe(5);
+ });
+
+ it("backs off on consecutive failures and resets on success", async () => {
+ const fetchMock = vi.fn(async () => new Response("nope", { status: 500 }));
+ vi.stubGlobal("fetch", fetchMock);
+ const { otlp } = await loadTelemetry({ endpoint: ENDPOINT });
+ const [first, second] = APP_CONSTANTS.TELEMETRY_BACKOFF_MS;
+
+ otlp.enqueueSpan(span("s1"));
+ otlp.enqueueSpan(span("s2"));
+ await vi.advanceTimersByTimeAsync(first!);
+ expect(otlp.getTelemetryStats().failures).toBe(1);
+
+ // The retry of what is still queued now waits the second backoff step.
+ otlp.enqueueSpan(span("s3"));
+ await vi.advanceTimersByTimeAsync(first!);
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ await vi.advanceTimersByTimeAsync(second! - first!);
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+});
diff --git a/__tests__/telemetry/resilience.spec.ts b/__tests__/telemetry/resilience.spec.ts
new file mode 100644
index 0000000..a2e9c40
--- /dev/null
+++ b/__tests__/telemetry/resilience.spec.ts
@@ -0,0 +1,72 @@
+import { loadTelemetry, mockFetchOk } from "./helpers";
+import { APP_CONSTANTS } from "@/lib/constants";
+
+const ENDPOINT = "http://homelab:5080/api/default";
+
+describe("telemetry never breaks a request", () => {
+ afterEach(() => {
+ vi.unstubAllEnvs();
+ vi.unstubAllGlobals();
+ });
+
+ it("swallows a rejecting fetch", async () => {
+ vi.useFakeTimers();
+ vi.stubGlobal("fetch", vi.fn(async () => { throw new Error("ECONNREFUSED"); }));
+ const { span, otlp } = await loadTelemetry({ endpoint: ENDPOINT });
+
+ span.startSpan("probe").end();
+ await expect(
+ vi.advanceTimersByTimeAsync(APP_CONSTANTS.TELEMETRY_FLUSH_INTERVAL_MS),
+ ).resolves.not.toThrow();
+ expect(otlp.getTelemetryStats().failures).toBe(1);
+ vi.useRealTimers();
+ });
+
+ it("swallows a synchronous throw from a hostile attrs object", async () => {
+ mockFetchOk();
+ const { span } = await loadTelemetry({ endpoint: ENDPOINT });
+
+ const hostile = {
+ get "jobsync.surface"() {
+ throw new TypeError("no");
+ },
+ };
+
+ expect(() => span.startSpan("probe", hostile as any)).not.toThrow();
+ const live = span.startSpan("probe");
+ expect(() => live.setAttrs(hostile as any)).not.toThrow();
+ expect(() => live.end(hostile as any)).not.toThrow();
+ });
+
+ it("drops a record whose attribute cannot be serialized without losing the batch", async () => {
+ const fetchMock = mockFetchOk();
+ vi.useFakeTimers();
+ const { span, otlp } = await loadTelemetry({ endpoint: ENDPOINT });
+
+ const circular: Record<string, unknown> = {};
+ circular.self = circular;
+ span.startSpan("bad").end({ "jobsync.payload": circular });
+ span.startSpan("good").end();
+
+ await vi.advanceTimersByTimeAsync(APP_CONSTANTS.TELEMETRY_FLUSH_INTERVAL_MS);
+ const body = JSON.parse(fetchMock.mock.calls[0]![1]!.body as string);
+ const names = body.resourceSpans[0].scopeSpans[0].spans.map(
+ (s: any) => s.name,
+ );
+ expect(names).toEqual(["good"]);
+ expect(otlp.getTelemetryStats().failures).toBe(0);
+ vi.useRealTimers();
+ });
+
+ it("propagates the wrapped function's own error while still ending the span", async () => {
+ mockFetchOk();
+ const { span, otlp } = await loadTelemetry({ endpoint: ENDPOINT });
+
+ await expect(
+ span.withSpan("boom", {}, async () => {
+ throw new Error("application failure");
+ }),
+ ).rejects.toThrow("application failure");
+ expect(otlp.getTelemetryStats().queuedSpans).toBe(1);
+ });
+});
diff --git a/src/lib/constants.ts b/src/lib/constants.ts
index 889d0a8..0a0d973 100644
--- a/src/lib/constants.ts
+++ b/src/lib/constants.ts
@@ -194,6 +194,22 @@ export const APP_CONSTANTS = {
// Must not be "ai-panel-width" — that key belongs to the three AI sheets.
AGENT_CHAT_PANEL_WIDTH_KEY: "agent-chat-width",
+ // Telemetry (OTLP export)
+ // Flush cadence. A batch is never serialized in the enqueueing caller's
+ // stack — reaching the batch size only arms the timer.
+ TELEMETRY_FLUSH_INTERVAL_MS: 5_000,
+ // Consecutive-failure backoff, resets to the first entry on any success.
+ // A collector down for a day is retried a few dozen times, not 17,000.
+ TELEMETRY_BACKOFF_MS: [5_000, 30_000, 60_000],
+ // Bounded queue, drops oldest — mirrors MAX_LOGS_PER_RUN in
+ // automation-logger.ts. A dead collector cannot grow memory without limit.
+ TELEMETRY_MAX_QUEUE: 1_000,
+ TELEMETRY_BATCH_SIZE: 200,
+ TELEMETRY_EXPORT_TIMEOUT_MS: 5_000,
+ // A resume-review prompt runs 10-50 KB; this caps what leaves the process.
+ // Not the same thing as jobsync.input.truncated — see spec section 8.
+ TELEMETRY_MAX_ATTR_CHARS: 32_000,
+
// File uploads
UPLOADS_DIR: process.env.NODE_ENV !== "production" ? "data" : "/data",
diff --git a/src/lib/telemetry/config.ts b/src/lib/telemetry/config.ts
new file mode 100644
index 0000000..d90a3c3
--- /dev/null
+++ b/src/lib/telemetry/config.ts
@@ -0,0 +1,37 @@
+// Read once, at module load. That is what makes "disabled" genuinely free:
+// startSpan and log return immediately, no queue exists, fetch is never
+// referenced. Nothing else in the app reads process.env for telemetry.
+
+function parseHeaders(raw: string | undefined): Record<string, string> {
+ const headers: Record<string, string> = {};
+ for (const pair of (raw ?? "").split(",")) {
+ const trimmed = pair.trim();
+ if (!trimmed) continue;
+ // First "=" only: an Authorization value is base64 and carries its own.
+ const separator = trimmed.indexOf("=");
+ if (separator <= 0) continue;
+ headers[trimmed.slice(0, separator).trim()] = trimmed
+ .slice(separator + 1)
+ .trim();
+ }
+ return headers;
+}
+
+const endpoint = (process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "")
+ .trim()
+ .replace(/\/+$/, "");
+
+export const TELEMETRY_ENABLED = endpoint.length > 0;
+
+// Two independent stages: content capture has no effect without an endpoint.
+export const CAPTURE_CONTENT =
+ TELEMETRY_ENABLED && process.env.OTEL_CAPTURE_CONTENT === "true";
+
+export const SERVICE_NAME = process.env.OTEL_SERVICE_NAME || "jobsync";
+
+// The base/suffix split is the OTel convention, and is what keeps the backend
+// swappable for Seq, Tempo or SigNoz by changing one URL.
+export const TRACES_URL = TELEMETRY_ENABLED ? `${endpoint}/v1/traces` : "";
+export const LOGS_URL = TELEMETRY_ENABLED ? `${endpoint}/v1/logs` : "";
+
+export const OTLP_HEADERS = parseHeaders(process.env.OTEL_EXPORTER_OTLP_HEADERS);
diff --git a/src/lib/telemetry/context.ts b/src/lib/telemetry/context.ts
new file mode 100644
index 0000000..2e3c063
--- /dev/null
+++ b/src/lib/telemetry/context.ts
@@ -0,0 +1,22 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+
+export type ActiveSpan = { traceId: string; spanId: string };
+
+// Unconditional globalThis stash, unlike db.ts's and automation-logger.ts's
+// dev-only guards. Those exist to survive hot reload; this one exists because
+// Next can emit a module into more than one server bundle, and two ALS
+// instances would silently orphan child spans instead of merely duplicating
+// work — the waterfall would arrive flat with nothing reporting it.
+const globalForTelemetry = globalThis as typeof globalThis & {
+ telemetryContextGlobal?: AsyncLocalStorage<ActiveSpan>;
+};
+
+export const activeSpanStorage =
+ globalForTelemetry.telemetryContextGlobal ??
+ new AsyncLocalStorage<ActiveSpan>();
+
+globalForTelemetry.telemetryContextGlobal = activeSpanStorage;
+
+export function currentSpan(): ActiveSpan | undefined {
+ return activeSpanStorage.getStore();
+}
diff --git a/src/lib/telemetry/genai.ts b/src/lib/telemetry/genai.ts
new file mode 100644
index 0000000..2a80e56
--- /dev/null
+++ b/src/lib/telemetry/genai.ts
@@ -0,0 +1,2 @@
+// Filled in by Phase 3 — OTel GenAI semantic-convention mapping.
+export {};
diff --git a/src/lib/telemetry/index.ts b/src/lib/telemetry/index.ts
new file mode 100644
index 0000000..d514f42
--- /dev/null
+++ b/src/lib/telemetry/index.ts
@@ -0,0 +1,8 @@
+export {
+ CAPTURE_CONTENT,
+ SERVICE_NAME,
+ TELEMETRY_ENABLED,
+} from "./config";
+export { currentSpan, type ActiveSpan } from "./context";
+export { runInSpan, startSpan, withSpan, type Span } from "./span";
+export { type Attrs } from "./otlp";
diff --git a/src/lib/telemetry/logger.ts b/src/lib/telemetry/logger.ts
new file mode 100644
index 0000000..8bd13c2
--- /dev/null
+++ b/src/lib/telemetry/logger.ts
@@ -0,0 +1,2 @@
+// Filled in by Phase 2 — log.info / warn / error over console.*.
+export {};
diff --git a/src/lib/telemetry/otlp.ts b/src/lib/telemetry/otlp.ts
new file mode 100644
index 0000000..f605e4a
--- /dev/null
+++ b/src/lib/telemetry/otlp.ts
@@ -0,0 +1,290 @@
+import { APP_CONSTANTS } from "@/lib/constants";
+import {
+ LOGS_URL,
+ OTLP_HEADERS,
+ SERVICE_NAME,
+ TELEMETRY_ENABLED,
+ TRACES_URL,
+} from "./config";
+
+export type Attrs = Record<string, unknown>;
+
+export type QueuedSpan = {
+ name: string;
+ traceId: string;
+ spanId: string;
+ parentSpanId?: string;
+ startedAt: number;
+ endedAt: number;
+ attrs: Attrs;
+ error?: boolean;
+ errorMessage?: string;
+};
+
+export type QueuedLog = {
+ timestamp: number;
+ severityNumber: number;
+ severityText: string;
+ body: string;
+ attrs: Attrs;
+ traceId?: string;
+ spanId?: string;
+};
+
+type TelemetryState = {
+ spans: QueuedSpan[];
+ logs: QueuedLog[];
+ timer: ReturnType<typeof setTimeout> | null;
+ inFlight: boolean;
+ failures: number;
+ dropped: number;
+};
+
+// Same globalThis pattern as db.ts, but unconditional: a second copy of this
+// module in another server bundle would mean a second timer and a second
+// partial batch for no benefit.
+const globalForTelemetry = globalThis as typeof globalThis & {
+ telemetryStateGlobal?: TelemetryState;
+};
+
+function state(): TelemetryState {
+ if (!globalForTelemetry.telemetryStateGlobal) {
+ globalForTelemetry.telemetryStateGlobal = {
+ spans: [],
+ logs: [],
+ timer: null,
+ inFlight: false,
+ failures: 0,
+ dropped: 0,
+ };
+ }
+ return globalForTelemetry.telemetryStateGlobal;
+}
+
+// BigInt literals need ES2020; the project targets ES2017, so build the
+// multiplier with the constructor instead.
+function nanos(ms: number): string {
+ return String(BigInt(Math.trunc(ms)) * BigInt(1_000_000));
+}
+
+function encodeValue(value: unknown): unknown {
+ if (typeof value === "string") return { stringValue: value };
+ if (typeof value === "boolean") return { boolValue: value };
+ if (typeof value === "number") {
+ return Number.isInteger(value)
+ ? { intValue: String(value) }
+ : { doubleValue: value };
+ }
+ return { stringValue: JSON.stringify(value) };
+}
+
+function toKeyValues(attrs: Attrs): unknown[] {
+ const out: unknown[] = [];
+ const capped: string[] = [];
+ for (const [key, raw] of Object.entries(attrs)) {
+ if (raw === undefined || raw === null) continue;
+ const encoded = encodeValue(raw) as { stringValue?: string };
+ if (
+ typeof encoded.stringValue === "string" &&
+ encoded.stringValue.length > APP_CONSTANTS.TELEMETRY_MAX_ATTR_CHARS
+ ) {
+ encoded.stringValue = encoded.stringValue.slice(
+ 0,
+ APP_CONSTANTS.TELEMETRY_MAX_ATTR_CHARS,
+ );
+ capped.push(key);
+ }
+ out.push({ key, value: encoded });
+ }
+ // Marks that TELEMETRY clipped the attribute, not that the app clipped the
+ // model input — jobsync.input.truncated is the other fact entirely.
+ if (capped.length) {
+ out.push({
+ key: "jobsync.telemetry.capped_attrs",
+ value: { stringValue: capped.join(",") },
+ });
+ }
+ return out;
+}
+
+function resource() {
+ return {
+ attributes: [{ key: "service.name", value: { stringValue: SERVICE_NAME } }],
+ };
+}
+
+function encodeSpan(span: QueuedSpan) {
+ return {
+ traceId: span.traceId,
+ spanId: span.spanId,
+ ...(span.parentSpanId ? { parentSpanId: span.parentSpanId } : {}),
+ name: span.name,
+ kind: 1,
+ startTimeUnixNano: nanos(span.startedAt),
+ endTimeUnixNano: nanos(span.endedAt),
+ attributes: toKeyValues(span.attrs),
+ status: span.error
+ ? { code: 2, message: span.errorMessage ?? "" }
+ : { code: 0 },
+ };
+}
+
+function encodeLog(record: QueuedLog) {
+ return {
+ timeUnixNano: nanos(record.timestamp),
+ severityNumber: record.severityNumber,
+ severityText: record.severityText,
+ body: { stringValue: record.body },
+ attributes: toKeyValues(record.attrs),
+ ...(record.traceId ? { traceId: record.traceId } : {}),
+ ...(record.spanId ? { spanId: record.spanId } : {}),
+ };
+}
+
+// A single record that cannot be encoded (a circular attribute value) must
+// not take the rest of the batch down with it.
+function encodeEach<T>(items: T[], encode: (item: T) => unknown): unknown[] {
+ const out: unknown[] = [];
+ for (const item of items) {
+ try {
+ out.push(encode(item));
+ } catch {
+ // Dropped silently: telemetry never reports on telemetry.
+ }
+ }
+ return out;
+}
+
+export function buildTracePayload(spans: QueuedSpan[]) {
+ return {
+ resourceSpans: [
+ {
+ resource: resource(),
+ scopeSpans: [
+ { scope: { name: "jobsync" }, spans: encodeEach(spans, encodeSpan) },
+ ],
+ },
+ ],
+ };
+}
+
+export function buildLogPayload(records: QueuedLog[]) {
+ return {
+ resourceLogs: [
+ {
+ resource: resource(),
+ scopeLogs: [
+ {
+ scope: { name: "jobsync" },
+ logRecords: encodeEach(records, encodeLog),
+ },
+ ],
+ },
+ ],
+ };
+}
+
+function backoffDelay(failures: number): number {
+ const table = APP_CONSTANTS.TELEMETRY_BACKOFF_MS;
+ return table[Math.min(failures, table.length - 1)]!;
+}
+
+function ensureTimer(): void {
+ const current = state();
+ if (current.timer) return;
+ const timer = setTimeout(() => {
+ state().timer = null;
+ void flush();
+ }, backoffDelay(current.failures));
+ // Never hold the process open for a flush.
+ timer.unref?.();
+ current.timer = timer;
+}
+
+async function post(url: string, payload: unknown): Promise<boolean> {
+ try {
+ const response = await fetch(url, {
+ method: "POST",
+ headers: { "Content-Type": "application/json", ...OTLP_HEADERS },
+ body: JSON.stringify(payload),
+ signal: AbortSignal.timeout(APP_CONSTANTS.TELEMETRY_EXPORT_TIMEOUT_MS),
+ });
+ return response.ok;
+ } catch {
+ return false;
+ }
+}
+
+async function flush(): Promise<void> {
+ const current = state();
+ // A collector hanging until the POST timeout would otherwise stack
+ // in-flight requests, since the timeout is not below the flush interval.
+ if (current.inFlight) {
+ if (current.spans.length || current.logs.length) ensureTimer();
+ return;
+ }
+
+ const spans = current.spans.splice(0, APP_CONSTANTS.TELEMETRY_BATCH_SIZE);
+ const logs = current.logs.splice(0, APP_CONSTANTS.TELEMETRY_BATCH_SIZE);
+ if (!spans.length && !logs.length) return;
+
+ current.inFlight = true;
+ try {
+ const results = await Promise.all([
+ spans.length ? post(TRACES_URL, buildTracePayload(spans)) : true,
+ logs.length ? post(LOGS_URL, buildLogPayload(logs)) : true,
+ ]);
+ // Counted, never logged per occurrence: an hour of downtime must not
+ // produce an hour of error spam in the container's own stdout.
+ current.failures = results.every(Boolean) ? 0 : current.failures + 1;
+ } catch {
+ current.failures += 1;
+ } finally {
+ current.inFlight = false;
+ if (current.spans.length || current.logs.length) ensureTimer();
+ }
+}
+
+function enqueue<T>(queue: T[], record: T): void {
+ queue.push(record);
+ while (queue.length > APP_CONSTANTS.TELEMETRY_MAX_QUEUE) {
+ queue.shift();
+ state().dropped += 1;
+ }
+ ensureTimer();
+}
+
+export function enqueueSpan(span: QueuedSpan): void {
+ if (!TELEMETRY_ENABLED) return;
+ try {
+ enqueue(state().spans, span);
+ } catch {
+ // Enqueueing must never throw into a request.
+ }
+}
+
+export function enqueueLog(record: QueuedLog): void {
+ if (!TELEMETRY_ENABLED) return;
+ try {
+ enqueue(state().logs, record);
+ } catch {
+ // Enqueueing must never throw into a request.
+ }
+}
+
+export function getTelemetryStats() {
+ const current = state();
+ return {
+ queuedSpans: current.spans.length,
+ queuedLogs: current.logs.length,
+ dropped: current.dropped,
+ failures: current.failures,
+ };
+}
+
+// Test-only: the state lives on globalThis and survives vi.resetModules().
+export function resetTelemetryStateForTests(): void {
+ const current = state();
+ if (current.timer) clearTimeout(current.timer);
+ globalForTelemetry.telemetryStateGlobal = undefined;
+}
diff --git a/src/lib/telemetry/span.ts b/src/lib/telemetry/span.ts
new file mode 100644
index 0000000..2a967d8
--- /dev/null
+++ b/src/lib/telemetry/span.ts
@@ -0,0 +1,115 @@
+import { randomBytes } from "node:crypto";
+import { TELEMETRY_ENABLED } from "./config";
+import { activeSpanStorage, currentSpan } from "./context";
+import { enqueueSpan, type Attrs } from "./otlp";
+
+export type Span = {
+ traceId: string;
+ spanId: string;
+ setAttrs(attrs: Attrs): void;
+ setError(error: unknown): void;
+ end(attrs?: Attrs): void;
+};
+
+// One shared object rather than a fresh no-op per call: with telemetry off,
+// an instrumented request must allocate nothing at all.
+const NOOP_SPAN: Span = Object.freeze({
+ traceId: "",
+ spanId: "",
+ setAttrs() {},
+ setError() {},
+ end() {},
+});
+
+function merge(target: Attrs, source: Attrs | undefined): void {
+ if (!source) return;
+ for (const [key, value] of Object.entries(source)) {
+ if (value === undefined || value === null) continue;
+ target[key] = value;
+ }
+}
+
+export function startSpan(name: string, attrs?: Attrs): Span {
+ if (!TELEMETRY_ENABLED) return NOOP_SPAN;
+ try {
+ const parent = currentSpan();
+ const traceId = parent?.traceId ?? randomBytes(16).toString("hex");
+ const spanId = randomBytes(8).toString("hex");
+ const startedAt = Date.now();
+ const collected: Attrs = {};
+ merge(collected, attrs);
+ let ended = false;
+ let error = false;
+ let errorMessage: string | undefined;
+
+ return {
+ traceId,
+ spanId,
+ setAttrs(next) {
+ try {
+ merge(collected, next);
+ } catch {
+ // A hostile attrs object degrades telemetry, never the request.
+ }
+ },
+ setError(cause) {
+ try {
+ error = true;
+ errorMessage =
+ cause instanceof Error ? cause.message : String(cause);
+ } catch {
+ error = true;
+ }
+ },
+ end(next) {
+ if (ended) return;
+ ended = true;
+ try {
+ merge(collected, next);
+ } catch {
+ // Keep whatever was collected before the bad object.
+ }
+ try {
+ enqueueSpan({
+ name,
+ traceId,
+ spanId,
+ parentSpanId: parent?.spanId,
+ startedAt,
+ endedAt: Date.now(),
+ attrs: collected,
+ error,
+ errorMessage,
+ });
+ } catch {
+ // Enqueueing must never throw into a request.
+ }
+ },
+ };
+ } catch {
+ return NOOP_SPAN;
+ }
+}
+
+// Separate from startSpan because the chat turn starts its span in one scope,
+// makes it ambient in a second, and ends it in a third.
+export function runInSpan<T>(span: Span, fn: () => T): T {
+ if (!TELEMETRY_ENABLED || !span.spanId) return fn();
+ return activeSpanStorage.run({ traceId: span.traceId, spanId: span.spanId }, fn);
+}
+
+export async function withSpan<T>(
+ name: string,
+ attrs: Attrs,
+ fn: (span: Span) => Promise<T>,
+): Promise<T> {
+ const span = startSpan(name, attrs);
+ try {
+ return await runInSpan(span, () => fn(span));
+ } catch (error) {
+ span.setError(error);
+ throw error;
+ } finally {
+ span.end();
+ }
+}