Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.
TypeScript ai-agentai-resume-analyzerapplication-trackerapplication-tracking-systematscareerjob-applicationjob-automationjob-huntingjob-searchjob-seeker-toolsjob-trackerjob-trackingjobsearchjobseekersmcpollamaresumeresume-reviewself-hosted
feat(telemetry): add zero-dependency OTLP span exporter and context propagation
Khuram Niaz5423e14826c7
diff --git a/__tests__/telemetry/disabled.spec.ts b/__tests__/telemetry/disabled.spec.tsnew file mode 100644index 0000000..8847621--- /dev/null+++ b/__tests__/telemetry/disabled.spec.ts@@ -0,0 +1,68 @@+import { loadTelemetry, mockFetchOk } from "./helpers";++describe("telemetry config", () => {+ afterEach(() => {+ vi.unstubAllEnvs();+ vi.unstubAllGlobals();+ });++ it("is disabled when no endpoint is set", async () => {+ const { config } = await loadTelemetry();+ expect(config.TELEMETRY_ENABLED).toBe(false);+ expect(config.CAPTURE_CONTENT).toBe(false);+ expect(config.TRACES_URL).toBe("");+ });++ it("derives the two OTLP urls from the base endpoint, trailing slash and all", async () => {+ const { config } = await loadTelemetry({+ endpoint: "http://homelab:5080/api/default/",+ });+ expect(config.TELEMETRY_ENABLED).toBe(true);+ expect(config.TRACES_URL).toBe("http://homelab:5080/api/default/v1/traces");+ expect(config.LOGS_URL).toBe("http://homelab:5080/api/default/v1/logs");+ });++ it("keeps content capture off unless the endpoint is also set", async () => {+ const off = await loadTelemetry({ captureContent: true });+ expect(off.config.CAPTURE_CONTENT).toBe(false);++ const on = await loadTelemetry({+ endpoint: "http://homelab:5080/api/default",+ captureContent: true,+ });+ expect(on.config.CAPTURE_CONTENT).toBe(true);+ });++ it("splits header pairs on the first '=' so base64 padding survives", async () => {+ const { config } = await loadTelemetry({+ endpoint: "http://homelab:5080/api/default",+ headers: "Authorization=Basic dXNlckBleGFtcGxlLmNvbTp0b2tlbg==,X-Extra=1",+ });+ expect(config.OTLP_HEADERS).toEqual({+ Authorization: "Basic dXNlckBleGFtcGxlLmNvbTp0b2tlbg==",+ "X-Extra": "1",+ });+ });+ it("allocates nothing and never touches fetch when disabled", async () => {+ const fetchMock = mockFetchOk();+ const { span, otlp } = await loadTelemetry();++ const a = span.startSpan("agent.chat.turn", { "jobsync.surface": "agent.chat" });+ const b = span.startSpan("agent.nested.review_resume");+ a.setAttrs({ "gen_ai.usage.input_tokens": 10 });+ a.setError(new Error("ignored"));+ a.end();+ b.end();++ // The same frozen no-op, not two objects.+ expect(a).toBe(b);+ expect(otlp.getTelemetryStats().queuedSpans).toBe(0);+ expect(fetchMock).not.toHaveBeenCalled();+ });++ it("runInSpan still runs the function when disabled", async () => {+ const { span } = await loadTelemetry();+ const noop = span.startSpan("x");+ expect(span.runInSpan(noop, () => "ran")).toBe("ran");+ });+});diff --git a/__tests__/telemetry/helpers.ts b/__tests__/telemetry/helpers.tsnew file mode 100644index 0000000..248e35b--- /dev/null+++ b/__tests__/telemetry/helpers.ts@@ -0,0 +1,39 @@+import { vi } from "vitest";++type TelemetryEnv = {+ endpoint?: string;+ captureContent?: boolean;+ headers?: string;+ serviceName?: string;+};++// config.ts reads process.env once at module load, so every scenario needs a+// fresh module graph. The ALS and the queue live on globalThis and survive+// resetModules, hence the explicit queue reset below.+export async function loadTelemetry(env: TelemetryEnv = {}) {+ vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", env.endpoint ?? "");+ vi.stubEnv("OTEL_CAPTURE_CONTENT", env.captureContent ? "true" : "false");+ vi.stubEnv("OTEL_EXPORTER_OTLP_HEADERS", env.headers ?? "");+ vi.stubEnv("OTEL_SERVICE_NAME", env.serviceName ?? "jobsync-test");+ vi.resetModules();++ const otlp = await import("@/lib/telemetry/otlp");+ otlp.resetTelemetryStateForTests();++ return {+ config: await import("@/lib/telemetry/config"),+ span: await import("@/lib/telemetry/span"),+ genai: await import("@/lib/telemetry/genai"),+ logger: await import("@/lib/telemetry/logger"),+ otlp,+ };+}++export function mockFetchOk() {+ // Typed like fetch so specs can read the url and body off mock.calls.+ const fetchMock = vi.fn(+ async (_url: string, _init?: RequestInit) => new Response("{}", { status: 200 }),+ );+ vi.stubGlobal("fetch", fetchMock);+ return fetchMock;+}diff --git a/__tests__/telemetry/otlpPayload.spec.ts b/__tests__/telemetry/otlpPayload.spec.tsnew file mode 100644index 0000000..b2d9fee--- /dev/null+++ b/__tests__/telemetry/otlpPayload.spec.ts@@ -0,0 +1,121 @@+import { loadTelemetry } from "./helpers";+import { APP_CONSTANTS } from "@/lib/constants";++const ENDPOINT = "http://homelab:5080/api/default";++function span(overrides: Record<string, unknown> = {}) {+ return {+ name: "agent.chat.turn",+ traceId: "5b8aa5a2d2c872e8321cf37308d69df2",+ spanId: "051581bf3cb55c13",+ startedAt: 1_756_500_000_000,+ endedAt: 1_756_500_001_000,+ attrs: {},+ ...overrides,+ } as any;+}++describe("otlp payload shape", () => {+ afterEach(() => {+ vi.unstubAllEnvs();+ vi.unstubAllGlobals();+ });++ it("nests resourceSpans / scopeSpans and carries service.name", async () => {+ const { otlp } = await loadTelemetry({+ endpoint: ENDPOINT,+ serviceName: "jobsync-test",+ });+ const payload = otlp.buildTracePayload([span()]) as any;++ expect(payload.resourceSpans).toHaveLength(1);+ expect(payload.resourceSpans[0].resource.attributes).toContainEqual({+ key: "service.name",+ value: { stringValue: "jobsync-test" },+ });+ const encoded = payload.resourceSpans[0].scopeSpans[0].spans[0];+ expect(encoded.traceId).toBe("5b8aa5a2d2c872e8321cf37308d69df2");+ expect(encoded.spanId).toBe("051581bf3cb55c13");+ expect(encoded.parentSpanId).toBeUndefined();+ expect(encoded.kind).toBe(1);+ expect(encoded.startTimeUnixNano).toBe("1756500000000000000");+ expect(encoded.endTimeUnixNano).toBe("1756500001000000000");+ });++ it("types attribute values: string, int, double, bool, and JSON for objects", async () => {+ const { otlp } = await loadTelemetry({ endpoint: ENDPOINT });+ const payload = otlp.buildTracePayload([+ span({+ attrs: {+ "gen_ai.request.model": "qwen3.5:9b",+ "gen_ai.usage.input_tokens": 1234,+ "gen_ai.request.temperature": 0.1,+ "jobsync.prefix_changed": true,+ "gen_ai.prompt": { role: "user", content: "hi" },+ },+ }),+ ]) as any;+ const attrs = payload.resourceSpans[0].scopeSpans[0].spans[0].attributes;+ const byKey = Object.fromEntries(attrs.map((a: any) => [a.key, a.value]));++ expect(byKey["gen_ai.request.model"]).toEqual({ stringValue: "qwen3.5:9b" });+ expect(byKey["gen_ai.usage.input_tokens"]).toEqual({ intValue: "1234" });+ expect(byKey["gen_ai.request.temperature"]).toEqual({ doubleValue: 0.1 });+ expect(byKey["jobsync.prefix_changed"]).toEqual({ boolValue: true });+ expect(byKey["gen_ai.prompt"]).toEqual({+ stringValue: '{"role":"user","content":"hi"}',+ });+ });++ it("caps an oversized value and names the capped key", async () => {+ const { otlp } = await loadTelemetry({ endpoint: ENDPOINT });+ const huge = "x".repeat(APP_CONSTANTS.TELEMETRY_MAX_ATTR_CHARS + 500);+ const payload = otlp.buildTracePayload([+ span({ attrs: { "gen_ai.prompt": huge } }),+ ]) as any;+ const attrs = payload.resourceSpans[0].scopeSpans[0].spans[0].attributes;+ const byKey = Object.fromEntries(attrs.map((a: any) => [a.key, a.value]));++ expect(byKey["gen_ai.prompt"].stringValue).toHaveLength(+ APP_CONSTANTS.TELEMETRY_MAX_ATTR_CHARS,+ );+ // The telemetry cap is a different fact from jobsync.input.truncated:+ // the model still saw the whole thing.+ expect(byKey["jobsync.telemetry.capped_attrs"]).toEqual({+ stringValue: "gen_ai.prompt",+ });+ });++ it("sets an error status when the span recorded one", async () => {+ const { otlp } = await loadTelemetry({ endpoint: ENDPOINT });+ const payload = otlp.buildTracePayload([+ span({ error: true, errorMessage: "boom" }),+ ]) as any;+ expect(payload.resourceSpans[0].scopeSpans[0].spans[0].status).toEqual({+ code: 2,+ message: "boom",+ });+ });++ it("builds a log payload with severity, body and trace correlation", async () => {+ const { otlp } = await loadTelemetry({ endpoint: ENDPOINT });+ const payload = otlp.buildLogPayload([+ {+ timestamp: 1_756_500_000_000,+ severityNumber: 17,+ severityText: "ERROR",+ body: "scrape failed",+ attrs: { "automation.id": "a1" },+ traceId: "5b8aa5a2d2c872e8321cf37308d69df2",+ spanId: "051581bf3cb55c13",+ },+ ]) as any;+ const record = payload.resourceLogs[0].scopeLogs[0].logRecords[0];++ expect(record.timeUnixNano).toBe("1756500000000000000");+ expect(record.severityNumber).toBe(17);+ expect(record.severityText).toBe("ERROR");+ expect(record.body).toEqual({ stringValue: "scrape failed" });+ expect(record.traceId).toBe("5b8aa5a2d2c872e8321cf37308d69df2");+ });+});diff --git a/__tests__/telemetry/queue.spec.ts b/__tests__/telemetry/queue.spec.tsnew file mode 100644index 0000000..f8aec50--- /dev/null+++ b/__tests__/telemetry/queue.spec.ts@@ -0,0 +1,70 @@+import { loadTelemetry, mockFetchOk } from "./helpers";+import { APP_CONSTANTS } from "@/lib/constants";++const ENDPOINT = "http://homelab:5080/api/default";++function span(name: string) {+ return {+ name,+ traceId: "a".repeat(32),+ spanId: "b".repeat(16),+ startedAt: 1_756_500_000_000,+ endedAt: 1_756_500_000_100,+ attrs: {},+ } as any;+}++describe("otlp queue and flush timer", () => {+ beforeEach(() => vi.useFakeTimers());+ afterEach(() => {+ vi.useRealTimers();+ vi.unstubAllEnvs();+ vi.unstubAllGlobals();+ });++ it("enqueueing schedules a flush instead of performing one", async () => {+ const fetchMock = mockFetchOk();+ const { otlp } = await loadTelemetry({ endpoint: ENDPOINT });++ for (let i = 0; i < APP_CONSTANTS.TELEMETRY_BATCH_SIZE + 10; i++) {+ otlp.enqueueSpan(span(`s${i}`));+ }+ // The whole point: no serialization and no POST in the caller's stack.+ expect(fetchMock).not.toHaveBeenCalled();++ await vi.advanceTimersByTimeAsync(APP_CONSTANTS.TELEMETRY_FLUSH_INTERVAL_MS);+ expect(fetchMock).toHaveBeenCalledTimes(1);+ expect(fetchMock.mock.calls[0]![0]).toBe(`${ENDPOINT}/v1/traces`);+ });++ it("drops the oldest span past the queue cap", async () => {+ mockFetchOk();+ const { otlp } = await loadTelemetry({ endpoint: ENDPOINT });++ for (let i = 0; i < APP_CONSTANTS.TELEMETRY_MAX_QUEUE + 5; i++) {+ otlp.enqueueSpan(span(`s${i}`));+ }+ const stats = otlp.getTelemetryStats();+ expect(stats.queuedSpans).toBe(APP_CONSTANTS.TELEMETRY_MAX_QUEUE);+ expect(stats.dropped).toBe(5);+ });++ it("backs off on consecutive failures and resets on success", async () => {+ const fetchMock = vi.fn(async () => new Response("nope", { status: 500 }));+ vi.stubGlobal("fetch", fetchMock);+ const { otlp } = await loadTelemetry({ endpoint: ENDPOINT });+ const [first, second] = APP_CONSTANTS.TELEMETRY_BACKOFF_MS;++ otlp.enqueueSpan(span("s1"));+ otlp.enqueueSpan(span("s2"));+ await vi.advanceTimersByTimeAsync(first!);+ expect(otlp.getTelemetryStats().failures).toBe(1);++ // The retry of what is still queued now waits the second backoff step.+ otlp.enqueueSpan(span("s3"));+ await vi.advanceTimersByTimeAsync(first!);+ expect(fetchMock).toHaveBeenCalledTimes(1);+ await vi.advanceTimersByTimeAsync(second! - first!);+ expect(fetchMock).toHaveBeenCalledTimes(2);+ });+});diff --git a/__tests__/telemetry/resilience.spec.ts b/__tests__/telemetry/resilience.spec.tsnew file mode 100644index 0000000..a2e9c40--- /dev/null+++ b/__tests__/telemetry/resilience.spec.ts@@ -0,0 +1,72 @@+import { loadTelemetry, mockFetchOk } from "./helpers";+import { APP_CONSTANTS } from "@/lib/constants";++const ENDPOINT = "http://homelab:5080/api/default";++describe("telemetry never breaks a request", () => {+ afterEach(() => {+ vi.unstubAllEnvs();+ vi.unstubAllGlobals();+ });++ it("swallows a rejecting fetch", async () => {+ vi.useFakeTimers();+ vi.stubGlobal("fetch", vi.fn(async () => { throw new Error("ECONNREFUSED"); }));+ const { span, otlp } = await loadTelemetry({ endpoint: ENDPOINT });++ span.startSpan("probe").end();+ await expect(+ vi.advanceTimersByTimeAsync(APP_CONSTANTS.TELEMETRY_FLUSH_INTERVAL_MS),+ ).resolves.not.toThrow();+ expect(otlp.getTelemetryStats().failures).toBe(1);+ vi.useRealTimers();+ });++ it("swallows a synchronous throw from a hostile attrs object", async () => {+ mockFetchOk();+ const { span } = await loadTelemetry({ endpoint: ENDPOINT });++ const hostile = {+ get "jobsync.surface"() {+ throw new TypeError("no");+ },+ };++ expect(() => span.startSpan("probe", hostile as any)).not.toThrow();+ const live = span.startSpan("probe");+ expect(() => live.setAttrs(hostile as any)).not.toThrow();+ expect(() => live.end(hostile as any)).not.toThrow();+ });++ it("drops a record whose attribute cannot be serialized without losing the batch", async () => {+ const fetchMock = mockFetchOk();+ vi.useFakeTimers();+ const { span, otlp } = await loadTelemetry({ endpoint: ENDPOINT });++ const circular: Record<string, unknown> = {};+ circular.self = circular;+ span.startSpan("bad").end({ "jobsync.payload": circular });+ span.startSpan("good").end();++ await vi.advanceTimersByTimeAsync(APP_CONSTANTS.TELEMETRY_FLUSH_INTERVAL_MS);+ const body = JSON.parse(fetchMock.mock.calls[0]![1]!.body as string);+ const names = body.resourceSpans[0].scopeSpans[0].spans.map(+ (s: any) => s.name,+ );+ expect(names).toEqual(["good"]);+ expect(otlp.getTelemetryStats().failures).toBe(0);+ vi.useRealTimers();+ });++ it("propagates the wrapped function's own error while still ending the span", async () => {+ mockFetchOk();+ const { span, otlp } = await loadTelemetry({ endpoint: ENDPOINT });++ await expect(+ span.withSpan("boom", {}, async () => {+ throw new Error("application failure");+ }),+ ).rejects.toThrow("application failure");+ expect(otlp.getTelemetryStats().queuedSpans).toBe(1);+ });+});diff --git a/src/lib/constants.ts b/src/lib/constants.tsindex 889d0a8..0a0d973 100644--- a/src/lib/constants.ts+++ b/src/lib/constants.ts@@ -194,6 +194,22 @@ export const APP_CONSTANTS = {// Must not be "ai-panel-width" — that key belongs to the three AI sheets.AGENT_CHAT_PANEL_WIDTH_KEY: "agent-chat-width",+ // Telemetry (OTLP export)+ // Flush cadence. A batch is never serialized in the enqueueing caller's+ // stack — reaching the batch size only arms the timer.+ TELEMETRY_FLUSH_INTERVAL_MS: 5_000,+ // Consecutive-failure backoff, resets to the first entry on any success.+ // A collector down for a day is retried a few dozen times, not 17,000.+ TELEMETRY_BACKOFF_MS: [5_000, 30_000, 60_000],+ // Bounded queue, drops oldest — mirrors MAX_LOGS_PER_RUN in+ // automation-logger.ts. A dead collector cannot grow memory without limit.+ TELEMETRY_MAX_QUEUE: 1_000,+ TELEMETRY_BATCH_SIZE: 200,+ TELEMETRY_EXPORT_TIMEOUT_MS: 5_000,+ // A resume-review prompt runs 10-50 KB; this caps what leaves the process.+ // Not the same thing as jobsync.input.truncated — see spec section 8.+ TELEMETRY_MAX_ATTR_CHARS: 32_000,+// File uploadsUPLOADS_DIR: process.env.NODE_ENV !== "production" ? "data" : "/data",diff --git a/src/lib/telemetry/config.ts b/src/lib/telemetry/config.tsnew file mode 100644index 0000000..d90a3c3--- /dev/null+++ b/src/lib/telemetry/config.ts@@ -0,0 +1,37 @@+// Read once, at module load. That is what makes "disabled" genuinely free:+// startSpan and log return immediately, no queue exists, fetch is never+// referenced. Nothing else in the app reads process.env for telemetry.++function parseHeaders(raw: string | undefined): Record<string, string> {+ const headers: Record<string, string> = {};+ for (const pair of (raw ?? "").split(",")) {+ const trimmed = pair.trim();+ if (!trimmed) continue;+ // First "=" only: an Authorization value is base64 and carries its own.+ const separator = trimmed.indexOf("=");+ if (separator <= 0) continue;+ headers[trimmed.slice(0, separator).trim()] = trimmed+ .slice(separator + 1)+ .trim();+ }+ return headers;+}++const endpoint = (process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "")+ .trim()+ .replace(/\/+$/, "");++export const TELEMETRY_ENABLED = endpoint.length > 0;++// Two independent stages: content capture has no effect without an endpoint.+export const CAPTURE_CONTENT =+ TELEMETRY_ENABLED && process.env.OTEL_CAPTURE_CONTENT === "true";++export const SERVICE_NAME = process.env.OTEL_SERVICE_NAME || "jobsync";++// The base/suffix split is the OTel convention, and is what keeps the backend+// swappable for Seq, Tempo or SigNoz by changing one URL.+export const TRACES_URL = TELEMETRY_ENABLED ? `${endpoint}/v1/traces` : "";+export const LOGS_URL = TELEMETRY_ENABLED ? `${endpoint}/v1/logs` : "";++export const OTLP_HEADERS = parseHeaders(process.env.OTEL_EXPORTER_OTLP_HEADERS);diff --git a/src/lib/telemetry/context.ts b/src/lib/telemetry/context.tsnew file mode 100644index 0000000..2e3c063--- /dev/null+++ b/src/lib/telemetry/context.ts@@ -0,0 +1,22 @@+import { AsyncLocalStorage } from "node:async_hooks";++export type ActiveSpan = { traceId: string; spanId: string };++// Unconditional globalThis stash, unlike db.ts's and automation-logger.ts's+// dev-only guards. Those exist to survive hot reload; this one exists because+// Next can emit a module into more than one server bundle, and two ALS+// instances would silently orphan child spans instead of merely duplicating+// work — the waterfall would arrive flat with nothing reporting it.+const globalForTelemetry = globalThis as typeof globalThis & {+ telemetryContextGlobal?: AsyncLocalStorage<ActiveSpan>;+};++export const activeSpanStorage =+ globalForTelemetry.telemetryContextGlobal ??+ new AsyncLocalStorage<ActiveSpan>();++globalForTelemetry.telemetryContextGlobal = activeSpanStorage;++export function currentSpan(): ActiveSpan | undefined {+ return activeSpanStorage.getStore();+}diff --git a/src/lib/telemetry/genai.ts b/src/lib/telemetry/genai.tsnew file mode 100644index 0000000..2a80e56--- /dev/null+++ b/src/lib/telemetry/genai.ts@@ -0,0 +1,2 @@+// Filled in by Phase 3 — OTel GenAI semantic-convention mapping.+export {};diff --git a/src/lib/telemetry/index.ts b/src/lib/telemetry/index.tsnew file mode 100644index 0000000..d514f42--- /dev/null+++ b/src/lib/telemetry/index.ts@@ -0,0 +1,8 @@+export {+ CAPTURE_CONTENT,+ SERVICE_NAME,+ TELEMETRY_ENABLED,+} from "./config";+export { currentSpan, type ActiveSpan } from "./context";+export { runInSpan, startSpan, withSpan, type Span } from "./span";+export { type Attrs } from "./otlp";diff --git a/src/lib/telemetry/logger.ts b/src/lib/telemetry/logger.tsnew file mode 100644index 0000000..8bd13c2--- /dev/null+++ b/src/lib/telemetry/logger.ts@@ -0,0 +1,2 @@+// Filled in by Phase 2 — log.info / warn / error over console.*.+export {};diff --git a/src/lib/telemetry/otlp.ts b/src/lib/telemetry/otlp.tsnew file mode 100644index 0000000..f605e4a--- /dev/null+++ b/src/lib/telemetry/otlp.ts@@ -0,0 +1,290 @@+import { APP_CONSTANTS } from "@/lib/constants";+import {+ LOGS_URL,+ OTLP_HEADERS,+ SERVICE_NAME,+ TELEMETRY_ENABLED,+ TRACES_URL,+} from "./config";++export type Attrs = Record<string, unknown>;++export type QueuedSpan = {+ name: string;+ traceId: string;+ spanId: string;+ parentSpanId?: string;+ startedAt: number;+ endedAt: number;+ attrs: Attrs;+ error?: boolean;+ errorMessage?: string;+};++export type QueuedLog = {+ timestamp: number;+ severityNumber: number;+ severityText: string;+ body: string;+ attrs: Attrs;+ traceId?: string;+ spanId?: string;+};++type TelemetryState = {+ spans: QueuedSpan[];+ logs: QueuedLog[];+ timer: ReturnType<typeof setTimeout> | null;+ inFlight: boolean;+ failures: number;+ dropped: number;+};++// Same globalThis pattern as db.ts, but unconditional: a second copy of this+// module in another server bundle would mean a second timer and a second+// partial batch for no benefit.+const globalForTelemetry = globalThis as typeof globalThis & {+ telemetryStateGlobal?: TelemetryState;+};++function state(): TelemetryState {+ if (!globalForTelemetry.telemetryStateGlobal) {+ globalForTelemetry.telemetryStateGlobal = {+ spans: [],+ logs: [],+ timer: null,+ inFlight: false,+ failures: 0,+ dropped: 0,+ };+ }+ return globalForTelemetry.telemetryStateGlobal;+}++// BigInt literals need ES2020; the project targets ES2017, so build the+// multiplier with the constructor instead.+function nanos(ms: number): string {+ return String(BigInt(Math.trunc(ms)) * BigInt(1_000_000));+}++function encodeValue(value: unknown): unknown {+ if (typeof value === "string") return { stringValue: value };+ if (typeof value === "boolean") return { boolValue: value };+ if (typeof value === "number") {+ return Number.isInteger(value)+ ? { intValue: String(value) }+ : { doubleValue: value };+ }+ return { stringValue: JSON.stringify(value) };+}++function toKeyValues(attrs: Attrs): unknown[] {+ const out: unknown[] = [];+ const capped: string[] = [];+ for (const [key, raw] of Object.entries(attrs)) {+ if (raw === undefined || raw === null) continue;+ const encoded = encodeValue(raw) as { stringValue?: string };+ if (+ typeof encoded.stringValue === "string" &&+ encoded.stringValue.length > APP_CONSTANTS.TELEMETRY_MAX_ATTR_CHARS+ ) {+ encoded.stringValue = encoded.stringValue.slice(+ 0,+ APP_CONSTANTS.TELEMETRY_MAX_ATTR_CHARS,+ );+ capped.push(key);+ }+ out.push({ key, value: encoded });+ }+ // Marks that TELEMETRY clipped the attribute, not that the app clipped the+ // model input — jobsync.input.truncated is the other fact entirely.+ if (capped.length) {+ out.push({+ key: "jobsync.telemetry.capped_attrs",+ value: { stringValue: capped.join(",") },+ });+ }+ return out;+}++function resource() {+ return {+ attributes: [{ key: "service.name", value: { stringValue: SERVICE_NAME } }],+ };+}++function encodeSpan(span: QueuedSpan) {+ return {+ traceId: span.traceId,+ spanId: span.spanId,+ ...(span.parentSpanId ? { parentSpanId: span.parentSpanId } : {}),+ name: span.name,+ kind: 1,+ startTimeUnixNano: nanos(span.startedAt),+ endTimeUnixNano: nanos(span.endedAt),+ attributes: toKeyValues(span.attrs),+ status: span.error+ ? { code: 2, message: span.errorMessage ?? "" }+ : { code: 0 },+ };+}++function encodeLog(record: QueuedLog) {+ return {+ timeUnixNano: nanos(record.timestamp),+ severityNumber: record.severityNumber,+ severityText: record.severityText,+ body: { stringValue: record.body },+ attributes: toKeyValues(record.attrs),+ ...(record.traceId ? { traceId: record.traceId } : {}),+ ...(record.spanId ? { spanId: record.spanId } : {}),+ };+}++// A single record that cannot be encoded (a circular attribute value) must+// not take the rest of the batch down with it.+function encodeEach<T>(items: T[], encode: (item: T) => unknown): unknown[] {+ const out: unknown[] = [];+ for (const item of items) {+ try {+ out.push(encode(item));+ } catch {+ // Dropped silently: telemetry never reports on telemetry.+ }+ }+ return out;+}++export function buildTracePayload(spans: QueuedSpan[]) {+ return {+ resourceSpans: [+ {+ resource: resource(),+ scopeSpans: [+ { scope: { name: "jobsync" }, spans: encodeEach(spans, encodeSpan) },+ ],+ },+ ],+ };+}++export function buildLogPayload(records: QueuedLog[]) {+ return {+ resourceLogs: [+ {+ resource: resource(),+ scopeLogs: [+ {+ scope: { name: "jobsync" },+ logRecords: encodeEach(records, encodeLog),+ },+ ],+ },+ ],+ };+}++function backoffDelay(failures: number): number {+ const table = APP_CONSTANTS.TELEMETRY_BACKOFF_MS;+ return table[Math.min(failures, table.length - 1)]!;+}++function ensureTimer(): void {+ const current = state();+ if (current.timer) return;+ const timer = setTimeout(() => {+ state().timer = null;+ void flush();+ }, backoffDelay(current.failures));+ // Never hold the process open for a flush.+ timer.unref?.();+ current.timer = timer;+}++async function post(url: string, payload: unknown): Promise<boolean> {+ try {+ const response = await fetch(url, {+ method: "POST",+ headers: { "Content-Type": "application/json", ...OTLP_HEADERS },+ body: JSON.stringify(payload),+ signal: AbortSignal.timeout(APP_CONSTANTS.TELEMETRY_EXPORT_TIMEOUT_MS),+ });+ return response.ok;+ } catch {+ return false;+ }+}++async function flush(): Promise<void> {+ const current = state();+ // A collector hanging until the POST timeout would otherwise stack+ // in-flight requests, since the timeout is not below the flush interval.+ if (current.inFlight) {+ if (current.spans.length || current.logs.length) ensureTimer();+ return;+ }++ const spans = current.spans.splice(0, APP_CONSTANTS.TELEMETRY_BATCH_SIZE);+ const logs = current.logs.splice(0, APP_CONSTANTS.TELEMETRY_BATCH_SIZE);+ if (!spans.length && !logs.length) return;++ current.inFlight = true;+ try {+ const results = await Promise.all([+ spans.length ? post(TRACES_URL, buildTracePayload(spans)) : true,+ logs.length ? post(LOGS_URL, buildLogPayload(logs)) : true,+ ]);+ // Counted, never logged per occurrence: an hour of downtime must not+ // produce an hour of error spam in the container's own stdout.+ current.failures = results.every(Boolean) ? 0 : current.failures + 1;+ } catch {+ current.failures += 1;+ } finally {+ current.inFlight = false;+ if (current.spans.length || current.logs.length) ensureTimer();+ }+}++function enqueue<T>(queue: T[], record: T): void {+ queue.push(record);+ while (queue.length > APP_CONSTANTS.TELEMETRY_MAX_QUEUE) {+ queue.shift();+ state().dropped += 1;+ }+ ensureTimer();+}++export function enqueueSpan(span: QueuedSpan): void {+ if (!TELEMETRY_ENABLED) return;+ try {+ enqueue(state().spans, span);+ } catch {+ // Enqueueing must never throw into a request.+ }+}++export function enqueueLog(record: QueuedLog): void {+ if (!TELEMETRY_ENABLED) return;+ try {+ enqueue(state().logs, record);+ } catch {+ // Enqueueing must never throw into a request.+ }+}++export function getTelemetryStats() {+ const current = state();+ return {+ queuedSpans: current.spans.length,+ queuedLogs: current.logs.length,+ dropped: current.dropped,+ failures: current.failures,+ };+}++// Test-only: the state lives on globalThis and survives vi.resetModules().+export function resetTelemetryStateForTests(): void {+ const current = state();+ if (current.timer) clearTimeout(current.timer);+ globalForTelemetry.telemetryStateGlobal = undefined;+}diff --git a/src/lib/telemetry/span.ts b/src/lib/telemetry/span.tsnew file mode 100644index 0000000..2a967d8--- /dev/null+++ b/src/lib/telemetry/span.ts@@ -0,0 +1,115 @@+import { randomBytes } from "node:crypto";+import { TELEMETRY_ENABLED } from "./config";+import { activeSpanStorage, currentSpan } from "./context";+import { enqueueSpan, type Attrs } from "./otlp";++export type Span = {+ traceId: string;+ spanId: string;+ setAttrs(attrs: Attrs): void;+ setError(error: unknown): void;+ end(attrs?: Attrs): void;+};++// One shared object rather than a fresh no-op per call: with telemetry off,+// an instrumented request must allocate nothing at all.+const NOOP_SPAN: Span = Object.freeze({+ traceId: "",+ spanId: "",+ setAttrs() {},+ setError() {},+ end() {},+});++function merge(target: Attrs, source: Attrs | undefined): void {+ if (!source) return;+ for (const [key, value] of Object.entries(source)) {+ if (value === undefined || value === null) continue;+ target[key] = value;+ }+}++export function startSpan(name: string, attrs?: Attrs): Span {+ if (!TELEMETRY_ENABLED) return NOOP_SPAN;+ try {+ const parent = currentSpan();+ const traceId = parent?.traceId ?? randomBytes(16).toString("hex");+ const spanId = randomBytes(8).toString("hex");+ const startedAt = Date.now();+ const collected: Attrs = {};+ merge(collected, attrs);+ let ended = false;+ let error = false;+ let errorMessage: string | undefined;++ return {+ traceId,+ spanId,+ setAttrs(next) {+ try {+ merge(collected, next);+ } catch {+ // A hostile attrs object degrades telemetry, never the request.+ }+ },+ setError(cause) {+ try {+ error = true;+ errorMessage =+ cause instanceof Error ? cause.message : String(cause);+ } catch {+ error = true;+ }+ },+ end(next) {+ if (ended) return;+ ended = true;+ try {+ merge(collected, next);+ } catch {+ // Keep whatever was collected before the bad object.+ }+ try {+ enqueueSpan({+ name,+ traceId,+ spanId,+ parentSpanId: parent?.spanId,+ startedAt,+ endedAt: Date.now(),+ attrs: collected,+ error,+ errorMessage,+ });+ } catch {+ // Enqueueing must never throw into a request.+ }+ },+ };+ } catch {+ return NOOP_SPAN;+ }+}++// Separate from startSpan because the chat turn starts its span in one scope,+// makes it ambient in a second, and ends it in a third.+export function runInSpan<T>(span: Span, fn: () => T): T {+ if (!TELEMETRY_ENABLED || !span.spanId) return fn();+ return activeSpanStorage.run({ traceId: span.traceId, spanId: span.spanId }, fn);+}++export async function withSpan<T>(+ name: string,+ attrs: Attrs,+ fn: (span: Span) => Promise<T>,+): Promise<T> {+ const span = startSpan(name, attrs);+ try {+ return await runInSpan(span, () => fn(span));+ } catch (error) {+ span.setError(error);+ throw error;+ } finally {+ span.end();+ }+}