Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
Add server observability tracing and metrics (#1697)
Julius Marminge752f96e9ae72
diff --git a/README.md b/README.mdindex d7b2fccb8..d3a54a1b9 100644--- a/README.md+++ b/README.md@@ -45,6 +45,8 @@ We are very very early in this project. Expect bugs.We are not accepting contributions yet.+Observability guide: [docs/observability.md](./docs/observability.md)+## If you REALLY want to contribute still.... read this firstRead [CONTRIBUTING.md](./CONTRIBUTING.md) before opening an issue or PR.diff --git a/apps/server/src/cli-config.test.ts b/apps/server/src/cli-config.test.tsindex 27bc60b1b..d3753988d 100644--- a/apps/server/src/cli-config.test.ts+++ b/apps/server/src/cli-config.test.ts@@ -9,6 +9,18 @@ import { deriveServerPaths } from "./config";import { resolveServerConfig } from "./cli";it.layer(NodeServices.layer)("cli config resolution", (it) => {+ const defaultObservabilityConfig = {+ traceMinLevel: "Info",+ traceTimingEnabled: true,+ traceBatchWindowMs: 200,+ traceMaxBytes: 10 * 1024 * 1024,+ traceMaxFiles: 10,+ otlpTracesUrl: undefined,+ otlpMetricsUrl: undefined,+ otlpExportIntervalMs: 10_000,+ otlpServiceName: "t3-server",+ } as const;+const openBootstrapFd = Effect.fn(function* (payload: Record<string, unknown>) {const fs = yield* FileSystem.FileSystem;const filePath = yield* fs.makeTempFileScoped({ prefix: "t3-bootstrap-", suffix: ".ndjson" });@@ -62,6 +74,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {expect(resolved).toEqual({logLevel: "Warn",+ ...defaultObservabilityConfig,mode: "desktop",port: 4001,cwd: process.cwd(),@@ -123,6 +136,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {expect(resolved).toEqual({logLevel: "Debug",+ ...defaultObservabilityConfig,mode: "web",port: 8788,cwd: process.cwd(),@@ -187,6 +201,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {expect(resolved).toEqual({logLevel: "Info",+ ...defaultObservabilityConfig,mode: "desktop",port: 4888,cwd: process.cwd(),@@ -241,6 +256,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {resolved.attachmentsDir,resolved.worktreesDir,path.dirname(resolved.serverLogPath),+ path.dirname(resolved.serverTracePath),]) {expect(yield* fs.exists(directory)).toBe(true);}@@ -300,6 +316,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {expect(resolved).toEqual({logLevel: "Debug",+ ...defaultObservabilityConfig,mode: "web",port: 8788,cwd: process.cwd(),diff --git a/apps/server/src/cli.ts b/apps/server/src/cli.tsindex 2f76d25b7..af3202954 100644--- a/apps/server/src/cli.ts+++ b/apps/server/src/cli.ts@@ -1,5 +1,5 @@import { NetService } from "@t3tools/shared/Net";-import { Config, Effect, LogLevel, Option, Schema } from "effect";+import { Config, Effect, FileSystem, LogLevel, Option, Path, Schema } from "effect";import { Command, Flag, GlobalFlag } from "effect/unstable/cli";import {@@ -81,6 +81,27 @@ const logWebSocketEventsFlag = Flag.boolean("log-websocket-events").pipe(const EnvServerConfig = Config.all({logLevel: Config.logLevel("T3CODE_LOG_LEVEL").pipe(Config.withDefault("Info")),+ traceMinLevel: Config.logLevel("T3CODE_TRACE_MIN_LEVEL").pipe(Config.withDefault("Info")),+ traceTimingEnabled: Config.boolean("T3CODE_TRACE_TIMING_ENABLED").pipe(Config.withDefault(true)),+ traceFile: Config.string("T3CODE_TRACE_FILE").pipe(+ Config.option,+ Config.map(Option.getOrUndefined),+ ),+ traceMaxBytes: Config.int("T3CODE_TRACE_MAX_BYTES").pipe(Config.withDefault(10 * 1024 * 1024)),+ traceMaxFiles: Config.int("T3CODE_TRACE_MAX_FILES").pipe(Config.withDefault(10)),+ traceBatchWindowMs: Config.int("T3CODE_TRACE_BATCH_WINDOW_MS").pipe(Config.withDefault(200)),+ otlpTracesUrl: Config.string("T3CODE_OTLP_TRACES_URL").pipe(+ Config.option,+ Config.map(Option.getOrUndefined),+ ),+ otlpMetricsUrl: Config.string("T3CODE_OTLP_METRICS_URL").pipe(+ Config.option,+ Config.map(Option.getOrUndefined),+ ),+ otlpExportIntervalMs: Config.int("T3CODE_OTLP_EXPORT_INTERVAL_MS").pipe(+ Config.withDefault(10_000),+ ),+ otlpServiceName: Config.string("T3CODE_OTLP_SERVICE_NAME").pipe(Config.withDefault("t3-server")),mode: Config.schema(RuntimeMode, "T3CODE_MODE").pipe(Config.option,Config.map(Option.getOrUndefined),@@ -137,6 +158,8 @@ export const resolveServerConfig = () =>Effect.gen(function* () {const { findAvailablePort } = yield* NetService;+ const path = yield* Path.Path;+ const fs = yield* FileSystem.FileSystem;const env = yield* EnvServerConfig;const bootstrapFd = Option.getOrUndefined(flags.bootstrapFd) ?? env.bootstrapFd;const bootstrapEnvelope =@@ -190,6 +213,8 @@ export const resolveServerConfig = ();const derivedPaths = yield* deriveServerPaths(baseDir, devUrl);yield* ensureServerDirectories(derivedPaths);+ const serverTracePath = env.traceFile ?? derivedPaths.serverTracePath;+ yield* fs.makeDirectory(path.dirname(serverTracePath), { recursive: true });const noBrowser = resolveBooleanFlag(flags.noBrowser,Option.getOrElse(@@ -248,11 +273,21 @@ export const resolveServerConfig = (const config: ServerConfigShape = {logLevel,+ traceMinLevel: env.traceMinLevel,+ traceTimingEnabled: env.traceTimingEnabled,+ traceBatchWindowMs: env.traceBatchWindowMs,+ traceMaxBytes: env.traceMaxBytes,+ traceMaxFiles: env.traceMaxFiles,+ otlpTracesUrl: env.otlpTracesUrl,+ otlpMetricsUrl: env.otlpMetricsUrl,+ otlpExportIntervalMs: env.otlpExportIntervalMs,+ otlpServiceName: env.otlpServiceName,mode,port,cwd: process.cwd(),baseDir,...derivedPaths,+ serverTracePath,host,staticDir,devUrl,diff --git a/apps/server/src/config.ts b/apps/server/src/config.tsindex 21cd6f315..9ceea4c13 100644--- a/apps/server/src/config.ts+++ b/apps/server/src/config.ts@@ -25,6 +25,7 @@ export interface ServerDerivedPaths {readonly attachmentsDir: string;readonly logsDir: string;readonly serverLogPath: string;+ readonly serverTracePath: string;readonly providerLogsDir: string;readonly providerEventLogPath: string;readonly terminalLogsDir: string;@@ -36,6 +37,15 @@ export interface ServerDerivedPaths {*/export interface ServerConfigShape extends ServerDerivedPaths {readonly logLevel: LogLevel.LogLevel;+ readonly traceMinLevel: LogLevel.LogLevel;+ readonly traceTimingEnabled: boolean;+ readonly traceBatchWindowMs: number;+ readonly traceMaxBytes: number;+ readonly traceMaxFiles: number;+ readonly otlpTracesUrl: string | undefined;+ readonly otlpMetricsUrl: string | undefined;+ readonly otlpExportIntervalMs: number;+ readonly otlpServiceName: string;readonly mode: RuntimeMode;readonly port: number;readonly host: string | undefined;@@ -68,6 +78,7 @@ export const deriveServerPaths = Effect.fn(function* (attachmentsDir,logsDir,serverLogPath: join(logsDir, "server.log"),+ serverTracePath: join(logsDir, "server.trace.ndjson"),providerLogsDir,providerEventLogPath: join(providerLogsDir, "events.log"),terminalLogsDir: join(logsDir, "terminals"),@@ -117,6 +128,15 @@ export class ServerConfig extends ServiceMap.Service<ServerConfig, ServerConfigSreturn {logLevel: "Error",+ traceMinLevel: "Info",+ traceTimingEnabled: true,+ traceBatchWindowMs: 200,+ traceMaxBytes: 10 * 1024 * 1024,+ traceMaxFiles: 10,+ otlpTracesUrl: undefined,+ otlpMetricsUrl: undefined,+ otlpExportIntervalMs: 10_000,+ otlpServiceName: "t3-server",cwd,baseDir,...derivedPaths,diff --git a/apps/server/src/git/Layers/GitCore.ts b/apps/server/src/git/Layers/GitCore.tsindex b01406411..1178a4b67 100644--- a/apps/server/src/git/Layers/GitCore.ts+++ b/apps/server/src/git/Layers/GitCore.ts@@ -20,6 +20,8 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";import { GitCommandError, type GitBranch } from "@t3tools/contracts";import { dedupeRemoteBranchesWithLocalMatches } from "@t3tools/shared/git";+import { compactTraceAttributes } from "../../observability/Attributes.ts";+import { gitCommandDuration, gitCommandsTotal, withMetrics } from "../../observability/Metrics.ts";import {GitCore,type ExecuteGitProgress,@@ -372,6 +374,18 @@ interface Trace2Monitor {readonly flush: Effect.Effect<void, never>;}+const nowUnixNano = (): bigint => BigInt(Date.now()) * 1_000_000n;++const addCurrentSpanEvent = (name: string, attributes: Record<string, unknown>) =>+ Effect.currentSpan.pipe(+ Effect.tap((span) =>+ Effect.sync(() => {+ span.event(name, nowUnixNano(), compactTraceAttributes(attributes));+ }),+ ),+ Effect.catch(() => Effect.void),+ );+function trace2ChildKey(record: Record<string, unknown>): string | null {const childId = record.child_id;if (typeof childId === "number" || typeof childId === "string") {@@ -444,6 +458,9 @@ const createTrace2Monitor = Effect.fn("createTrace2Monitor")(function* (if (event === "child_start") {hookStartByChildKey.set(childKey, { hookName, startedAtMs: Date.now() });+ yield* addCurrentSpanEvent("git.hook.started", {+ hookName,+ });if (progress.onHookStarted) {yield* progress.onHookStarted(hookName);}@@ -452,12 +469,19 @@ const createTrace2Monitor = Effect.fn("createTrace2Monitor")(function* (if (event === "child_exit") {hookStartByChildKey.delete(childKey);+ const code = traceRecord.success.code;+ const exitCode = typeof code === "number" && Number.isInteger(code) ? code : null;+ const durationMs = started ? Math.max(0, Date.now() - started.startedAtMs) : null;+ yield* addCurrentSpanEvent("git.hook.finished", {+ hookName: started?.hookName ?? hookName,+ exitCode,+ durationMs,+ });if (progress.onHookFinished) {- const code = traceRecord.success.code;yield* progress.onHookFinished({hookName: started?.hookName ?? hookName,- exitCode: typeof code === "number" && Number.isInteger(code) ? code : null,- durationMs: started ? Math.max(0, Date.now() - started.startedAtMs) : null,+ exitCode,+ durationMs,});}}@@ -609,13 +633,13 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: {const path = yield* Path.Path;const { worktreesDir } = yield* ServerConfig;- let execute: GitCoreShape["execute"];+ let executeRaw: GitCoreShape["execute"];if (options?.executeOverride) {- execute = options.executeOverride;+ executeRaw = options.executeOverride;} else {const commandSpawner = yield* ChildProcessSpawner.ChildProcessSpawner;- execute = Effect.fnUntraced(function* (input) {+ executeRaw = Effect.fnUntraced(function* (input) {const commandInput = {...input,args: [...input.args],@@ -716,6 +740,25 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: {});}+ const execute: GitCoreShape["execute"] = (input) =>+ executeRaw(input).pipe(+ withMetrics({+ counter: gitCommandsTotal,+ timer: gitCommandDuration,+ attributes: {+ operation: input.operation,+ },+ }),+ Effect.withSpan(input.operation, {+ kind: "client",+ attributes: {+ "git.operation": input.operation,+ "git.cwd": input.cwd,+ "git.args_count": input.args.length,+ },+ }),+ );+const executeGit = (operation: string,cwd: string,diff --git a/apps/server/src/observability/Attributes.test.ts b/apps/server/src/observability/Attributes.test.tsnew file mode 100644index 000000000..4b495598e--- /dev/null+++ b/apps/server/src/observability/Attributes.test.ts@@ -0,0 +1,46 @@+import { assert, describe, it } from "@effect/vitest";++import { compactTraceAttributes, normalizeModelMetricLabel } from "./Attributes.ts";++describe("Attributes", () => {+ it("normalizes circular arrays, maps, and sets without recursing forever", () => {+ const array: Array<unknown> = ["alpha"];+ array.push(array);++ const map = new Map<string, unknown>();+ map.set("self", map);++ const set = new Set<unknown>();+ set.add(set);++ assert.deepStrictEqual(+ compactTraceAttributes({+ array,+ map,+ set,+ }),+ {+ array: ["alpha", "[Circular]"],+ map: { self: "[Circular]" },+ set: ["[Circular]"],+ },+ );+ });++ it("normalizes invalid dates without throwing", () => {+ assert.deepStrictEqual(+ compactTraceAttributes({+ invalidDate: new Date("not-a-real-date"),+ }),+ {+ invalidDate: "Invalid Date",+ },+ );+ });++ it("groups GPT-family models under a shared metric label", () => {+ assert.strictEqual(normalizeModelMetricLabel("gpt-4o"), "gpt");+ assert.strictEqual(normalizeModelMetricLabel("gpt-5.4"), "gpt");+ assert.strictEqual(normalizeModelMetricLabel("claude-sonnet-4"), "claude");+ });+});diff --git a/apps/server/src/observability/Attributes.ts b/apps/server/src/observability/Attributes.tsnew file mode 100644index 000000000..2251fcfea--- /dev/null+++ b/apps/server/src/observability/Attributes.ts@@ -0,0 +1,128 @@+import { Cause, Exit } from "effect";++export type MetricAttributeValue = string;+export type MetricAttributes = Readonly<Record<string, MetricAttributeValue>>;+export type TraceAttributes = Readonly<Record<string, unknown>>;+export type ObservabilityOutcome = "success" | "failure" | "interrupt";++function isPlainObject(value: unknown): value is Record<string, unknown> {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}++function markSeen(value: object, seen: WeakSet<object>): boolean {+ if (seen.has(value)) {+ return true;+ }+ seen.add(value);+ return false;+}++function normalizeJsonValue(value: unknown, seen: WeakSet<object> = new WeakSet()): unknown {+ if (+ value === null ||+ value === undefined ||+ typeof value === "string" ||+ typeof value === "number" ||+ typeof value === "boolean"+ ) {+ return value ?? null;+ }+ if (typeof value === "bigint") {+ return value.toString();+ }+ if (value instanceof Date) {+ return Number.isNaN(value.getTime()) ? "Invalid Date" : value.toISOString();+ }+ if (value instanceof Error) {+ return {+ name: value.name,+ message: value.message,+ ...(value.stack ? { stack: value.stack } : {}),+ };+ }+ if (Array.isArray(value)) {+ if (markSeen(value, seen)) {+ return "[Circular]";+ }+ return value.map((entry) => normalizeJsonValue(entry, seen));+ }+ if (value instanceof Map) {+ if (markSeen(value, seen)) {+ return "[Circular]";+ }+ return Object.fromEntries(+ Array.from(value.entries(), ([key, entryValue]) => [+ String(key),+ normalizeJsonValue(entryValue, seen),+ ]),+ );+ }+ if (value instanceof Set) {+ if (markSeen(value, seen)) {+ return "[Circular]";+ }+ return Array.from(value.values(), (entry) => normalizeJsonValue(entry, seen));+ }+ if (!isPlainObject(value)) {+ return String(value);+ }+ if (markSeen(value, seen)) {+ return "[Circular]";+ }+ return Object.fromEntries(+ Object.entries(value).map(([key, entryValue]) => [key, normalizeJsonValue(entryValue, seen)]),+ );+}++export function compactTraceAttributes(+ attributes: Readonly<Record<string, unknown>>,+): TraceAttributes {+ return Object.fromEntries(+ Object.entries(attributes)+ .filter(([, value]) => value !== undefined)+ .map(([key, value]) => [key, normalizeJsonValue(value)]),+ );+}++export function compactMetricAttributes(+ attributes: Readonly<Record<string, unknown>>,+): MetricAttributes {+ return Object.fromEntries(+ Object.entries(attributes).flatMap(([key, value]) => {+ if (value === undefined || value === null) {+ return [];+ }+ if (typeof value === "string") {+ return [[key, value]];+ }+ if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") {+ return [[key, String(value)]];+ }+ return [];+ }),+ );+}++export function outcomeFromExit(exit: Exit.Exit<unknown, unknown>): ObservabilityOutcome {+ if (Exit.isSuccess(exit)) {+ return "success";+ }+ return Cause.hasInterruptsOnly(exit.cause) ? "interrupt" : "failure";+}++export function normalizeModelMetricLabel(model: string | null | undefined): string | undefined {+ const normalized = model?.trim().toLowerCase();+ if (!normalized) {+ return undefined;+ }+ if (normalized.includes("gpt")) {+ return "gpt";+ }+ if (normalized.includes("claude")) {+ return "claude";+ }+ if (normalized.includes("gemini")) {+ return "gemini";+ }+ return "other";+}diff --git a/apps/server/src/observability/Layers/Observability.ts b/apps/server/src/observability/Layers/Observability.tsnew file mode 100644index 000000000..29b9d7eac--- /dev/null+++ b/apps/server/src/observability/Layers/Observability.ts@@ -0,0 +1,64 @@+import { Effect, Layer, References, Tracer } from "effect";+import { OtlpMetrics, OtlpSerialization, OtlpTracer } from "effect/unstable/observability";++import { ServerConfig } from "../../config.ts";+import { ServerLoggerLive } from "../../serverLogger.ts";+import { makeLocalFileTracer } from "../LocalFileTracer.ts";++const otlpSerializationLayer = OtlpSerialization.layerJson;++export const ObservabilityLive = Layer.unwrap(+ Effect.gen(function* () {+ const config = yield* ServerConfig;++ const traceReferencesLayer = Layer.mergeAll(+ Layer.succeed(Tracer.MinimumTraceLevel, config.traceMinLevel),+ Layer.succeed(References.TracerTimingEnabled, config.traceTimingEnabled),+ );++ const tracerLayer = Layer.effect(+ Tracer.Tracer,+ Effect.gen(function* () {+ const delegate =+ config.otlpTracesUrl === undefined+ ? undefined+ : yield* OtlpTracer.make({+ url: config.otlpTracesUrl,+ exportInterval: `${config.otlpExportIntervalMs} millis`,+ resource: {+ serviceName: config.otlpServiceName,+ attributes: {+ "service.runtime": "t3-server",+ "service.mode": config.mode,+ },+ },+ });++ return yield* makeLocalFileTracer({+ filePath: config.serverTracePath,+ maxBytes: config.traceMaxBytes,+ maxFiles: config.traceMaxFiles,+ batchWindowMs: config.traceBatchWindowMs,+ ...(delegate ? { delegate } : {}),+ });+ }),+ ).pipe(Layer.provideMerge(otlpSerializationLayer));++ const metricsLayer =+ config.otlpMetricsUrl === undefined+ ? Layer.empty+ : OtlpMetrics.layer({+ url: config.otlpMetricsUrl,+ exportInterval: `${config.otlpExportIntervalMs} millis`,+ resource: {+ serviceName: config.otlpServiceName,+ attributes: {+ "service.runtime": "t3-server",+ "service.mode": config.mode,+ },+ },+ }).pipe(Layer.provideMerge(otlpSerializationLayer));++ return Layer.mergeAll(ServerLoggerLive, traceReferencesLayer, tracerLayer, metricsLayer);+ }),+);diff --git a/apps/server/src/observability/LocalFileTracer.test.ts b/apps/server/src/observability/LocalFileTracer.test.tsnew file mode 100644index 000000000..1bc5a3430--- /dev/null+++ b/apps/server/src/observability/LocalFileTracer.test.ts@@ -0,0 +1,117 @@+import fs from "node:fs";+import os from "node:os";+import path from "node:path";++import { assert, describe, it } from "@effect/vitest";+import { Effect, Layer, Logger, References, Tracer } from "effect";++import type { TraceRecord } from "./TraceRecord.ts";+import { makeLocalFileTracer } from "./LocalFileTracer.ts";++const makeTestLayer = (tracePath: string) =>+ Layer.mergeAll(+ Layer.effect(+ Tracer.Tracer,+ makeLocalFileTracer({+ filePath: tracePath,+ maxBytes: 1024 * 1024,+ maxFiles: 2,+ batchWindowMs: 10_000,+ }),+ ),+ Logger.layer([Logger.tracerLogger], { mergeWithExisting: false }),+ Layer.succeed(References.MinimumLogLevel, "Info"),+ );++const readTraceRecords = (tracePath: string): Array<TraceRecord> =>+ fs+ .readFileSync(tracePath, "utf8")+ .trim()+ .split("\n")+ .filter((line) => line.length > 0)+ .map((line) => JSON.parse(line) as TraceRecord);++describe("LocalFileTracer", () => {+ it.effect("writes nested spans to disk and captures log messages as span events", () =>+ Effect.gen(function* () {+ const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-local-tracer-"));+ const tracePath = path.join(tempDir, "server.trace.ndjson");++ try {+ yield* Effect.scoped(+ Effect.gen(function* () {+ const program = Effect.gen(function* () {+ yield* Effect.annotateCurrentSpan({+ "demo.parent": true,+ });+ yield* Effect.logInfo("parent event");+ yield* Effect.gen(function* () {+ yield* Effect.annotateCurrentSpan({+ "demo.child": true,+ });+ yield* Effect.logInfo("child event");+ }).pipe(Effect.withSpan("child-span"));+ }).pipe(Effect.withSpan("parent-span"));++ yield* program.pipe(Effect.provide(makeTestLayer(tracePath)));+ }),+ );++ const records = readTraceRecords(tracePath);+ assert.equal(records.length, 2);++ const parent = records.find((record) => record.name === "parent-span");+ const child = records.find((record) => record.name === "child-span");++ assert.notEqual(parent, undefined);+ assert.notEqual(child, undefined);+ if (!parent || !child) {+ return;+ }++ assert.equal(child.parentSpanId, parent.spanId);+ assert.equal(parent.attributes["demo.parent"], true);+ assert.equal(child.attributes["demo.child"], true);+ assert.equal(+ parent.events.some((event) => event.name === "parent event"),+ true,+ );+ assert.equal(+ child.events.some((event) => event.name === "child event"),+ true,+ );+ assert.equal(+ child.events.some((event) => event.attributes["effect.logLevel"] === "INFO"),+ true,+ );+ } finally {+ fs.rmSync(tempDir, { recursive: true, force: true });+ }+ }),+ );++ it.effect("serializes interrupted spans with an interrupted exit status", () =>+ Effect.gen(function* () {+ const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-local-tracer-"));+ const tracePath = path.join(tempDir, "server.trace.ndjson");++ try {+ yield* Effect.scoped(+ Effect.exit(+ Effect.interrupt.pipe(+ Effect.withSpan("interrupt-span"),+ Effect.provide(makeTestLayer(tracePath)),+ ),+ ),+ );++ const records = readTraceRecords(tracePath);+ assert.equal(records.length, 1);+ assert.equal(records[0]?.name, "interrupt-span");+ assert.equal(records[0]?.exit._tag, "Interrupted");+ } finally {+ fs.rmSync(tempDir, { recursive: true, force: true });+ }+ }),+ );+});diff --git a/apps/server/src/observability/LocalFileTracer.ts b/apps/server/src/observability/LocalFileTracer.tsnew file mode 100644index 000000000..7f75eb0bf--- /dev/null+++ b/apps/server/src/observability/LocalFileTracer.ts@@ -0,0 +1,104 @@+import type * as Exit from "effect/Exit";+import { Effect, Option, Tracer } from "effect";++import { spanToTraceRecord } from "./TraceRecord.ts";+import { makeTraceSink } from "./TraceSink.ts";++export interface LocalFileTracerOptions {+ readonly filePath: string;+ readonly maxBytes: number;+ readonly maxFiles: number;+ readonly batchWindowMs: number;+ readonly delegate?: Tracer.Tracer;+}++class LocalFileSpan implements Tracer.Span {+ readonly _tag = "Span";+ readonly name: string;+ readonly spanId: string;+ readonly traceId: string;+ readonly parent: Option.Option<Tracer.AnySpan>;+ readonly annotations: Tracer.Span["annotations"];+ readonly links: Array<Tracer.SpanLink>;+ readonly sampled: boolean;+ readonly kind: Tracer.SpanKind;++ status: Tracer.SpanStatus;+ attributes: Map<string, unknown>;+ events: Array<[name: string, startTime: bigint, attributes: Record<string, unknown>]>;++ constructor(+ options: Parameters<Tracer.Tracer["span"]>[0],+ private readonly delegate: Tracer.Span,+ private readonly push: (record: ReturnType<typeof spanToTraceRecord>) => void,+ ) {+ this.name = delegate.name;+ this.spanId = delegate.spanId;+ this.traceId = delegate.traceId;+ this.parent = options.parent;+ this.annotations = options.annotations;+ this.links = [...options.links];+ this.sampled = delegate.sampled;+ this.kind = delegate.kind;+ this.status = {+ _tag: "Started",+ startTime: options.startTime,+ };+ this.attributes = new Map();+ this.events = [];+ }++ end(endTime: bigint, exit: Exit.Exit<unknown, unknown>): void {+ this.status = {+ _tag: "Ended",+ startTime: this.status.startTime,+ endTime,+ exit,+ };+ this.delegate.end(endTime, exit);++ if (this.sampled) {+ this.push(spanToTraceRecord(this));+ }+ }++ attribute(key: string, value: unknown): void {+ this.attributes.set(key, value);+ this.delegate.attribute(key, value);+ }++ event(name: string, startTime: bigint, attributes?: Record<string, unknown>): void {+ const nextAttributes = attributes ?? {};+ this.events.push([name, startTime, nextAttributes]);+ this.delegate.event(name, startTime, nextAttributes);+ }++ addLinks(links: ReadonlyArray<Tracer.SpanLink>): void {+ this.links.push(...links);+ this.delegate.addLinks(links);+ }+}++export const makeLocalFileTracer = Effect.fn("makeLocalFileTracer")(function* (+ options: LocalFileTracerOptions,+) {+ const sink = yield* makeTraceSink({+ filePath: options.filePath,+ maxBytes: options.maxBytes,+ maxFiles: options.maxFiles,+ batchWindowMs: options.batchWindowMs,+ });++ const delegate =+ options.delegate ??+ Tracer.make({+ span: (spanOptions) => new Tracer.NativeSpan(spanOptions),+ });++ return Tracer.make({+ span(spanOptions) {+ return new LocalFileSpan(spanOptions, delegate.span(spanOptions), sink.push);+ },+ ...(delegate.context ? { context: delegate.context } : {}),+ });+});diff --git a/apps/server/src/observability/Metrics.test.ts b/apps/server/src/observability/Metrics.test.tsnew file mode 100644index 000000000..4604f43b6--- /dev/null+++ b/apps/server/src/observability/Metrics.test.ts@@ -0,0 +1,111 @@+import { assert, describe, it } from "@effect/vitest";+import { Effect, Metric } from "effect";++import { withMetrics } from "./Metrics.ts";++const hasMetricSnapshot = (+ snapshots: ReadonlyArray<Metric.Metric.Snapshot>,+ id: string,+ attributes: Readonly<Record<string, string>>,+) =>+ snapshots.some(+ (snapshot) =>+ snapshot.id === id &&+ Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),+ );++describe("withMetrics", () => {+ it.effect("supports pipe-style usage", () =>+ Effect.gen(function* () {+ const counter = Metric.counter("with_metrics_pipe_total");+ const timer = Metric.timer("with_metrics_pipe_duration");++ const result = yield* Effect.succeed("ok").pipe(+ withMetrics({+ counter,+ timer,+ attributes: {+ operation: "pipe",+ },+ }),+ );++ assert.equal(result, "ok");++ const snapshots = yield* Metric.snapshot;+ assert.equal(+ hasMetricSnapshot(snapshots, "with_metrics_pipe_total", {+ operation: "pipe",+ outcome: "success",+ }),+ true,+ );+ assert.equal(+ hasMetricSnapshot(snapshots, "with_metrics_pipe_duration", {+ operation: "pipe",+ }),+ true,+ );+ }),+ );++ it.effect("supports direct invocation", () =>+ Effect.gen(function* () {+ const counter = Metric.counter("with_metrics_direct_total");++ yield* withMetrics(Effect.fail("boom"), {+ counter,+ attributes: {+ operation: "direct",+ },+ }).pipe(Effect.exit);++ const snapshots = yield* Metric.snapshot;+ assert.equal(+ hasMetricSnapshot(snapshots, "with_metrics_direct_total", {+ operation: "direct",+ outcome: "failure",+ }),+ true,+ );+ }),+ );++ it.effect("evaluates attributes lazily after the wrapped effect runs", () =>+ Effect.gen(function* () {+ const counter = Metric.counter("with_metrics_lazy_total");+ const timer = Metric.timer("with_metrics_lazy_duration");+ let provider = "unknown";++ yield* Effect.sync(() => {+ provider = "codex";+ }).pipe(+ withMetrics({+ counter,+ timer,+ attributes: () => ({+ provider,+ operation: "lazy",+ }),+ }),+ );++ const snapshots = yield* Metric.snapshot;+ assert.equal(+ hasMetricSnapshot(snapshots, "with_metrics_lazy_total", {+ provider: "codex",+ operation: "lazy",+ outcome: "success",+ }),+ true,+ );+ assert.equal(+ hasMetricSnapshot(snapshots, "with_metrics_lazy_duration", {+ provider: "codex",+ operation: "lazy",+ }),+ true,+ );+ }),+ );+});diff --git a/apps/server/src/observability/Metrics.ts b/apps/server/src/observability/Metrics.tsnew file mode 100644index 000000000..3e527c7cb--- /dev/null+++ b/apps/server/src/observability/Metrics.ts@@ -0,0 +1,160 @@+import { Duration, Effect, Exit, Metric } from "effect";+import { dual } from "effect/Function";++import {+ compactMetricAttributes,+ normalizeModelMetricLabel,+ outcomeFromExit,+} from "./Attributes.ts";++export const rpcRequestsTotal = Metric.counter("t3_rpc_requests_total", {+ description: "Total RPC requests handled by the websocket RPC server.",+});++export const rpcRequestDuration = Metric.timer("t3_rpc_request_duration", {+ description: "RPC request handling duration.",+});++export const orchestrationCommandsTotal = Metric.counter("t3_orchestration_commands_total", {+ description: "Total orchestration commands dispatched.",+});++export const orchestrationCommandDuration = Metric.timer("t3_orchestration_command_duration", {+ description: "Orchestration command dispatch duration.",+});++export const orchestrationCommandAckDuration = Metric.timer(+ "t3_orchestration_command_ack_duration",+ {+ description:+ "Time from orchestration command dispatch to the first committed domain event emitted for that command.",+ },+);++export const orchestrationEventsProcessedTotal = Metric.counter(+ "t3_orchestration_events_processed_total",+ {+ description: "Total orchestration intent events processed by runtime reactors.",+ },+);++export const providerSessionsTotal = Metric.counter("t3_provider_sessions_total", {+ description: "Total provider session lifecycle operations.",+});++export const providerTurnsTotal = Metric.counter("t3_provider_turns_total", {+ description: "Total provider turn lifecycle operations.",+});++export const providerTurnDuration = Metric.timer("t3_provider_turn_duration", {+ description: "Provider turn request duration.",+});++export const providerRuntimeEventsTotal = Metric.counter("t3_provider_runtime_events_total", {+ description: "Total canonical provider runtime events processed.",+});++export const gitCommandsTotal = Metric.counter("t3_git_commands_total", {+ description: "Total git commands executed by the server runtime.",+});++export const gitCommandDuration = Metric.timer("t3_git_command_duration", {+ description: "Git command execution duration.",+});++export const terminalSessionsTotal = Metric.counter("t3_terminal_sessions_total", {+ description: "Total terminal sessions started.",+});++export const terminalRestartsTotal = Metric.counter("t3_terminal_restarts_total", {+ description: "Total terminal restart requests handled.",+});++export const metricAttributes = (+ attributes: Readonly<Record<string, unknown>>,+): ReadonlyArray<[string, string]> => Object.entries(compactMetricAttributes(attributes));++export const increment = (+ metric: Metric.Metric<number, unknown>,+ attributes: Readonly<Record<string, unknown>>,+ amount = 1,+) => Metric.update(Metric.withAttributes(metric, metricAttributes(attributes)), amount);++export interface WithMetricsOptions {+ readonly counter?: Metric.Metric<number, unknown>;+ readonly timer?: Metric.Metric<Duration.Duration, unknown>;+ readonly attributes?:+ | Readonly<Record<string, unknown>>+ | (() => Readonly<Record<string, unknown>>);+ readonly outcomeAttributes?: (+ outcome: ReturnType<typeof outcomeFromExit>,+ ) => Readonly<Record<string, unknown>>;+}++const withMetricsImpl = <A, E, R>(+ effect: Effect.Effect<A, E, R>,+ options: WithMetricsOptions,+): Effect.Effect<A, E, R> =>+ Effect.gen(function* () {+ const startedAt = Date.now();+ const exit = yield* Effect.exit(effect);+ const duration = Duration.millis(Math.max(0, Date.now() - startedAt));+ const baseAttributes =+ typeof options.attributes === "function" ? options.attributes() : (options.attributes ?? {});++ if (options.timer) {+ yield* Metric.update(+ Metric.withAttributes(options.timer, metricAttributes(baseAttributes)),+ duration,+ );+ }++ if (options.counter) {+ const outcome = outcomeFromExit(exit);+ yield* Metric.update(+ Metric.withAttributes(+ options.counter,+ metricAttributes({+ ...baseAttributes,+ outcome,+ ...(options.outcomeAttributes ? options.outcomeAttributes(outcome) : {}),+ }),+ ),+ 1,+ );+ }++ if (Exit.isSuccess(exit)) {+ return exit.value;+ }+ return yield* Effect.failCause(exit.cause);+ });++export const withMetrics: {+ <A, E, R>(+ options: WithMetricsOptions,+ ): (effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;+ <A, E, R>(effect: Effect.Effect<A, E, R>, options: WithMetricsOptions): Effect.Effect<A, E, R>;+} = dual(2, withMetricsImpl);++export const providerMetricAttributes = (+ provider: string,+ extra?: Readonly<Record<string, unknown>>,+) =>+ compactMetricAttributes({+ provider,+ ...extra,+ });++export const providerTurnMetricAttributes = (input: {+ readonly provider: string;+ readonly model: string | null | undefined;+ readonly extra?: Readonly<Record<string, unknown>>;+}) => {+ const modelFamily = normalizeModelMetricLabel(input.model);+ return compactMetricAttributes({+ provider: input.provider,+ ...(modelFamily ? { modelFamily } : {}),+ ...input.extra,+ });+};diff --git a/apps/server/src/observability/RpcInstrumentation.test.ts b/apps/server/src/observability/RpcInstrumentation.test.tsnew file mode 100644index 000000000..d29b05f3c--- /dev/null+++ b/apps/server/src/observability/RpcInstrumentation.test.ts@@ -0,0 +1,161 @@+import { assert, describe, it } from "@effect/vitest";+import { Effect, Exit, Metric, Stream } from "effect";++import {+ observeRpcEffect,+ observeRpcStream,+ observeRpcStreamEffect,+} from "./RpcInstrumentation.ts";++const hasMetricSnapshot = (+ snapshots: ReadonlyArray<Metric.Metric.Snapshot>,+ id: string,+ attributes: Readonly<Record<string, string>>,+) =>+ snapshots.some(+ (snapshot) =>+ snapshot.id === id &&+ Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),+ );++describe("RpcInstrumentation", () => {+ it.effect("records success metrics for unary RPC handlers", () =>+ Effect.gen(function* () {+ yield* observeRpcEffect("rpc.instrumentation.success", Effect.succeed("ok"), {+ "rpc.aggregate": "test",+ }).pipe(Effect.withSpan("rpc.instrumentation.success.span"));++ const snapshots = yield* Metric.snapshot;++ assert.equal(+ hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {+ method: "rpc.instrumentation.success",+ outcome: "success",+ }),+ true,+ );+ assert.equal(+ hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {+ method: "rpc.instrumentation.success",+ }),+ true,+ );+ }),+ );++ it.effect("records failure outcomes for unary RPC handlers", () =>+ Effect.gen(function* () {+ yield* Effect.exit(+ observeRpcEffect("rpc.instrumentation.failure", Effect.fail("boom"), {+ "rpc.aggregate": "test",+ }).pipe(Effect.withSpan("rpc.instrumentation.failure.span")),+ );++ const snapshots = yield* Metric.snapshot;++ assert.equal(+ hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {+ method: "rpc.instrumentation.failure",+ outcome: "failure",+ }),+ true,+ );+ assert.equal(+ hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {+ method: "rpc.instrumentation.failure",+ }),+ true,+ );+ }),+ );++ it.effect("records subscription activation metrics for stream RPC handlers", () =>+ Effect.gen(function* () {+ const events = yield* Stream.runCollect(+ observeRpcStreamEffect(+ "rpc.instrumentation.stream",+ Effect.succeed(Stream.make("a", "b")),+ { "rpc.aggregate": "test" },+ ).pipe(Stream.withSpan("rpc.instrumentation.stream.span")),+ );++ assert.deepStrictEqual(Array.from(events), ["a", "b"]);++ const snapshots = yield* Metric.snapshot;++ assert.equal(+ hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {+ method: "rpc.instrumentation.stream",+ outcome: "success",+ }),+ true,+ );+ assert.equal(+ hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {+ method: "rpc.instrumentation.stream",+ }),+ true,+ );+ }),+ );++ it.effect("records failure outcomes for direct stream RPC handlers during consumption", () =>+ Effect.gen(function* () {+ const exit = yield* Stream.runCollect(+ observeRpcStream(+ "rpc.instrumentation.stream.failure",+ Stream.make("a").pipe(Stream.concat(Stream.fail("boom"))),+ { "rpc.aggregate": "test" },+ ).pipe(Stream.withSpan("rpc.instrumentation.stream.failure.span")),+ ).pipe(Effect.exit);++ assert.equal(Exit.isFailure(exit), true);++ const snapshots = yield* Metric.snapshot;++ assert.equal(+ hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {+ method: "rpc.instrumentation.stream.failure",+ outcome: "failure",+ }),+ true,+ );+ assert.equal(+ hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {+ method: "rpc.instrumentation.stream.failure",+ }),+ true,+ );+ }),+ );++ it.effect("records failure outcomes when a stream RPC effect produces a failing stream", () =>+ Effect.gen(function* () {+ const exit = yield* Stream.runCollect(+ observeRpcStreamEffect(+ "rpc.instrumentation.stream.effect.failure",+ Effect.succeed(Stream.fail("boom")),+ { "rpc.aggregate": "test" },+ ).pipe(Stream.withSpan("rpc.instrumentation.stream.effect.failure.span")),+ ).pipe(Effect.exit);++ assert.equal(Exit.isFailure(exit), true);++ const snapshots = yield* Metric.snapshot;++ assert.equal(+ hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {+ method: "rpc.instrumentation.stream.effect.failure",+ outcome: "failure",+ }),+ true,+ );+ assert.equal(+ hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {+ method: "rpc.instrumentation.stream.effect.failure",+ }),+ true,+ );+ }),+ );+});diff --git a/apps/server/src/observability/RpcInstrumentation.ts b/apps/server/src/observability/RpcInstrumentation.tsnew file mode 100644index 000000000..a3ac29aa0--- /dev/null+++ b/apps/server/src/observability/RpcInstrumentation.ts@@ -0,0 +1,89 @@+import { Duration, Effect, Exit, Metric, Stream } from "effect";++import { outcomeFromExit } from "./Attributes.ts";+import { metricAttributes, rpcRequestDuration, rpcRequestsTotal, withMetrics } from "./Metrics.ts";++const annotateRpcSpan = (+ method: string,+ traceAttributes?: Readonly<Record<string, unknown>>,+): Effect.Effect<void, never, never> =>+ Effect.annotateCurrentSpan({+ "rpc.method": method,+ ...traceAttributes,+ });++const recordRpcStreamMetrics = <E>(+ method: string,+ startedAt: number,+ exit: Exit.Exit<unknown, E>,+): Effect.Effect<void, never, never> =>+ Effect.gen(function* () {+ yield* Metric.update(+ Metric.withAttributes(rpcRequestDuration, metricAttributes({ method })),+ Duration.millis(Math.max(0, Date.now() - startedAt)),+ );+ yield* Metric.update(+ Metric.withAttributes(+ rpcRequestsTotal,+ metricAttributes({+ method,+ outcome: outcomeFromExit(exit),+ }),+ ),+ 1,+ );+ });++export const observeRpcEffect = <A, E, R>(+ method: string,+ effect: Effect.Effect<A, E, R>,+ traceAttributes?: Readonly<Record<string, unknown>>,+): Effect.Effect<A, E, R> =>+ Effect.gen(function* () {+ yield* annotateRpcSpan(method, traceAttributes);++ return yield* effect.pipe(+ withMetrics({+ counter: rpcRequestsTotal,+ timer: rpcRequestDuration,+ attributes: {+ method,+ },+ }),+ );+ });++export const observeRpcStream = <A, E, R>(+ method: string,+ stream: Stream.Stream<A, E, R>,+ traceAttributes?: Readonly<Record<string, unknown>>,+): Stream.Stream<A, E, R> =>+ Stream.unwrap(+ Effect.gen(function* () {+ yield* annotateRpcSpan(method, traceAttributes);+ const startedAt = Date.now();+ return stream.pipe(Stream.onExit((exit) => recordRpcStreamMetrics(method, startedAt, exit)));+ }),+ );++export const observeRpcStreamEffect = <A, StreamError, StreamContext, EffectError, EffectContext>(+ method: string,+ effect: Effect.Effect<Stream.Stream<A, StreamError, StreamContext>, EffectError, EffectContext>,+ traceAttributes?: Readonly<Record<string, unknown>>,+): Stream.Stream<A, StreamError | EffectError, StreamContext | EffectContext> =>+ Stream.unwrap(+ Effect.gen(function* () {+ yield* annotateRpcSpan(method, traceAttributes);+ const startedAt = Date.now();+ const exit = yield* Effect.exit(effect);++ if (Exit.isFailure(exit)) {+ yield* recordRpcStreamMetrics(method, startedAt, exit);+ return yield* Effect.failCause(exit.cause);+ }++ return exit.value.pipe(+ Stream.onExit((streamExit) => recordRpcStreamMetrics(method, startedAt, streamExit)),+ );+ }),+ );diff --git a/apps/server/src/observability/TraceRecord.ts b/apps/server/src/observability/TraceRecord.tsnew file mode 100644index 000000000..3aee1b267--- /dev/null+++ b/apps/server/src/observability/TraceRecord.ts@@ -0,0 +1,100 @@+import { Cause, Exit, Option, Tracer } from "effect";++import { compactTraceAttributes } from "./Attributes.ts";++export interface TraceRecord {+ readonly type: "effect-span";+ readonly name: string;+ readonly traceId: string;+ readonly spanId: string;+ readonly parentSpanId?: string;+ readonly sampled: boolean;+ readonly kind: Tracer.SpanKind;+ readonly startTimeUnixNano: string;+ readonly endTimeUnixNano: string;+ readonly durationMs: number;+ readonly attributes: Readonly<Record<string, unknown>>;+ readonly events: ReadonlyArray<{+ readonly name: string;+ readonly timeUnixNano: string;+ readonly attributes: Readonly<Record<string, unknown>>;+ }>;+ readonly links: ReadonlyArray<{+ readonly traceId: string;+ readonly spanId: string;+ readonly attributes: Readonly<Record<string, unknown>>;+ }>;+ readonly exit:+ | {+ readonly _tag: "Success";+ }+ | {+ readonly _tag: "Interrupted";+ readonly cause: string;+ }+ | {+ readonly _tag: "Failure";+ readonly cause: string;+ };+}++interface SerializableSpan {+ readonly name: string;+ readonly traceId: string;+ readonly spanId: string;+ readonly parent: Option.Option<Tracer.AnySpan>;+ readonly status: Tracer.SpanStatus;+ readonly sampled: boolean;+ readonly kind: Tracer.SpanKind;+ readonly attributes: ReadonlyMap<string, unknown>;+ readonly links: ReadonlyArray<Tracer.SpanLink>;+ readonly events: ReadonlyArray<+ readonly [name: string, startTime: bigint, attributes: Record<string, unknown>]+ >;+}++function formatTraceExit(exit: Exit.Exit<unknown, unknown>): TraceRecord["exit"] {+ if (Exit.isSuccess(exit)) {+ return { _tag: "Success" };+ }+ if (Cause.hasInterruptsOnly(exit.cause)) {+ return {+ _tag: "Interrupted",+ cause: Cause.pretty(exit.cause),+ };+ }+ return {+ _tag: "Failure",+ cause: Cause.pretty(exit.cause),+ };+}++export function spanToTraceRecord(span: SerializableSpan): TraceRecord {+ const status = span.status as Extract<Tracer.SpanStatus, { _tag: "Ended" }>;+ const parentSpanId = Option.getOrUndefined(span.parent)?.spanId;++ return {+ type: "effect-span",+ name: span.name,+ traceId: span.traceId,+ spanId: span.spanId,+ ...(parentSpanId ? { parentSpanId } : {}),+ sampled: span.sampled,+ kind: span.kind,+ startTimeUnixNano: String(status.startTime),+ endTimeUnixNano: String(status.endTime),+ durationMs: Number(status.endTime - status.startTime) / 1_000_000,+ attributes: compactTraceAttributes(Object.fromEntries(span.attributes)),+ events: span.events.map(([name, startTime, attributes]) => ({+ name,+ timeUnixNano: String(startTime),+ attributes: compactTraceAttributes(attributes),+ })),+ links: span.links.map((link) => ({+ traceId: link.span.traceId,+ spanId: link.span.spanId,+ attributes: compactTraceAttributes(link.attributes),+ })),+ exit: formatTraceExit(status.exit),+ };+}diff --git a/apps/server/src/observability/TraceSink.test.ts b/apps/server/src/observability/TraceSink.test.tsnew file mode 100644index 000000000..f4db90516--- /dev/null+++ b/apps/server/src/observability/TraceSink.test.ts@@ -0,0 +1,152 @@+import fs from "node:fs";+import os from "node:os";+import path from "node:path";++import { assert, describe, it } from "@effect/vitest";+import { Effect } from "effect";++import type { TraceRecord } from "./TraceRecord.ts";+import { makeTraceSink } from "./TraceSink.ts";++const makeRecord = (name: string, suffix = ""): TraceRecord => ({+ type: "effect-span",+ name,+ traceId: `trace-${name}-${suffix}`,+ spanId: `span-${name}-${suffix}`,+ sampled: true,+ kind: "internal",+ startTimeUnixNano: "1",+ endTimeUnixNano: "2",+ durationMs: 1,+ attributes: {+ payload: suffix,+ },+ events: [],+ links: [],+ exit: {+ _tag: "Success",+ },+});++describe("TraceSink", () => {+ it.effect("flushes buffered records on close", () =>+ Effect.scoped(+ Effect.gen(function* () {+ const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-trace-sink-"));+ const tracePath = path.join(tempDir, "server.trace.ndjson");++ try {+ const sink = yield* makeTraceSink({+ filePath: tracePath,+ maxBytes: 1024,+ maxFiles: 2,+ batchWindowMs: 10_000,+ });++ sink.push(makeRecord("alpha"));+ sink.push(makeRecord("beta"));+ yield* sink.close();++ const lines = fs+ .readFileSync(tracePath, "utf8")+ .trim()+ .split("\n")+ .map((line) => JSON.parse(line) as TraceRecord);++ assert.equal(lines.length, 2);+ assert.equal(lines[0]?.name, "alpha");+ assert.equal(lines[1]?.name, "beta");+ } finally {+ fs.rmSync(tempDir, { recursive: true, force: true });+ }+ }),+ ),+ );++ it.effect("rotates the trace file when the configured max size is exceeded", () =>+ Effect.scoped(+ Effect.gen(function* () {+ const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-trace-sink-"));+ const tracePath = path.join(tempDir, "server.trace.ndjson");++ try {+ const sink = yield* makeTraceSink({+ filePath: tracePath,+ maxBytes: 180,+ maxFiles: 2,+ batchWindowMs: 10_000,+ });++ for (let index = 0; index < 8; index += 1) {+ sink.push(makeRecord("rotate", `${index}-${"x".repeat(48)}`));+ yield* sink.flush;+ }+ yield* sink.close();++ const matchingFiles = fs+ .readdirSync(tempDir)+ .filter(+ (entry) =>+ entry === "server.trace.ndjson" || entry.startsWith("server.trace.ndjson."),+ )+ .toSorted();++ assert.equal(+ matchingFiles.some((entry) => entry === "server.trace.ndjson.1"),+ true,+ );+ assert.equal(+ matchingFiles.some((entry) => entry === "server.trace.ndjson.3"),+ false,+ );+ } finally {+ fs.rmSync(tempDir, { recursive: true, force: true });+ }+ }),+ ),+ );++ it.effect("drops only the invalid record when serialization fails", () =>+ Effect.scoped(+ Effect.gen(function* () {+ const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-trace-sink-"));+ const tracePath = path.join(tempDir, "server.trace.ndjson");++ try {+ const sink = yield* makeTraceSink({+ filePath: tracePath,+ maxBytes: 1024,+ maxFiles: 2,+ batchWindowMs: 10_000,+ });++ const circular: Array<unknown> = [];+ circular.push(circular);++ sink.push(makeRecord("alpha"));+ sink.push({+ ...makeRecord("invalid"),+ attributes: {+ circular,+ },+ } as TraceRecord);+ sink.push(makeRecord("beta"));+ yield* sink.close();++ const lines = fs+ .readFileSync(tracePath, "utf8")+ .trim()+ .split("\n")+ .map((line) => JSON.parse(line) as TraceRecord);++ assert.deepStrictEqual(+ lines.map((line) => line.name),+ ["alpha", "beta"],+ );+ } finally {+ fs.rmSync(tempDir, { recursive: true, force: true });+ }+ }),+ ),+ );+});diff --git a/apps/server/src/observability/TraceSink.ts b/apps/server/src/observability/TraceSink.tsnew file mode 100644index 000000000..1bd00b473--- /dev/null+++ b/apps/server/src/observability/TraceSink.ts@@ -0,0 +1,68 @@+import { RotatingFileSink } from "@t3tools/shared/logging";+import { Effect } from "effect";++import type { TraceRecord } from "./TraceRecord.ts";++const FLUSH_BUFFER_THRESHOLD = 32;++export interface TraceSinkOptions {+ readonly filePath: string;+ readonly maxBytes: number;+ readonly maxFiles: number;+ readonly batchWindowMs: number;+}++export interface TraceSink {+ readonly filePath: string;+ push: (record: TraceRecord) => void;+ flush: Effect.Effect<void>;+ close: () => Effect.Effect<void>;+}++export const makeTraceSink = Effect.fn("makeTraceSink")(function* (options: TraceSinkOptions) {+ const sink = new RotatingFileSink({+ filePath: options.filePath,+ maxBytes: options.maxBytes,+ maxFiles: options.maxFiles,+ });++ let buffer: Array<string> = [];++ const flushUnsafe = () => {+ if (buffer.length === 0) {+ return;+ }++ const chunk = buffer.join("");+ buffer = [];++ try {+ sink.write(chunk);+ } catch {+ buffer.unshift(chunk);+ }+ };++ const flush = Effect.sync(flushUnsafe).pipe(Effect.withTracerEnabled(false));++ yield* Effect.addFinalizer(() => flush.pipe(Effect.ignore));+ yield* Effect.forkScoped(+ Effect.sleep(`${options.batchWindowMs} millis`).pipe(Effect.andThen(flush), Effect.forever),+ );++ return {+ filePath: options.filePath,+ push(record) {+ try {+ buffer.push(`${JSON.stringify(record)}\n`);+ if (buffer.length >= FLUSH_BUFFER_THRESHOLD) {+ flushUnsafe();+ }+ } catch {+ return;+ }+ },+ flush,+ close: () => flush,+ } satisfies TraceSink;+});diff --git a/apps/server/src/open.ts b/apps/server/src/open.tsindex cdacda7c0..58074ceef 100644--- a/apps/server/src/open.ts+++ b/apps/server/src/open.ts@@ -233,10 +233,15 @@ export class Open extends ServiceMap.Service<Open, OpenShape>()("t3/open") {}// Implementations// ==============================-export const resolveEditorLaunch = Effect.fnUntraced(function* (+export const resolveEditorLaunch = Effect.fn("resolveEditorLaunch")(function* (input: OpenInEditorInput,platform: NodeJS.Platform = process.platform,): Effect.fn.Return<EditorLaunch, OpenError> {+ yield* Effect.annotateCurrentSpan({+ "open.editor": input.editor,+ "open.cwd": input.cwd,+ "open.platform": platform,+ });const editorDef = EDITORS.find((editor) => editor.id === input.editor);if (!editorDef) {return yield* new OpenError({ message: `Unknown editor: ${input.editor}` });diff --git a/apps/server/src/orchestration/Layers/CheckpointReactor.ts b/apps/server/src/orchestration/Layers/CheckpointReactor.tsindex 28609cecb..03abebaf3 100644--- a/apps/server/src/orchestration/Layers/CheckpointReactor.ts+++ b/apps/server/src/orchestration/Layers/CheckpointReactor.ts@@ -119,7 +119,7 @@ const make = Effect.gen(function* () {createdAt: input.createdAt,});- const resolveSessionRuntimeForThread = Effect.fnUntraced(function* (+ const resolveSessionRuntimeForThread = Effect.fn("resolveSessionRuntimeForThread")(function* (threadId: ThreadId,): Effect.fn.Return<Option.Option<{ readonly threadId: ThreadId; readonly cwd: string }>> {const readModel = yield* orchestrationEngine.getReadModel();@@ -153,7 +153,7 @@ const make = Effect.gen(function* () {// active provider session CWD and falling back to the thread/project config.// Returns undefined when no CWD can be determined or the workspace is not// a git repository.- const resolveCheckpointCwd = Effect.fnUntraced(function* (input: {+ const resolveCheckpointCwd = Effect.fn("resolveCheckpointCwd")(function* (input: {readonly threadId: ThreadId;readonly thread: { readonly projectId: ProjectId; readonly worktreePath: string | null };readonly projects: ReadonlyArray<{ readonly id: ProjectId; readonly workspaceRoot: string }>;@@ -188,7 +188,7 @@ const make = Effect.gen(function* () {// Shared tail for both capture paths: creates the git checkpoint ref, diffs// it against the previous turn, then dispatches the domain events to update// the orchestration read model.- const captureAndDispatchCheckpoint = Effect.fnUntraced(function* (input: {+ const captureAndDispatchCheckpoint = Effect.fn("captureAndDispatchCheckpoint")(function* (input: {readonly threadId: ThreadId;readonly turnId: TurnId;readonly thread: {@@ -321,70 +321,70 @@ const make = Effect.gen(function* () {});// Captures a real git checkpoint when a turn completes via a runtime event.- const captureCheckpointFromTurnCompletion = Effect.fnUntraced(function* (- event: Extract<ProviderRuntimeEvent, { type: "turn.completed" }>,- ) {- const turnId = toTurnId(event.turnId);- if (!turnId) {- return;- }-- const readModel = yield* orchestrationEngine.getReadModel();- const thread = readModel.threads.find((entry) => entry.id === event.threadId);- if (!thread) {- return;- }+ const captureCheckpointFromTurnCompletion = Effect.fn("captureCheckpointFromTurnCompletion")(+ function* (event: Extract<ProviderRuntimeEvent, { type: "turn.completed" }>) {+ const turnId = toTurnId(event.turnId);+ if (!turnId) {+ return;+ }- // When a primary turn is active, only that turn may produce completion checkpoints.- if (thread.session?.activeTurnId && !sameId(thread.session.activeTurnId, turnId)) {- return;- }+ const readModel = yield* orchestrationEngine.getReadModel();+ const thread = readModel.threads.find((entry) => entry.id === event.threadId);+ if (!thread) {+ return;+ }- // Only skip if a real (non-placeholder) checkpoint already exists for this turn.- // ProviderRuntimeIngestion may insert placeholder entries with status "missing"- // before this reactor runs; those must not prevent real git capture.- if (- thread.checkpoints.some(- (checkpoint) => checkpoint.turnId === turnId && checkpoint.status !== "missing",- )- ) {- return;- }+ // When a primary turn is active, only that turn may produce completion checkpoints.+ if (thread.session?.activeTurnId && !sameId(thread.session.activeTurnId, turnId)) {+ return;+ }- const checkpointCwd = yield* resolveCheckpointCwd({- threadId: thread.id,- thread,- projects: readModel.projects,- preferSessionRuntime: true,- });- if (!checkpointCwd) {- return;- }+ // Only skip if a real (non-placeholder) checkpoint already exists for this turn.+ // ProviderRuntimeIngestion may insert placeholder entries with status "missing"+ // before this reactor runs; those must not prevent real git capture.+ if (+ thread.checkpoints.some(+ (checkpoint) => checkpoint.turnId === turnId && checkpoint.status !== "missing",+ )+ ) {+ return;+ }- // If a placeholder checkpoint exists for this turn, reuse its turn count- // instead of incrementing past it.- const existingPlaceholder = thread.checkpoints.find(- (checkpoint) => checkpoint.turnId === turnId && checkpoint.status === "missing",- );- const currentTurnCount = thread.checkpoints.reduce(- (maxTurnCount, checkpoint) => Math.max(maxTurnCount, checkpoint.checkpointTurnCount),- 0,- );- const nextTurnCount = existingPlaceholder- ? existingPlaceholder.checkpointTurnCount- : currentTurnCount + 1;+ const checkpointCwd = yield* resolveCheckpointCwd({+ threadId: thread.id,+ thread,+ projects: readModel.projects,+ preferSessionRuntime: true,+ });+ if (!checkpointCwd) {+ return;+ }- yield* captureAndDispatchCheckpoint({- threadId: thread.id,- turnId,- thread,- cwd: checkpointCwd,- turnCount: nextTurnCount,- status: checkpointStatusFromRuntime(event.payload.state),- assistantMessageId: undefined,- createdAt: event.createdAt,- });- });+ // If a placeholder checkpoint exists for this turn, reuse its turn count+ // instead of incrementing past it.+ const existingPlaceholder = thread.checkpoints.find(+ (checkpoint) => checkpoint.turnId === turnId && checkpoint.status === "missing",+ );+ const currentTurnCount = thread.checkpoints.reduce(+ (maxTurnCount, checkpoint) => Math.max(maxTurnCount, checkpoint.checkpointTurnCount),+ 0,+ );+ const nextTurnCount = existingPlaceholder+ ? existingPlaceholder.checkpointTurnCount+ : currentTurnCount + 1;++ yield* captureAndDispatchCheckpoint({+ threadId: thread.id,+ turnId,+ thread,+ cwd: checkpointCwd,+ turnCount: nextTurnCount,+ status: checkpointStatusFromRuntime(event.payload.state),+ assistantMessageId: undefined,+ createdAt: event.createdAt,+ });+ },+ );// Captures a real git checkpoint when a placeholder checkpoint (status "missing")// is detected via a domain event. This replaces the placeholder with a real@@ -394,7 +394,7 @@ const make = Effect.gen(function* () {// events from the Codex runtime. This handler fires when the corresponding// domain event arrives, allowing the reactor to capture the actual filesystem// state into a git ref and dispatch a replacement checkpoint.- const captureCheckpointFromPlaceholder = Effect.fnUntraced(function* (+ const captureCheckpointFromPlaceholder = Effect.fn("captureCheckpointFromPlaceholder")(function* (event: Extract<OrchestrationEvent, { type: "thread.turn-diff-completed" }>,) {const { threadId, turnId, checkpointTurnCount, status } = event.payload;@@ -448,57 +448,59 @@ const make = Effect.gen(function* () {});});- const ensurePreTurnBaselineFromTurnStart = Effect.fnUntraced(function* (- event: Extract<ProviderRuntimeEvent, { type: "turn.started" }>,- ) {- const turnId = toTurnId(event.turnId);- if (!turnId) {- return;- }+ const ensurePreTurnBaselineFromTurnStart = Effect.fn("ensurePreTurnBaselineFromTurnStart")(+ function* (event: Extract<ProviderRuntimeEvent, { type: "turn.started" }>) {+ const turnId = toTurnId(event.turnId);+ if (!turnId) {+ return;+ }- const readModel = yield* orchestrationEngine.getReadModel();- const thread = readModel.threads.find((entry) => entry.id === event.threadId);- if (!thread) {- return;- }+ const readModel = yield* orchestrationEngine.getReadModel();+ const thread = readModel.threads.find((entry) => entry.id === event.threadId);+ if (!thread) {+ return;+ }- const checkpointCwd = yield* resolveCheckpointCwd({- threadId: thread.id,- thread,- projects: readModel.projects,- preferSessionRuntime: false,- });- if (!checkpointCwd) {- return;- }+ const checkpointCwd = yield* resolveCheckpointCwd({+ threadId: thread.id,+ thread,+ projects: readModel.projects,+ preferSessionRuntime: false,+ });+ if (!checkpointCwd) {+ return;+ }- const currentTurnCount = thread.checkpoints.reduce(- (maxTurnCount, checkpoint) => Math.max(maxTurnCount, checkpoint.checkpointTurnCount),- 0,- );- const baselineCheckpointRef = checkpointRefForThreadTurn(thread.id, currentTurnCount);- const baselineExists = yield* checkpointStore.hasCheckpointRef({- cwd: checkpointCwd,- checkpointRef: baselineCheckpointRef,- });- if (baselineExists) {- return;- }+ const currentTurnCount = thread.checkpoints.reduce(+ (maxTurnCount, checkpoint) => Math.max(maxTurnCount, checkpoint.checkpointTurnCount),+ 0,+ );+ const baselineCheckpointRef = checkpointRefForThreadTurn(thread.id, currentTurnCount);+ const baselineExists = yield* checkpointStore.hasCheckpointRef({+ cwd: checkpointCwd,+ checkpointRef: baselineCheckpointRef,+ });+ if (baselineExists) {+ return;+ }- yield* checkpointStore.captureCheckpoint({- cwd: checkpointCwd,- checkpointRef: baselineCheckpointRef,- });- yield* receiptBus.publish({- type: "checkpoint.baseline.captured",- threadId: thread.id,- checkpointTurnCount: currentTurnCount,- checkpointRef: baselineCheckpointRef,- createdAt: event.createdAt,- });- });+ yield* checkpointStore.captureCheckpoint({+ cwd: checkpointCwd,+ checkpointRef: baselineCheckpointRef,+ });+ yield* receiptBus.publish({+ type: "checkpoint.baseline.captured",+ threadId: thread.id,+ checkpointTurnCount: currentTurnCount,+ checkpointRef: baselineCheckpointRef,+ createdAt: event.createdAt,+ });+ },+ );- const ensurePreTurnBaselineFromDomainTurnStart = Effect.fnUntraced(function* (+ const ensurePreTurnBaselineFromDomainTurnStart = Effect.fn(+ "ensurePreTurnBaselineFromDomainTurnStart",+ )(function* (event: Extract<OrchestrationEvent,{ type: "thread.turn-start-requested" | "thread.message-sent" }@@ -557,7 +559,7 @@ const make = Effect.gen(function* () {});});- const handleRevertRequested = Effect.fnUntraced(function* (+ const handleRevertRequested = Effect.fn("handleRevertRequested")(function* (event: Extract<OrchestrationEvent, { type: "thread.checkpoint-revert-requested" }>,) {const now = new Date().toISOString();@@ -685,7 +687,7 @@ const make = Effect.gen(function* () {);});- const processDomainEvent = Effect.fnUntraced(function* (event: OrchestrationEvent) {+ const processDomainEvent = Effect.fn("processDomainEvent")(function* (event: OrchestrationEvent) {if (event.type === "thread.turn-start-requested" || event.type === "thread.message-sent") {yield* ensurePreTurnBaselineFromDomainTurnStart(event);return;@@ -724,7 +726,9 @@ const make = Effect.gen(function* () {}});- const processRuntimeEvent = Effect.fnUntraced(function* (event: ProviderRuntimeEvent) {+ const processRuntimeEvent = Effect.fn("processRuntimeEvent")(function* (+ event: ProviderRuntimeEvent,+ ) {if (event.type === "turn.started") {yield* ensurePreTurnBaselineFromTurnStart(event);return;diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.tsindex 16bbd98d6..5a0a6113f 100644--- a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts+++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts@@ -8,7 +8,7 @@ import {TurnId,type OrchestrationEvent,} from "@t3tools/contracts";-import { Effect, Layer, ManagedRuntime, Option, Queue, Stream } from "effect";+import { Effect, Layer, ManagedRuntime, Metric, Option, Queue, Stream } from "effect";import { describe, expect, it } from "vitest";import { PersistenceSqlError } from "../../persistence/Errors.ts";@@ -62,6 +62,17 @@ function now() {return new Date().toISOString();}+const hasMetricSnapshot = (+ snapshots: ReadonlyArray<Metric.Metric.Snapshot>,+ id: string,+ attributes: Readonly<Record<string, string>>,+) =>+ snapshots.some(+ (snapshot) =>+ snapshot.id === id &&+ Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),+ );+describe("OrchestrationEngine", () => {it("bootstraps the in-memory read model from persisted projections", async () => {const failOnHistoricalReplayStore: OrchestrationEventStoreShape = {@@ -408,6 +419,95 @@ describe("OrchestrationEngine", () => {await system.dispose();});+ it("records command ack duration using the first committed event type", async () => {+ const system = await createOrchestrationSystem();+ const { engine } = system;+ const createdAt = now();++ await system.run(+ engine.dispatch({+ type: "project.create",+ commandId: CommandId.makeUnsafe("cmd-project-ack-create"),+ projectId: asProjectId("project-ack"),+ title: "Ack Project",+ workspaceRoot: "/tmp/project-ack",+ defaultModelSelection: {+ provider: "codex",+ model: "gpt-5-codex",+ },+ createdAt,+ }),+ );++ await system.run(+ engine.dispatch({+ type: "thread.create",+ commandId: CommandId.makeUnsafe("cmd-thread-ack-create"),+ threadId: ThreadId.makeUnsafe("thread-ack"),+ projectId: asProjectId("project-ack"),+ title: "Ack Thread",+ modelSelection: {+ provider: "codex",+ model: "gpt-5-codex",+ },+ interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,+ runtimeMode: "full-access",+ branch: null,+ worktreePath: null,+ createdAt,+ }),+ );++ const snapshots = await system.run(Metric.snapshot);+ expect(+ hasMetricSnapshot(snapshots, "t3_orchestration_command_ack_duration", {+ commandType: "thread.create",+ aggregateKind: "thread",+ ackEventType: "thread.created",+ }),+ ).toBe(true);++ await system.dispose();+ });++ it("records failed command dispatches as metric failures", async () => {+ const system = await createOrchestrationSystem();+ const { engine } = system;+ const createdAt = now();++ await expect(+ system.run(+ engine.dispatch({+ type: "thread.create",+ commandId: CommandId.makeUnsafe("cmd-thread-missing-project"),+ threadId: ThreadId.makeUnsafe("thread-missing-project"),+ projectId: asProjectId("project-missing"),+ title: "Missing Project Thread",+ modelSelection: {+ provider: "codex",+ model: "gpt-5-codex",+ },+ interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,+ runtimeMode: "full-access",+ branch: null,+ worktreePath: null,+ createdAt,+ }),+ ),+ ).rejects.toThrow("does not exist");++ const snapshots = await system.run(Metric.snapshot);+ expect(+ hasMetricSnapshot(snapshots, "t3_orchestration_commands_total", {+ commandType: "thread.create",+ aggregateKind: "thread",+ outcome: "failure",+ }),+ ).toBe(true);++ await system.dispose();+ });+it("stores completed checkpoint summaries even when no files changed", async () => {const system = await createOrchestrationSystem();const { engine } = system;diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.tsindex bd3581deb..ddd1718fa 100644--- a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts+++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts@@ -5,9 +5,28 @@ import type {ThreadId,} from "@t3tools/contracts";import { OrchestrationCommand } from "@t3tools/contracts";-import { Deferred, Effect, Layer, Option, PubSub, Queue, Schema, Stream } from "effect";+import {+ Cause,+ Deferred,+ Duration,+ Effect,+ Exit,+ Layer,+ Metric,+ Option,+ PubSub,+ Queue,+ Schema,+ Stream,+} from "effect";import * as SqlClient from "effect/unstable/sql/SqlClient";+import {+ metricAttributes,+ orchestrationCommandAckDuration,+ orchestrationCommandsTotal,+ orchestrationCommandDuration,+} from "../../observability/Metrics.ts";import { toPersistenceSqlError } from "../../persistence/Errors.ts";import { OrchestrationEventStore } from "../../persistence/Services/OrchestrationEventStore.ts";import { OrchestrationCommandReceiptRepository } from "../../persistence/Services/OrchestrationCommandReceipts.ts";@@ -28,6 +47,7 @@ import {interface CommandEnvelope {command: OrchestrationCommand;result: Deferred.Deferred<{ sequence: number }, OrchestrationDispatchError>;+ startedAtMs: number;}function commandToAggregateRef(command: OrchestrationCommand): {@@ -64,6 +84,12 @@ const makeOrchestrationEngine = Effect.gen(function* () {const processEnvelope = (envelope: CommandEnvelope): Effect.Effect<void> => {const dispatchStartSequence = readModel.snapshotSequence;+ const processingStartedAtMs = Date.now();+ const aggregateRef = commandToAggregateRef(envelope.command);+ const baseMetricAttributes = {+ commandType: envelope.command.type,+ aggregateKind: aggregateRef.aggregateKind,+ } as const;const reconcileReadModelAfterDispatchFailure = Effect.gen(function* () {const persistedEvents = yield* Stream.runCollect(eventStore.readFromSequence(dispatchStartSequence),@@ -83,113 +109,160 @@ const makeOrchestrationEngine = Effect.gen(function* () {}});- return Effect.gen(function* () {- const existingReceipt = yield* commandReceiptRepository.getByCommandId({- commandId: envelope.command.commandId,- });- if (Option.isSome(existingReceipt)) {- if (existingReceipt.value.status === "accepted") {- yield* Deferred.succeed(envelope.result, {- sequence: existingReceipt.value.resultSequence,- });- return;- }- yield* Deferred.fail(- envelope.result,- new OrchestrationCommandPreviouslyRejectedError({+ return Effect.exit(+ Effect.gen(function* () {+ yield* Effect.annotateCurrentSpan({+ "orchestration.command_id": envelope.command.commandId,+ "orchestration.command_type": envelope.command.type,+ "orchestration.aggregate_kind": aggregateRef.aggregateKind,+ "orchestration.aggregate_id": aggregateRef.aggregateId,+ });++ const existingReceipt = yield* commandReceiptRepository.getByCommandId({+ commandId: envelope.command.commandId,+ });+ if (Option.isSome(existingReceipt)) {+ if (existingReceipt.value.status === "accepted") {+ return {+ sequence: existingReceipt.value.resultSequence,+ };+ }+ return yield* new OrchestrationCommandPreviouslyRejectedError({commandId: envelope.command.commandId,detail: existingReceipt.value.error ?? "Previously rejected.",- }),- );- return;- }+ });+ }- const eventBase = yield* decideOrchestrationCommand({- command: envelope.command,- readModel,- });- const eventBases = Array.isArray(eventBase) ? eventBase : [eventBase];- const committedCommand = yield* sql- .withTransaction(- Effect.gen(function* () {- const committedEvents: OrchestrationEvent[] = [];- let nextReadModel = readModel;-- for (const nextEvent of eventBases) {- const savedEvent = yield* eventStore.append(nextEvent);- nextReadModel = yield* projectEvent(nextReadModel, savedEvent);- yield* projectionPipeline.projectEvent(savedEvent);- committedEvents.push(savedEvent);- }+ const eventBase = yield* decideOrchestrationCommand({+ command: envelope.command,+ readModel,+ });+ const eventBases = Array.isArray(eventBase) ? eventBase : [eventBase];+ const committedCommand = yield* sql+ .withTransaction(+ Effect.gen(function* () {+ const committedEvents: OrchestrationEvent[] = [];+ let nextReadModel = readModel;- const lastSavedEvent = committedEvents.at(-1) ?? null;- if (lastSavedEvent === null) {- return yield* new OrchestrationCommandInvariantError({- commandType: envelope.command.type,- detail: "Command produced no events.",- });- }+ for (const nextEvent of eventBases) {+ const savedEvent = yield* eventStore.append(nextEvent);+ nextReadModel = yield* projectEvent(nextReadModel, savedEvent);+ yield* projectionPipeline.projectEvent(savedEvent);+ committedEvents.push(savedEvent);+ }- yield* commandReceiptRepository.upsert({- commandId: envelope.command.commandId,- aggregateKind: lastSavedEvent.aggregateKind,- aggregateId: lastSavedEvent.aggregateId,- acceptedAt: lastSavedEvent.occurredAt,- resultSequence: lastSavedEvent.sequence,- status: "accepted",- error: null,- });+ const lastSavedEvent = committedEvents.at(-1) ?? null;+ if (lastSavedEvent === null) {+ return yield* new OrchestrationCommandInvariantError({+ commandType: envelope.command.type,+ detail: "Command produced no events.",+ });+ }- return {- committedEvents,- lastSequence: lastSavedEvent.sequence,- nextReadModel,- } as const;- }),- )- .pipe(- Effect.catchTag("SqlError", (sqlError) =>- Effect.fail(- toPersistenceSqlError("OrchestrationEngine.processEnvelope:transaction")(sqlError),+ yield* commandReceiptRepository.upsert({+ commandId: envelope.command.commandId,+ aggregateKind: lastSavedEvent.aggregateKind,+ aggregateId: lastSavedEvent.aggregateId,+ acceptedAt: lastSavedEvent.occurredAt,+ resultSequence: lastSavedEvent.sequence,+ status: "accepted",+ error: null,+ });++ return {+ committedEvents,+ lastSequence: lastSavedEvent.sequence,+ nextReadModel,+ } as const;+ }),+ )+ .pipe(+ Effect.catchTag("SqlError", (sqlError) =>+ Effect.fail(+ toPersistenceSqlError("OrchestrationEngine.processEnvelope:transaction")(sqlError),+ ),),- ),- );+ );- readModel = committedCommand.nextReadModel;- for (const event of committedCommand.committedEvents) {- yield* PubSub.publish(eventPubSub, event);- }- yield* Deferred.succeed(envelope.result, { sequence: committedCommand.lastSequence });- }).pipe(- Effect.catch((error) =>- Effect.gen(function* () {- yield* reconcileReadModelAfterDispatchFailure.pipe(- Effect.catch(() =>- Effect.logWarning(- "failed to reconcile orchestration read model after dispatch failure",- ).pipe(- Effect.annotateLogs({- commandId: envelope.command.commandId,- snapshotSequence: readModel.snapshotSequence,+ readModel = committedCommand.nextReadModel;+ for (const [index, event] of committedCommand.committedEvents.entries()) {+ yield* PubSub.publish(eventPubSub, event);+ if (index === 0) {+ yield* Metric.update(+ Metric.withAttributes(+ orchestrationCommandAckDuration,+ metricAttributes({+ ...baseMetricAttributes,+ ackEventType: event.type,}),),+ Duration.millis(Math.max(0, Date.now() - envelope.startedAtMs)),+ );+ }+ }+ return { sequence: committedCommand.lastSequence };+ }).pipe(Effect.withSpan(`orchestration.command.${envelope.command.type}`)),+ ).pipe(+ Effect.flatMap((exit) =>+ Effect.gen(function* () {+ const outcome = Exit.isSuccess(exit)+ ? "success"+ : Cause.hasInterruptsOnly(exit.cause)+ ? "interrupt"+ : "failure";+ yield* Metric.update(+ Metric.withAttributes(+ orchestrationCommandDuration,+ metricAttributes(baseMetricAttributes),+ ),+ Duration.millis(Math.max(0, Date.now() - processingStartedAtMs)),+ );+ yield* Metric.update(+ Metric.withAttributes(+ orchestrationCommandsTotal,+ metricAttributes({+ ...baseMetricAttributes,+ outcome,+ }),),+ 1,);- if (Schema.is(OrchestrationCommandInvariantError)(error)) {- const aggregateRef = commandToAggregateRef(envelope.command);- yield* commandReceiptRepository- .upsert({- commandId: envelope.command.commandId,- aggregateKind: aggregateRef.aggregateKind,- aggregateId: aggregateRef.aggregateId,- acceptedAt: new Date().toISOString(),- resultSequence: readModel.snapshotSequence,- status: "rejected",- error: error.message,- })- .pipe(Effect.catch(() => Effect.void));+ if (Exit.isSuccess(exit)) {+ yield* Deferred.succeed(envelope.result, exit.value);+ return;}++ const error = Cause.squash(exit.cause) as OrchestrationDispatchError;+ if (!Schema.is(OrchestrationCommandPreviouslyRejectedError)(error)) {+ yield* reconcileReadModelAfterDispatchFailure.pipe(+ Effect.catch(() =>+ Effect.logWarning(+ "failed to reconcile orchestration read model after dispatch failure",+ ).pipe(+ Effect.annotateLogs({+ commandId: envelope.command.commandId,+ snapshotSequence: readModel.snapshotSequence,+ }),+ ),+ ),+ );++ if (Schema.is(OrchestrationCommandInvariantError)(error)) {+ yield* commandReceiptRepository+ .upsert({+ commandId: envelope.command.commandId,+ aggregateKind: aggregateRef.aggregateKind,+ aggregateId: aggregateRef.aggregateId,+ acceptedAt: new Date().toISOString(),+ resultSequence: readModel.snapshotSequence,+ status: "rejected",+ error: error.message,+ })+ .pipe(Effect.catch(() => Effect.void));+ }+ }+yield* Deferred.fail(envelope.result, error);}),),@@ -214,7 +287,7 @@ const makeOrchestrationEngine = Effect.gen(function* () {const dispatch: OrchestrationEngineShape["dispatch"] = (command) =>Effect.gen(function* () {const result = yield* Deferred.make<{ sequence: number }, OrchestrationDispatchError>();- yield* Queue.offer(commandQueue, { command, result });+ yield* Queue.offer(commandQueue, { command, result, startedAtMs: Date.now() });return yield* Deferred.await(result);});diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.tsindex 8e2a19bcd..419e3f3bf 100644--- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts+++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts@@ -16,6 +16,7 @@ import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker";import { resolveThreadWorkspaceCwd } from "../../checkpointing/Utils.ts";import { GitCore } from "../../git/Services/GitCore.ts";+import { increment, orchestrationEventsProcessedTotal } from "../../observability/Metrics.ts";import { ProviderAdapterRequestError, ProviderServiceError } from "../../provider/Errors.ts";import { TextGeneration } from "../../git/Services/TextGeneration.ts";import { ProviderService } from "../../provider/Services/ProviderService.ts";@@ -212,12 +213,12 @@ const make = Effect.gen(function* () {createdAt: input.createdAt,});- const resolveThread = Effect.fnUntraced(function* (threadId: ThreadId) {+ const resolveThread = Effect.fn("resolveThread")(function* (threadId: ThreadId) {const readModel = yield* orchestrationEngine.getReadModel();return readModel.threads.find((entry) => entry.id === threadId);});- const ensureSessionForThread = Effect.fnUntraced(function* (+ const ensureSessionForThread = Effect.fn("ensureSessionForThread")(function* (threadId: ThreadId,createdAt: string,options?: {@@ -357,7 +358,7 @@ const make = Effect.gen(function* () {return startedSession.threadId;});- const sendTurnForThread = Effect.fnUntraced(function* (input: {+ const sendTurnForThread = Effect.fn("sendTurnForThread")(function* (input: {readonly threadId: ThreadId;readonly messageText: string;readonly attachments?: ReadonlyArray<ChatAttachment>;@@ -409,7 +410,9 @@ const make = Effect.gen(function* () {});});- const maybeGenerateAndRenameWorktreeBranchForFirstTurn = Effect.fnUntraced(function* (input: {+ const maybeGenerateAndRenameWorktreeBranchForFirstTurn = Effect.fn(+ "maybeGenerateAndRenameWorktreeBranchForFirstTurn",+ )(function* (input: {readonly threadId: ThreadId;readonly branch: string | null;readonly worktreePath: string | null;@@ -461,50 +464,52 @@ const make = Effect.gen(function* () {);});- const maybeGenerateThreadTitleForFirstTurn = Effect.fnUntraced(function* (input: {- readonly threadId: ThreadId;- readonly cwd: string;- readonly messageText: string;- readonly attachments?: ReadonlyArray<ChatAttachment>;- readonly titleSeed?: string;- }) {- const attachments = input.attachments ?? [];- yield* Effect.gen(function* () {- const { textGenerationModelSelection: modelSelection } =- yield* serverSettingsService.getSettings;-- const generated = yield* textGeneration.generateThreadTitle({- cwd: input.cwd,- message: input.messageText,- ...(attachments.length > 0 ? { attachments } : {}),- modelSelection,- });- if (!generated) return;+ const maybeGenerateThreadTitleForFirstTurn = Effect.fn("maybeGenerateThreadTitleForFirstTurn")(+ function* (input: {+ readonly threadId: ThreadId;+ readonly cwd: string;+ readonly messageText: string;+ readonly attachments?: ReadonlyArray<ChatAttachment>;+ readonly titleSeed?: string;+ }) {+ const attachments = input.attachments ?? [];+ yield* Effect.gen(function* () {+ const { textGenerationModelSelection: modelSelection } =+ yield* serverSettingsService.getSettings;++ const generated = yield* textGeneration.generateThreadTitle({+ cwd: input.cwd,+ message: input.messageText,+ ...(attachments.length > 0 ? { attachments } : {}),+ modelSelection,+ });+ if (!generated) return;- const thread = yield* resolveThread(input.threadId);- if (!thread) return;- if (!canReplaceThreadTitle(thread.title, input.titleSeed)) {- return;- }+ const thread = yield* resolveThread(input.threadId);+ if (!thread) return;+ if (!canReplaceThreadTitle(thread.title, input.titleSeed)) {+ return;+ }- yield* orchestrationEngine.dispatch({- type: "thread.meta.update",- commandId: serverCommandId("thread-title-rename"),- threadId: input.threadId,- title: generated.title,- });- }).pipe(- Effect.catchCause((cause) =>- Effect.logWarning("provider command reactor failed to generate or rename thread title", {+ yield* orchestrationEngine.dispatch({+ type: "thread.meta.update",+ commandId: serverCommandId("thread-title-rename"),threadId: input.threadId,- cwd: input.cwd,- cause: Cause.pretty(cause),- }),- ),- );- });+ title: generated.title,+ });+ }).pipe(+ Effect.catchCause((cause) =>+ Effect.logWarning("provider command reactor failed to generate or rename thread title", {+ threadId: input.threadId,+ cwd: input.cwd,+ cause: Cause.pretty(cause),+ }),+ ),+ );+ },+ );- const processTurnStartRequested = Effect.fnUntraced(function* (+ const processTurnStartRequested = Effect.fn("processTurnStartRequested")(function* (event: Extract<ProviderIntentEvent, { type: "thread.turn-start-requested" }>,) {const key = turnStartKeyForEvent(event);@@ -583,7 +588,7 @@ const make = Effect.gen(function* () {);});- const processTurnInterruptRequested = Effect.fnUntraced(function* (+ const processTurnInterruptRequested = Effect.fn("processTurnInterruptRequested")(function* (event: Extract<ProviderIntentEvent, { type: "thread.turn-interrupt-requested" }>,) {const thread = yield* resolveThread(event.payload.threadId);@@ -606,7 +611,7 @@ const make = Effect.gen(function* () {yield* providerService.interruptTurn({ threadId: event.payload.threadId });});- const processApprovalResponseRequested = Effect.fnUntraced(function* (+ const processApprovalResponseRequested = Effect.fn("processApprovalResponseRequested")(function* (event: Extract<ProviderIntentEvent, { type: "thread.approval-response-requested" }>,) {const thread = yield* resolveThread(event.payload.threadId);@@ -653,50 +658,52 @@ const make = Effect.gen(function* () {);});- const processUserInputResponseRequested = Effect.fnUntraced(function* (- event: Extract<ProviderIntentEvent, { type: "thread.user-input-response-requested" }>,- ) {- const thread = yield* resolveThread(event.payload.threadId);- if (!thread) {- return;- }- const hasSession = thread.session && thread.session.status !== "stopped";- if (!hasSession) {- return yield* appendProviderFailureActivity({- threadId: event.payload.threadId,- kind: "provider.user-input.respond.failed",- summary: "Provider user input response failed",- detail: "No active provider session is bound to this thread.",- turnId: null,- createdAt: event.payload.createdAt,- requestId: event.payload.requestId,- });- }+ const processUserInputResponseRequested = Effect.fn("processUserInputResponseRequested")(+ function* (+ event: Extract<ProviderIntentEvent, { type: "thread.user-input-response-requested" }>,+ ) {+ const thread = yield* resolveThread(event.payload.threadId);+ if (!thread) {+ return;+ }+ const hasSession = thread.session && thread.session.status !== "stopped";+ if (!hasSession) {+ return yield* appendProviderFailureActivity({+ threadId: event.payload.threadId,+ kind: "provider.user-input.respond.failed",+ summary: "Provider user input response failed",+ detail: "No active provider session is bound to this thread.",+ turnId: null,+ createdAt: event.payload.createdAt,+ requestId: event.payload.requestId,+ });+ }- yield* providerService- .respondToUserInput({- threadId: event.payload.threadId,- requestId: event.payload.requestId,- answers: event.payload.answers,- })- .pipe(- Effect.catchCause((cause) =>- appendProviderFailureActivity({- threadId: event.payload.threadId,- kind: "provider.user-input.respond.failed",- summary: "Provider user input response failed",- detail: isUnknownPendingUserInputRequestError(cause)- ? stalePendingRequestDetail("user-input", event.payload.requestId)- : Cause.pretty(cause),- turnId: null,- createdAt: event.payload.createdAt,- requestId: event.payload.requestId,- }),- ),- );- });+ yield* providerService+ .respondToUserInput({+ threadId: event.payload.threadId,+ requestId: event.payload.requestId,+ answers: event.payload.answers,+ })+ .pipe(+ Effect.catchCause((cause) =>+ appendProviderFailureActivity({+ threadId: event.payload.threadId,+ kind: "provider.user-input.respond.failed",+ summary: "Provider user input response failed",+ detail: isUnknownPendingUserInputRequestError(cause)+ ? stalePendingRequestDetail("user-input", event.payload.requestId)+ : Cause.pretty(cause),+ turnId: null,+ createdAt: event.payload.createdAt,+ requestId: event.payload.requestId,+ }),+ ),+ );+ },+ );- const processSessionStopRequested = Effect.fnUntraced(function* (+ const processSessionStopRequested = Effect.fn("processSessionStopRequested")(function* (event: Extract<ProviderIntentEvent, { type: "thread.session-stop-requested" }>,) {const thread = yield* resolveThread(event.payload.threadId);@@ -727,6 +734,14 @@ const make = Effect.gen(function* () {const processDomainEvent = Effect.fn("processDomainEvent")(function* (event: ProviderIntentEvent,) {+ yield* Effect.annotateCurrentSpan({+ "orchestration.event_type": event.type,+ "orchestration.thread_id": event.payload.threadId,+ ...(event.commandId ? { "orchestration.command_id": event.commandId } : {}),+ });+ yield* increment(orchestrationEventsProcessedTotal, {+ eventType: event.type,+ });switch (event.type) {case "thread.runtime-mode-set": {const thread = yield* resolveThread(event.payload.threadId);diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.tsindex b479d1c28..8d3fb5d75 100644--- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts+++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts@@ -524,7 +524,7 @@ const make = Effect.fn("make")(function* () {lookup: () => Effect.succeed({ text: "", createdAt: "" }),});- const isGitRepoForThread = Effect.fnUntraced(function* (threadId: ThreadId) {+ const isGitRepoForThread = Effect.fn("isGitRepoForThread")(function* (threadId: ThreadId) {const readModel = yield* orchestrationEngine.getReadModel();const thread = readModel.threads.find((entry) => entry.id === threadId);if (!thread) {@@ -796,9 +796,9 @@ const make = Effect.fn("make")(function* () {).pipe(Effect.asVoid);});- const getSourceProposedPlanReferenceForPendingTurnStart = Effect.fnUntraced(function* (- threadId: ThreadId,- ) {+ const getSourceProposedPlanReferenceForPendingTurnStart = Effect.fn(+ "getSourceProposedPlanReferenceForPendingTurnStart",+ )(function* (threadId: ThreadId) {const pendingTurnStart = yield* projectionTurnRepository.getPendingTurnStartByThreadId({threadId,});@@ -818,16 +818,17 @@ const make = Effect.fn("make")(function* () {} as const;});- const getExpectedProviderTurnIdForThread = Effect.fnUntraced(function* (threadId: ThreadId) {- const sessions = yield* providerService.listSessions();- const session = sessions.find((entry) => entry.threadId === threadId);- return session?.activeTurnId;- });-- const getSourceProposedPlanReferenceForAcceptedTurnStart = Effect.fnUntraced(function* (- threadId: ThreadId,- eventTurnId: TurnId | undefined,- ) {+ const getExpectedProviderTurnIdForThread = Effect.fn("getExpectedProviderTurnIdForThread")(+ function* (threadId: ThreadId) {+ const sessions = yield* providerService.listSessions();+ const session = sessions.find((entry) => entry.threadId === threadId);+ return session?.activeTurnId;+ },+ );++ const getSourceProposedPlanReferenceForAcceptedTurnStart = Effect.fn(+ "getSourceProposedPlanReferenceForAcceptedTurnStart",+ )(function* (threadId: ThreadId, eventTurnId: TurnId | undefined) {if (eventTurnId === undefined) {return null;}@@ -840,34 +841,36 @@ const make = Effect.fn("make")(function* () {return yield* getSourceProposedPlanReferenceForPendingTurnStart(threadId);});- const markSourceProposedPlanImplemented = Effect.fnUntraced(function* (- sourceThreadId: ThreadId,- sourcePlanId: OrchestrationProposedPlanId,- implementationThreadId: ThreadId,- implementedAt: string,- ) {- const readModel = yield* orchestrationEngine.getReadModel();- const sourceThread = readModel.threads.find((entry) => entry.id === sourceThreadId);- const sourcePlan = sourceThread?.proposedPlans.find((entry) => entry.id === sourcePlanId);- if (!sourceThread || !sourcePlan || sourcePlan.implementedAt !== null) {- return;- }+ const markSourceProposedPlanImplemented = Effect.fn("markSourceProposedPlanImplemented")(+ function* (+ sourceThreadId: ThreadId,+ sourcePlanId: OrchestrationProposedPlanId,+ implementationThreadId: ThreadId,+ implementedAt: string,+ ) {+ const readModel = yield* orchestrationEngine.getReadModel();+ const sourceThread = readModel.threads.find((entry) => entry.id === sourceThreadId);+ const sourcePlan = sourceThread?.proposedPlans.find((entry) => entry.id === sourcePlanId);+ if (!sourceThread || !sourcePlan || sourcePlan.implementedAt !== null) {+ return;+ }- yield* orchestrationEngine.dispatch({- type: "thread.proposed-plan.upsert",- commandId: CommandId.makeUnsafe(- `provider:source-proposed-plan-implemented:${implementationThreadId}:${crypto.randomUUID()}`,- ),- threadId: sourceThread.id,- proposedPlan: {- ...sourcePlan,- implementedAt,- implementationThreadId,- updatedAt: implementedAt,- },- createdAt: implementedAt,- });- });+ yield* orchestrationEngine.dispatch({+ type: "thread.proposed-plan.upsert",+ commandId: CommandId.makeUnsafe(+ `provider:source-proposed-plan-implemented:${implementationThreadId}:${crypto.randomUUID()}`,+ ),+ threadId: sourceThread.id,+ proposedPlan: {+ ...sourcePlan,+ implementedAt,+ implementationThreadId,+ updatedAt: implementedAt,+ },+ createdAt: implementedAt,+ });+ },+ );const processRuntimeEvent = Effect.fn("processRuntimeEvent")(function* (event: ProviderRuntimeEvent,diff --git a/apps/server/src/persistence/Layers/Sqlite.ts b/apps/server/src/persistence/Layers/Sqlite.tsindex 07cbf4479..58556099d 100644--- a/apps/server/src/persistence/Layers/Sqlite.ts+++ b/apps/server/src/persistence/Layers/Sqlite.ts@@ -6,6 +6,7 @@ import { ServerConfig } from "../../config.ts";type RuntimeSqliteLayerConfig = {readonly filename: string;+ readonly spanAttributes?: Record<string, unknown>;};type Loader = {@@ -41,7 +42,16 @@ export const makeSqlitePersistenceLive = Effect.fn("makeSqlitePersistenceLive")(const path = yield* Path.Path;yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true });- return Layer.provideMerge(setup, makeRuntimeSqliteLayer({ filename: dbPath }));+ return Layer.provideMerge(+ setup,+ makeRuntimeSqliteLayer({+ filename: dbPath,+ spanAttributes: {+ "db.name": path.basename(dbPath),+ "service.name": "t3-server",+ },+ }),+ );}, Layer.unwrap);export const SqlitePersistenceMemory = Layer.provideMerge(diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.tsindex 651a61164..cd6c81405 100644--- a/apps/server/src/provider/Layers/ProviderService.test.ts+++ b/apps/server/src/provider/Layers/ProviderService.test.ts@@ -20,7 +20,7 @@ import {import { it, assert, vi } from "@effect/vitest";import { assertFailure } from "@effect/vitest/utils";-import { Effect, Fiber, Layer, Option, PubSub, Ref, Stream } from "effect";+import { Effect, Fiber, Layer, Metric, Option, PubSub, Ref, Stream } from "effect";import * as SqlClient from "effect/unstable/sql/SqlClient";import {@@ -230,6 +230,17 @@ function makeFakeCodexAdapter(provider: ProviderKind = "codex") {const sleep = (ms: number) =>Effect.promise(() => new Promise<void>((resolve) => setTimeout(resolve, ms)));+const hasMetricSnapshot = (+ snapshots: ReadonlyArray<Metric.Metric.Snapshot>,+ id: string,+ attributes: Readonly<Record<string, string>>,+) =>+ snapshots.some(+ (snapshot) =>+ snapshot.id === id &&+ Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),+ );+function makeProviderServiceLayer() {const codex = makeFakeCodexAdapter();const claude = makeFakeCodexAdapter("claudeAgent");@@ -1060,6 +1071,120 @@ fanout.layer("ProviderServiceLive fanout", (it) => {);}),);++ it.effect("records provider metrics with the routed provider label", () =>+ Effect.gen(function* () {+ const provider = yield* ProviderService;++ const session = yield* provider.startSession(asThreadId("thread-metrics"), {+ provider: "claudeAgent",+ threadId: asThreadId("thread-metrics"),+ cwd: "/tmp/project",+ runtimeMode: "full-access",+ });++ yield* provider.interruptTurn({ threadId: session.threadId });+ yield* provider.respondToRequest({+ threadId: session.threadId,+ requestId: asRequestId("req-metrics-1"),+ decision: "accept",+ });+ yield* provider.respondToUserInput({+ threadId: session.threadId,+ requestId: asRequestId("req-metrics-2"),+ answers: {+ sandbox_mode: "workspace-write",+ },+ });+ yield* provider.rollbackConversation({+ threadId: session.threadId,+ numTurns: 1,+ });+ yield* provider.stopSession({ threadId: session.threadId });++ const snapshots = yield* Metric.snapshot;++ assert.equal(+ hasMetricSnapshot(snapshots, "t3_provider_turns_total", {+ provider: "claudeAgent",+ operation: "interrupt",+ outcome: "success",+ }),+ true,+ );+ assert.equal(+ hasMetricSnapshot(snapshots, "t3_provider_turns_total", {+ provider: "claudeAgent",+ operation: "approval-response",+ outcome: "success",+ }),+ true,+ );+ assert.equal(+ hasMetricSnapshot(snapshots, "t3_provider_turns_total", {+ provider: "claudeAgent",+ operation: "user-input-response",+ outcome: "success",+ }),+ true,+ );+ assert.equal(+ hasMetricSnapshot(snapshots, "t3_provider_turns_total", {+ provider: "claudeAgent",+ operation: "rollback",+ outcome: "success",+ }),+ true,+ );+ assert.equal(+ hasMetricSnapshot(snapshots, "t3_provider_sessions_total", {+ provider: "claudeAgent",+ operation: "stop",+ outcome: "success",+ }),+ true,+ );+ }),+ );++ it.effect(+ "records sendTurn metrics with the resolved provider when modelSelection is omitted",+ () =>+ Effect.gen(function* () {+ const provider = yield* ProviderService;++ const session = yield* provider.startSession(asThreadId("thread-send-metrics"), {+ provider: "claudeAgent",+ threadId: asThreadId("thread-send-metrics"),+ cwd: "/tmp/project-send-metrics",+ runtimeMode: "full-access",+ });++ yield* provider.sendTurn({+ threadId: session.threadId,+ input: "hello",+ attachments: [],+ });++ const snapshots = yield* Metric.snapshot;++ assert.equal(+ hasMetricSnapshot(snapshots, "t3_provider_turns_total", {+ provider: "claudeAgent",+ operation: "send",+ outcome: "success",+ }),+ true,+ );+ assert.equal(+ hasMetricSnapshot(snapshots, "t3_provider_turn_duration", {+ provider: "claudeAgent",+ operation: "send",+ }),+ true,+ );+ }),+ );});const validation = makeProviderServiceLayer();diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.tsindex f34882959..da2c293fa 100644--- a/apps/server/src/provider/Layers/ProviderService.ts+++ b/apps/server/src/provider/Layers/ProviderService.ts@@ -24,6 +24,16 @@ import {} from "@t3tools/contracts";import { Effect, Layer, Option, PubSub, Queue, Schema, SchemaIssue, Stream } from "effect";+import {+ increment,+ providerMetricAttributes,+ providerRuntimeEventsTotal,+ providerSessionsTotal,+ providerTurnDuration,+ providerTurnsTotal,+ providerTurnMetricAttributes,+ withMetrics,+} from "../../observability/Metrics.ts";import { ProviderValidationError } from "../Errors.ts";import { ProviderAdapterRegistry } from "../Services/ProviderAdapterRegistry.ts";import { ProviderService, type ProviderServiceShape } from "../Services/ProviderService.ts";@@ -179,7 +189,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (const providers = yield* registry.listProviders();const adapters = yield* Effect.forEach(providers, (provider) => registry.getByProvider(provider));const processRuntimeEvent = (event: ProviderRuntimeEvent): Effect.Effect<void> =>- publishRuntimeEvent(event);+ increment(providerRuntimeEventsTotal, {+ provider: event.provider,+ eventType: event.type,+ }).pipe(Effect.andThen(publishRuntimeEvent(event)));const worker = Effect.forever(Queue.take(runtimeEventQueue).pipe(Effect.flatMap(processRuntimeEvent)),@@ -196,58 +209,72 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (readonly binding: ProviderRuntimeBinding;readonly operation: string;}) {- const adapter = yield* registry.getByProvider(input.binding.provider);- const hasResumeCursor =- input.binding.resumeCursor !== null && input.binding.resumeCursor !== undefined;- const hasActiveSession = yield* adapter.hasSession(input.binding.threadId);- if (hasActiveSession) {- const activeSessions = yield* adapter.listSessions();- const existing = activeSessions.find(- (session) => session.threadId === input.binding.threadId,- );- if (existing) {- yield* upsertSessionBinding(existing, input.binding.threadId);- yield* analytics.record("provider.session.recovered", {- provider: existing.provider,- strategy: "adopt-existing",- hasResumeCursor: existing.resumeCursor !== undefined,- });- return { adapter, session: existing } as const;+ yield* Effect.annotateCurrentSpan({+ "provider.operation": "recover-session",+ "provider.kind": input.binding.provider,+ "provider.thread_id": input.binding.threadId,+ });+ return yield* Effect.gen(function* () {+ const adapter = yield* registry.getByProvider(input.binding.provider);+ const hasResumeCursor =+ input.binding.resumeCursor !== null && input.binding.resumeCursor !== undefined;+ const hasActiveSession = yield* adapter.hasSession(input.binding.threadId);+ if (hasActiveSession) {+ const activeSessions = yield* adapter.listSessions();+ const existing = activeSessions.find(+ (session) => session.threadId === input.binding.threadId,+ );+ if (existing) {+ yield* upsertSessionBinding(existing, input.binding.threadId);+ yield* analytics.record("provider.session.recovered", {+ provider: existing.provider,+ strategy: "adopt-existing",+ hasResumeCursor: existing.resumeCursor !== undefined,+ });+ return { adapter, session: existing } as const;+ }}- }- if (!hasResumeCursor) {- return yield* toValidationError(- input.operation,- `Cannot recover thread '${input.binding.threadId}' because no provider resume state is persisted.`,- );- }+ if (!hasResumeCursor) {+ return yield* toValidationError(+ input.operation,+ `Cannot recover thread '${input.binding.threadId}' because no provider resume state is persisted.`,+ );+ }- const persistedCwd = readPersistedCwd(input.binding.runtimePayload);- const persistedModelSelection = readPersistedModelSelection(input.binding.runtimePayload);+ const persistedCwd = readPersistedCwd(input.binding.runtimePayload);+ const persistedModelSelection = readPersistedModelSelection(input.binding.runtimePayload);- const resumed = yield* adapter.startSession({- threadId: input.binding.threadId,- provider: input.binding.provider,- ...(persistedCwd ? { cwd: persistedCwd } : {}),- ...(persistedModelSelection ? { modelSelection: persistedModelSelection } : {}),- ...(hasResumeCursor ? { resumeCursor: input.binding.resumeCursor } : {}),- runtimeMode: input.binding.runtimeMode ?? "full-access",- });- if (resumed.provider !== adapter.provider) {- return yield* toValidationError(- input.operation,- `Adapter/provider mismatch while recovering thread '${input.binding.threadId}'. Expected '${adapter.provider}', received '${resumed.provider}'.`,- );- }+ const resumed = yield* adapter.startSession({+ threadId: input.binding.threadId,+ provider: input.binding.provider,+ ...(persistedCwd ? { cwd: persistedCwd } : {}),+ ...(persistedModelSelection ? { modelSelection: persistedModelSelection } : {}),+ ...(hasResumeCursor ? { resumeCursor: input.binding.resumeCursor } : {}),+ runtimeMode: input.binding.runtimeMode ?? "full-access",+ });+ if (resumed.provider !== adapter.provider) {+ return yield* toValidationError(+ input.operation,+ `Adapter/provider mismatch while recovering thread '${input.binding.threadId}'. Expected '${adapter.provider}', received '${resumed.provider}'.`,+ );+ }- yield* upsertSessionBinding(resumed, input.binding.threadId);- yield* analytics.record("provider.session.recovered", {- provider: resumed.provider,- strategy: "resume-thread",- hasResumeCursor: resumed.resumeCursor !== undefined,- });- return { adapter, session: resumed } as const;+ yield* upsertSessionBinding(resumed, input.binding.threadId);+ yield* analytics.record("provider.session.recovered", {+ provider: resumed.provider,+ strategy: "resume-thread",+ hasResumeCursor: resumed.resumeCursor !== undefined,+ });+ return { adapter, session: resumed } as const;+ }).pipe(+ withMetrics({+ counter: providerSessionsTotal,+ attributes: providerMetricAttributes(input.binding.provider, {+ operation: "recover",+ }),+ }),+ );});const resolveRoutableSession = Effect.fn("resolveRoutableSession")(function* (input: {@@ -291,52 +318,69 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (threadId,provider: parsed.provider ?? "codex",};- const settings = yield* serverSettings.getSettings.pipe(- Effect.mapError((error) =>- toValidationError(- "ProviderService.startSession",- `Failed to load provider settings: ${error.message}`,- error,+ yield* Effect.annotateCurrentSpan({+ "provider.operation": "start-session",+ "provider.kind": input.provider,+ "provider.thread_id": threadId,+ "provider.runtime_mode": input.runtimeMode,+ });+ return yield* Effect.gen(function* () {+ const settings = yield* serverSettings.getSettings.pipe(+ Effect.mapError((error) =>+ toValidationError(+ "ProviderService.startSession",+ `Failed to load provider settings: ${error.message}`,+ error,+ ),),- ),- );- if (!settings.providers[input.provider].enabled) {- return yield* toValidationError(- "ProviderService.startSession",- `Provider '${input.provider}' is disabled in T3 Code settings.`,);- }- const persistedBinding = Option.getOrUndefined(yield* directory.getBinding(threadId));- const effectiveResumeCursor =- input.resumeCursor ??- (persistedBinding?.provider === input.provider ? persistedBinding.resumeCursor : undefined);- const adapter = yield* registry.getByProvider(input.provider);- const session = yield* adapter.startSession({- ...input,- ...(effectiveResumeCursor !== undefined ? { resumeCursor: effectiveResumeCursor } : {}),- });+ if (!settings.providers[input.provider].enabled) {+ return yield* toValidationError(+ "ProviderService.startSession",+ `Provider '${input.provider}' is disabled in T3 Code settings.`,+ );+ }+ const persistedBinding = Option.getOrUndefined(yield* directory.getBinding(threadId));+ const effectiveResumeCursor =+ input.resumeCursor ??+ (persistedBinding?.provider === input.provider+ ? persistedBinding.resumeCursor+ : undefined);+ const adapter = yield* registry.getByProvider(input.provider);+ const session = yield* adapter.startSession({+ ...input,+ ...(effectiveResumeCursor !== undefined ? { resumeCursor: effectiveResumeCursor } : {}),+ });- if (session.provider !== adapter.provider) {- return yield* toValidationError(- "ProviderService.startSession",- `Adapter/provider mismatch: requested '${adapter.provider}', received '${session.provider}'.`,- );- }+ if (session.provider !== adapter.provider) {+ return yield* toValidationError(+ "ProviderService.startSession",+ `Adapter/provider mismatch: requested '${adapter.provider}', received '${session.provider}'.`,+ );+ }- yield* upsertSessionBinding(session, threadId, {- modelSelection: input.modelSelection,- });- yield* analytics.record("provider.session.started", {- provider: session.provider,- runtimeMode: input.runtimeMode,- hasResumeCursor: session.resumeCursor !== undefined,- hasCwd: typeof input.cwd === "string" && input.cwd.trim().length > 0,- hasModel:- typeof input.modelSelection?.model === "string" &&- input.modelSelection.model.trim().length > 0,- });+ yield* upsertSessionBinding(session, threadId, {+ modelSelection: input.modelSelection,+ });+ yield* analytics.record("provider.session.started", {+ provider: session.provider,+ runtimeMode: input.runtimeMode,+ hasResumeCursor: session.resumeCursor !== undefined,+ hasCwd: typeof input.cwd === "string" && input.cwd.trim().length > 0,+ hasModel:+ typeof input.modelSelection?.model === "string" &&+ input.modelSelection.model.trim().length > 0,+ });- return session;+ return session;+ }).pipe(+ withMetrics({+ counter: providerSessionsTotal,+ attributes: providerMetricAttributes(input.provider, {+ operation: "start",+ }),+ }),+ );},);@@ -357,32 +401,61 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ("Either input text or at least one attachment is required",);}- const routed = yield* resolveRoutableSession({- threadId: input.threadId,- operation: "ProviderService.sendTurn",- allowRecovery: true,- });- const turn = yield* routed.adapter.sendTurn(input);- yield* directory.upsert({- threadId: input.threadId,- provider: routed.adapter.provider,- status: "running",- ...(turn.resumeCursor !== undefined ? { resumeCursor: turn.resumeCursor } : {}),- runtimePayload: {- ...(input.modelSelection !== undefined ? { modelSelection: input.modelSelection } : {}),- activeTurnId: turn.turnId,- lastRuntimeEvent: "provider.sendTurn",- lastRuntimeEventAt: new Date().toISOString(),- },- });- yield* analytics.record("provider.turn.sent", {- provider: routed.adapter.provider,- model: input.modelSelection?.model,- interactionMode: input.interactionMode,- attachmentCount: input.attachments.length,- hasInput: typeof input.input === "string" && input.input.trim().length > 0,+ yield* Effect.annotateCurrentSpan({+ "provider.operation": "send-turn",+ "provider.thread_id": input.threadId,+ "provider.interaction_mode": input.interactionMode,+ "provider.attachment_count": input.attachments.length,});- return turn;+ let metricProvider = "unknown";+ let metricModel = input.modelSelection?.model;+ return yield* Effect.gen(function* () {+ const routed = yield* resolveRoutableSession({+ threadId: input.threadId,+ operation: "ProviderService.sendTurn",+ allowRecovery: true,+ });+ metricProvider = routed.adapter.provider;+ metricModel = input.modelSelection?.model;+ yield* Effect.annotateCurrentSpan({+ "provider.kind": routed.adapter.provider,+ ...(input.modelSelection?.model ? { "provider.model": input.modelSelection.model } : {}),+ });+ const turn = yield* routed.adapter.sendTurn(input);+ yield* directory.upsert({+ threadId: input.threadId,+ provider: routed.adapter.provider,+ status: "running",+ ...(turn.resumeCursor !== undefined ? { resumeCursor: turn.resumeCursor } : {}),+ runtimePayload: {+ ...(input.modelSelection !== undefined ? { modelSelection: input.modelSelection } : {}),+ activeTurnId: turn.turnId,+ lastRuntimeEvent: "provider.sendTurn",+ lastRuntimeEventAt: new Date().toISOString(),+ },+ });+ yield* analytics.record("provider.turn.sent", {+ provider: routed.adapter.provider,+ model: input.modelSelection?.model,+ interactionMode: input.interactionMode,+ attachmentCount: input.attachments.length,+ hasInput: typeof input.input === "string" && input.input.trim().length > 0,+ });+ return turn;+ }).pipe(+ withMetrics({+ counter: providerTurnsTotal,+ timer: providerTurnDuration,+ attributes: () =>+ providerTurnMetricAttributes({+ provider: metricProvider,+ model: metricModel,+ extra: {+ operation: "send",+ },+ }),+ }),+ );});const interruptTurn: ProviderServiceShape["interruptTurn"] = Effect.fn("interruptTurn")(@@ -392,15 +465,33 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (schema: ProviderInterruptTurnInput,payload: rawInput,});- const routed = yield* resolveRoutableSession({- threadId: input.threadId,- operation: "ProviderService.interruptTurn",- allowRecovery: true,- });- yield* routed.adapter.interruptTurn(routed.threadId, input.turnId);- yield* analytics.record("provider.turn.interrupted", {- provider: routed.adapter.provider,- });+ let metricProvider = "unknown";+ return yield* Effect.gen(function* () {+ const routed = yield* resolveRoutableSession({+ threadId: input.threadId,+ operation: "ProviderService.interruptTurn",+ allowRecovery: true,+ });+ metricProvider = routed.adapter.provider;+ yield* Effect.annotateCurrentSpan({+ "provider.operation": "interrupt-turn",+ "provider.kind": routed.adapter.provider,+ "provider.thread_id": input.threadId,+ "provider.turn_id": input.turnId,+ });+ yield* routed.adapter.interruptTurn(routed.threadId, input.turnId);+ yield* analytics.record("provider.turn.interrupted", {+ provider: routed.adapter.provider,+ });+ }).pipe(+ withMetrics({+ counter: providerTurnsTotal,+ outcomeAttributes: () =>+ providerMetricAttributes(metricProvider, {+ operation: "interrupt",+ }),+ }),+ );},);@@ -411,16 +502,34 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (schema: ProviderRespondToRequestInput,payload: rawInput,});- const routed = yield* resolveRoutableSession({- threadId: input.threadId,- operation: "ProviderService.respondToRequest",- allowRecovery: true,- });- yield* routed.adapter.respondToRequest(routed.threadId, input.requestId, input.decision);- yield* analytics.record("provider.request.responded", {- provider: routed.adapter.provider,- decision: input.decision,- });+ let metricProvider = "unknown";+ return yield* Effect.gen(function* () {+ const routed = yield* resolveRoutableSession({+ threadId: input.threadId,+ operation: "ProviderService.respondToRequest",+ allowRecovery: true,+ });+ metricProvider = routed.adapter.provider;+ yield* Effect.annotateCurrentSpan({+ "provider.operation": "respond-to-request",+ "provider.kind": routed.adapter.provider,+ "provider.thread_id": input.threadId,+ "provider.request_id": input.requestId,+ });+ yield* routed.adapter.respondToRequest(routed.threadId, input.requestId, input.decision);+ yield* analytics.record("provider.request.responded", {+ provider: routed.adapter.provider,+ decision: input.decision,+ });+ }).pipe(+ withMetrics({+ counter: providerTurnsTotal,+ outcomeAttributes: () =>+ providerMetricAttributes(metricProvider, {+ operation: "approval-response",+ }),+ }),+ );},);@@ -432,12 +541,30 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (schema: ProviderRespondToUserInputInput,payload: rawInput,});- const routed = yield* resolveRoutableSession({- threadId: input.threadId,- operation: "ProviderService.respondToUserInput",- allowRecovery: true,- });- yield* routed.adapter.respondToUserInput(routed.threadId, input.requestId, input.answers);+ let metricProvider = "unknown";+ return yield* Effect.gen(function* () {+ const routed = yield* resolveRoutableSession({+ threadId: input.threadId,+ operation: "ProviderService.respondToUserInput",+ allowRecovery: true,+ });+ metricProvider = routed.adapter.provider;+ yield* Effect.annotateCurrentSpan({+ "provider.operation": "respond-to-user-input",+ "provider.kind": routed.adapter.provider,+ "provider.thread_id": input.threadId,+ "provider.request_id": input.requestId,+ });+ yield* routed.adapter.respondToUserInput(routed.threadId, input.requestId, input.answers);+ }).pipe(+ withMetrics({+ counter: providerTurnsTotal,+ outcomeAttributes: () =>+ providerMetricAttributes(metricProvider, {+ operation: "user-input-response",+ }),+ }),+ );});const stopSession: ProviderServiceShape["stopSession"] = Effect.fn("stopSession")(@@ -447,18 +574,35 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (schema: ProviderStopSessionInput,payload: rawInput,});- const routed = yield* resolveRoutableSession({- threadId: input.threadId,- operation: "ProviderService.stopSession",- allowRecovery: false,- });- if (routed.isActive) {- yield* routed.adapter.stopSession(routed.threadId);- }- yield* directory.remove(input.threadId);- yield* analytics.record("provider.session.stopped", {- provider: routed.adapter.provider,- });+ let metricProvider = "unknown";+ return yield* Effect.gen(function* () {+ const routed = yield* resolveRoutableSession({+ threadId: input.threadId,+ operation: "ProviderService.stopSession",+ allowRecovery: false,+ });+ metricProvider = routed.adapter.provider;+ yield* Effect.annotateCurrentSpan({+ "provider.operation": "stop-session",+ "provider.kind": routed.adapter.provider,+ "provider.thread_id": input.threadId,+ });+ if (routed.isActive) {+ yield* routed.adapter.stopSession(routed.threadId);+ }+ yield* directory.remove(input.threadId);+ yield* analytics.record("provider.session.stopped", {+ provider: routed.adapter.provider,+ });+ }).pipe(+ withMetrics({+ counter: providerSessionsTotal,+ outcomeAttributes: () =>+ providerMetricAttributes(metricProvider, {+ operation: "stop",+ }),+ }),+ );},);@@ -524,16 +668,34 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (if (input.numTurns === 0) {return;}- const routed = yield* resolveRoutableSession({- threadId: input.threadId,- operation: "ProviderService.rollbackConversation",- allowRecovery: true,- });- yield* routed.adapter.rollbackThread(routed.threadId, input.numTurns);- yield* analytics.record("provider.conversation.rolled_back", {- provider: routed.adapter.provider,- turns: input.numTurns,- });+ let metricProvider = "unknown";+ return yield* Effect.gen(function* () {+ const routed = yield* resolveRoutableSession({+ threadId: input.threadId,+ operation: "ProviderService.rollbackConversation",+ allowRecovery: true,+ });+ metricProvider = routed.adapter.provider;+ yield* Effect.annotateCurrentSpan({+ "provider.operation": "rollback-conversation",+ "provider.kind": routed.adapter.provider,+ "provider.thread_id": input.threadId,+ "provider.rollback_turns": input.numTurns,+ });+ yield* routed.adapter.rollbackThread(routed.threadId, input.numTurns);+ yield* analytics.record("provider.conversation.rolled_back", {+ provider: routed.adapter.provider,+ turns: input.numTurns,+ });+ }).pipe(+ withMetrics({+ counter: providerTurnsTotal,+ outcomeAttributes: () =>+ providerMetricAttributes(metricProvider, {+ operation: "rollback",+ }),+ }),+ );});const runStopAll = Effect.fn("runStopAll")(function* () {diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.tsindex 4367ca40a..0e8b909c9 100644--- a/apps/server/src/server.test.ts+++ b/apps/server/src/server.test.ts@@ -139,8 +139,17 @@ const buildAppUnderTest = (options?: {const baseDir = options?.config?.baseDir ?? tempBaseDir;const devUrl = options?.config?.devUrl;const derivedPaths = yield* deriveServerPaths(baseDir, devUrl);- const config = {+ const config: ServerConfigShape = {logLevel: "Info",+ traceMinLevel: "Info",+ traceTimingEnabled: true,+ traceBatchWindowMs: 200,+ traceMaxBytes: 10 * 1024 * 1024,+ traceMaxFiles: 10,+ otlpTracesUrl: undefined,+ otlpMetricsUrl: undefined,+ otlpExportIntervalMs: 10_000,+ otlpServiceName: "t3-server",mode: "web",port: 0,host: "127.0.0.1",@@ -154,7 +163,7 @@ const buildAppUnderTest = (options?: {autoBootstrapProjectFromCwd: false,logWebSocketEvents: false,...options?.config,- } satisfies ServerConfigShape;+ };const layerConfig = Layer.succeed(ServerConfig, config);const appLayer = HttpRouter.serve(makeRoutesLayer, {@@ -533,6 +542,10 @@ it.layer(NodeServices.layer)("server router seam", (it) => {} as const;yield* buildAppUnderTest({+ config: {+ otlpTracesUrl: "http://localhost:4318/v1/traces",+ otlpMetricsUrl: "http://localhost:4318/v1/metrics",+ },layers: {keybindings: {loadConfigState: Effect.succeed({@@ -561,6 +574,12 @@ it.layer(NodeServices.layer)("server router seam", (it) => {assert.deepEqual(first.config.keybindings, []);assert.deepEqual(first.config.issues, []);assert.deepEqual(first.config.providers, providers);+ assert.equal(first.config.observability.logsDirectoryPath.endsWith("/logs"), true);+ assert.equal(first.config.observability.localTracingEnabled, true);+ assert.equal(first.config.observability.otlpTracesUrl, "http://localhost:4318/v1/traces");+ assert.equal(first.config.observability.otlpTracesEnabled, true);+ assert.equal(first.config.observability.otlpMetricsUrl, "http://localhost:4318/v1/metrics");+ assert.equal(first.config.observability.otlpMetricsEnabled, true);assert.deepEqual(first.config.settings, DEFAULT_SERVER_SETTINGS);}assert.deepEqual(second, {diff --git a/apps/server/src/server.ts b/apps/server/src/server.tsindex 4e296e817..f630444a6 100644--- a/apps/server/src/server.ts+++ b/apps/server/src/server.ts@@ -29,7 +29,6 @@ import { RoutingTextGenerationLive } from "./git/Layers/RoutingTextGeneration";import { TerminalManagerLive } from "./terminal/Layers/Manager";import { GitManagerLive } from "./git/Layers/GitManager";import { KeybindingsLive } from "./keybindings";-import { ServerLoggerLive } from "./serverLogger";import { ServerRuntimeStartup, ServerRuntimeStartupLive } from "./serverRuntimeStartup";import { OrchestrationReactorLive } from "./orchestration/Layers/OrchestrationReactor";import { RuntimeReceiptBusLive } from "./orchestration/Layers/RuntimeReceiptBus";@@ -42,6 +41,7 @@ import { ProjectFaviconResolverLive } from "./project/Layers/ProjectFaviconResolimport { WorkspaceEntriesLive } from "./workspace/Layers/WorkspaceEntries";import { WorkspaceFileSystemLive } from "./workspace/Layers/WorkspaceFileSystem";import { WorkspacePathsLive } from "./workspace/Layers/WorkspacePaths";+import { ObservabilityLive } from "./observability/Layers/Observability";const PtyAdapterLive = Layer.unwrap(Effect.gen(function* () {@@ -231,7 +231,7 @@ export const makeServerLayer = Layer.unwrap(return serverApplicationLayer.pipe(Layer.provideMerge(RuntimeServicesLive),Layer.provideMerge(HttpServerLive),- Layer.provide(ServerLoggerLive),+ Layer.provide(ObservabilityLive),Layer.provideMerge(FetchHttpClient.layer),Layer.provideMerge(PlatformServicesLive),);diff --git a/apps/server/src/serverLogger.ts b/apps/server/src/serverLogger.tsindex aea53aacf..ea098dcbb 100644--- a/apps/server/src/serverLogger.ts+++ b/apps/server/src/serverLogger.ts@@ -4,11 +4,8 @@ import { ServerConfig } from "./config";export const ServerLoggerLive = Effect.gen(function* () {const config = yield* ServerConfig;- const { serverLogPath } = config;-- const fileLogger = Logger.formatSimple.pipe(Logger.toFile(serverLogPath));const minimumLogLevelLayer = Layer.succeed(References.MinimumLogLevel, config.logLevel);- const loggerLayer = Logger.layer([Logger.consolePretty(), fileLogger], {+ const loggerLayer = Logger.layer([Logger.consolePretty(), Logger.tracerLogger], {mergeWithExisting: false,});diff --git a/apps/server/src/serverRuntimeStartup.ts b/apps/server/src/serverRuntimeStartup.tsindex 026145eca..7c9231ac9 100644--- a/apps/server/src/serverRuntimeStartup.ts+++ b/apps/server/src/serverRuntimeStartup.ts@@ -141,6 +141,8 @@ export const recordStartupHeartbeat = Effect.gen(function* () {});export const launchStartupHeartbeat = recordStartupHeartbeat.pipe(+ Effect.annotateSpans({ "startup.phase": "heartbeat.record" }),+ Effect.withSpan("server.startup.heartbeat.record"),Effect.ignoreCause({ log: true }),Effect.forkScoped,Effect.asVoid,@@ -248,7 +250,14 @@ const maybeOpenBrowser = Effect.gen(function* () {);});+const runStartupPhase = <A, E, R>(phase: string, effect: Effect.Effect<A, E, R>) =>+ effect.pipe(+ Effect.annotateSpans({ "startup.phase": phase }),+ Effect.withSpan(`server.startup.${phase}`),+ );+const makeServerRuntimeStartup = Effect.gen(function* () {+ const serverConfig = yield* ServerConfig;const keybindings = yield* Keybindings;const orchestrationReactor = yield* OrchestrationReactor;const lifecycleEvents = yield* ServerLifecycleEvents;@@ -262,46 +271,65 @@ const makeServerRuntimeStartup = Effect.gen(function* () {const startup = Effect.gen(function* () {yield* Effect.logDebug("startup phase: starting keybindings runtime");- yield* keybindings.start.pipe(- Effect.catch((error) =>- Effect.logWarning("failed to start keybindings runtime", {- path: error.configPath,- detail: error.detail,- cause: error.cause,- }),+ yield* runStartupPhase(+ "keybindings.start",+ keybindings.start.pipe(+ Effect.catch((error) =>+ Effect.logWarning("failed to start keybindings runtime", {+ path: error.configPath,+ detail: error.detail,+ cause: error.cause,+ }),+ ),+ Effect.forkScoped,),- Effect.forkScoped,);yield* Effect.logDebug("startup phase: starting server settings runtime");- yield* serverSettings.start.pipe(- Effect.catch((error) =>- Effect.logWarning("failed to start server settings runtime", {- path: error.settingsPath,- detail: error.detail,- cause: error.cause,- }),+ yield* runStartupPhase(+ "settings.start",+ serverSettings.start.pipe(+ Effect.catch((error) =>+ Effect.logWarning("failed to start server settings runtime", {+ path: error.settingsPath,+ detail: error.detail,+ cause: error.cause,+ }),+ ),+ Effect.forkScoped,),- Effect.forkScoped,);yield* Effect.logDebug("startup phase: starting orchestration reactors");- yield* orchestrationReactor.start().pipe(Scope.provide(reactorScope));+ yield* runStartupPhase(+ "reactors.start",+ orchestrationReactor.start().pipe(Scope.provide(reactorScope)),+ );yield* Effect.logDebug("startup phase: preparing welcome payload");- const welcome = yield* autoBootstrapWelcome;+ const welcome = yield* runStartupPhase("welcome.prepare", autoBootstrapWelcome);yield* Effect.logDebug("startup phase: publishing welcome event", {cwd: welcome.cwd,projectName: welcome.projectName,bootstrapProjectId: welcome.bootstrapProjectId,bootstrapThreadId: welcome.bootstrapThreadId,});- yield* lifecycleEvents.publish({- version: 1,- type: "welcome",- payload: welcome,- });- });+ yield* runStartupPhase(+ "welcome.publish",+ lifecycleEvents.publish({+ version: 1,+ type: "welcome",+ payload: welcome,+ }),+ );+ }).pipe(+ Effect.annotateSpans({+ "server.mode": serverConfig.mode,+ "server.port": serverConfig.port,+ "server.host": serverConfig.host ?? "default",+ }),+ Effect.withSpan("server.startup", { kind: "server", root: true }),+ );yield* Effect.forkScoped(Effect.gen(function* () {@@ -319,18 +347,21 @@ const makeServerRuntimeStartup = Effect.gen(function* () {yield* Effect.logDebug("Accepting commands");yield* commandGate.signalCommandReady;yield* Effect.logDebug("startup phase: waiting for http listener");- yield* Deferred.await(httpListening);+ yield* runStartupPhase("http.wait", Deferred.await(httpListening));yield* Effect.logDebug("startup phase: publishing ready event");- yield* lifecycleEvents.publish({- version: 1,- type: "ready",- payload: { at: new Date().toISOString() },- });+ yield* runStartupPhase(+ "ready.publish",+ lifecycleEvents.publish({+ version: 1,+ type: "ready",+ payload: { at: new Date().toISOString() },+ }),+ );yield* Effect.logDebug("startup phase: recording startup heartbeat");yield* launchStartupHeartbeat;yield* Effect.logDebug("startup phase: browser open check");- yield* maybeOpenBrowser;+ yield* runStartupPhase("browser.open", maybeOpenBrowser);yield* Effect.logDebug("startup phase: complete");}),);diff --git a/apps/server/src/telemetry/Layers/AnalyticsService.ts b/apps/server/src/telemetry/Layers/AnalyticsService.tsindex 06ecfd023..e933576df 100644--- a/apps/server/src/telemetry/Layers/AnalyticsService.ts+++ b/apps/server/src/telemetry/Layers/AnalyticsService.ts@@ -125,17 +125,19 @@ const makeAnalyticsService = Effect.gen(function* () {}}).pipe(Effect.catch((cause) => Effect.logError("Failed to flush telemetry", { cause })));- const record: AnalyticsServiceShape["record"] = Effect.fnUntraced(function* (event, properties) {- if (!telemetryConfig.enabled || !identifier) return;-- const enqueueResult = yield* enqueueBufferedEvent(event, properties);- if (enqueueResult.dropped) {- yield* Effect.logDebug("analytics buffer full; dropping oldest event", {- size: enqueueResult.size,- event,- });- }- });+ const record: AnalyticsServiceShape["record"] = Effect.fn("record")(+ function* (event, properties) {+ if (!telemetryConfig.enabled || !identifier) return;++ const enqueueResult = yield* enqueueBufferedEvent(event, properties);+ if (enqueueResult.dropped) {+ yield* Effect.logDebug("analytics buffer full; dropping oldest event", {+ size: enqueueResult.size,+ event,+ });+ }+ },+ );yield* Effect.forever(Effect.sleep(1000).pipe(Effect.flatMap(() => flush)), {disableYield: true,diff --git a/apps/server/src/terminal/Layers/Manager.ts b/apps/server/src/terminal/Layers/Manager.tsindex 7ef18b165..a076e7600 100644--- a/apps/server/src/terminal/Layers/Manager.ts+++ b/apps/server/src/terminal/Layers/Manager.ts@@ -23,6 +23,11 @@ import {} from "effect";import { ServerConfig } from "../../config";+import {+ increment,+ terminalRestartsTotal,+ terminalSessionsTotal,+} from "../../observability/Metrics";import { runProcess } from "../../processRunner";import {TerminalCwdError,@@ -1294,6 +1299,12 @@ export const makeTerminalManagerWithOptions = Effect.fn("makeTerminalManagerWitheventType: "started" | "restarted",) {yield* stopProcess(session);+ yield* Effect.annotateCurrentSpan({+ "terminal.thread_id": session.threadId,+ "terminal.id": session.terminalId,+ "terminal.event_type": eventType,+ "terminal.cwd": input.cwd,+ });yield* modifyManagerState((state) => {session.status = "starting";@@ -1314,45 +1325,49 @@ export const makeTerminalManagerWithOptions = Effect.fn("makeTerminalManagerWithlet startedShell: string | null = null;const startResult = yield* Effect.result(- Effect.gen(function* () {- const shellCandidates = resolveShellCandidates(shellResolver);- const terminalEnv = createTerminalSpawnEnv(process.env, session.runtimeEnv);- const spawnResult = yield* trySpawn(shellCandidates, terminalEnv, session);- ptyProcess = spawnResult.process;- startedShell = spawnResult.shellLabel;-- const processPid = ptyProcess.pid;- const unsubscribeData = ptyProcess.onData((data) => {- if (!enqueueProcessEvent(session, processPid, { type: "output", data })) {- return;- }- runFork(drainProcessEvents(session, processPid));- });- const unsubscribeExit = ptyProcess.onExit((event) => {- if (!enqueueProcessEvent(session, processPid, { type: "exit", event })) {- return;- }- runFork(drainProcessEvents(session, processPid));- });-- yield* modifyManagerState((state) => {- session.process = ptyProcess;- session.pid = processPid;- session.status = "running";- session.updatedAt = new Date().toISOString();- session.unsubscribeData = unsubscribeData;- session.unsubscribeExit = unsubscribeExit;- return [undefined, state] as const;- });-- yield* publishEvent({- type: eventType,- threadId: session.threadId,- terminalId: session.terminalId,- createdAt: new Date().toISOString(),- snapshot: snapshot(session),- });- }),+ increment(terminalSessionsTotal, { lifecycle: eventType }).pipe(+ Effect.andThen(+ Effect.gen(function* () {+ const shellCandidates = resolveShellCandidates(shellResolver);+ const terminalEnv = createTerminalSpawnEnv(process.env, session.runtimeEnv);+ const spawnResult = yield* trySpawn(shellCandidates, terminalEnv, session);+ ptyProcess = spawnResult.process;+ startedShell = spawnResult.shellLabel;++ const processPid = ptyProcess.pid;+ const unsubscribeData = ptyProcess.onData((data) => {+ if (!enqueueProcessEvent(session, processPid, { type: "output", data })) {+ return;+ }+ runFork(drainProcessEvents(session, processPid));+ });+ const unsubscribeExit = ptyProcess.onExit((event) => {+ if (!enqueueProcessEvent(session, processPid, { type: "exit", event })) {+ return;+ }+ runFork(drainProcessEvents(session, processPid));+ });++ yield* modifyManagerState((state) => {+ session.process = ptyProcess;+ session.pid = processPid;+ session.status = "running";+ session.updatedAt = new Date().toISOString();+ session.unsubscribeData = unsubscribeData;+ session.unsubscribeExit = unsubscribeExit;+ return [undefined, state] as const;+ });++ yield* publishEvent({+ type: eventType,+ threadId: session.threadId,+ terminalId: session.terminalId,+ createdAt: new Date().toISOString(),+ snapshot: snapshot(session),+ });+ }),+ ),+ ),);if (startResult._tag === "Success") {@@ -1722,6 +1737,7 @@ export const makeTerminalManagerWithOptions = Effect.fn("makeTerminalManagerWithwithThreadLock(input.threadId,Effect.gen(function* () {+ yield* increment(terminalRestartsTotal, { scope: "thread" });const terminalId = input.terminalId ?? DEFAULT_TERMINAL_ID;yield* assertValidCwd(input.cwd);diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.tsindex e467ea445..de263ecb9 100644--- a/apps/server/src/ws.ts+++ b/apps/server/src/ws.ts@@ -28,6 +28,11 @@ import { Open, resolveAvailableEditors } from "./open";import { normalizeDispatchCommand } from "./orchestration/Normalizer";import { OrchestrationEngineService } from "./orchestration/Services/OrchestrationEngine";import { ProjectionSnapshotQuery } from "./orchestration/Services/ProjectionSnapshotQuery";+import {+ observeRpcEffect,+ observeRpcStream,+ observeRpcStreamEffect,+} from "./observability/RpcInstrumentation";import { ProviderRegistry } from "./provider/Services/ProviderRegistry";import { ServerLifecycleEvents } from "./serverLifecycleEvents";import { ServerRuntimeStartup } from "./serverRuntimeStartup";@@ -67,85 +72,114 @@ const WsRpcLayer = WsRpcGroup.toLayer(issues: keybindingsConfig.issues,providers,availableEditors: resolveAvailableEditors(),+ observability: {+ logsDirectoryPath: config.logsDir,+ localTracingEnabled: true,+ ...(config.otlpTracesUrl !== undefined ? { otlpTracesUrl: config.otlpTracesUrl } : {}),+ otlpTracesEnabled: config.otlpTracesUrl !== undefined,+ ...(config.otlpMetricsUrl !== undefined ? { otlpMetricsUrl: config.otlpMetricsUrl } : {}),+ otlpMetricsEnabled: config.otlpMetricsUrl !== undefined,+ },settings,};});return WsRpcGroup.of({[ORCHESTRATION_WS_METHODS.getSnapshot]: (_input) =>- projectionSnapshotQuery.getSnapshot().pipe(- Effect.mapError(- (cause) =>- new OrchestrationGetSnapshotError({- message: "Failed to load orchestration snapshot",- cause,- }),+ observeRpcEffect(+ ORCHESTRATION_WS_METHODS.getSnapshot,+ projectionSnapshotQuery.getSnapshot().pipe(+ Effect.mapError(+ (cause) =>+ new OrchestrationGetSnapshotError({+ message: "Failed to load orchestration snapshot",+ cause,+ }),+ ),),+ { "rpc.aggregate": "orchestration" },),[ORCHESTRATION_WS_METHODS.dispatchCommand]: (command) =>- Effect.gen(function* () {- const normalizedCommand = yield* normalizeDispatchCommand(command);- const result = yield* startup.enqueueCommand(- orchestrationEngine.dispatch(normalizedCommand),- );- if (normalizedCommand.type === "thread.archive") {- yield* terminalManager.close({ threadId: normalizedCommand.threadId }).pipe(- Effect.catch((error) =>- Effect.logWarning("failed to close thread terminals after archive", {- threadId: normalizedCommand.threadId,- error: error.message,- }),- ),+ observeRpcEffect(+ ORCHESTRATION_WS_METHODS.dispatchCommand,+ Effect.gen(function* () {+ const normalizedCommand = yield* normalizeDispatchCommand(command);+ const result = yield* startup.enqueueCommand(+ orchestrationEngine.dispatch(normalizedCommand),);- }- return result;- }).pipe(- Effect.mapError((cause) =>- Schema.is(OrchestrationDispatchCommandError)(cause)- ? cause- : new OrchestrationDispatchCommandError({- message: "Failed to dispatch orchestration command",- cause,- }),+ if (normalizedCommand.type === "thread.archive") {+ yield* terminalManager.close({ threadId: normalizedCommand.threadId }).pipe(+ Effect.catch((error) =>+ Effect.logWarning("failed to close thread terminals after archive", {+ threadId: normalizedCommand.threadId,+ error: error.message,+ }),+ ),+ );+ }+ return result;+ }).pipe(+ Effect.mapError((cause) =>+ Schema.is(OrchestrationDispatchCommandError)(cause)+ ? cause+ : new OrchestrationDispatchCommandError({+ message: "Failed to dispatch orchestration command",+ cause,+ }),+ ),),+ { "rpc.aggregate": "orchestration" },),[ORCHESTRATION_WS_METHODS.getTurnDiff]: (input) =>- checkpointDiffQuery.getTurnDiff(input).pipe(- Effect.mapError(- (cause) =>- new OrchestrationGetTurnDiffError({- message: "Failed to load turn diff",- cause,- }),+ observeRpcEffect(+ ORCHESTRATION_WS_METHODS.getTurnDiff,+ checkpointDiffQuery.getTurnDiff(input).pipe(+ Effect.mapError(+ (cause) =>+ new OrchestrationGetTurnDiffError({+ message: "Failed to load turn diff",+ cause,+ }),+ ),),+ { "rpc.aggregate": "orchestration" },),[ORCHESTRATION_WS_METHODS.getFullThreadDiff]: (input) =>- checkpointDiffQuery.getFullThreadDiff(input).pipe(- Effect.mapError(- (cause) =>- new OrchestrationGetFullThreadDiffError({- message: "Failed to load full thread diff",- cause,- }),+ observeRpcEffect(+ ORCHESTRATION_WS_METHODS.getFullThreadDiff,+ checkpointDiffQuery.getFullThreadDiff(input).pipe(+ Effect.mapError(+ (cause) =>+ new OrchestrationGetFullThreadDiffError({+ message: "Failed to load full thread diff",+ cause,+ }),+ ),),+ { "rpc.aggregate": "orchestration" },),[ORCHESTRATION_WS_METHODS.replayEvents]: (input) =>- Stream.runCollect(- orchestrationEngine.readEvents(- clamp(input.fromSequenceExclusive, { maximum: Number.MAX_SAFE_INTEGER, minimum: 0 }),- ),- ).pipe(- Effect.map((events) => Array.from(events)),- Effect.mapError(- (cause) =>- new OrchestrationReplayEventsError({- message: "Failed to replay orchestration events",- cause,- }),+ observeRpcEffect(+ ORCHESTRATION_WS_METHODS.replayEvents,+ Stream.runCollect(+ orchestrationEngine.readEvents(+ clamp(input.fromSequenceExclusive, { maximum: Number.MAX_SAFE_INTEGER, minimum: 0 }),+ ),+ ).pipe(+ Effect.map((events) => Array.from(events)),+ Effect.mapError(+ (cause) =>+ new OrchestrationReplayEventsError({+ message: "Failed to replay orchestration events",+ cause,+ }),+ ),),+ { "rpc.aggregate": "orchestration" },),[WS_METHODS.subscribeOrchestrationDomainEvents]: (_input) =>- Stream.unwrap(+ observeRpcStreamEffect(+ WS_METHODS.subscribeOrchestrationDomainEvents,Effect.gen(function* () {const snapshot = yield* orchestrationEngine.getReadModel();const fromSequenceExclusive = snapshot.snapshotSequence;@@ -200,82 +234,167 @@ const WsRpcLayer = WsRpcGroup.toLayer(Stream.flatMap((events) => Stream.fromIterable(events)),);}),+ { "rpc.aggregate": "orchestration" },),- [WS_METHODS.serverGetConfig]: (_input) => loadServerConfig,+ [WS_METHODS.serverGetConfig]: (_input) =>+ observeRpcEffect(WS_METHODS.serverGetConfig, loadServerConfig, {+ "rpc.aggregate": "server",+ }),[WS_METHODS.serverRefreshProviders]: (_input) =>- providerRegistry.refresh().pipe(Effect.map((providers) => ({ providers }))),+ observeRpcEffect(+ WS_METHODS.serverRefreshProviders,+ providerRegistry.refresh().pipe(Effect.map((providers) => ({ providers }))),+ { "rpc.aggregate": "server" },+ ),[WS_METHODS.serverUpsertKeybinding]: (rule) =>- Effect.gen(function* () {- const keybindingsConfig = yield* keybindings.upsertKeybindingRule(rule);- return { keybindings: keybindingsConfig, issues: [] };+ observeRpcEffect(+ WS_METHODS.serverUpsertKeybinding,+ Effect.gen(function* () {+ const keybindingsConfig = yield* keybindings.upsertKeybindingRule(rule);+ return { keybindings: keybindingsConfig, issues: [] };+ }),+ { "rpc.aggregate": "server" },+ ),+ [WS_METHODS.serverGetSettings]: (_input) =>+ observeRpcEffect(WS_METHODS.serverGetSettings, serverSettings.getSettings, {+ "rpc.aggregate": "server",+ }),+ [WS_METHODS.serverUpdateSettings]: ({ patch }) =>+ observeRpcEffect(WS_METHODS.serverUpdateSettings, serverSettings.updateSettings(patch), {+ "rpc.aggregate": "server",}),- [WS_METHODS.serverGetSettings]: (_input) => serverSettings.getSettings,- [WS_METHODS.serverUpdateSettings]: ({ patch }) => serverSettings.updateSettings(patch),[WS_METHODS.projectsSearchEntries]: (input) =>- workspaceEntries.search(input).pipe(- Effect.mapError(- (cause) =>- new ProjectSearchEntriesError({- message: `Failed to search workspace entries: ${cause.detail}`,- cause,- }),+ observeRpcEffect(+ WS_METHODS.projectsSearchEntries,+ workspaceEntries.search(input).pipe(+ Effect.mapError(+ (cause) =>+ new ProjectSearchEntriesError({+ message: `Failed to search workspace entries: ${cause.detail}`,+ cause,+ }),+ ),),+ { "rpc.aggregate": "workspace" },),[WS_METHODS.projectsWriteFile]: (input) =>- workspaceFileSystem.writeFile(input).pipe(- Effect.mapError((cause) => {- const message = Schema.is(WorkspacePathOutsideRootError)(cause)- ? "Workspace file path must stay within the project root."- : "Failed to write workspace file";- return new ProjectWriteFileError({- message,- cause,- });- }),+ observeRpcEffect(+ WS_METHODS.projectsWriteFile,+ workspaceFileSystem.writeFile(input).pipe(+ Effect.mapError((cause) => {+ const message = Schema.is(WorkspacePathOutsideRootError)(cause)+ ? "Workspace file path must stay within the project root."+ : "Failed to write workspace file";+ return new ProjectWriteFileError({+ message,+ cause,+ });+ }),+ ),+ { "rpc.aggregate": "workspace" },),- [WS_METHODS.shellOpenInEditor]: (input) => open.openInEditor(input),- [WS_METHODS.gitStatus]: (input) => gitManager.status(input),- [WS_METHODS.gitPull]: (input) => git.pullCurrentBranch(input.cwd),+ [WS_METHODS.shellOpenInEditor]: (input) =>+ observeRpcEffect(WS_METHODS.shellOpenInEditor, open.openInEditor(input), {+ "rpc.aggregate": "workspace",+ }),+ [WS_METHODS.gitStatus]: (input) =>+ observeRpcEffect(WS_METHODS.gitStatus, gitManager.status(input), {+ "rpc.aggregate": "git",+ }),+ [WS_METHODS.gitPull]: (input) =>+ observeRpcEffect(WS_METHODS.gitPull, git.pullCurrentBranch(input.cwd), {+ "rpc.aggregate": "git",+ }),[WS_METHODS.gitRunStackedAction]: (input) =>- Stream.callback<GitActionProgressEvent, GitManagerServiceError>((queue) =>- gitManager- .runStackedAction(input, {- actionId: input.actionId,- progressReporter: {- publish: (event) => Queue.offer(queue, event).pipe(Effect.asVoid),- },- })- .pipe(- Effect.matchCauseEffect({- onFailure: (cause) => Queue.failCause(queue, cause),- onSuccess: () => Queue.end(queue).pipe(Effect.asVoid),- }),- ),+ observeRpcStream(+ WS_METHODS.gitRunStackedAction,+ Stream.callback<GitActionProgressEvent, GitManagerServiceError>((queue) =>+ gitManager+ .runStackedAction(input, {+ actionId: input.actionId,+ progressReporter: {+ publish: (event) => Queue.offer(queue, event).pipe(Effect.asVoid),+ },+ })+ .pipe(+ Effect.matchCauseEffect({+ onFailure: (cause) => Queue.failCause(queue, cause),+ onSuccess: () => Queue.end(queue).pipe(Effect.asVoid),+ }),+ ),+ ),+ { "rpc.aggregate": "git" },),- [WS_METHODS.gitResolvePullRequest]: (input) => gitManager.resolvePullRequest(input),+ [WS_METHODS.gitResolvePullRequest]: (input) =>+ observeRpcEffect(WS_METHODS.gitResolvePullRequest, gitManager.resolvePullRequest(input), {+ "rpc.aggregate": "git",+ }),[WS_METHODS.gitPreparePullRequestThread]: (input) =>- gitManager.preparePullRequestThread(input),- [WS_METHODS.gitListBranches]: (input) => git.listBranches(input),- [WS_METHODS.gitCreateWorktree]: (input) => git.createWorktree(input),- [WS_METHODS.gitRemoveWorktree]: (input) => git.removeWorktree(input),- [WS_METHODS.gitCreateBranch]: (input) => git.createBranch(input),- [WS_METHODS.gitCheckout]: (input) => Effect.scoped(git.checkoutBranch(input)),- [WS_METHODS.gitInit]: (input) => git.initRepo(input),- [WS_METHODS.terminalOpen]: (input) => terminalManager.open(input),- [WS_METHODS.terminalWrite]: (input) => terminalManager.write(input),- [WS_METHODS.terminalResize]: (input) => terminalManager.resize(input),- [WS_METHODS.terminalClear]: (input) => terminalManager.clear(input),- [WS_METHODS.terminalRestart]: (input) => terminalManager.restart(input),- [WS_METHODS.terminalClose]: (input) => terminalManager.close(input),+ observeRpcEffect(+ WS_METHODS.gitPreparePullRequestThread,+ gitManager.preparePullRequestThread(input),+ { "rpc.aggregate": "git" },+ ),+ [WS_METHODS.gitListBranches]: (input) =>+ observeRpcEffect(WS_METHODS.gitListBranches, git.listBranches(input), {+ "rpc.aggregate": "git",+ }),+ [WS_METHODS.gitCreateWorktree]: (input) =>+ observeRpcEffect(WS_METHODS.gitCreateWorktree, git.createWorktree(input), {+ "rpc.aggregate": "git",+ }),+ [WS_METHODS.gitRemoveWorktree]: (input) =>+ observeRpcEffect(WS_METHODS.gitRemoveWorktree, git.removeWorktree(input), {+ "rpc.aggregate": "git",+ }),+ [WS_METHODS.gitCreateBranch]: (input) =>+ observeRpcEffect(WS_METHODS.gitCreateBranch, git.createBranch(input), {+ "rpc.aggregate": "git",+ }),+ [WS_METHODS.gitCheckout]: (input) =>+ observeRpcEffect(WS_METHODS.gitCheckout, Effect.scoped(git.checkoutBranch(input)), {+ "rpc.aggregate": "git",+ }),+ [WS_METHODS.gitInit]: (input) =>+ observeRpcEffect(WS_METHODS.gitInit, git.initRepo(input), { "rpc.aggregate": "git" }),+ [WS_METHODS.terminalOpen]: (input) =>+ observeRpcEffect(WS_METHODS.terminalOpen, terminalManager.open(input), {+ "rpc.aggregate": "terminal",+ }),+ [WS_METHODS.terminalWrite]: (input) =>+ observeRpcEffect(WS_METHODS.terminalWrite, terminalManager.write(input), {+ "rpc.aggregate": "terminal",+ }),+ [WS_METHODS.terminalResize]: (input) =>+ observeRpcEffect(WS_METHODS.terminalResize, terminalManager.resize(input), {+ "rpc.aggregate": "terminal",+ }),+ [WS_METHODS.terminalClear]: (input) =>+ observeRpcEffect(WS_METHODS.terminalClear, terminalManager.clear(input), {+ "rpc.aggregate": "terminal",+ }),+ [WS_METHODS.terminalRestart]: (input) =>+ observeRpcEffect(WS_METHODS.terminalRestart, terminalManager.restart(input), {+ "rpc.aggregate": "terminal",+ }),+ [WS_METHODS.terminalClose]: (input) =>+ observeRpcEffect(WS_METHODS.terminalClose, terminalManager.close(input), {+ "rpc.aggregate": "terminal",+ }),[WS_METHODS.subscribeTerminalEvents]: (_input) =>- Stream.callback<TerminalEvent>((queue) =>- Effect.acquireRelease(- terminalManager.subscribe((event) => Queue.offer(queue, event)),- (unsubscribe) => Effect.sync(unsubscribe),+ observeRpcStream(+ WS_METHODS.subscribeTerminalEvents,+ Stream.callback<TerminalEvent>((queue) =>+ Effect.acquireRelease(+ terminalManager.subscribe((event) => Queue.offer(queue, event)),+ (unsubscribe) => Effect.sync(unsubscribe),+ ),),+ { "rpc.aggregate": "terminal" },),[WS_METHODS.subscribeServerConfig]: (_input) =>- Stream.unwrap(+ observeRpcStreamEffect(+ WS_METHODS.subscribeServerConfig,Effect.gen(function* () {const keybindingsUpdates = keybindings.streamChanges.pipe(Stream.map((event) => ({@@ -310,9 +429,11 @@ const WsRpcLayer = WsRpcGroup.toLayer(Stream.merge(keybindingsUpdates, Stream.merge(providerStatuses, settingsUpdates)),);}),+ { "rpc.aggregate": "server" },),[WS_METHODS.subscribeServerLifecycle]: (_input) =>- Stream.unwrap(+ observeRpcStreamEffect(+ WS_METHODS.subscribeServerLifecycle,Effect.gen(function* () {const snapshot = yield* lifecycleEvents.snapshot;const snapshotEvents = Array.from(snapshot.events).toSorted(@@ -323,6 +444,7 @@ const WsRpcLayer = WsRpcGroup.toLayer();return Stream.concat(Stream.fromIterable(snapshotEvents), liveEvents);}),+ { "rpc.aggregate": "server" },),});}),@@ -330,9 +452,13 @@ const WsRpcLayer = WsRpcGroup.toLayer(export const websocketRpcRouteLayer = Layer.unwrap(Effect.gen(function* () {- const rpcWebSocketHttpEffect = yield* RpcServer.toHttpEffectWebsocket(WsRpcGroup).pipe(- Effect.provide(Layer.mergeAll(WsRpcLayer, RpcSerialization.layerJson)),- );+ const rpcWebSocketHttpEffect = yield* RpcServer.toHttpEffectWebsocket(WsRpcGroup, {+ spanPrefix: "ws.rpc",+ spanAttributes: {+ "rpc.transport": "websocket",+ "rpc.system": "effect-rpc",+ },+ }).pipe(Effect.provide(Layer.mergeAll(WsRpcLayer, RpcSerialization.layerJson)));return HttpRouter.add("GET","/ws",diff --git a/apps/web/src/components/ChatView.browser.tsx b/apps/web/src/components/ChatView.browser.tsxindex b0263cbf4..f9f1362d0 100644--- a/apps/web/src/components/ChatView.browser.tsx+++ b/apps/web/src/components/ChatView.browser.tsx@@ -135,6 +135,12 @@ function createBaseServerConfig(): ServerConfig {},],availableEditors: [],+ observability: {+ logsDirectoryPath: "/repo/project/.t3/logs",+ localTracingEnabled: true,+ otlpTracesEnabled: false,+ otlpMetricsEnabled: false,+ },settings: {...DEFAULT_SERVER_SETTINGS,...DEFAULT_CLIENT_SETTINGS,diff --git a/apps/web/src/components/KeybindingsToast.browser.tsx b/apps/web/src/components/KeybindingsToast.browser.tsxindex b8398e7b8..de7d84b4e 100644--- a/apps/web/src/components/KeybindingsToast.browser.tsx+++ b/apps/web/src/components/KeybindingsToast.browser.tsx@@ -56,6 +56,12 @@ function createBaseServerConfig(): ServerConfig {},],availableEditors: [],+ observability: {+ logsDirectoryPath: "/repo/project/.t3/logs",+ localTracingEnabled: true,+ otlpTracesEnabled: false,+ otlpMetricsEnabled: false,+ },settings: {enableAssistantStreaming: false,defaultThreadEnvMode: "local" as const,diff --git a/apps/web/src/components/settings/SettingsPanels.browser.tsx b/apps/web/src/components/settings/SettingsPanels.browser.tsxnew file mode 100644index 000000000..090f6f12a--- /dev/null+++ b/apps/web/src/components/settings/SettingsPanels.browser.tsx@@ -0,0 +1,91 @@+import "../../index.css";++import { DEFAULT_SERVER_SETTINGS, type NativeApi, type ServerConfig } from "@t3tools/contracts";+import { page } from "vitest/browser";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { render } from "vitest-browser-react";++import { __resetNativeApiForTests } from "../../nativeApi";+import { AppAtomRegistryProvider } from "../../rpc/atomRegistry";+import { resetServerStateForTests, setServerConfigSnapshot } from "../../rpc/serverState";+import { GeneralSettingsPanel } from "./SettingsPanels";++function createBaseServerConfig(): ServerConfig {+ return {+ cwd: "/repo/project",+ keybindingsConfigPath: "/repo/project/.t3code-keybindings.json",+ keybindings: [],+ issues: [],+ providers: [],+ availableEditors: ["cursor"],+ observability: {+ logsDirectoryPath: "/repo/project/.t3/logs",+ localTracingEnabled: true,+ otlpTracesUrl: "http://localhost:4318/v1/traces",+ otlpTracesEnabled: true,+ otlpMetricsEnabled: false,+ },+ settings: DEFAULT_SERVER_SETTINGS,+ };+}++describe("GeneralSettingsPanel observability", () => {+ beforeEach(() => {+ resetServerStateForTests();+ __resetNativeApiForTests();+ localStorage.clear();+ document.body.innerHTML = "";+ });++ afterEach(() => {+ resetServerStateForTests();+ __resetNativeApiForTests();+ document.body.innerHTML = "";+ });++ it("shows diagnostics inside About with a single logs-folder action", async () => {+ setServerConfigSnapshot(createBaseServerConfig());++ await render(+ <AppAtomRegistryProvider>+ <GeneralSettingsPanel />+ </AppAtomRegistryProvider>,+ );++ await expect.element(page.getByText("About")).toBeInTheDocument();+ await expect.element(page.getByText("Diagnostics")).toBeInTheDocument();+ await expect.element(page.getByText("Open logs folder")).toBeInTheDocument();+ await expect+ .element(page.getByText("/repo/project/.t3/logs", { exact: true }))+ .toBeInTheDocument();+ await expect+ .element(+ page.getByText(+ "Local trace file. OTLP exporting traces to http://localhost:4318/v1/traces.",+ ),+ )+ .toBeInTheDocument();+ });++ it("opens the logs folder in the preferred editor", async () => {+ const openInEditor = vi.fn<NativeApi["shell"]["openInEditor"]>().mockResolvedValue(undefined);+ window.nativeApi = {+ shell: {+ openInEditor,+ },+ } as unknown as NativeApi;++ setServerConfigSnapshot(createBaseServerConfig());++ await render(+ <AppAtomRegistryProvider>+ <GeneralSettingsPanel />+ </AppAtomRegistryProvider>,+ );++ const openLogsButton = page.getByText("Open logs folder");+ await openLogsButton.click();++ expect(openInEditor).toHaveBeenCalledWith("/repo/project/.t3/logs", "cursor");+ });+});diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsxindex e3e620030..d534eefaa 100644--- a/apps/web/src/components/settings/SettingsPanels.tsx+++ b/apps/web/src/components/settings/SettingsPanels.tsx@@ -61,6 +61,7 @@ import { ProjectFavicon } from "../ProjectFavicon";import {useServerAvailableEditors,useServerKeybindingsConfigPath,+ useServerObservability,useServerProviders,} from "../../rpc/serverState";@@ -518,8 +519,13 @@ export function GeneralSettingsPanel() {const { theme, setTheme } = useTheme();const settings = useSettings();const { updateSettings } = useUpdateSettings();- const [isOpeningKeybindings, setIsOpeningKeybindings] = useState(false);- const [openKeybindingsError, setOpenKeybindingsError] = useState<string | null>(null);+ const [openingPathByTarget, setOpeningPathByTarget] = useState({+ keybindings: false,+ logsDirectory: false,+ });+ const [openPathErrorByTarget, setOpenPathErrorByTarget] = useState<+ Partial<Record<"keybindings" | "logsDirectory", string | null>>+ >({});const [openProviderDetails, setOpenProviderDetails] = useState<Record<ProviderKind, boolean>>({codex: Boolean(settings.providers.codex.binaryPath !== DEFAULT_UNIFIED_SETTINGS.providers.codex.binaryPath ||@@ -561,8 +567,21 @@ export function GeneralSettingsPanel() {const keybindingsConfigPath = useServerKeybindingsConfigPath();const availableEditors = useServerAvailableEditors();+ const observability = useServerObservability();const serverProviders = useServerProviders();const codexHomePath = settings.providers.codex.homePath;+ const logsDirectoryPath = observability?.logsDirectoryPath ?? null;+ const diagnosticsDescription = (() => {+ const exports: string[] = [];+ if (observability?.otlpTracesEnabled && observability.otlpTracesUrl) {+ exports.push(`traces to ${observability.otlpTracesUrl}`);+ }+ if (observability?.otlpMetricsEnabled && observability.otlpMetricsUrl) {+ exports.push(`metrics to ${observability.otlpMetricsUrl}`);+ }+ const mode = observability?.localTracingEnabled ? "Local trace file" : "Terminal logs only";+ return exports.length > 0 ? `${mode}. OTLP exporting ${exports.join(" and ")}.` : `${mode}.`;+ })();const textGenerationModelSelection = resolveAppModelSelectionState(settings, serverProviders);const textGenProvider = textGenerationModelSelection.provider;@@ -579,27 +598,49 @@ export function GeneralSettingsPanel() {DEFAULT_UNIFIED_SETTINGS.textGenerationModelSelection ?? null,);+ const openInPreferredEditor = useCallback(+ (target: "keybindings" | "logsDirectory", path: string | null, failureMessage: string) => {+ if (!path) return;+ setOpenPathErrorByTarget((existing) => ({ ...existing, [target]: null }));+ setOpeningPathByTarget((existing) => ({ ...existing, [target]: true }));++ const editor = resolveAndPersistPreferredEditor(availableEditors ?? []);+ if (!editor) {+ setOpenPathErrorByTarget((existing) => ({+ ...existing,+ [target]: "No available editors found.",+ }));+ setOpeningPathByTarget((existing) => ({ ...existing, [target]: false }));+ return;+ }++ void ensureNativeApi()+ .shell.openInEditor(path, editor)+ .catch((error) => {+ setOpenPathErrorByTarget((existing) => ({+ ...existing,+ [target]: error instanceof Error ? error.message : failureMessage,+ }));+ })+ .finally(() => {+ setOpeningPathByTarget((existing) => ({ ...existing, [target]: false }));+ });+ },+ [availableEditors],+ );+const openKeybindingsFile = useCallback(() => {- if (!keybindingsConfigPath) return;- setOpenKeybindingsError(null);- setIsOpeningKeybindings(true);- const editor = resolveAndPersistPreferredEditor(availableEditors ?? []);- if (!editor) {- setOpenKeybindingsError("No available editors found.");- setIsOpeningKeybindings(false);- return;- }- void ensureNativeApi()- .shell.openInEditor(keybindingsConfigPath, editor)- .catch((error) => {- setOpenKeybindingsError(- error instanceof Error ? error.message : "Unable to open keybindings file.",- );- })- .finally(() => {- setIsOpeningKeybindings(false);- });- }, [availableEditors, keybindingsConfigPath]);+ openInPreferredEditor("keybindings", keybindingsConfigPath, "Unable to open keybindings file.");+ }, [keybindingsConfigPath, openInPreferredEditor]);++ const openLogsDirectory = useCallback(() => {+ openInPreferredEditor("logsDirectory", logsDirectoryPath, "Unable to open logs folder.");+ }, [logsDirectoryPath, openInPreferredEditor]);++ const openKeybindingsError = openPathErrorByTarget.keybindings ?? null;+ const openDiagnosticsError = openPathErrorByTarget.logsDirectory ?? null;+ const isOpeningKeybindings = openingPathByTarget.keybindings;+ const isOpeningLogsDirectory = openingPathByTarget.logsDirectory;const addCustomModel = useCallback((provider: ProviderKind) => {@@ -1408,6 +1449,30 @@ export function GeneralSettingsPanel() {description="Current version of the application."/>)}+ <SettingsRow+ title="Diagnostics"+ description={diagnosticsDescription}+ status={+ <>+ <span className="block break-all font-mono text-[11px] text-foreground">+ {logsDirectoryPath ?? "Resolving logs directory..."}+ </span>+ {openDiagnosticsError ? (+ <span className="mt-1 block text-destructive">{openDiagnosticsError}</span>+ ) : null}+ </>+ }+ control={+ <Button+ size="xs"+ variant="outline"+ disabled={!logsDirectoryPath || isOpeningLogsDirectory}+ onClick={openLogsDirectory}+ >+ {isOpeningLogsDirectory ? "Opening..." : "Open logs folder"}+ </Button>+ }+ /></SettingsSection></SettingsPageContainer>);diff --git a/apps/web/src/rpc/serverState.test.ts b/apps/web/src/rpc/serverState.test.tsindex 8da5846fe..721ce25fb 100644--- a/apps/web/src/rpc/serverState.test.ts+++ b/apps/web/src/rpc/serverState.test.ts@@ -57,6 +57,12 @@ const baseServerConfig: ServerConfig = {issues: [],providers: defaultProviders,availableEditors: ["cursor"],+ observability: {+ logsDirectoryPath: "/tmp/workspace/.config/logs",+ localTracingEnabled: true,+ otlpTracesEnabled: false,+ otlpMetricsEnabled: false,+ },settings: DEFAULT_SERVER_SETTINGS,};diff --git a/apps/web/src/rpc/serverState.ts b/apps/web/src/rpc/serverState.tsindex 2a6939406..3e8dbab28 100644--- a/apps/web/src/rpc/serverState.ts+++ b/apps/web/src/rpc/serverState.ts@@ -50,6 +50,7 @@ const selectAvailableEditors = (config: ServerConfig | null): ReadonlyArray<Editconst selectKeybindings = (config: ServerConfig | null) => config?.keybindings ?? EMPTY_KEYBINDINGS;const selectKeybindingsConfigPath = (config: ServerConfig | null) =>config?.keybindingsConfigPath ?? null;+const selectObservability = (config: ServerConfig | null) => config?.observability ?? null;const selectProviders = (config: ServerConfig | null) =>config?.providers ?? EMPTY_SERVER_PROVIDERS;const selectSettings = (config: ServerConfig | null): ServerSettings =>@@ -281,6 +282,10 @@ export function useServerKeybindingsConfigPath(): string | null {return useAtomValue(serverConfigAtom, selectKeybindingsConfigPath);}+export function useServerObservability(): ServerConfig["observability"] | null {+ return useAtomValue(serverConfigAtom, selectObservability);+}+export function useServerWelcomeSubscription(listener: (payload: ServerLifecycleWelcomePayload) => void,): void {diff --git a/apps/web/src/wsNativeApi.test.ts b/apps/web/src/wsNativeApi.test.tsindex c9bcc7ffd..47f78f967 100644--- a/apps/web/src/wsNativeApi.test.ts+++ b/apps/web/src/wsNativeApi.test.ts@@ -159,6 +159,12 @@ const baseServerConfig: ServerConfig = {issues: [],providers: defaultProviders,availableEditors: ["cursor"],+ observability: {+ logsDirectoryPath: "/tmp/workspace/.config/logs",+ localTracingEnabled: true,+ otlpTracesEnabled: false,+ otlpMetricsEnabled: false,+ },settings: DEFAULT_SERVER_SETTINGS,};diff --git a/docs/observability.md b/docs/observability.mdnew file mode 100644index 000000000..dde109357--- /dev/null+++ b/docs/observability.md@@ -0,0 +1,519 @@+# Observability++T3 Code has one server-side observability model:++- pretty logs go to stdout for humans+- completed spans go to a local NDJSON trace file+- traces and metrics can also be exported over OTLP to a real backend like Grafana LGTM++The local trace file is the persisted source of truth. There is no separate persisted server log file anymore.++## Where To Find Things++### Logs++Logs are human-facing only:++- destination: stdout+- format: `Logger.consolePretty()`+- persistence: none++If you want a log message to show up in the trace file, emit it inside an active span with `Effect.log...`. `Logger.tracerLogger` will attach it as a span event.++### Traces++Completed spans are written as NDJSON records to `serverTracePath` (by default, `~/.t3/userdata/logs/server.trace.ndjson`).++Important fields in each record:++- `name`: span name+- `traceId`, `spanId`, `parentSpanId`: correlation+- `durationMs`: elapsed time+- `attributes`: structured context+- `events`: embedded logs and custom events+- `exit`: `Success`, `Failure`, or `Interrupted`++The schema lives in `apps/server/src/observability/TraceRecord.ts`.++### Metrics++Metrics are not written to a local file.++- local persistence: none+- remote export: OTLP only, when configured+- current definitions: `apps/server/src/observability/Metrics.ts`++If OTLP is not configured, metrics still exist in-process, but you will not have a local artifact to inspect.++### Related Artifacts++Provider event NDJSON files still exist for provider runtime streams. Those are separate from the main server trace file.++## Run The Server In Instrumented Mode++There are two useful modes:++- local-only: stdout + local `server.trace.ndjson`+- full local observability: stdout + local trace file + OTLP export to Grafana/Tempo/Prometheus++The local trace file is always on. OTLP export is opt-in.++### Option 1: Local Traces Only++You do not need any extra env vars. Just run the app normally and inspect `server.trace.ndjson`.++Examples:++```bash+npx t3+```++```bash+bun dev+```++```bash+bun dev:desktop+```++### Option 2: Run With A Local LGTM Stack++#### 1. Start Grafana LGTM++```bash+docker run --name lgtm \+ -p 3000:3000 \+ -p 4317:4317 \+ -p 4318:4318 \+ --rm -ti \+ grafana/otel-lgtm+```++Then open `http://localhost:3000`.++Default Grafana login:++- username: `admin`+- password: `admin`++#### 2. Export OTLP env vars++```bash+export T3CODE_OTLP_TRACES_URL=http://localhost:4318/v1/traces+export T3CODE_OTLP_METRICS_URL=http://localhost:4318/v1/metrics+export T3CODE_OTLP_SERVICE_NAME=t3-local+```++Optional:++```bash+export T3CODE_TRACE_MIN_LEVEL=Info+export T3CODE_TRACE_TIMING_ENABLED=true+```++#### 3. Launch the app from that same shell++CLI:++```bash+npx t3+```++Monorepo web/server dev:++```bash+bun dev+```++Monorepo desktop dev:++```bash+bun dev:desktop+```++Packaged desktop app:++Launch the actual app executable from the same shell so the desktop app and embedded backend inherit `T3CODE_OTLP_*`.++macOS app bundle example:++```bash+T3CODE_OTLP_TRACES_URL=http://localhost:4318/v1/traces \+T3CODE_OTLP_METRICS_URL=http://localhost:4318/v1/metrics \+T3CODE_OTLP_SERVICE_NAME=t3-desktop \+"/Applications/T3 Code.app/Contents/MacOS/T3 Code"+```++Direct binary example:++```bash+T3CODE_OTLP_TRACES_URL=http://localhost:4318/v1/traces \+T3CODE_OTLP_METRICS_URL=http://localhost:4318/v1/metrics \+T3CODE_OTLP_SERVICE_NAME=t3-desktop \+./path/to/your/desktop-app-binary+```++Do not rely on launching from Finder, Spotlight, the dock, or the Start menu after setting shell env vars. Those launches usually will not pick them up.++#### 4. Fully restart after changing env++The backend reads observability config at process start. If you change OTLP env vars, stop the app completely and start it again.++## How To Use Traces And Metrics To Debug The Server++### Start With The Local Trace File++The trace file is the fastest way to inspect raw span data.++Tail it:++```bash+tail -f "$T3CODE_HOME/userdata/logs/server.trace.ndjson"+```++In monorepo dev, use:++```bash+tail -f ./dev/logs/server.trace.ndjson+```++Show failed spans:++```bash+jq -c 'select(.exit._tag != "Success") | {+ name,+ durationMs,+ exit,+ attributes+}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"+```++Show slow spans:++```bash+jq -c 'select(.durationMs > 1000) | {+ name,+ durationMs,+ traceId,+ spanId+}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"+```++Inspect embedded log events:++```bash+jq -c 'select(any(.events[]?; .attributes["effect.logLevel"] != null)) | {+ name,+ durationMs,+ events: [+ .events[]+ | select(.attributes["effect.logLevel"] != null)+ | {+ message: .name,+ level: .attributes["effect.logLevel"]+ }+ ]+}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"+```++Follow one trace:++```bash+jq -r 'select(.traceId == "TRACE_ID_HERE") | [+ .name,+ .spanId,+ (.parentSpanId // "-"),+ .durationMs+] | @tsv' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"+```++Filter orchestration commands:++```bash+jq -c 'select(.attributes["orchestration.command_type"] != null) | {+ name,+ durationMs,+ commandType: .attributes["orchestration.command_type"],+ aggregateKind: .attributes["orchestration.aggregate_kind"]+}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"+```++Filter git activity:++```bash+jq -c 'select(.attributes["git.operation"] != null) | {+ name,+ durationMs,+ operation: .attributes["git.operation"],+ cwd: .attributes["git.cwd"],+ hookEvents: [+ .events[]+ | select(.name == "git.hook.started" or .name == "git.hook.finished")+ ]+}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"+```++### Use Tempo When You Need A Real Trace Viewer++Tempo is better than raw NDJSON when you want to:++- search across many traces+- inspect parent/child relationships visually+- compare many slow traces+- drill into one failing request without hand-joining by `traceId`++Recommended flow in Grafana:++1. Open `Explore`.+2. Pick the `Tempo` data source.+3. Set the time range to something recent like `Last 15 minutes`.+4. Start broad. Do not begin with a very narrow query.+5. Look for spans from your configured service name, then narrow by span name or attributes.++Good first searches:++- service name such as `t3-local`, `t3-dev`, or `t3-desktop`+- span names like `sql.execute`, `git.runCommand`, `provider.sendTurn`+- orchestration spans with attributes like `orchestration.command_type`++Once you know traces are arriving, narrower TraceQL queries like `name = "sql.execute"` become useful.++### Use Metrics To See Systemic Problems++Traces are best for one request. Metrics are best for trends.++Good metric families to watch:++- `t3_rpc_request_duration`+- `t3_orchestration_command_duration`+- `t3_orchestration_command_ack_duration`+- `t3_provider_turn_duration`+- `t3_git_command_duration`+- `t3_db_query_duration`++Counters tell you volume and failure rate:++- `t3_rpc_requests_total`+- `t3_orchestration_commands_total`+- `t3_provider_turns_total`+- `t3_git_commands_total`+- `t3_db_queries_total`++Use metrics when the question is:++- "is this always slow?"+- "did this get worse after a change?"+- "which command type is failing most often?"++Use traces when the question is:++- "what happened in this specific request?"+- "which child span caused this one slow interaction?"+- "what logs were emitted inside the failing flow?"++### What The New Ack Metric Means++`t3_orchestration_command_ack_duration` measures:++- start: command dispatch enters the orchestration engine+- end: the first committed domain event for that command is published by the server++That is a server-side acknowledgment metric. It does not measure:++- websocket transit to the browser+- client receipt+- React render time++If you need those later, add client-side instrumentation or a dedicated server fanout metric.++## Common Workflows++### "Why did this request fail?"++1. Start with the local NDJSON file.+2. Find spans where `exit._tag != "Success"`.+3. Group by `traceId`.+4. Inspect sibling spans and span events.+5. If needed, move to Tempo for the full trace tree.++### "Why is the UI feeling slow?"++1. Search for slow top-level spans in the trace file or Tempo.+2. Check child spans for sqlite, git, provider, or terminal work.+3. Look at the matching duration metrics to see whether the slowness is systemic.++### "Did this command take too long to acknowledge?"++1. Check `t3_orchestration_command_ack_duration` by `commandType`.+2. If it is high, inspect the corresponding orchestration trace.+3. Look at child spans for projection, sqlite, provider, or git work.++### "Are git hooks causing latency?"++1. Filter `git.operation` spans.+2. Inspect `git.hook.started` and `git.hook.finished` events.+3. Compare hook timing to the enclosing git span duration.++### "Why do I have spans locally but nothing in Grafana?"++Usually one of these is true:++- `T3CODE_OTLP_TRACES_URL` was not set+- the app was launched from a different environment than the one where you exported the vars+- the app was not fully restarted after changing env+- Grafana is looking at the wrong time range or service name++If the local NDJSON file is updating, local tracing is working. The problem is almost always OTLP export configuration or process startup.++## How To Think About Adding Tracing To Future Code++### Prefer Boundaries Over Tiny Helpers++Good span boundaries:++- RPC methods+- orchestration command handling+- provider adapter calls+- external process calls+- persistence writes+- queue handoffs++Avoid tracing every tiny helper. Most helpers should inherit the active span rather than create a new one.++### Reuse `Effect.fn(...)` Where It Already Exists++The codebase already uses `Effect.fn("name")` heavily. That should usually be your first tracing boundary.++For ad hoc work:++```ts+import { Effect } from "effect";++const runThing = Effect.gen(function* () {+ yield* Effect.annotateCurrentSpan({+ "thing.id": "abc123",+ "thing.kind": "example",+ });++ yield* Effect.logInfo("starting thing");+ return yield* doWork();+}).pipe(Effect.withSpan("thing.run"));+```++### Put High-Cardinality Detail On Spans++Use span annotations for IDs, paths, and other detailed context:++```ts+yield *+ Effect.annotateCurrentSpan({+ "provider.thread_id": input.threadId,+ "provider.request_id": input.requestId,+ "git.cwd": input.cwd,+ });+```++### Keep Metric Labels Low Cardinality++Good metric labels:++- operation kind+- method name+- provider kind+- aggregate kind+- outcome++Bad metric labels:++- raw thread IDs+- command IDs+- file paths+- cwd+- full prompts+- full model strings when a normalized family label would do++Detailed context belongs on spans, not metrics.++### Use Logs As Span Events++Logs inside a span become part of the trace story:++```ts+yield * Effect.logInfo("starting provider turn");+yield * Effect.logDebug("waiting for approval response");+```++Those messages show up as span events because `Logger.tracerLogger` is installed.++### Use The Pipeable Metrics API++`withMetrics(...)` is the default way to attach a counter and timer to an effect:++```ts+import { someCounter, someDuration, withMetrics } from "../observability/Metrics.ts";++const program = doWork().pipe(+ withMetrics({+ counter: someCounter,+ timer: someDuration,+ attributes: {+ operation: "work",+ },+ }),+);+```++## Detailed API Reference++### Runtime Wiring++The server observability layer is assembled in `apps/server/src/observability/Layers/Observability.ts`.++It provides:++- pretty stdout logger+- `Logger.tracerLogger`+- local NDJSON tracer+- optional OTLP trace exporter+- optional OTLP metrics exporter+- Effect trace-level and timing refs++### Env Vars++Local trace file:++- `T3CODE_TRACE_FILE`: override trace file path+- `T3CODE_TRACE_MAX_BYTES`: per-file rotation size, default `10485760`+- `T3CODE_TRACE_MAX_FILES`: rotated file count, default `10`+- `T3CODE_TRACE_BATCH_WINDOW_MS`: flush window, default `200`+- `T3CODE_TRACE_MIN_LEVEL`: minimum trace level, default `Info`+- `T3CODE_TRACE_TIMING_ENABLED`: enable timing metadata, default `true`++OTLP export:++- `T3CODE_OTLP_TRACES_URL`: OTLP trace endpoint+- `T3CODE_OTLP_METRICS_URL`: OTLP metric endpoint+- `T3CODE_OTLP_EXPORT_INTERVAL_MS`: export interval, default `10000`+- `T3CODE_OTLP_SERVICE_NAME`: service name, default `t3-server`++If the OTLP URLs are unset, local tracing still works and metrics stay in-process only.++### What Is Instrumented Today++Current high-value span and metric boundaries include:++- Effect RPC websocket request spans from `effect/rpc`+- RPC request metrics in `apps/server/src/observability/RpcInstrumentation.ts`+- startup phases+- orchestration command processing+- orchestration command acknowledgment latency+- provider session and turn operations+- git command execution and git hook events+- terminal session lifecycle+- sqlite query execution++### Current Constraints++- logs outside spans are not persisted+- metrics are not snapshotted locally+- the old `serverLogPath` still exists in config for compatibility, but the trace file is the persisted artifact that mattersdiff --git a/packages/contracts/src/server.ts b/packages/contracts/src/server.tsindex d4038deb3..776a0a89e 100644--- a/packages/contracts/src/server.ts+++ b/packages/contracts/src/server.ts@@ -72,6 +72,16 @@ export type ServerProvider = typeof ServerProvider.Type;export const ServerProviders = Schema.Array(ServerProvider);export type ServerProviders = typeof ServerProviders.Type;+export const ServerObservability = Schema.Struct({+ logsDirectoryPath: TrimmedNonEmptyString,+ localTracingEnabled: Schema.Boolean,+ otlpTracesUrl: Schema.optional(TrimmedNonEmptyString),+ otlpTracesEnabled: Schema.Boolean,+ otlpMetricsUrl: Schema.optional(TrimmedNonEmptyString),+ otlpMetricsEnabled: Schema.Boolean,+});+export type ServerObservability = typeof ServerObservability.Type;+export const ServerConfig = Schema.Struct({cwd: TrimmedNonEmptyString,keybindingsConfigPath: TrimmedNonEmptyString,@@ -79,6 +89,7 @@ export const ServerConfig = Schema.Struct({issues: ServerConfigIssues,providers: ServerProviders,availableEditors: Schema.Array(EditorId),+ observability: ServerObservability,settings: ServerSettings,});export type ServerConfig = typeof ServerConfig.Type;diff --git a/turbo.json b/turbo.jsonindex 9359376f5..47d709138 100644--- a/turbo.json+++ b/turbo.json@@ -11,7 +11,17 @@"T3CODE_NO_BROWSER","T3CODE_HOME","T3CODE_AUTH_TOKEN",- "T3CODE_DESKTOP_WS_URL"+ "T3CODE_DESKTOP_WS_URL",+ "T3CODE_TRACE_MIN_LEVEL",+ "T3CODE_TRACE_TIMING_ENABLED",+ "T3CODE_TRACE_FILE",+ "T3CODE_TRACE_MAX_BYTES",+ "T3CODE_TRACE_MAX_FILES",+ "T3CODE_TRACE_BATCH_WINDOW_MS",+ "T3CODE_OTLP_TRACES_URL",+ "T3CODE_OTLP_METRICS_URL",+ "T3CODE_OTLP_EXPORT_INTERVAL_MS",+ "T3CODE_OTLP_SERVICE_NAME"],"tasks": {"build": {