Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

Add server observability tracing and metrics (#1697)

Julius Marminge752f96e9ae72
diff --git a/README.md b/README.md
index d7b2fccb8..d3a54a1b9 100644
--- a/README.md
+++ b/README.md
@@ -45,6 +45,8 @@ We are very very early in this project. Expect bugs.
We are not accepting contributions yet.
+Observability guide: [docs/observability.md](./docs/observability.md)
+
## If you REALLY want to contribute still.... read this first
Read [CONTRIBUTING.md](./CONTRIBUTING.md) before opening an issue or PR.
diff --git a/apps/server/src/cli-config.test.ts b/apps/server/src/cli-config.test.ts
index 27bc60b1b..d3753988d 100644
--- a/apps/server/src/cli-config.test.ts
+++ b/apps/server/src/cli-config.test.ts
@@ -9,6 +9,18 @@ import { deriveServerPaths } from "./config";
import { resolveServerConfig } from "./cli";
it.layer(NodeServices.layer)("cli config resolution", (it) => {
+ const defaultObservabilityConfig = {
+ traceMinLevel: "Info",
+ traceTimingEnabled: true,
+ traceBatchWindowMs: 200,
+ traceMaxBytes: 10 * 1024 * 1024,
+ traceMaxFiles: 10,
+ otlpTracesUrl: undefined,
+ otlpMetricsUrl: undefined,
+ otlpExportIntervalMs: 10_000,
+ otlpServiceName: "t3-server",
+ } as const;
+
const openBootstrapFd = Effect.fn(function* (payload: Record<string, unknown>) {
const fs = yield* FileSystem.FileSystem;
const filePath = yield* fs.makeTempFileScoped({ prefix: "t3-bootstrap-", suffix: ".ndjson" });
@@ -62,6 +74,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
expect(resolved).toEqual({
logLevel: "Warn",
+ ...defaultObservabilityConfig,
mode: "desktop",
port: 4001,
cwd: process.cwd(),
@@ -123,6 +136,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
expect(resolved).toEqual({
logLevel: "Debug",
+ ...defaultObservabilityConfig,
mode: "web",
port: 8788,
cwd: process.cwd(),
@@ -187,6 +201,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
expect(resolved).toEqual({
logLevel: "Info",
+ ...defaultObservabilityConfig,
mode: "desktop",
port: 4888,
cwd: process.cwd(),
@@ -241,6 +256,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
resolved.attachmentsDir,
resolved.worktreesDir,
path.dirname(resolved.serverLogPath),
+ path.dirname(resolved.serverTracePath),
]) {
expect(yield* fs.exists(directory)).toBe(true);
}
@@ -300,6 +316,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
expect(resolved).toEqual({
logLevel: "Debug",
+ ...defaultObservabilityConfig,
mode: "web",
port: 8788,
cwd: process.cwd(),
diff --git a/apps/server/src/cli.ts b/apps/server/src/cli.ts
index 2f76d25b7..af3202954 100644
--- a/apps/server/src/cli.ts
+++ b/apps/server/src/cli.ts
@@ -1,5 +1,5 @@
import { NetService } from "@t3tools/shared/Net";
-import { Config, Effect, LogLevel, Option, Schema } from "effect";
+import { Config, Effect, FileSystem, LogLevel, Option, Path, Schema } from "effect";
import { Command, Flag, GlobalFlag } from "effect/unstable/cli";
import {
@@ -81,6 +81,27 @@ const logWebSocketEventsFlag = Flag.boolean("log-websocket-events").pipe(
const EnvServerConfig = Config.all({
logLevel: Config.logLevel("T3CODE_LOG_LEVEL").pipe(Config.withDefault("Info")),
+ traceMinLevel: Config.logLevel("T3CODE_TRACE_MIN_LEVEL").pipe(Config.withDefault("Info")),
+ traceTimingEnabled: Config.boolean("T3CODE_TRACE_TIMING_ENABLED").pipe(Config.withDefault(true)),
+ traceFile: Config.string("T3CODE_TRACE_FILE").pipe(
+ Config.option,
+ Config.map(Option.getOrUndefined),
+ ),
+ traceMaxBytes: Config.int("T3CODE_TRACE_MAX_BYTES").pipe(Config.withDefault(10 * 1024 * 1024)),
+ traceMaxFiles: Config.int("T3CODE_TRACE_MAX_FILES").pipe(Config.withDefault(10)),
+ traceBatchWindowMs: Config.int("T3CODE_TRACE_BATCH_WINDOW_MS").pipe(Config.withDefault(200)),
+ otlpTracesUrl: Config.string("T3CODE_OTLP_TRACES_URL").pipe(
+ Config.option,
+ Config.map(Option.getOrUndefined),
+ ),
+ otlpMetricsUrl: Config.string("T3CODE_OTLP_METRICS_URL").pipe(
+ Config.option,
+ Config.map(Option.getOrUndefined),
+ ),
+ otlpExportIntervalMs: Config.int("T3CODE_OTLP_EXPORT_INTERVAL_MS").pipe(
+ Config.withDefault(10_000),
+ ),
+ otlpServiceName: Config.string("T3CODE_OTLP_SERVICE_NAME").pipe(Config.withDefault("t3-server")),
mode: Config.schema(RuntimeMode, "T3CODE_MODE").pipe(
Config.option,
Config.map(Option.getOrUndefined),
@@ -137,6 +158,8 @@ export const resolveServerConfig = (
) =>
Effect.gen(function* () {
const { findAvailablePort } = yield* NetService;
+ const path = yield* Path.Path;
+ const fs = yield* FileSystem.FileSystem;
const env = yield* EnvServerConfig;
const bootstrapFd = Option.getOrUndefined(flags.bootstrapFd) ?? env.bootstrapFd;
const bootstrapEnvelope =
@@ -190,6 +213,8 @@ export const resolveServerConfig = (
);
const derivedPaths = yield* deriveServerPaths(baseDir, devUrl);
yield* ensureServerDirectories(derivedPaths);
+ const serverTracePath = env.traceFile ?? derivedPaths.serverTracePath;
+ yield* fs.makeDirectory(path.dirname(serverTracePath), { recursive: true });
const noBrowser = resolveBooleanFlag(
flags.noBrowser,
Option.getOrElse(
@@ -248,11 +273,21 @@ export const resolveServerConfig = (
const config: ServerConfigShape = {
logLevel,
+ traceMinLevel: env.traceMinLevel,
+ traceTimingEnabled: env.traceTimingEnabled,
+ traceBatchWindowMs: env.traceBatchWindowMs,
+ traceMaxBytes: env.traceMaxBytes,
+ traceMaxFiles: env.traceMaxFiles,
+ otlpTracesUrl: env.otlpTracesUrl,
+ otlpMetricsUrl: env.otlpMetricsUrl,
+ otlpExportIntervalMs: env.otlpExportIntervalMs,
+ otlpServiceName: env.otlpServiceName,
mode,
port,
cwd: process.cwd(),
baseDir,
...derivedPaths,
+ serverTracePath,
host,
staticDir,
devUrl,
diff --git a/apps/server/src/config.ts b/apps/server/src/config.ts
index 21cd6f315..9ceea4c13 100644
--- a/apps/server/src/config.ts
+++ b/apps/server/src/config.ts
@@ -25,6 +25,7 @@ export interface ServerDerivedPaths {
readonly attachmentsDir: string;
readonly logsDir: string;
readonly serverLogPath: string;
+ readonly serverTracePath: string;
readonly providerLogsDir: string;
readonly providerEventLogPath: string;
readonly terminalLogsDir: string;
@@ -36,6 +37,15 @@ export interface ServerDerivedPaths {
*/
export interface ServerConfigShape extends ServerDerivedPaths {
readonly logLevel: LogLevel.LogLevel;
+ readonly traceMinLevel: LogLevel.LogLevel;
+ readonly traceTimingEnabled: boolean;
+ readonly traceBatchWindowMs: number;
+ readonly traceMaxBytes: number;
+ readonly traceMaxFiles: number;
+ readonly otlpTracesUrl: string | undefined;
+ readonly otlpMetricsUrl: string | undefined;
+ readonly otlpExportIntervalMs: number;
+ readonly otlpServiceName: string;
readonly mode: RuntimeMode;
readonly port: number;
readonly host: string | undefined;
@@ -68,6 +78,7 @@ export const deriveServerPaths = Effect.fn(function* (
attachmentsDir,
logsDir,
serverLogPath: join(logsDir, "server.log"),
+ serverTracePath: join(logsDir, "server.trace.ndjson"),
providerLogsDir,
providerEventLogPath: join(providerLogsDir, "events.log"),
terminalLogsDir: join(logsDir, "terminals"),
@@ -117,6 +128,15 @@ export class ServerConfig extends ServiceMap.Service<ServerConfig, ServerConfigS
return {
logLevel: "Error",
+ traceMinLevel: "Info",
+ traceTimingEnabled: true,
+ traceBatchWindowMs: 200,
+ traceMaxBytes: 10 * 1024 * 1024,
+ traceMaxFiles: 10,
+ otlpTracesUrl: undefined,
+ otlpMetricsUrl: undefined,
+ otlpExportIntervalMs: 10_000,
+ otlpServiceName: "t3-server",
cwd,
baseDir,
...derivedPaths,
diff --git a/apps/server/src/git/Layers/GitCore.ts b/apps/server/src/git/Layers/GitCore.ts
index b01406411..1178a4b67 100644
--- a/apps/server/src/git/Layers/GitCore.ts
+++ b/apps/server/src/git/Layers/GitCore.ts
@@ -20,6 +20,8 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
import { GitCommandError, type GitBranch } from "@t3tools/contracts";
import { dedupeRemoteBranchesWithLocalMatches } from "@t3tools/shared/git";
+import { compactTraceAttributes } from "../../observability/Attributes.ts";
+import { gitCommandDuration, gitCommandsTotal, withMetrics } from "../../observability/Metrics.ts";
import {
GitCore,
type ExecuteGitProgress,
@@ -372,6 +374,18 @@ interface Trace2Monitor {
readonly flush: Effect.Effect<void, never>;
}
+const nowUnixNano = (): bigint => BigInt(Date.now()) * 1_000_000n;
+
+const addCurrentSpanEvent = (name: string, attributes: Record<string, unknown>) =>
+ Effect.currentSpan.pipe(
+ Effect.tap((span) =>
+ Effect.sync(() => {
+ span.event(name, nowUnixNano(), compactTraceAttributes(attributes));
+ }),
+ ),
+ Effect.catch(() => Effect.void),
+ );
+
function trace2ChildKey(record: Record<string, unknown>): string | null {
const childId = record.child_id;
if (typeof childId === "number" || typeof childId === "string") {
@@ -444,6 +458,9 @@ const createTrace2Monitor = Effect.fn("createTrace2Monitor")(function* (
if (event === "child_start") {
hookStartByChildKey.set(childKey, { hookName, startedAtMs: Date.now() });
+ yield* addCurrentSpanEvent("git.hook.started", {
+ hookName,
+ });
if (progress.onHookStarted) {
yield* progress.onHookStarted(hookName);
}
@@ -452,12 +469,19 @@ const createTrace2Monitor = Effect.fn("createTrace2Monitor")(function* (
if (event === "child_exit") {
hookStartByChildKey.delete(childKey);
+ const code = traceRecord.success.code;
+ const exitCode = typeof code === "number" && Number.isInteger(code) ? code : null;
+ const durationMs = started ? Math.max(0, Date.now() - started.startedAtMs) : null;
+ yield* addCurrentSpanEvent("git.hook.finished", {
+ hookName: started?.hookName ?? hookName,
+ exitCode,
+ durationMs,
+ });
if (progress.onHookFinished) {
- const code = traceRecord.success.code;
yield* progress.onHookFinished({
hookName: started?.hookName ?? hookName,
- exitCode: typeof code === "number" && Number.isInteger(code) ? code : null,
- durationMs: started ? Math.max(0, Date.now() - started.startedAtMs) : null,
+ exitCode,
+ durationMs,
});
}
}
@@ -609,13 +633,13 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: {
const path = yield* Path.Path;
const { worktreesDir } = yield* ServerConfig;
- let execute: GitCoreShape["execute"];
+ let executeRaw: GitCoreShape["execute"];
if (options?.executeOverride) {
- execute = options.executeOverride;
+ executeRaw = options.executeOverride;
} else {
const commandSpawner = yield* ChildProcessSpawner.ChildProcessSpawner;
- execute = Effect.fnUntraced(function* (input) {
+ executeRaw = Effect.fnUntraced(function* (input) {
const commandInput = {
...input,
args: [...input.args],
@@ -716,6 +740,25 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: {
});
}
+ const execute: GitCoreShape["execute"] = (input) =>
+ executeRaw(input).pipe(
+ withMetrics({
+ counter: gitCommandsTotal,
+ timer: gitCommandDuration,
+ attributes: {
+ operation: input.operation,
+ },
+ }),
+ Effect.withSpan(input.operation, {
+ kind: "client",
+ attributes: {
+ "git.operation": input.operation,
+ "git.cwd": input.cwd,
+ "git.args_count": input.args.length,
+ },
+ }),
+ );
+
const executeGit = (
operation: string,
cwd: string,
diff --git a/apps/server/src/observability/Attributes.test.ts b/apps/server/src/observability/Attributes.test.ts
new file mode 100644
index 000000000..4b495598e
--- /dev/null
+++ b/apps/server/src/observability/Attributes.test.ts
@@ -0,0 +1,46 @@
+import { assert, describe, it } from "@effect/vitest";
+
+import { compactTraceAttributes, normalizeModelMetricLabel } from "./Attributes.ts";
+
+describe("Attributes", () => {
+ it("normalizes circular arrays, maps, and sets without recursing forever", () => {
+ const array: Array<unknown> = ["alpha"];
+ array.push(array);
+
+ const map = new Map<string, unknown>();
+ map.set("self", map);
+
+ const set = new Set<unknown>();
+ set.add(set);
+
+ assert.deepStrictEqual(
+ compactTraceAttributes({
+ array,
+ map,
+ set,
+ }),
+ {
+ array: ["alpha", "[Circular]"],
+ map: { self: "[Circular]" },
+ set: ["[Circular]"],
+ },
+ );
+ });
+
+ it("normalizes invalid dates without throwing", () => {
+ assert.deepStrictEqual(
+ compactTraceAttributes({
+ invalidDate: new Date("not-a-real-date"),
+ }),
+ {
+ invalidDate: "Invalid Date",
+ },
+ );
+ });
+
+ it("groups GPT-family models under a shared metric label", () => {
+ assert.strictEqual(normalizeModelMetricLabel("gpt-4o"), "gpt");
+ assert.strictEqual(normalizeModelMetricLabel("gpt-5.4"), "gpt");
+ assert.strictEqual(normalizeModelMetricLabel("claude-sonnet-4"), "claude");
+ });
+});
diff --git a/apps/server/src/observability/Attributes.ts b/apps/server/src/observability/Attributes.ts
new file mode 100644
index 000000000..2251fcfea
--- /dev/null
+++ b/apps/server/src/observability/Attributes.ts
@@ -0,0 +1,128 @@
+import { Cause, Exit } from "effect";
+
+export type MetricAttributeValue = string;
+export type MetricAttributes = Readonly<Record<string, MetricAttributeValue>>;
+export type TraceAttributes = Readonly<Record<string, unknown>>;
+export type ObservabilityOutcome = "success" | "failure" | "interrupt";
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function markSeen(value: object, seen: WeakSet<object>): boolean {
+ if (seen.has(value)) {
+ return true;
+ }
+ seen.add(value);
+ return false;
+}
+
+function normalizeJsonValue(value: unknown, seen: WeakSet<object> = new WeakSet()): unknown {
+ if (
+ value === null ||
+ value === undefined ||
+ typeof value === "string" ||
+ typeof value === "number" ||
+ typeof value === "boolean"
+ ) {
+ return value ?? null;
+ }
+ if (typeof value === "bigint") {
+ return value.toString();
+ }
+ if (value instanceof Date) {
+ return Number.isNaN(value.getTime()) ? "Invalid Date" : value.toISOString();
+ }
+ if (value instanceof Error) {
+ return {
+ name: value.name,
+ message: value.message,
+ ...(value.stack ? { stack: value.stack } : {}),
+ };
+ }
+ if (Array.isArray(value)) {
+ if (markSeen(value, seen)) {
+ return "[Circular]";
+ }
+ return value.map((entry) => normalizeJsonValue(entry, seen));
+ }
+ if (value instanceof Map) {
+ if (markSeen(value, seen)) {
+ return "[Circular]";
+ }
+ return Object.fromEntries(
+ Array.from(value.entries(), ([key, entryValue]) => [
+ String(key),
+ normalizeJsonValue(entryValue, seen),
+ ]),
+ );
+ }
+ if (value instanceof Set) {
+ if (markSeen(value, seen)) {
+ return "[Circular]";
+ }
+ return Array.from(value.values(), (entry) => normalizeJsonValue(entry, seen));
+ }
+ if (!isPlainObject(value)) {
+ return String(value);
+ }
+ if (markSeen(value, seen)) {
+ return "[Circular]";
+ }
+ return Object.fromEntries(
+ Object.entries(value).map(([key, entryValue]) => [key, normalizeJsonValue(entryValue, seen)]),
+ );
+}
+
+export function compactTraceAttributes(
+ attributes: Readonly<Record<string, unknown>>,
+): TraceAttributes {
+ return Object.fromEntries(
+ Object.entries(attributes)
+ .filter(([, value]) => value !== undefined)
+ .map(([key, value]) => [key, normalizeJsonValue(value)]),
+ );
+}
+
+export function compactMetricAttributes(
+ attributes: Readonly<Record<string, unknown>>,
+): MetricAttributes {
+ return Object.fromEntries(
+ Object.entries(attributes).flatMap(([key, value]) => {
+ if (value === undefined || value === null) {
+ return [];
+ }
+ if (typeof value === "string") {
+ return [[key, value]];
+ }
+ if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") {
+ return [[key, String(value)]];
+ }
+ return [];
+ }),
+ );
+}
+
+export function outcomeFromExit(exit: Exit.Exit<unknown, unknown>): ObservabilityOutcome {
+ if (Exit.isSuccess(exit)) {
+ return "success";
+ }
+ return Cause.hasInterruptsOnly(exit.cause) ? "interrupt" : "failure";
+}
+
+export function normalizeModelMetricLabel(model: string | null | undefined): string | undefined {
+ const normalized = model?.trim().toLowerCase();
+ if (!normalized) {
+ return undefined;
+ }
+ if (normalized.includes("gpt")) {
+ return "gpt";
+ }
+ if (normalized.includes("claude")) {
+ return "claude";
+ }
+ if (normalized.includes("gemini")) {
+ return "gemini";
+ }
+ return "other";
+}
diff --git a/apps/server/src/observability/Layers/Observability.ts b/apps/server/src/observability/Layers/Observability.ts
new file mode 100644
index 000000000..29b9d7eac
--- /dev/null
+++ b/apps/server/src/observability/Layers/Observability.ts
@@ -0,0 +1,64 @@
+import { Effect, Layer, References, Tracer } from "effect";
+import { OtlpMetrics, OtlpSerialization, OtlpTracer } from "effect/unstable/observability";
+
+import { ServerConfig } from "../../config.ts";
+import { ServerLoggerLive } from "../../serverLogger.ts";
+import { makeLocalFileTracer } from "../LocalFileTracer.ts";
+
+const otlpSerializationLayer = OtlpSerialization.layerJson;
+
+export const ObservabilityLive = Layer.unwrap(
+ Effect.gen(function* () {
+ const config = yield* ServerConfig;
+
+ const traceReferencesLayer = Layer.mergeAll(
+ Layer.succeed(Tracer.MinimumTraceLevel, config.traceMinLevel),
+ Layer.succeed(References.TracerTimingEnabled, config.traceTimingEnabled),
+ );
+
+ const tracerLayer = Layer.effect(
+ Tracer.Tracer,
+ Effect.gen(function* () {
+ const delegate =
+ config.otlpTracesUrl === undefined
+ ? undefined
+ : yield* OtlpTracer.make({
+ url: config.otlpTracesUrl,
+ exportInterval: `${config.otlpExportIntervalMs} millis`,
+ resource: {
+ serviceName: config.otlpServiceName,
+ attributes: {
+ "service.runtime": "t3-server",
+ "service.mode": config.mode,
+ },
+ },
+ });
+
+ return yield* makeLocalFileTracer({
+ filePath: config.serverTracePath,
+ maxBytes: config.traceMaxBytes,
+ maxFiles: config.traceMaxFiles,
+ batchWindowMs: config.traceBatchWindowMs,
+ ...(delegate ? { delegate } : {}),
+ });
+ }),
+ ).pipe(Layer.provideMerge(otlpSerializationLayer));
+
+ const metricsLayer =
+ config.otlpMetricsUrl === undefined
+ ? Layer.empty
+ : OtlpMetrics.layer({
+ url: config.otlpMetricsUrl,
+ exportInterval: `${config.otlpExportIntervalMs} millis`,
+ resource: {
+ serviceName: config.otlpServiceName,
+ attributes: {
+ "service.runtime": "t3-server",
+ "service.mode": config.mode,
+ },
+ },
+ }).pipe(Layer.provideMerge(otlpSerializationLayer));
+
+ return Layer.mergeAll(ServerLoggerLive, traceReferencesLayer, tracerLayer, metricsLayer);
+ }),
+);
diff --git a/apps/server/src/observability/LocalFileTracer.test.ts b/apps/server/src/observability/LocalFileTracer.test.ts
new file mode 100644
index 000000000..1bc5a3430
--- /dev/null
+++ b/apps/server/src/observability/LocalFileTracer.test.ts
@@ -0,0 +1,117 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { assert, describe, it } from "@effect/vitest";
+import { Effect, Layer, Logger, References, Tracer } from "effect";
+
+import type { TraceRecord } from "./TraceRecord.ts";
+import { makeLocalFileTracer } from "./LocalFileTracer.ts";
+
+const makeTestLayer = (tracePath: string) =>
+ Layer.mergeAll(
+ Layer.effect(
+ Tracer.Tracer,
+ makeLocalFileTracer({
+ filePath: tracePath,
+ maxBytes: 1024 * 1024,
+ maxFiles: 2,
+ batchWindowMs: 10_000,
+ }),
+ ),
+ Logger.layer([Logger.tracerLogger], { mergeWithExisting: false }),
+ Layer.succeed(References.MinimumLogLevel, "Info"),
+ );
+
+const readTraceRecords = (tracePath: string): Array<TraceRecord> =>
+ fs
+ .readFileSync(tracePath, "utf8")
+ .trim()
+ .split("\n")
+ .filter((line) => line.length > 0)
+ .map((line) => JSON.parse(line) as TraceRecord);
+
+describe("LocalFileTracer", () => {
+ it.effect("writes nested spans to disk and captures log messages as span events", () =>
+ Effect.gen(function* () {
+ const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-local-tracer-"));
+ const tracePath = path.join(tempDir, "server.trace.ndjson");
+
+ try {
+ yield* Effect.scoped(
+ Effect.gen(function* () {
+ const program = Effect.gen(function* () {
+ yield* Effect.annotateCurrentSpan({
+ "demo.parent": true,
+ });
+ yield* Effect.logInfo("parent event");
+ yield* Effect.gen(function* () {
+ yield* Effect.annotateCurrentSpan({
+ "demo.child": true,
+ });
+ yield* Effect.logInfo("child event");
+ }).pipe(Effect.withSpan("child-span"));
+ }).pipe(Effect.withSpan("parent-span"));
+
+ yield* program.pipe(Effect.provide(makeTestLayer(tracePath)));
+ }),
+ );
+
+ const records = readTraceRecords(tracePath);
+ assert.equal(records.length, 2);
+
+ const parent = records.find((record) => record.name === "parent-span");
+ const child = records.find((record) => record.name === "child-span");
+
+ assert.notEqual(parent, undefined);
+ assert.notEqual(child, undefined);
+ if (!parent || !child) {
+ return;
+ }
+
+ assert.equal(child.parentSpanId, parent.spanId);
+ assert.equal(parent.attributes["demo.parent"], true);
+ assert.equal(child.attributes["demo.child"], true);
+ assert.equal(
+ parent.events.some((event) => event.name === "parent event"),
+ true,
+ );
+ assert.equal(
+ child.events.some((event) => event.name === "child event"),
+ true,
+ );
+ assert.equal(
+ child.events.some((event) => event.attributes["effect.logLevel"] === "INFO"),
+ true,
+ );
+ } finally {
+ fs.rmSync(tempDir, { recursive: true, force: true });
+ }
+ }),
+ );
+
+ it.effect("serializes interrupted spans with an interrupted exit status", () =>
+ Effect.gen(function* () {
+ const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-local-tracer-"));
+ const tracePath = path.join(tempDir, "server.trace.ndjson");
+
+ try {
+ yield* Effect.scoped(
+ Effect.exit(
+ Effect.interrupt.pipe(
+ Effect.withSpan("interrupt-span"),
+ Effect.provide(makeTestLayer(tracePath)),
+ ),
+ ),
+ );
+
+ const records = readTraceRecords(tracePath);
+ assert.equal(records.length, 1);
+ assert.equal(records[0]?.name, "interrupt-span");
+ assert.equal(records[0]?.exit._tag, "Interrupted");
+ } finally {
+ fs.rmSync(tempDir, { recursive: true, force: true });
+ }
+ }),
+ );
+});
diff --git a/apps/server/src/observability/LocalFileTracer.ts b/apps/server/src/observability/LocalFileTracer.ts
new file mode 100644
index 000000000..7f75eb0bf
--- /dev/null
+++ b/apps/server/src/observability/LocalFileTracer.ts
@@ -0,0 +1,104 @@
+import type * as Exit from "effect/Exit";
+import { Effect, Option, Tracer } from "effect";
+
+import { spanToTraceRecord } from "./TraceRecord.ts";
+import { makeTraceSink } from "./TraceSink.ts";
+
+export interface LocalFileTracerOptions {
+ readonly filePath: string;
+ readonly maxBytes: number;
+ readonly maxFiles: number;
+ readonly batchWindowMs: number;
+ readonly delegate?: Tracer.Tracer;
+}
+
+class LocalFileSpan implements Tracer.Span {
+ readonly _tag = "Span";
+ readonly name: string;
+ readonly spanId: string;
+ readonly traceId: string;
+ readonly parent: Option.Option<Tracer.AnySpan>;
+ readonly annotations: Tracer.Span["annotations"];
+ readonly links: Array<Tracer.SpanLink>;
+ readonly sampled: boolean;
+ readonly kind: Tracer.SpanKind;
+
+ status: Tracer.SpanStatus;
+ attributes: Map<string, unknown>;
+ events: Array<[name: string, startTime: bigint, attributes: Record<string, unknown>]>;
+
+ constructor(
+ options: Parameters<Tracer.Tracer["span"]>[0],
+ private readonly delegate: Tracer.Span,
+ private readonly push: (record: ReturnType<typeof spanToTraceRecord>) => void,
+ ) {
+ this.name = delegate.name;
+ this.spanId = delegate.spanId;
+ this.traceId = delegate.traceId;
+ this.parent = options.parent;
+ this.annotations = options.annotations;
+ this.links = [...options.links];
+ this.sampled = delegate.sampled;
+ this.kind = delegate.kind;
+ this.status = {
+ _tag: "Started",
+ startTime: options.startTime,
+ };
+ this.attributes = new Map();
+ this.events = [];
+ }
+
+ end(endTime: bigint, exit: Exit.Exit<unknown, unknown>): void {
+ this.status = {
+ _tag: "Ended",
+ startTime: this.status.startTime,
+ endTime,
+ exit,
+ };
+ this.delegate.end(endTime, exit);
+
+ if (this.sampled) {
+ this.push(spanToTraceRecord(this));
+ }
+ }
+
+ attribute(key: string, value: unknown): void {
+ this.attributes.set(key, value);
+ this.delegate.attribute(key, value);
+ }
+
+ event(name: string, startTime: bigint, attributes?: Record<string, unknown>): void {
+ const nextAttributes = attributes ?? {};
+ this.events.push([name, startTime, nextAttributes]);
+ this.delegate.event(name, startTime, nextAttributes);
+ }
+
+ addLinks(links: ReadonlyArray<Tracer.SpanLink>): void {
+ this.links.push(...links);
+ this.delegate.addLinks(links);
+ }
+}
+
+export const makeLocalFileTracer = Effect.fn("makeLocalFileTracer")(function* (
+ options: LocalFileTracerOptions,
+) {
+ const sink = yield* makeTraceSink({
+ filePath: options.filePath,
+ maxBytes: options.maxBytes,
+ maxFiles: options.maxFiles,
+ batchWindowMs: options.batchWindowMs,
+ });
+
+ const delegate =
+ options.delegate ??
+ Tracer.make({
+ span: (spanOptions) => new Tracer.NativeSpan(spanOptions),
+ });
+
+ return Tracer.make({
+ span(spanOptions) {
+ return new LocalFileSpan(spanOptions, delegate.span(spanOptions), sink.push);
+ },
+ ...(delegate.context ? { context: delegate.context } : {}),
+ });
+});
diff --git a/apps/server/src/observability/Metrics.test.ts b/apps/server/src/observability/Metrics.test.ts
new file mode 100644
index 000000000..4604f43b6
--- /dev/null
+++ b/apps/server/src/observability/Metrics.test.ts
@@ -0,0 +1,111 @@
+import { assert, describe, it } from "@effect/vitest";
+import { Effect, Metric } from "effect";
+
+import { withMetrics } from "./Metrics.ts";
+
+const hasMetricSnapshot = (
+ snapshots: ReadonlyArray<Metric.Metric.Snapshot>,
+ id: string,
+ attributes: Readonly<Record<string, string>>,
+) =>
+ snapshots.some(
+ (snapshot) =>
+ snapshot.id === id &&
+ Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),
+ );
+
+describe("withMetrics", () => {
+ it.effect("supports pipe-style usage", () =>
+ Effect.gen(function* () {
+ const counter = Metric.counter("with_metrics_pipe_total");
+ const timer = Metric.timer("with_metrics_pipe_duration");
+
+ const result = yield* Effect.succeed("ok").pipe(
+ withMetrics({
+ counter,
+ timer,
+ attributes: {
+ operation: "pipe",
+ },
+ }),
+ );
+
+ assert.equal(result, "ok");
+
+ const snapshots = yield* Metric.snapshot;
+ assert.equal(
+ hasMetricSnapshot(snapshots, "with_metrics_pipe_total", {
+ operation: "pipe",
+ outcome: "success",
+ }),
+ true,
+ );
+ assert.equal(
+ hasMetricSnapshot(snapshots, "with_metrics_pipe_duration", {
+ operation: "pipe",
+ }),
+ true,
+ );
+ }),
+ );
+
+ it.effect("supports direct invocation", () =>
+ Effect.gen(function* () {
+ const counter = Metric.counter("with_metrics_direct_total");
+
+ yield* withMetrics(Effect.fail("boom"), {
+ counter,
+ attributes: {
+ operation: "direct",
+ },
+ }).pipe(Effect.exit);
+
+ const snapshots = yield* Metric.snapshot;
+ assert.equal(
+ hasMetricSnapshot(snapshots, "with_metrics_direct_total", {
+ operation: "direct",
+ outcome: "failure",
+ }),
+ true,
+ );
+ }),
+ );
+
+ it.effect("evaluates attributes lazily after the wrapped effect runs", () =>
+ Effect.gen(function* () {
+ const counter = Metric.counter("with_metrics_lazy_total");
+ const timer = Metric.timer("with_metrics_lazy_duration");
+ let provider = "unknown";
+
+ yield* Effect.sync(() => {
+ provider = "codex";
+ }).pipe(
+ withMetrics({
+ counter,
+ timer,
+ attributes: () => ({
+ provider,
+ operation: "lazy",
+ }),
+ }),
+ );
+
+ const snapshots = yield* Metric.snapshot;
+ assert.equal(
+ hasMetricSnapshot(snapshots, "with_metrics_lazy_total", {
+ provider: "codex",
+ operation: "lazy",
+ outcome: "success",
+ }),
+ true,
+ );
+ assert.equal(
+ hasMetricSnapshot(snapshots, "with_metrics_lazy_duration", {
+ provider: "codex",
+ operation: "lazy",
+ }),
+ true,
+ );
+ }),
+ );
+});
diff --git a/apps/server/src/observability/Metrics.ts b/apps/server/src/observability/Metrics.ts
new file mode 100644
index 000000000..3e527c7cb
--- /dev/null
+++ b/apps/server/src/observability/Metrics.ts
@@ -0,0 +1,160 @@
+import { Duration, Effect, Exit, Metric } from "effect";
+import { dual } from "effect/Function";
+
+import {
+ compactMetricAttributes,
+ normalizeModelMetricLabel,
+ outcomeFromExit,
+} from "./Attributes.ts";
+
+export const rpcRequestsTotal = Metric.counter("t3_rpc_requests_total", {
+ description: "Total RPC requests handled by the websocket RPC server.",
+});
+
+export const rpcRequestDuration = Metric.timer("t3_rpc_request_duration", {
+ description: "RPC request handling duration.",
+});
+
+export const orchestrationCommandsTotal = Metric.counter("t3_orchestration_commands_total", {
+ description: "Total orchestration commands dispatched.",
+});
+
+export const orchestrationCommandDuration = Metric.timer("t3_orchestration_command_duration", {
+ description: "Orchestration command dispatch duration.",
+});
+
+export const orchestrationCommandAckDuration = Metric.timer(
+ "t3_orchestration_command_ack_duration",
+ {
+ description:
+ "Time from orchestration command dispatch to the first committed domain event emitted for that command.",
+ },
+);
+
+export const orchestrationEventsProcessedTotal = Metric.counter(
+ "t3_orchestration_events_processed_total",
+ {
+ description: "Total orchestration intent events processed by runtime reactors.",
+ },
+);
+
+export const providerSessionsTotal = Metric.counter("t3_provider_sessions_total", {
+ description: "Total provider session lifecycle operations.",
+});
+
+export const providerTurnsTotal = Metric.counter("t3_provider_turns_total", {
+ description: "Total provider turn lifecycle operations.",
+});
+
+export const providerTurnDuration = Metric.timer("t3_provider_turn_duration", {
+ description: "Provider turn request duration.",
+});
+
+export const providerRuntimeEventsTotal = Metric.counter("t3_provider_runtime_events_total", {
+ description: "Total canonical provider runtime events processed.",
+});
+
+export const gitCommandsTotal = Metric.counter("t3_git_commands_total", {
+ description: "Total git commands executed by the server runtime.",
+});
+
+export const gitCommandDuration = Metric.timer("t3_git_command_duration", {
+ description: "Git command execution duration.",
+});
+
+export const terminalSessionsTotal = Metric.counter("t3_terminal_sessions_total", {
+ description: "Total terminal sessions started.",
+});
+
+export const terminalRestartsTotal = Metric.counter("t3_terminal_restarts_total", {
+ description: "Total terminal restart requests handled.",
+});
+
+export const metricAttributes = (
+ attributes: Readonly<Record<string, unknown>>,
+): ReadonlyArray<[string, string]> => Object.entries(compactMetricAttributes(attributes));
+
+export const increment = (
+ metric: Metric.Metric<number, unknown>,
+ attributes: Readonly<Record<string, unknown>>,
+ amount = 1,
+) => Metric.update(Metric.withAttributes(metric, metricAttributes(attributes)), amount);
+
+export interface WithMetricsOptions {
+ readonly counter?: Metric.Metric<number, unknown>;
+ readonly timer?: Metric.Metric<Duration.Duration, unknown>;
+ readonly attributes?:
+ | Readonly<Record<string, unknown>>
+ | (() => Readonly<Record<string, unknown>>);
+ readonly outcomeAttributes?: (
+ outcome: ReturnType<typeof outcomeFromExit>,
+ ) => Readonly<Record<string, unknown>>;
+}
+
+const withMetricsImpl = <A, E, R>(
+ effect: Effect.Effect<A, E, R>,
+ options: WithMetricsOptions,
+): Effect.Effect<A, E, R> =>
+ Effect.gen(function* () {
+ const startedAt = Date.now();
+ const exit = yield* Effect.exit(effect);
+ const duration = Duration.millis(Math.max(0, Date.now() - startedAt));
+ const baseAttributes =
+ typeof options.attributes === "function" ? options.attributes() : (options.attributes ?? {});
+
+ if (options.timer) {
+ yield* Metric.update(
+ Metric.withAttributes(options.timer, metricAttributes(baseAttributes)),
+ duration,
+ );
+ }
+
+ if (options.counter) {
+ const outcome = outcomeFromExit(exit);
+ yield* Metric.update(
+ Metric.withAttributes(
+ options.counter,
+ metricAttributes({
+ ...baseAttributes,
+ outcome,
+ ...(options.outcomeAttributes ? options.outcomeAttributes(outcome) : {}),
+ }),
+ ),
+ 1,
+ );
+ }
+
+ if (Exit.isSuccess(exit)) {
+ return exit.value;
+ }
+ return yield* Effect.failCause(exit.cause);
+ });
+
+export const withMetrics: {
+ <A, E, R>(
+ options: WithMetricsOptions,
+ ): (effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
+ <A, E, R>(effect: Effect.Effect<A, E, R>, options: WithMetricsOptions): Effect.Effect<A, E, R>;
+} = dual(2, withMetricsImpl);
+
+export const providerMetricAttributes = (
+ provider: string,
+ extra?: Readonly<Record<string, unknown>>,
+) =>
+ compactMetricAttributes({
+ provider,
+ ...extra,
+ });
+
+export const providerTurnMetricAttributes = (input: {
+ readonly provider: string;
+ readonly model: string | null | undefined;
+ readonly extra?: Readonly<Record<string, unknown>>;
+}) => {
+ const modelFamily = normalizeModelMetricLabel(input.model);
+ return compactMetricAttributes({
+ provider: input.provider,
+ ...(modelFamily ? { modelFamily } : {}),
+ ...input.extra,
+ });
+};
diff --git a/apps/server/src/observability/RpcInstrumentation.test.ts b/apps/server/src/observability/RpcInstrumentation.test.ts
new file mode 100644
index 000000000..d29b05f3c
--- /dev/null
+++ b/apps/server/src/observability/RpcInstrumentation.test.ts
@@ -0,0 +1,161 @@
+import { assert, describe, it } from "@effect/vitest";
+import { Effect, Exit, Metric, Stream } from "effect";
+
+import {
+ observeRpcEffect,
+ observeRpcStream,
+ observeRpcStreamEffect,
+} from "./RpcInstrumentation.ts";
+
+const hasMetricSnapshot = (
+ snapshots: ReadonlyArray<Metric.Metric.Snapshot>,
+ id: string,
+ attributes: Readonly<Record<string, string>>,
+) =>
+ snapshots.some(
+ (snapshot) =>
+ snapshot.id === id &&
+ Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),
+ );
+
+describe("RpcInstrumentation", () => {
+ it.effect("records success metrics for unary RPC handlers", () =>
+ Effect.gen(function* () {
+ yield* observeRpcEffect("rpc.instrumentation.success", Effect.succeed("ok"), {
+ "rpc.aggregate": "test",
+ }).pipe(Effect.withSpan("rpc.instrumentation.success.span"));
+
+ const snapshots = yield* Metric.snapshot;
+
+ assert.equal(
+ hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {
+ method: "rpc.instrumentation.success",
+ outcome: "success",
+ }),
+ true,
+ );
+ assert.equal(
+ hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {
+ method: "rpc.instrumentation.success",
+ }),
+ true,
+ );
+ }),
+ );
+
+ it.effect("records failure outcomes for unary RPC handlers", () =>
+ Effect.gen(function* () {
+ yield* Effect.exit(
+ observeRpcEffect("rpc.instrumentation.failure", Effect.fail("boom"), {
+ "rpc.aggregate": "test",
+ }).pipe(Effect.withSpan("rpc.instrumentation.failure.span")),
+ );
+
+ const snapshots = yield* Metric.snapshot;
+
+ assert.equal(
+ hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {
+ method: "rpc.instrumentation.failure",
+ outcome: "failure",
+ }),
+ true,
+ );
+ assert.equal(
+ hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {
+ method: "rpc.instrumentation.failure",
+ }),
+ true,
+ );
+ }),
+ );
+
+ it.effect("records subscription activation metrics for stream RPC handlers", () =>
+ Effect.gen(function* () {
+ const events = yield* Stream.runCollect(
+ observeRpcStreamEffect(
+ "rpc.instrumentation.stream",
+ Effect.succeed(Stream.make("a", "b")),
+ { "rpc.aggregate": "test" },
+ ).pipe(Stream.withSpan("rpc.instrumentation.stream.span")),
+ );
+
+ assert.deepStrictEqual(Array.from(events), ["a", "b"]);
+
+ const snapshots = yield* Metric.snapshot;
+
+ assert.equal(
+ hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {
+ method: "rpc.instrumentation.stream",
+ outcome: "success",
+ }),
+ true,
+ );
+ assert.equal(
+ hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {
+ method: "rpc.instrumentation.stream",
+ }),
+ true,
+ );
+ }),
+ );
+
+ it.effect("records failure outcomes for direct stream RPC handlers during consumption", () =>
+ Effect.gen(function* () {
+ const exit = yield* Stream.runCollect(
+ observeRpcStream(
+ "rpc.instrumentation.stream.failure",
+ Stream.make("a").pipe(Stream.concat(Stream.fail("boom"))),
+ { "rpc.aggregate": "test" },
+ ).pipe(Stream.withSpan("rpc.instrumentation.stream.failure.span")),
+ ).pipe(Effect.exit);
+
+ assert.equal(Exit.isFailure(exit), true);
+
+ const snapshots = yield* Metric.snapshot;
+
+ assert.equal(
+ hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {
+ method: "rpc.instrumentation.stream.failure",
+ outcome: "failure",
+ }),
+ true,
+ );
+ assert.equal(
+ hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {
+ method: "rpc.instrumentation.stream.failure",
+ }),
+ true,
+ );
+ }),
+ );
+
+ it.effect("records failure outcomes when a stream RPC effect produces a failing stream", () =>
+ Effect.gen(function* () {
+ const exit = yield* Stream.runCollect(
+ observeRpcStreamEffect(
+ "rpc.instrumentation.stream.effect.failure",
+ Effect.succeed(Stream.fail("boom")),
+ { "rpc.aggregate": "test" },
+ ).pipe(Stream.withSpan("rpc.instrumentation.stream.effect.failure.span")),
+ ).pipe(Effect.exit);
+
+ assert.equal(Exit.isFailure(exit), true);
+
+ const snapshots = yield* Metric.snapshot;
+
+ assert.equal(
+ hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {
+ method: "rpc.instrumentation.stream.effect.failure",
+ outcome: "failure",
+ }),
+ true,
+ );
+ assert.equal(
+ hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {
+ method: "rpc.instrumentation.stream.effect.failure",
+ }),
+ true,
+ );
+ }),
+ );
+});
diff --git a/apps/server/src/observability/RpcInstrumentation.ts b/apps/server/src/observability/RpcInstrumentation.ts
new file mode 100644
index 000000000..a3ac29aa0
--- /dev/null
+++ b/apps/server/src/observability/RpcInstrumentation.ts
@@ -0,0 +1,89 @@
+import { Duration, Effect, Exit, Metric, Stream } from "effect";
+
+import { outcomeFromExit } from "./Attributes.ts";
+import { metricAttributes, rpcRequestDuration, rpcRequestsTotal, withMetrics } from "./Metrics.ts";
+
+const annotateRpcSpan = (
+ method: string,
+ traceAttributes?: Readonly<Record<string, unknown>>,
+): Effect.Effect<void, never, never> =>
+ Effect.annotateCurrentSpan({
+ "rpc.method": method,
+ ...traceAttributes,
+ });
+
+const recordRpcStreamMetrics = <E>(
+ method: string,
+ startedAt: number,
+ exit: Exit.Exit<unknown, E>,
+): Effect.Effect<void, never, never> =>
+ Effect.gen(function* () {
+ yield* Metric.update(
+ Metric.withAttributes(rpcRequestDuration, metricAttributes({ method })),
+ Duration.millis(Math.max(0, Date.now() - startedAt)),
+ );
+ yield* Metric.update(
+ Metric.withAttributes(
+ rpcRequestsTotal,
+ metricAttributes({
+ method,
+ outcome: outcomeFromExit(exit),
+ }),
+ ),
+ 1,
+ );
+ });
+
+export const observeRpcEffect = <A, E, R>(
+ method: string,
+ effect: Effect.Effect<A, E, R>,
+ traceAttributes?: Readonly<Record<string, unknown>>,
+): Effect.Effect<A, E, R> =>
+ Effect.gen(function* () {
+ yield* annotateRpcSpan(method, traceAttributes);
+
+ return yield* effect.pipe(
+ withMetrics({
+ counter: rpcRequestsTotal,
+ timer: rpcRequestDuration,
+ attributes: {
+ method,
+ },
+ }),
+ );
+ });
+
+export const observeRpcStream = <A, E, R>(
+ method: string,
+ stream: Stream.Stream<A, E, R>,
+ traceAttributes?: Readonly<Record<string, unknown>>,
+): Stream.Stream<A, E, R> =>
+ Stream.unwrap(
+ Effect.gen(function* () {
+ yield* annotateRpcSpan(method, traceAttributes);
+ const startedAt = Date.now();
+ return stream.pipe(Stream.onExit((exit) => recordRpcStreamMetrics(method, startedAt, exit)));
+ }),
+ );
+
+export const observeRpcStreamEffect = <A, StreamError, StreamContext, EffectError, EffectContext>(
+ method: string,
+ effect: Effect.Effect<Stream.Stream<A, StreamError, StreamContext>, EffectError, EffectContext>,
+ traceAttributes?: Readonly<Record<string, unknown>>,
+): Stream.Stream<A, StreamError | EffectError, StreamContext | EffectContext> =>
+ Stream.unwrap(
+ Effect.gen(function* () {
+ yield* annotateRpcSpan(method, traceAttributes);
+ const startedAt = Date.now();
+ const exit = yield* Effect.exit(effect);
+
+ if (Exit.isFailure(exit)) {
+ yield* recordRpcStreamMetrics(method, startedAt, exit);
+ return yield* Effect.failCause(exit.cause);
+ }
+
+ return exit.value.pipe(
+ Stream.onExit((streamExit) => recordRpcStreamMetrics(method, startedAt, streamExit)),
+ );
+ }),
+ );
diff --git a/apps/server/src/observability/TraceRecord.ts b/apps/server/src/observability/TraceRecord.ts
new file mode 100644
index 000000000..3aee1b267
--- /dev/null
+++ b/apps/server/src/observability/TraceRecord.ts
@@ -0,0 +1,100 @@
+import { Cause, Exit, Option, Tracer } from "effect";
+
+import { compactTraceAttributes } from "./Attributes.ts";
+
+export interface TraceRecord {
+ readonly type: "effect-span";
+ readonly name: string;
+ readonly traceId: string;
+ readonly spanId: string;
+ readonly parentSpanId?: string;
+ readonly sampled: boolean;
+ readonly kind: Tracer.SpanKind;
+ readonly startTimeUnixNano: string;
+ readonly endTimeUnixNano: string;
+ readonly durationMs: number;
+ readonly attributes: Readonly<Record<string, unknown>>;
+ readonly events: ReadonlyArray<{
+ readonly name: string;
+ readonly timeUnixNano: string;
+ readonly attributes: Readonly<Record<string, unknown>>;
+ }>;
+ readonly links: ReadonlyArray<{
+ readonly traceId: string;
+ readonly spanId: string;
+ readonly attributes: Readonly<Record<string, unknown>>;
+ }>;
+ readonly exit:
+ | {
+ readonly _tag: "Success";
+ }
+ | {
+ readonly _tag: "Interrupted";
+ readonly cause: string;
+ }
+ | {
+ readonly _tag: "Failure";
+ readonly cause: string;
+ };
+}
+
+interface SerializableSpan {
+ readonly name: string;
+ readonly traceId: string;
+ readonly spanId: string;
+ readonly parent: Option.Option<Tracer.AnySpan>;
+ readonly status: Tracer.SpanStatus;
+ readonly sampled: boolean;
+ readonly kind: Tracer.SpanKind;
+ readonly attributes: ReadonlyMap<string, unknown>;
+ readonly links: ReadonlyArray<Tracer.SpanLink>;
+ readonly events: ReadonlyArray<
+ readonly [name: string, startTime: bigint, attributes: Record<string, unknown>]
+ >;
+}
+
+function formatTraceExit(exit: Exit.Exit<unknown, unknown>): TraceRecord["exit"] {
+ if (Exit.isSuccess(exit)) {
+ return { _tag: "Success" };
+ }
+ if (Cause.hasInterruptsOnly(exit.cause)) {
+ return {
+ _tag: "Interrupted",
+ cause: Cause.pretty(exit.cause),
+ };
+ }
+ return {
+ _tag: "Failure",
+ cause: Cause.pretty(exit.cause),
+ };
+}
+
+export function spanToTraceRecord(span: SerializableSpan): TraceRecord {
+ const status = span.status as Extract<Tracer.SpanStatus, { _tag: "Ended" }>;
+ const parentSpanId = Option.getOrUndefined(span.parent)?.spanId;
+
+ return {
+ type: "effect-span",
+ name: span.name,
+ traceId: span.traceId,
+ spanId: span.spanId,
+ ...(parentSpanId ? { parentSpanId } : {}),
+ sampled: span.sampled,
+ kind: span.kind,
+ startTimeUnixNano: String(status.startTime),
+ endTimeUnixNano: String(status.endTime),
+ durationMs: Number(status.endTime - status.startTime) / 1_000_000,
+ attributes: compactTraceAttributes(Object.fromEntries(span.attributes)),
+ events: span.events.map(([name, startTime, attributes]) => ({
+ name,
+ timeUnixNano: String(startTime),
+ attributes: compactTraceAttributes(attributes),
+ })),
+ links: span.links.map((link) => ({
+ traceId: link.span.traceId,
+ spanId: link.span.spanId,
+ attributes: compactTraceAttributes(link.attributes),
+ })),
+ exit: formatTraceExit(status.exit),
+ };
+}
diff --git a/apps/server/src/observability/TraceSink.test.ts b/apps/server/src/observability/TraceSink.test.ts
new file mode 100644
index 000000000..f4db90516
--- /dev/null
+++ b/apps/server/src/observability/TraceSink.test.ts
@@ -0,0 +1,152 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { assert, describe, it } from "@effect/vitest";
+import { Effect } from "effect";
+
+import type { TraceRecord } from "./TraceRecord.ts";
+import { makeTraceSink } from "./TraceSink.ts";
+
+const makeRecord = (name: string, suffix = ""): TraceRecord => ({
+ type: "effect-span",
+ name,
+ traceId: `trace-${name}-${suffix}`,
+ spanId: `span-${name}-${suffix}`,
+ sampled: true,
+ kind: "internal",
+ startTimeUnixNano: "1",
+ endTimeUnixNano: "2",
+ durationMs: 1,
+ attributes: {
+ payload: suffix,
+ },
+ events: [],
+ links: [],
+ exit: {
+ _tag: "Success",
+ },
+});
+
+describe("TraceSink", () => {
+ it.effect("flushes buffered records on close", () =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-trace-sink-"));
+ const tracePath = path.join(tempDir, "server.trace.ndjson");
+
+ try {
+ const sink = yield* makeTraceSink({
+ filePath: tracePath,
+ maxBytes: 1024,
+ maxFiles: 2,
+ batchWindowMs: 10_000,
+ });
+
+ sink.push(makeRecord("alpha"));
+ sink.push(makeRecord("beta"));
+ yield* sink.close();
+
+ const lines = fs
+ .readFileSync(tracePath, "utf8")
+ .trim()
+ .split("\n")
+ .map((line) => JSON.parse(line) as TraceRecord);
+
+ assert.equal(lines.length, 2);
+ assert.equal(lines[0]?.name, "alpha");
+ assert.equal(lines[1]?.name, "beta");
+ } finally {
+ fs.rmSync(tempDir, { recursive: true, force: true });
+ }
+ }),
+ ),
+ );
+
+ it.effect("rotates the trace file when the configured max size is exceeded", () =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-trace-sink-"));
+ const tracePath = path.join(tempDir, "server.trace.ndjson");
+
+ try {
+ const sink = yield* makeTraceSink({
+ filePath: tracePath,
+ maxBytes: 180,
+ maxFiles: 2,
+ batchWindowMs: 10_000,
+ });
+
+ for (let index = 0; index < 8; index += 1) {
+ sink.push(makeRecord("rotate", `${index}-${"x".repeat(48)}`));
+ yield* sink.flush;
+ }
+ yield* sink.close();
+
+ const matchingFiles = fs
+ .readdirSync(tempDir)
+ .filter(
+ (entry) =>
+ entry === "server.trace.ndjson" || entry.startsWith("server.trace.ndjson."),
+ )
+ .toSorted();
+
+ assert.equal(
+ matchingFiles.some((entry) => entry === "server.trace.ndjson.1"),
+ true,
+ );
+ assert.equal(
+ matchingFiles.some((entry) => entry === "server.trace.ndjson.3"),
+ false,
+ );
+ } finally {
+ fs.rmSync(tempDir, { recursive: true, force: true });
+ }
+ }),
+ ),
+ );
+
+ it.effect("drops only the invalid record when serialization fails", () =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-trace-sink-"));
+ const tracePath = path.join(tempDir, "server.trace.ndjson");
+
+ try {
+ const sink = yield* makeTraceSink({
+ filePath: tracePath,
+ maxBytes: 1024,
+ maxFiles: 2,
+ batchWindowMs: 10_000,
+ });
+
+ const circular: Array<unknown> = [];
+ circular.push(circular);
+
+ sink.push(makeRecord("alpha"));
+ sink.push({
+ ...makeRecord("invalid"),
+ attributes: {
+ circular,
+ },
+ } as TraceRecord);
+ sink.push(makeRecord("beta"));
+ yield* sink.close();
+
+ const lines = fs
+ .readFileSync(tracePath, "utf8")
+ .trim()
+ .split("\n")
+ .map((line) => JSON.parse(line) as TraceRecord);
+
+ assert.deepStrictEqual(
+ lines.map((line) => line.name),
+ ["alpha", "beta"],
+ );
+ } finally {
+ fs.rmSync(tempDir, { recursive: true, force: true });
+ }
+ }),
+ ),
+ );
+});
diff --git a/apps/server/src/observability/TraceSink.ts b/apps/server/src/observability/TraceSink.ts
new file mode 100644
index 000000000..1bd00b473
--- /dev/null
+++ b/apps/server/src/observability/TraceSink.ts
@@ -0,0 +1,68 @@
+import { RotatingFileSink } from "@t3tools/shared/logging";
+import { Effect } from "effect";
+
+import type { TraceRecord } from "./TraceRecord.ts";
+
+const FLUSH_BUFFER_THRESHOLD = 32;
+
+export interface TraceSinkOptions {
+ readonly filePath: string;
+ readonly maxBytes: number;
+ readonly maxFiles: number;
+ readonly batchWindowMs: number;
+}
+
+export interface TraceSink {
+ readonly filePath: string;
+ push: (record: TraceRecord) => void;
+ flush: Effect.Effect<void>;
+ close: () => Effect.Effect<void>;
+}
+
+export const makeTraceSink = Effect.fn("makeTraceSink")(function* (options: TraceSinkOptions) {
+ const sink = new RotatingFileSink({
+ filePath: options.filePath,
+ maxBytes: options.maxBytes,
+ maxFiles: options.maxFiles,
+ });
+
+ let buffer: Array<string> = [];
+
+ const flushUnsafe = () => {
+ if (buffer.length === 0) {
+ return;
+ }
+
+ const chunk = buffer.join("");
+ buffer = [];
+
+ try {
+ sink.write(chunk);
+ } catch {
+ buffer.unshift(chunk);
+ }
+ };
+
+ const flush = Effect.sync(flushUnsafe).pipe(Effect.withTracerEnabled(false));
+
+ yield* Effect.addFinalizer(() => flush.pipe(Effect.ignore));
+ yield* Effect.forkScoped(
+ Effect.sleep(`${options.batchWindowMs} millis`).pipe(Effect.andThen(flush), Effect.forever),
+ );
+
+ return {
+ filePath: options.filePath,
+ push(record) {
+ try {
+ buffer.push(`${JSON.stringify(record)}\n`);
+ if (buffer.length >= FLUSH_BUFFER_THRESHOLD) {
+ flushUnsafe();
+ }
+ } catch {
+ return;
+ }
+ },
+ flush,
+ close: () => flush,
+ } satisfies TraceSink;
+});
diff --git a/apps/server/src/open.ts b/apps/server/src/open.ts
index cdacda7c0..58074ceef 100644
--- a/apps/server/src/open.ts
+++ b/apps/server/src/open.ts
@@ -233,10 +233,15 @@ export class Open extends ServiceMap.Service<Open, OpenShape>()("t3/open") {}
// Implementations
// ==============================
-export const resolveEditorLaunch = Effect.fnUntraced(function* (
+export const resolveEditorLaunch = Effect.fn("resolveEditorLaunch")(function* (
input: OpenInEditorInput,
platform: NodeJS.Platform = process.platform,
): Effect.fn.Return<EditorLaunch, OpenError> {
+ yield* Effect.annotateCurrentSpan({
+ "open.editor": input.editor,
+ "open.cwd": input.cwd,
+ "open.platform": platform,
+ });
const editorDef = EDITORS.find((editor) => editor.id === input.editor);
if (!editorDef) {
return yield* new OpenError({ message: `Unknown editor: ${input.editor}` });
diff --git a/apps/server/src/orchestration/Layers/CheckpointReactor.ts b/apps/server/src/orchestration/Layers/CheckpointReactor.ts
index 28609cecb..03abebaf3 100644
--- a/apps/server/src/orchestration/Layers/CheckpointReactor.ts
+++ b/apps/server/src/orchestration/Layers/CheckpointReactor.ts
@@ -119,7 +119,7 @@ const make = Effect.gen(function* () {
createdAt: input.createdAt,
});
- const resolveSessionRuntimeForThread = Effect.fnUntraced(function* (
+ const resolveSessionRuntimeForThread = Effect.fn("resolveSessionRuntimeForThread")(function* (
threadId: ThreadId,
): Effect.fn.Return<Option.Option<{ readonly threadId: ThreadId; readonly cwd: string }>> {
const readModel = yield* orchestrationEngine.getReadModel();
@@ -153,7 +153,7 @@ const make = Effect.gen(function* () {
// active provider session CWD and falling back to the thread/project config.
// Returns undefined when no CWD can be determined or the workspace is not
// a git repository.
- const resolveCheckpointCwd = Effect.fnUntraced(function* (input: {
+ const resolveCheckpointCwd = Effect.fn("resolveCheckpointCwd")(function* (input: {
readonly threadId: ThreadId;
readonly thread: { readonly projectId: ProjectId; readonly worktreePath: string | null };
readonly projects: ReadonlyArray<{ readonly id: ProjectId; readonly workspaceRoot: string }>;
@@ -188,7 +188,7 @@ const make = Effect.gen(function* () {
// Shared tail for both capture paths: creates the git checkpoint ref, diffs
// it against the previous turn, then dispatches the domain events to update
// the orchestration read model.
- const captureAndDispatchCheckpoint = Effect.fnUntraced(function* (input: {
+ const captureAndDispatchCheckpoint = Effect.fn("captureAndDispatchCheckpoint")(function* (input: {
readonly threadId: ThreadId;
readonly turnId: TurnId;
readonly thread: {
@@ -321,70 +321,70 @@ const make = Effect.gen(function* () {
});
// Captures a real git checkpoint when a turn completes via a runtime event.
- const captureCheckpointFromTurnCompletion = Effect.fnUntraced(function* (
- event: Extract<ProviderRuntimeEvent, { type: "turn.completed" }>,
- ) {
- const turnId = toTurnId(event.turnId);
- if (!turnId) {
- return;
- }
-
- const readModel = yield* orchestrationEngine.getReadModel();
- const thread = readModel.threads.find((entry) => entry.id === event.threadId);
- if (!thread) {
- return;
- }
+ const captureCheckpointFromTurnCompletion = Effect.fn("captureCheckpointFromTurnCompletion")(
+ function* (event: Extract<ProviderRuntimeEvent, { type: "turn.completed" }>) {
+ const turnId = toTurnId(event.turnId);
+ if (!turnId) {
+ return;
+ }
- // When a primary turn is active, only that turn may produce completion checkpoints.
- if (thread.session?.activeTurnId && !sameId(thread.session.activeTurnId, turnId)) {
- return;
- }
+ const readModel = yield* orchestrationEngine.getReadModel();
+ const thread = readModel.threads.find((entry) => entry.id === event.threadId);
+ if (!thread) {
+ return;
+ }
- // Only skip if a real (non-placeholder) checkpoint already exists for this turn.
- // ProviderRuntimeIngestion may insert placeholder entries with status "missing"
- // before this reactor runs; those must not prevent real git capture.
- if (
- thread.checkpoints.some(
- (checkpoint) => checkpoint.turnId === turnId && checkpoint.status !== "missing",
- )
- ) {
- return;
- }
+ // When a primary turn is active, only that turn may produce completion checkpoints.
+ if (thread.session?.activeTurnId && !sameId(thread.session.activeTurnId, turnId)) {
+ return;
+ }
- const checkpointCwd = yield* resolveCheckpointCwd({
- threadId: thread.id,
- thread,
- projects: readModel.projects,
- preferSessionRuntime: true,
- });
- if (!checkpointCwd) {
- return;
- }
+ // Only skip if a real (non-placeholder) checkpoint already exists for this turn.
+ // ProviderRuntimeIngestion may insert placeholder entries with status "missing"
+ // before this reactor runs; those must not prevent real git capture.
+ if (
+ thread.checkpoints.some(
+ (checkpoint) => checkpoint.turnId === turnId && checkpoint.status !== "missing",
+ )
+ ) {
+ return;
+ }
- // If a placeholder checkpoint exists for this turn, reuse its turn count
- // instead of incrementing past it.
- const existingPlaceholder = thread.checkpoints.find(
- (checkpoint) => checkpoint.turnId === turnId && checkpoint.status === "missing",
- );
- const currentTurnCount = thread.checkpoints.reduce(
- (maxTurnCount, checkpoint) => Math.max(maxTurnCount, checkpoint.checkpointTurnCount),
- 0,
- );
- const nextTurnCount = existingPlaceholder
- ? existingPlaceholder.checkpointTurnCount
- : currentTurnCount + 1;
+ const checkpointCwd = yield* resolveCheckpointCwd({
+ threadId: thread.id,
+ thread,
+ projects: readModel.projects,
+ preferSessionRuntime: true,
+ });
+ if (!checkpointCwd) {
+ return;
+ }
- yield* captureAndDispatchCheckpoint({
- threadId: thread.id,
- turnId,
- thread,
- cwd: checkpointCwd,
- turnCount: nextTurnCount,
- status: checkpointStatusFromRuntime(event.payload.state),
- assistantMessageId: undefined,
- createdAt: event.createdAt,
- });
- });
+ // If a placeholder checkpoint exists for this turn, reuse its turn count
+ // instead of incrementing past it.
+ const existingPlaceholder = thread.checkpoints.find(
+ (checkpoint) => checkpoint.turnId === turnId && checkpoint.status === "missing",
+ );
+ const currentTurnCount = thread.checkpoints.reduce(
+ (maxTurnCount, checkpoint) => Math.max(maxTurnCount, checkpoint.checkpointTurnCount),
+ 0,
+ );
+ const nextTurnCount = existingPlaceholder
+ ? existingPlaceholder.checkpointTurnCount
+ : currentTurnCount + 1;
+
+ yield* captureAndDispatchCheckpoint({
+ threadId: thread.id,
+ turnId,
+ thread,
+ cwd: checkpointCwd,
+ turnCount: nextTurnCount,
+ status: checkpointStatusFromRuntime(event.payload.state),
+ assistantMessageId: undefined,
+ createdAt: event.createdAt,
+ });
+ },
+ );
// Captures a real git checkpoint when a placeholder checkpoint (status "missing")
// is detected via a domain event. This replaces the placeholder with a real
@@ -394,7 +394,7 @@ const make = Effect.gen(function* () {
// events from the Codex runtime. This handler fires when the corresponding
// domain event arrives, allowing the reactor to capture the actual filesystem
// state into a git ref and dispatch a replacement checkpoint.
- const captureCheckpointFromPlaceholder = Effect.fnUntraced(function* (
+ const captureCheckpointFromPlaceholder = Effect.fn("captureCheckpointFromPlaceholder")(function* (
event: Extract<OrchestrationEvent, { type: "thread.turn-diff-completed" }>,
) {
const { threadId, turnId, checkpointTurnCount, status } = event.payload;
@@ -448,57 +448,59 @@ const make = Effect.gen(function* () {
});
});
- const ensurePreTurnBaselineFromTurnStart = Effect.fnUntraced(function* (
- event: Extract<ProviderRuntimeEvent, { type: "turn.started" }>,
- ) {
- const turnId = toTurnId(event.turnId);
- if (!turnId) {
- return;
- }
+ const ensurePreTurnBaselineFromTurnStart = Effect.fn("ensurePreTurnBaselineFromTurnStart")(
+ function* (event: Extract<ProviderRuntimeEvent, { type: "turn.started" }>) {
+ const turnId = toTurnId(event.turnId);
+ if (!turnId) {
+ return;
+ }
- const readModel = yield* orchestrationEngine.getReadModel();
- const thread = readModel.threads.find((entry) => entry.id === event.threadId);
- if (!thread) {
- return;
- }
+ const readModel = yield* orchestrationEngine.getReadModel();
+ const thread = readModel.threads.find((entry) => entry.id === event.threadId);
+ if (!thread) {
+ return;
+ }
- const checkpointCwd = yield* resolveCheckpointCwd({
- threadId: thread.id,
- thread,
- projects: readModel.projects,
- preferSessionRuntime: false,
- });
- if (!checkpointCwd) {
- return;
- }
+ const checkpointCwd = yield* resolveCheckpointCwd({
+ threadId: thread.id,
+ thread,
+ projects: readModel.projects,
+ preferSessionRuntime: false,
+ });
+ if (!checkpointCwd) {
+ return;
+ }
- const currentTurnCount = thread.checkpoints.reduce(
- (maxTurnCount, checkpoint) => Math.max(maxTurnCount, checkpoint.checkpointTurnCount),
- 0,
- );
- const baselineCheckpointRef = checkpointRefForThreadTurn(thread.id, currentTurnCount);
- const baselineExists = yield* checkpointStore.hasCheckpointRef({
- cwd: checkpointCwd,
- checkpointRef: baselineCheckpointRef,
- });
- if (baselineExists) {
- return;
- }
+ const currentTurnCount = thread.checkpoints.reduce(
+ (maxTurnCount, checkpoint) => Math.max(maxTurnCount, checkpoint.checkpointTurnCount),
+ 0,
+ );
+ const baselineCheckpointRef = checkpointRefForThreadTurn(thread.id, currentTurnCount);
+ const baselineExists = yield* checkpointStore.hasCheckpointRef({
+ cwd: checkpointCwd,
+ checkpointRef: baselineCheckpointRef,
+ });
+ if (baselineExists) {
+ return;
+ }
- yield* checkpointStore.captureCheckpoint({
- cwd: checkpointCwd,
- checkpointRef: baselineCheckpointRef,
- });
- yield* receiptBus.publish({
- type: "checkpoint.baseline.captured",
- threadId: thread.id,
- checkpointTurnCount: currentTurnCount,
- checkpointRef: baselineCheckpointRef,
- createdAt: event.createdAt,
- });
- });
+ yield* checkpointStore.captureCheckpoint({
+ cwd: checkpointCwd,
+ checkpointRef: baselineCheckpointRef,
+ });
+ yield* receiptBus.publish({
+ type: "checkpoint.baseline.captured",
+ threadId: thread.id,
+ checkpointTurnCount: currentTurnCount,
+ checkpointRef: baselineCheckpointRef,
+ createdAt: event.createdAt,
+ });
+ },
+ );
- const ensurePreTurnBaselineFromDomainTurnStart = Effect.fnUntraced(function* (
+ const ensurePreTurnBaselineFromDomainTurnStart = Effect.fn(
+ "ensurePreTurnBaselineFromDomainTurnStart",
+ )(function* (
event: Extract<
OrchestrationEvent,
{ type: "thread.turn-start-requested" | "thread.message-sent" }
@@ -557,7 +559,7 @@ const make = Effect.gen(function* () {
});
});
- const handleRevertRequested = Effect.fnUntraced(function* (
+ const handleRevertRequested = Effect.fn("handleRevertRequested")(function* (
event: Extract<OrchestrationEvent, { type: "thread.checkpoint-revert-requested" }>,
) {
const now = new Date().toISOString();
@@ -685,7 +687,7 @@ const make = Effect.gen(function* () {
);
});
- const processDomainEvent = Effect.fnUntraced(function* (event: OrchestrationEvent) {
+ const processDomainEvent = Effect.fn("processDomainEvent")(function* (event: OrchestrationEvent) {
if (event.type === "thread.turn-start-requested" || event.type === "thread.message-sent") {
yield* ensurePreTurnBaselineFromDomainTurnStart(event);
return;
@@ -724,7 +726,9 @@ const make = Effect.gen(function* () {
}
});
- const processRuntimeEvent = Effect.fnUntraced(function* (event: ProviderRuntimeEvent) {
+ const processRuntimeEvent = Effect.fn("processRuntimeEvent")(function* (
+ event: ProviderRuntimeEvent,
+ ) {
if (event.type === "turn.started") {
yield* ensurePreTurnBaselineFromTurnStart(event);
return;
diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts
index 16bbd98d6..5a0a6113f 100644
--- a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts
+++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts
@@ -8,7 +8,7 @@ import {
TurnId,
type OrchestrationEvent,
} from "@t3tools/contracts";
-import { Effect, Layer, ManagedRuntime, Option, Queue, Stream } from "effect";
+import { Effect, Layer, ManagedRuntime, Metric, Option, Queue, Stream } from "effect";
import { describe, expect, it } from "vitest";
import { PersistenceSqlError } from "../../persistence/Errors.ts";
@@ -62,6 +62,17 @@ function now() {
return new Date().toISOString();
}
+const hasMetricSnapshot = (
+ snapshots: ReadonlyArray<Metric.Metric.Snapshot>,
+ id: string,
+ attributes: Readonly<Record<string, string>>,
+) =>
+ snapshots.some(
+ (snapshot) =>
+ snapshot.id === id &&
+ Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),
+ );
+
describe("OrchestrationEngine", () => {
it("bootstraps the in-memory read model from persisted projections", async () => {
const failOnHistoricalReplayStore: OrchestrationEventStoreShape = {
@@ -408,6 +419,95 @@ describe("OrchestrationEngine", () => {
await system.dispose();
});
+ it("records command ack duration using the first committed event type", async () => {
+ const system = await createOrchestrationSystem();
+ const { engine } = system;
+ const createdAt = now();
+
+ await system.run(
+ engine.dispatch({
+ type: "project.create",
+ commandId: CommandId.makeUnsafe("cmd-project-ack-create"),
+ projectId: asProjectId("project-ack"),
+ title: "Ack Project",
+ workspaceRoot: "/tmp/project-ack",
+ defaultModelSelection: {
+ provider: "codex",
+ model: "gpt-5-codex",
+ },
+ createdAt,
+ }),
+ );
+
+ await system.run(
+ engine.dispatch({
+ type: "thread.create",
+ commandId: CommandId.makeUnsafe("cmd-thread-ack-create"),
+ threadId: ThreadId.makeUnsafe("thread-ack"),
+ projectId: asProjectId("project-ack"),
+ title: "Ack Thread",
+ modelSelection: {
+ provider: "codex",
+ model: "gpt-5-codex",
+ },
+ interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
+ runtimeMode: "full-access",
+ branch: null,
+ worktreePath: null,
+ createdAt,
+ }),
+ );
+
+ const snapshots = await system.run(Metric.snapshot);
+ expect(
+ hasMetricSnapshot(snapshots, "t3_orchestration_command_ack_duration", {
+ commandType: "thread.create",
+ aggregateKind: "thread",
+ ackEventType: "thread.created",
+ }),
+ ).toBe(true);
+
+ await system.dispose();
+ });
+
+ it("records failed command dispatches as metric failures", async () => {
+ const system = await createOrchestrationSystem();
+ const { engine } = system;
+ const createdAt = now();
+
+ await expect(
+ system.run(
+ engine.dispatch({
+ type: "thread.create",
+ commandId: CommandId.makeUnsafe("cmd-thread-missing-project"),
+ threadId: ThreadId.makeUnsafe("thread-missing-project"),
+ projectId: asProjectId("project-missing"),
+ title: "Missing Project Thread",
+ modelSelection: {
+ provider: "codex",
+ model: "gpt-5-codex",
+ },
+ interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
+ runtimeMode: "full-access",
+ branch: null,
+ worktreePath: null,
+ createdAt,
+ }),
+ ),
+ ).rejects.toThrow("does not exist");
+
+ const snapshots = await system.run(Metric.snapshot);
+ expect(
+ hasMetricSnapshot(snapshots, "t3_orchestration_commands_total", {
+ commandType: "thread.create",
+ aggregateKind: "thread",
+ outcome: "failure",
+ }),
+ ).toBe(true);
+
+ await system.dispose();
+ });
+
it("stores completed checkpoint summaries even when no files changed", async () => {
const system = await createOrchestrationSystem();
const { engine } = system;
diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts
index bd3581deb..ddd1718fa 100644
--- a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts
+++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts
@@ -5,9 +5,28 @@ import type {
ThreadId,
} from "@t3tools/contracts";
import { OrchestrationCommand } from "@t3tools/contracts";
-import { Deferred, Effect, Layer, Option, PubSub, Queue, Schema, Stream } from "effect";
+import {
+ Cause,
+ Deferred,
+ Duration,
+ Effect,
+ Exit,
+ Layer,
+ Metric,
+ Option,
+ PubSub,
+ Queue,
+ Schema,
+ Stream,
+} from "effect";
import * as SqlClient from "effect/unstable/sql/SqlClient";
+import {
+ metricAttributes,
+ orchestrationCommandAckDuration,
+ orchestrationCommandsTotal,
+ orchestrationCommandDuration,
+} from "../../observability/Metrics.ts";
import { toPersistenceSqlError } from "../../persistence/Errors.ts";
import { OrchestrationEventStore } from "../../persistence/Services/OrchestrationEventStore.ts";
import { OrchestrationCommandReceiptRepository } from "../../persistence/Services/OrchestrationCommandReceipts.ts";
@@ -28,6 +47,7 @@ import {
interface CommandEnvelope {
command: OrchestrationCommand;
result: Deferred.Deferred<{ sequence: number }, OrchestrationDispatchError>;
+ startedAtMs: number;
}
function commandToAggregateRef(command: OrchestrationCommand): {
@@ -64,6 +84,12 @@ const makeOrchestrationEngine = Effect.gen(function* () {
const processEnvelope = (envelope: CommandEnvelope): Effect.Effect<void> => {
const dispatchStartSequence = readModel.snapshotSequence;
+ const processingStartedAtMs = Date.now();
+ const aggregateRef = commandToAggregateRef(envelope.command);
+ const baseMetricAttributes = {
+ commandType: envelope.command.type,
+ aggregateKind: aggregateRef.aggregateKind,
+ } as const;
const reconcileReadModelAfterDispatchFailure = Effect.gen(function* () {
const persistedEvents = yield* Stream.runCollect(
eventStore.readFromSequence(dispatchStartSequence),
@@ -83,113 +109,160 @@ const makeOrchestrationEngine = Effect.gen(function* () {
}
});
- return Effect.gen(function* () {
- const existingReceipt = yield* commandReceiptRepository.getByCommandId({
- commandId: envelope.command.commandId,
- });
- if (Option.isSome(existingReceipt)) {
- if (existingReceipt.value.status === "accepted") {
- yield* Deferred.succeed(envelope.result, {
- sequence: existingReceipt.value.resultSequence,
- });
- return;
- }
- yield* Deferred.fail(
- envelope.result,
- new OrchestrationCommandPreviouslyRejectedError({
+ return Effect.exit(
+ Effect.gen(function* () {
+ yield* Effect.annotateCurrentSpan({
+ "orchestration.command_id": envelope.command.commandId,
+ "orchestration.command_type": envelope.command.type,
+ "orchestration.aggregate_kind": aggregateRef.aggregateKind,
+ "orchestration.aggregate_id": aggregateRef.aggregateId,
+ });
+
+ const existingReceipt = yield* commandReceiptRepository.getByCommandId({
+ commandId: envelope.command.commandId,
+ });
+ if (Option.isSome(existingReceipt)) {
+ if (existingReceipt.value.status === "accepted") {
+ return {
+ sequence: existingReceipt.value.resultSequence,
+ };
+ }
+ return yield* new OrchestrationCommandPreviouslyRejectedError({
commandId: envelope.command.commandId,
detail: existingReceipt.value.error ?? "Previously rejected.",
- }),
- );
- return;
- }
+ });
+ }
- const eventBase = yield* decideOrchestrationCommand({
- command: envelope.command,
- readModel,
- });
- const eventBases = Array.isArray(eventBase) ? eventBase : [eventBase];
- const committedCommand = yield* sql
- .withTransaction(
- Effect.gen(function* () {
- const committedEvents: OrchestrationEvent[] = [];
- let nextReadModel = readModel;
-
- for (const nextEvent of eventBases) {
- const savedEvent = yield* eventStore.append(nextEvent);
- nextReadModel = yield* projectEvent(nextReadModel, savedEvent);
- yield* projectionPipeline.projectEvent(savedEvent);
- committedEvents.push(savedEvent);
- }
+ const eventBase = yield* decideOrchestrationCommand({
+ command: envelope.command,
+ readModel,
+ });
+ const eventBases = Array.isArray(eventBase) ? eventBase : [eventBase];
+ const committedCommand = yield* sql
+ .withTransaction(
+ Effect.gen(function* () {
+ const committedEvents: OrchestrationEvent[] = [];
+ let nextReadModel = readModel;
- const lastSavedEvent = committedEvents.at(-1) ?? null;
- if (lastSavedEvent === null) {
- return yield* new OrchestrationCommandInvariantError({
- commandType: envelope.command.type,
- detail: "Command produced no events.",
- });
- }
+ for (const nextEvent of eventBases) {
+ const savedEvent = yield* eventStore.append(nextEvent);
+ nextReadModel = yield* projectEvent(nextReadModel, savedEvent);
+ yield* projectionPipeline.projectEvent(savedEvent);
+ committedEvents.push(savedEvent);
+ }
- yield* commandReceiptRepository.upsert({
- commandId: envelope.command.commandId,
- aggregateKind: lastSavedEvent.aggregateKind,
- aggregateId: lastSavedEvent.aggregateId,
- acceptedAt: lastSavedEvent.occurredAt,
- resultSequence: lastSavedEvent.sequence,
- status: "accepted",
- error: null,
- });
+ const lastSavedEvent = committedEvents.at(-1) ?? null;
+ if (lastSavedEvent === null) {
+ return yield* new OrchestrationCommandInvariantError({
+ commandType: envelope.command.type,
+ detail: "Command produced no events.",
+ });
+ }
- return {
- committedEvents,
- lastSequence: lastSavedEvent.sequence,
- nextReadModel,
- } as const;
- }),
- )
- .pipe(
- Effect.catchTag("SqlError", (sqlError) =>
- Effect.fail(
- toPersistenceSqlError("OrchestrationEngine.processEnvelope:transaction")(sqlError),
+ yield* commandReceiptRepository.upsert({
+ commandId: envelope.command.commandId,
+ aggregateKind: lastSavedEvent.aggregateKind,
+ aggregateId: lastSavedEvent.aggregateId,
+ acceptedAt: lastSavedEvent.occurredAt,
+ resultSequence: lastSavedEvent.sequence,
+ status: "accepted",
+ error: null,
+ });
+
+ return {
+ committedEvents,
+ lastSequence: lastSavedEvent.sequence,
+ nextReadModel,
+ } as const;
+ }),
+ )
+ .pipe(
+ Effect.catchTag("SqlError", (sqlError) =>
+ Effect.fail(
+ toPersistenceSqlError("OrchestrationEngine.processEnvelope:transaction")(sqlError),
+ ),
),
- ),
- );
+ );
- readModel = committedCommand.nextReadModel;
- for (const event of committedCommand.committedEvents) {
- yield* PubSub.publish(eventPubSub, event);
- }
- yield* Deferred.succeed(envelope.result, { sequence: committedCommand.lastSequence });
- }).pipe(
- Effect.catch((error) =>
- Effect.gen(function* () {
- yield* reconcileReadModelAfterDispatchFailure.pipe(
- Effect.catch(() =>
- Effect.logWarning(
- "failed to reconcile orchestration read model after dispatch failure",
- ).pipe(
- Effect.annotateLogs({
- commandId: envelope.command.commandId,
- snapshotSequence: readModel.snapshotSequence,
+ readModel = committedCommand.nextReadModel;
+ for (const [index, event] of committedCommand.committedEvents.entries()) {
+ yield* PubSub.publish(eventPubSub, event);
+ if (index === 0) {
+ yield* Metric.update(
+ Metric.withAttributes(
+ orchestrationCommandAckDuration,
+ metricAttributes({
+ ...baseMetricAttributes,
+ ackEventType: event.type,
}),
),
+ Duration.millis(Math.max(0, Date.now() - envelope.startedAtMs)),
+ );
+ }
+ }
+ return { sequence: committedCommand.lastSequence };
+ }).pipe(Effect.withSpan(`orchestration.command.${envelope.command.type}`)),
+ ).pipe(
+ Effect.flatMap((exit) =>
+ Effect.gen(function* () {
+ const outcome = Exit.isSuccess(exit)
+ ? "success"
+ : Cause.hasInterruptsOnly(exit.cause)
+ ? "interrupt"
+ : "failure";
+ yield* Metric.update(
+ Metric.withAttributes(
+ orchestrationCommandDuration,
+ metricAttributes(baseMetricAttributes),
+ ),
+ Duration.millis(Math.max(0, Date.now() - processingStartedAtMs)),
+ );
+ yield* Metric.update(
+ Metric.withAttributes(
+ orchestrationCommandsTotal,
+ metricAttributes({
+ ...baseMetricAttributes,
+ outcome,
+ }),
),
+ 1,
);
- if (Schema.is(OrchestrationCommandInvariantError)(error)) {
- const aggregateRef = commandToAggregateRef(envelope.command);
- yield* commandReceiptRepository
- .upsert({
- commandId: envelope.command.commandId,
- aggregateKind: aggregateRef.aggregateKind,
- aggregateId: aggregateRef.aggregateId,
- acceptedAt: new Date().toISOString(),
- resultSequence: readModel.snapshotSequence,
- status: "rejected",
- error: error.message,
- })
- .pipe(Effect.catch(() => Effect.void));
+ if (Exit.isSuccess(exit)) {
+ yield* Deferred.succeed(envelope.result, exit.value);
+ return;
}
+
+ const error = Cause.squash(exit.cause) as OrchestrationDispatchError;
+ if (!Schema.is(OrchestrationCommandPreviouslyRejectedError)(error)) {
+ yield* reconcileReadModelAfterDispatchFailure.pipe(
+ Effect.catch(() =>
+ Effect.logWarning(
+ "failed to reconcile orchestration read model after dispatch failure",
+ ).pipe(
+ Effect.annotateLogs({
+ commandId: envelope.command.commandId,
+ snapshotSequence: readModel.snapshotSequence,
+ }),
+ ),
+ ),
+ );
+
+ if (Schema.is(OrchestrationCommandInvariantError)(error)) {
+ yield* commandReceiptRepository
+ .upsert({
+ commandId: envelope.command.commandId,
+ aggregateKind: aggregateRef.aggregateKind,
+ aggregateId: aggregateRef.aggregateId,
+ acceptedAt: new Date().toISOString(),
+ resultSequence: readModel.snapshotSequence,
+ status: "rejected",
+ error: error.message,
+ })
+ .pipe(Effect.catch(() => Effect.void));
+ }
+ }
+
yield* Deferred.fail(envelope.result, error);
}),
),
@@ -214,7 +287,7 @@ const makeOrchestrationEngine = Effect.gen(function* () {
const dispatch: OrchestrationEngineShape["dispatch"] = (command) =>
Effect.gen(function* () {
const result = yield* Deferred.make<{ sequence: number }, OrchestrationDispatchError>();
- yield* Queue.offer(commandQueue, { command, result });
+ yield* Queue.offer(commandQueue, { command, result, startedAtMs: Date.now() });
return yield* Deferred.await(result);
});
diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts
index 8e2a19bcd..419e3f3bf 100644
--- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts
+++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts
@@ -16,6 +16,7 @@ import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker";
import { resolveThreadWorkspaceCwd } from "../../checkpointing/Utils.ts";
import { GitCore } from "../../git/Services/GitCore.ts";
+import { increment, orchestrationEventsProcessedTotal } from "../../observability/Metrics.ts";
import { ProviderAdapterRequestError, ProviderServiceError } from "../../provider/Errors.ts";
import { TextGeneration } from "../../git/Services/TextGeneration.ts";
import { ProviderService } from "../../provider/Services/ProviderService.ts";
@@ -212,12 +213,12 @@ const make = Effect.gen(function* () {
createdAt: input.createdAt,
});
- const resolveThread = Effect.fnUntraced(function* (threadId: ThreadId) {
+ const resolveThread = Effect.fn("resolveThread")(function* (threadId: ThreadId) {
const readModel = yield* orchestrationEngine.getReadModel();
return readModel.threads.find((entry) => entry.id === threadId);
});
- const ensureSessionForThread = Effect.fnUntraced(function* (
+ const ensureSessionForThread = Effect.fn("ensureSessionForThread")(function* (
threadId: ThreadId,
createdAt: string,
options?: {
@@ -357,7 +358,7 @@ const make = Effect.gen(function* () {
return startedSession.threadId;
});
- const sendTurnForThread = Effect.fnUntraced(function* (input: {
+ const sendTurnForThread = Effect.fn("sendTurnForThread")(function* (input: {
readonly threadId: ThreadId;
readonly messageText: string;
readonly attachments?: ReadonlyArray<ChatAttachment>;
@@ -409,7 +410,9 @@ const make = Effect.gen(function* () {
});
});
- const maybeGenerateAndRenameWorktreeBranchForFirstTurn = Effect.fnUntraced(function* (input: {
+ const maybeGenerateAndRenameWorktreeBranchForFirstTurn = Effect.fn(
+ "maybeGenerateAndRenameWorktreeBranchForFirstTurn",
+ )(function* (input: {
readonly threadId: ThreadId;
readonly branch: string | null;
readonly worktreePath: string | null;
@@ -461,50 +464,52 @@ const make = Effect.gen(function* () {
);
});
- const maybeGenerateThreadTitleForFirstTurn = Effect.fnUntraced(function* (input: {
- readonly threadId: ThreadId;
- readonly cwd: string;
- readonly messageText: string;
- readonly attachments?: ReadonlyArray<ChatAttachment>;
- readonly titleSeed?: string;
- }) {
- const attachments = input.attachments ?? [];
- yield* Effect.gen(function* () {
- const { textGenerationModelSelection: modelSelection } =
- yield* serverSettingsService.getSettings;
-
- const generated = yield* textGeneration.generateThreadTitle({
- cwd: input.cwd,
- message: input.messageText,
- ...(attachments.length > 0 ? { attachments } : {}),
- modelSelection,
- });
- if (!generated) return;
+ const maybeGenerateThreadTitleForFirstTurn = Effect.fn("maybeGenerateThreadTitleForFirstTurn")(
+ function* (input: {
+ readonly threadId: ThreadId;
+ readonly cwd: string;
+ readonly messageText: string;
+ readonly attachments?: ReadonlyArray<ChatAttachment>;
+ readonly titleSeed?: string;
+ }) {
+ const attachments = input.attachments ?? [];
+ yield* Effect.gen(function* () {
+ const { textGenerationModelSelection: modelSelection } =
+ yield* serverSettingsService.getSettings;
+
+ const generated = yield* textGeneration.generateThreadTitle({
+ cwd: input.cwd,
+ message: input.messageText,
+ ...(attachments.length > 0 ? { attachments } : {}),
+ modelSelection,
+ });
+ if (!generated) return;
- const thread = yield* resolveThread(input.threadId);
- if (!thread) return;
- if (!canReplaceThreadTitle(thread.title, input.titleSeed)) {
- return;
- }
+ const thread = yield* resolveThread(input.threadId);
+ if (!thread) return;
+ if (!canReplaceThreadTitle(thread.title, input.titleSeed)) {
+ return;
+ }
- yield* orchestrationEngine.dispatch({
- type: "thread.meta.update",
- commandId: serverCommandId("thread-title-rename"),
- threadId: input.threadId,
- title: generated.title,
- });
- }).pipe(
- Effect.catchCause((cause) =>
- Effect.logWarning("provider command reactor failed to generate or rename thread title", {
+ yield* orchestrationEngine.dispatch({
+ type: "thread.meta.update",
+ commandId: serverCommandId("thread-title-rename"),
threadId: input.threadId,
- cwd: input.cwd,
- cause: Cause.pretty(cause),
- }),
- ),
- );
- });
+ title: generated.title,
+ });
+ }).pipe(
+ Effect.catchCause((cause) =>
+ Effect.logWarning("provider command reactor failed to generate or rename thread title", {
+ threadId: input.threadId,
+ cwd: input.cwd,
+ cause: Cause.pretty(cause),
+ }),
+ ),
+ );
+ },
+ );
- const processTurnStartRequested = Effect.fnUntraced(function* (
+ const processTurnStartRequested = Effect.fn("processTurnStartRequested")(function* (
event: Extract<ProviderIntentEvent, { type: "thread.turn-start-requested" }>,
) {
const key = turnStartKeyForEvent(event);
@@ -583,7 +588,7 @@ const make = Effect.gen(function* () {
);
});
- const processTurnInterruptRequested = Effect.fnUntraced(function* (
+ const processTurnInterruptRequested = Effect.fn("processTurnInterruptRequested")(function* (
event: Extract<ProviderIntentEvent, { type: "thread.turn-interrupt-requested" }>,
) {
const thread = yield* resolveThread(event.payload.threadId);
@@ -606,7 +611,7 @@ const make = Effect.gen(function* () {
yield* providerService.interruptTurn({ threadId: event.payload.threadId });
});
- const processApprovalResponseRequested = Effect.fnUntraced(function* (
+ const processApprovalResponseRequested = Effect.fn("processApprovalResponseRequested")(function* (
event: Extract<ProviderIntentEvent, { type: "thread.approval-response-requested" }>,
) {
const thread = yield* resolveThread(event.payload.threadId);
@@ -653,50 +658,52 @@ const make = Effect.gen(function* () {
);
});
- const processUserInputResponseRequested = Effect.fnUntraced(function* (
- event: Extract<ProviderIntentEvent, { type: "thread.user-input-response-requested" }>,
- ) {
- const thread = yield* resolveThread(event.payload.threadId);
- if (!thread) {
- return;
- }
- const hasSession = thread.session && thread.session.status !== "stopped";
- if (!hasSession) {
- return yield* appendProviderFailureActivity({
- threadId: event.payload.threadId,
- kind: "provider.user-input.respond.failed",
- summary: "Provider user input response failed",
- detail: "No active provider session is bound to this thread.",
- turnId: null,
- createdAt: event.payload.createdAt,
- requestId: event.payload.requestId,
- });
- }
+ const processUserInputResponseRequested = Effect.fn("processUserInputResponseRequested")(
+ function* (
+ event: Extract<ProviderIntentEvent, { type: "thread.user-input-response-requested" }>,
+ ) {
+ const thread = yield* resolveThread(event.payload.threadId);
+ if (!thread) {
+ return;
+ }
+ const hasSession = thread.session && thread.session.status !== "stopped";
+ if (!hasSession) {
+ return yield* appendProviderFailureActivity({
+ threadId: event.payload.threadId,
+ kind: "provider.user-input.respond.failed",
+ summary: "Provider user input response failed",
+ detail: "No active provider session is bound to this thread.",
+ turnId: null,
+ createdAt: event.payload.createdAt,
+ requestId: event.payload.requestId,
+ });
+ }
- yield* providerService
- .respondToUserInput({
- threadId: event.payload.threadId,
- requestId: event.payload.requestId,
- answers: event.payload.answers,
- })
- .pipe(
- Effect.catchCause((cause) =>
- appendProviderFailureActivity({
- threadId: event.payload.threadId,
- kind: "provider.user-input.respond.failed",
- summary: "Provider user input response failed",
- detail: isUnknownPendingUserInputRequestError(cause)
- ? stalePendingRequestDetail("user-input", event.payload.requestId)
- : Cause.pretty(cause),
- turnId: null,
- createdAt: event.payload.createdAt,
- requestId: event.payload.requestId,
- }),
- ),
- );
- });
+ yield* providerService
+ .respondToUserInput({
+ threadId: event.payload.threadId,
+ requestId: event.payload.requestId,
+ answers: event.payload.answers,
+ })
+ .pipe(
+ Effect.catchCause((cause) =>
+ appendProviderFailureActivity({
+ threadId: event.payload.threadId,
+ kind: "provider.user-input.respond.failed",
+ summary: "Provider user input response failed",
+ detail: isUnknownPendingUserInputRequestError(cause)
+ ? stalePendingRequestDetail("user-input", event.payload.requestId)
+ : Cause.pretty(cause),
+ turnId: null,
+ createdAt: event.payload.createdAt,
+ requestId: event.payload.requestId,
+ }),
+ ),
+ );
+ },
+ );
- const processSessionStopRequested = Effect.fnUntraced(function* (
+ const processSessionStopRequested = Effect.fn("processSessionStopRequested")(function* (
event: Extract<ProviderIntentEvent, { type: "thread.session-stop-requested" }>,
) {
const thread = yield* resolveThread(event.payload.threadId);
@@ -727,6 +734,14 @@ const make = Effect.gen(function* () {
const processDomainEvent = Effect.fn("processDomainEvent")(function* (
event: ProviderIntentEvent,
) {
+ yield* Effect.annotateCurrentSpan({
+ "orchestration.event_type": event.type,
+ "orchestration.thread_id": event.payload.threadId,
+ ...(event.commandId ? { "orchestration.command_id": event.commandId } : {}),
+ });
+ yield* increment(orchestrationEventsProcessedTotal, {
+ eventType: event.type,
+ });
switch (event.type) {
case "thread.runtime-mode-set": {
const thread = yield* resolveThread(event.payload.threadId);
diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts
index b479d1c28..8d3fb5d75 100644
--- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts
+++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts
@@ -524,7 +524,7 @@ const make = Effect.fn("make")(function* () {
lookup: () => Effect.succeed({ text: "", createdAt: "" }),
});
- const isGitRepoForThread = Effect.fnUntraced(function* (threadId: ThreadId) {
+ const isGitRepoForThread = Effect.fn("isGitRepoForThread")(function* (threadId: ThreadId) {
const readModel = yield* orchestrationEngine.getReadModel();
const thread = readModel.threads.find((entry) => entry.id === threadId);
if (!thread) {
@@ -796,9 +796,9 @@ const make = Effect.fn("make")(function* () {
).pipe(Effect.asVoid);
});
- const getSourceProposedPlanReferenceForPendingTurnStart = Effect.fnUntraced(function* (
- threadId: ThreadId,
- ) {
+ const getSourceProposedPlanReferenceForPendingTurnStart = Effect.fn(
+ "getSourceProposedPlanReferenceForPendingTurnStart",
+ )(function* (threadId: ThreadId) {
const pendingTurnStart = yield* projectionTurnRepository.getPendingTurnStartByThreadId({
threadId,
});
@@ -818,16 +818,17 @@ const make = Effect.fn("make")(function* () {
} as const;
});
- const getExpectedProviderTurnIdForThread = Effect.fnUntraced(function* (threadId: ThreadId) {
- const sessions = yield* providerService.listSessions();
- const session = sessions.find((entry) => entry.threadId === threadId);
- return session?.activeTurnId;
- });
-
- const getSourceProposedPlanReferenceForAcceptedTurnStart = Effect.fnUntraced(function* (
- threadId: ThreadId,
- eventTurnId: TurnId | undefined,
- ) {
+ const getExpectedProviderTurnIdForThread = Effect.fn("getExpectedProviderTurnIdForThread")(
+ function* (threadId: ThreadId) {
+ const sessions = yield* providerService.listSessions();
+ const session = sessions.find((entry) => entry.threadId === threadId);
+ return session?.activeTurnId;
+ },
+ );
+
+ const getSourceProposedPlanReferenceForAcceptedTurnStart = Effect.fn(
+ "getSourceProposedPlanReferenceForAcceptedTurnStart",
+ )(function* (threadId: ThreadId, eventTurnId: TurnId | undefined) {
if (eventTurnId === undefined) {
return null;
}
@@ -840,34 +841,36 @@ const make = Effect.fn("make")(function* () {
return yield* getSourceProposedPlanReferenceForPendingTurnStart(threadId);
});
- const markSourceProposedPlanImplemented = Effect.fnUntraced(function* (
- sourceThreadId: ThreadId,
- sourcePlanId: OrchestrationProposedPlanId,
- implementationThreadId: ThreadId,
- implementedAt: string,
- ) {
- const readModel = yield* orchestrationEngine.getReadModel();
- const sourceThread = readModel.threads.find((entry) => entry.id === sourceThreadId);
- const sourcePlan = sourceThread?.proposedPlans.find((entry) => entry.id === sourcePlanId);
- if (!sourceThread || !sourcePlan || sourcePlan.implementedAt !== null) {
- return;
- }
+ const markSourceProposedPlanImplemented = Effect.fn("markSourceProposedPlanImplemented")(
+ function* (
+ sourceThreadId: ThreadId,
+ sourcePlanId: OrchestrationProposedPlanId,
+ implementationThreadId: ThreadId,
+ implementedAt: string,
+ ) {
+ const readModel = yield* orchestrationEngine.getReadModel();
+ const sourceThread = readModel.threads.find((entry) => entry.id === sourceThreadId);
+ const sourcePlan = sourceThread?.proposedPlans.find((entry) => entry.id === sourcePlanId);
+ if (!sourceThread || !sourcePlan || sourcePlan.implementedAt !== null) {
+ return;
+ }
- yield* orchestrationEngine.dispatch({
- type: "thread.proposed-plan.upsert",
- commandId: CommandId.makeUnsafe(
- `provider:source-proposed-plan-implemented:${implementationThreadId}:${crypto.randomUUID()}`,
- ),
- threadId: sourceThread.id,
- proposedPlan: {
- ...sourcePlan,
- implementedAt,
- implementationThreadId,
- updatedAt: implementedAt,
- },
- createdAt: implementedAt,
- });
- });
+ yield* orchestrationEngine.dispatch({
+ type: "thread.proposed-plan.upsert",
+ commandId: CommandId.makeUnsafe(
+ `provider:source-proposed-plan-implemented:${implementationThreadId}:${crypto.randomUUID()}`,
+ ),
+ threadId: sourceThread.id,
+ proposedPlan: {
+ ...sourcePlan,
+ implementedAt,
+ implementationThreadId,
+ updatedAt: implementedAt,
+ },
+ createdAt: implementedAt,
+ });
+ },
+ );
const processRuntimeEvent = Effect.fn("processRuntimeEvent")(function* (
event: ProviderRuntimeEvent,
diff --git a/apps/server/src/persistence/Layers/Sqlite.ts b/apps/server/src/persistence/Layers/Sqlite.ts
index 07cbf4479..58556099d 100644
--- a/apps/server/src/persistence/Layers/Sqlite.ts
+++ b/apps/server/src/persistence/Layers/Sqlite.ts
@@ -6,6 +6,7 @@ import { ServerConfig } from "../../config.ts";
type RuntimeSqliteLayerConfig = {
readonly filename: string;
+ readonly spanAttributes?: Record<string, unknown>;
};
type Loader = {
@@ -41,7 +42,16 @@ export const makeSqlitePersistenceLive = Effect.fn("makeSqlitePersistenceLive")(
const path = yield* Path.Path;
yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true });
- return Layer.provideMerge(setup, makeRuntimeSqliteLayer({ filename: dbPath }));
+ return Layer.provideMerge(
+ setup,
+ makeRuntimeSqliteLayer({
+ filename: dbPath,
+ spanAttributes: {
+ "db.name": path.basename(dbPath),
+ "service.name": "t3-server",
+ },
+ }),
+ );
}, Layer.unwrap);
export const SqlitePersistenceMemory = Layer.provideMerge(
diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.ts
index 651a61164..cd6c81405 100644
--- a/apps/server/src/provider/Layers/ProviderService.test.ts
+++ b/apps/server/src/provider/Layers/ProviderService.test.ts
@@ -20,7 +20,7 @@ import {
import { it, assert, vi } from "@effect/vitest";
import { assertFailure } from "@effect/vitest/utils";
-import { Effect, Fiber, Layer, Option, PubSub, Ref, Stream } from "effect";
+import { Effect, Fiber, Layer, Metric, Option, PubSub, Ref, Stream } from "effect";
import * as SqlClient from "effect/unstable/sql/SqlClient";
import {
@@ -230,6 +230,17 @@ function makeFakeCodexAdapter(provider: ProviderKind = "codex") {
const sleep = (ms: number) =>
Effect.promise(() => new Promise<void>((resolve) => setTimeout(resolve, ms)));
+const hasMetricSnapshot = (
+ snapshots: ReadonlyArray<Metric.Metric.Snapshot>,
+ id: string,
+ attributes: Readonly<Record<string, string>>,
+) =>
+ snapshots.some(
+ (snapshot) =>
+ snapshot.id === id &&
+ Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),
+ );
+
function makeProviderServiceLayer() {
const codex = makeFakeCodexAdapter();
const claude = makeFakeCodexAdapter("claudeAgent");
@@ -1060,6 +1071,120 @@ fanout.layer("ProviderServiceLive fanout", (it) => {
);
}),
);
+
+ it.effect("records provider metrics with the routed provider label", () =>
+ Effect.gen(function* () {
+ const provider = yield* ProviderService;
+
+ const session = yield* provider.startSession(asThreadId("thread-metrics"), {
+ provider: "claudeAgent",
+ threadId: asThreadId("thread-metrics"),
+ cwd: "/tmp/project",
+ runtimeMode: "full-access",
+ });
+
+ yield* provider.interruptTurn({ threadId: session.threadId });
+ yield* provider.respondToRequest({
+ threadId: session.threadId,
+ requestId: asRequestId("req-metrics-1"),
+ decision: "accept",
+ });
+ yield* provider.respondToUserInput({
+ threadId: session.threadId,
+ requestId: asRequestId("req-metrics-2"),
+ answers: {
+ sandbox_mode: "workspace-write",
+ },
+ });
+ yield* provider.rollbackConversation({
+ threadId: session.threadId,
+ numTurns: 1,
+ });
+ yield* provider.stopSession({ threadId: session.threadId });
+
+ const snapshots = yield* Metric.snapshot;
+
+ assert.equal(
+ hasMetricSnapshot(snapshots, "t3_provider_turns_total", {
+ provider: "claudeAgent",
+ operation: "interrupt",
+ outcome: "success",
+ }),
+ true,
+ );
+ assert.equal(
+ hasMetricSnapshot(snapshots, "t3_provider_turns_total", {
+ provider: "claudeAgent",
+ operation: "approval-response",
+ outcome: "success",
+ }),
+ true,
+ );
+ assert.equal(
+ hasMetricSnapshot(snapshots, "t3_provider_turns_total", {
+ provider: "claudeAgent",
+ operation: "user-input-response",
+ outcome: "success",
+ }),
+ true,
+ );
+ assert.equal(
+ hasMetricSnapshot(snapshots, "t3_provider_turns_total", {
+ provider: "claudeAgent",
+ operation: "rollback",
+ outcome: "success",
+ }),
+ true,
+ );
+ assert.equal(
+ hasMetricSnapshot(snapshots, "t3_provider_sessions_total", {
+ provider: "claudeAgent",
+ operation: "stop",
+ outcome: "success",
+ }),
+ true,
+ );
+ }),
+ );
+
+ it.effect(
+ "records sendTurn metrics with the resolved provider when modelSelection is omitted",
+ () =>
+ Effect.gen(function* () {
+ const provider = yield* ProviderService;
+
+ const session = yield* provider.startSession(asThreadId("thread-send-metrics"), {
+ provider: "claudeAgent",
+ threadId: asThreadId("thread-send-metrics"),
+ cwd: "/tmp/project-send-metrics",
+ runtimeMode: "full-access",
+ });
+
+ yield* provider.sendTurn({
+ threadId: session.threadId,
+ input: "hello",
+ attachments: [],
+ });
+
+ const snapshots = yield* Metric.snapshot;
+
+ assert.equal(
+ hasMetricSnapshot(snapshots, "t3_provider_turns_total", {
+ provider: "claudeAgent",
+ operation: "send",
+ outcome: "success",
+ }),
+ true,
+ );
+ assert.equal(
+ hasMetricSnapshot(snapshots, "t3_provider_turn_duration", {
+ provider: "claudeAgent",
+ operation: "send",
+ }),
+ true,
+ );
+ }),
+ );
});
const validation = makeProviderServiceLayer();
diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts
index f34882959..da2c293fa 100644
--- a/apps/server/src/provider/Layers/ProviderService.ts
+++ b/apps/server/src/provider/Layers/ProviderService.ts
@@ -24,6 +24,16 @@ import {
} from "@t3tools/contracts";
import { Effect, Layer, Option, PubSub, Queue, Schema, SchemaIssue, Stream } from "effect";
+import {
+ increment,
+ providerMetricAttributes,
+ providerRuntimeEventsTotal,
+ providerSessionsTotal,
+ providerTurnDuration,
+ providerTurnsTotal,
+ providerTurnMetricAttributes,
+ withMetrics,
+} from "../../observability/Metrics.ts";
import { ProviderValidationError } from "../Errors.ts";
import { ProviderAdapterRegistry } from "../Services/ProviderAdapterRegistry.ts";
import { ProviderService, type ProviderServiceShape } from "../Services/ProviderService.ts";
@@ -179,7 +189,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
const providers = yield* registry.listProviders();
const adapters = yield* Effect.forEach(providers, (provider) => registry.getByProvider(provider));
const processRuntimeEvent = (event: ProviderRuntimeEvent): Effect.Effect<void> =>
- publishRuntimeEvent(event);
+ increment(providerRuntimeEventsTotal, {
+ provider: event.provider,
+ eventType: event.type,
+ }).pipe(Effect.andThen(publishRuntimeEvent(event)));
const worker = Effect.forever(
Queue.take(runtimeEventQueue).pipe(Effect.flatMap(processRuntimeEvent)),
@@ -196,58 +209,72 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
readonly binding: ProviderRuntimeBinding;
readonly operation: string;
}) {
- const adapter = yield* registry.getByProvider(input.binding.provider);
- const hasResumeCursor =
- input.binding.resumeCursor !== null && input.binding.resumeCursor !== undefined;
- const hasActiveSession = yield* adapter.hasSession(input.binding.threadId);
- if (hasActiveSession) {
- const activeSessions = yield* adapter.listSessions();
- const existing = activeSessions.find(
- (session) => session.threadId === input.binding.threadId,
- );
- if (existing) {
- yield* upsertSessionBinding(existing, input.binding.threadId);
- yield* analytics.record("provider.session.recovered", {
- provider: existing.provider,
- strategy: "adopt-existing",
- hasResumeCursor: existing.resumeCursor !== undefined,
- });
- return { adapter, session: existing } as const;
+ yield* Effect.annotateCurrentSpan({
+ "provider.operation": "recover-session",
+ "provider.kind": input.binding.provider,
+ "provider.thread_id": input.binding.threadId,
+ });
+ return yield* Effect.gen(function* () {
+ const adapter = yield* registry.getByProvider(input.binding.provider);
+ const hasResumeCursor =
+ input.binding.resumeCursor !== null && input.binding.resumeCursor !== undefined;
+ const hasActiveSession = yield* adapter.hasSession(input.binding.threadId);
+ if (hasActiveSession) {
+ const activeSessions = yield* adapter.listSessions();
+ const existing = activeSessions.find(
+ (session) => session.threadId === input.binding.threadId,
+ );
+ if (existing) {
+ yield* upsertSessionBinding(existing, input.binding.threadId);
+ yield* analytics.record("provider.session.recovered", {
+ provider: existing.provider,
+ strategy: "adopt-existing",
+ hasResumeCursor: existing.resumeCursor !== undefined,
+ });
+ return { adapter, session: existing } as const;
+ }
}
- }
- if (!hasResumeCursor) {
- return yield* toValidationError(
- input.operation,
- `Cannot recover thread '${input.binding.threadId}' because no provider resume state is persisted.`,
- );
- }
+ if (!hasResumeCursor) {
+ return yield* toValidationError(
+ input.operation,
+ `Cannot recover thread '${input.binding.threadId}' because no provider resume state is persisted.`,
+ );
+ }
- const persistedCwd = readPersistedCwd(input.binding.runtimePayload);
- const persistedModelSelection = readPersistedModelSelection(input.binding.runtimePayload);
+ const persistedCwd = readPersistedCwd(input.binding.runtimePayload);
+ const persistedModelSelection = readPersistedModelSelection(input.binding.runtimePayload);
- const resumed = yield* adapter.startSession({
- threadId: input.binding.threadId,
- provider: input.binding.provider,
- ...(persistedCwd ? { cwd: persistedCwd } : {}),
- ...(persistedModelSelection ? { modelSelection: persistedModelSelection } : {}),
- ...(hasResumeCursor ? { resumeCursor: input.binding.resumeCursor } : {}),
- runtimeMode: input.binding.runtimeMode ?? "full-access",
- });
- if (resumed.provider !== adapter.provider) {
- return yield* toValidationError(
- input.operation,
- `Adapter/provider mismatch while recovering thread '${input.binding.threadId}'. Expected '${adapter.provider}', received '${resumed.provider}'.`,
- );
- }
+ const resumed = yield* adapter.startSession({
+ threadId: input.binding.threadId,
+ provider: input.binding.provider,
+ ...(persistedCwd ? { cwd: persistedCwd } : {}),
+ ...(persistedModelSelection ? { modelSelection: persistedModelSelection } : {}),
+ ...(hasResumeCursor ? { resumeCursor: input.binding.resumeCursor } : {}),
+ runtimeMode: input.binding.runtimeMode ?? "full-access",
+ });
+ if (resumed.provider !== adapter.provider) {
+ return yield* toValidationError(
+ input.operation,
+ `Adapter/provider mismatch while recovering thread '${input.binding.threadId}'. Expected '${adapter.provider}', received '${resumed.provider}'.`,
+ );
+ }
- yield* upsertSessionBinding(resumed, input.binding.threadId);
- yield* analytics.record("provider.session.recovered", {
- provider: resumed.provider,
- strategy: "resume-thread",
- hasResumeCursor: resumed.resumeCursor !== undefined,
- });
- return { adapter, session: resumed } as const;
+ yield* upsertSessionBinding(resumed, input.binding.threadId);
+ yield* analytics.record("provider.session.recovered", {
+ provider: resumed.provider,
+ strategy: "resume-thread",
+ hasResumeCursor: resumed.resumeCursor !== undefined,
+ });
+ return { adapter, session: resumed } as const;
+ }).pipe(
+ withMetrics({
+ counter: providerSessionsTotal,
+ attributes: providerMetricAttributes(input.binding.provider, {
+ operation: "recover",
+ }),
+ }),
+ );
});
const resolveRoutableSession = Effect.fn("resolveRoutableSession")(function* (input: {
@@ -291,52 +318,69 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
threadId,
provider: parsed.provider ?? "codex",
};
- const settings = yield* serverSettings.getSettings.pipe(
- Effect.mapError((error) =>
- toValidationError(
- "ProviderService.startSession",
- `Failed to load provider settings: ${error.message}`,
- error,
+ yield* Effect.annotateCurrentSpan({
+ "provider.operation": "start-session",
+ "provider.kind": input.provider,
+ "provider.thread_id": threadId,
+ "provider.runtime_mode": input.runtimeMode,
+ });
+ return yield* Effect.gen(function* () {
+ const settings = yield* serverSettings.getSettings.pipe(
+ Effect.mapError((error) =>
+ toValidationError(
+ "ProviderService.startSession",
+ `Failed to load provider settings: ${error.message}`,
+ error,
+ ),
),
- ),
- );
- if (!settings.providers[input.provider].enabled) {
- return yield* toValidationError(
- "ProviderService.startSession",
- `Provider '${input.provider}' is disabled in T3 Code settings.`,
);
- }
- const persistedBinding = Option.getOrUndefined(yield* directory.getBinding(threadId));
- const effectiveResumeCursor =
- input.resumeCursor ??
- (persistedBinding?.provider === input.provider ? persistedBinding.resumeCursor : undefined);
- const adapter = yield* registry.getByProvider(input.provider);
- const session = yield* adapter.startSession({
- ...input,
- ...(effectiveResumeCursor !== undefined ? { resumeCursor: effectiveResumeCursor } : {}),
- });
+ if (!settings.providers[input.provider].enabled) {
+ return yield* toValidationError(
+ "ProviderService.startSession",
+ `Provider '${input.provider}' is disabled in T3 Code settings.`,
+ );
+ }
+ const persistedBinding = Option.getOrUndefined(yield* directory.getBinding(threadId));
+ const effectiveResumeCursor =
+ input.resumeCursor ??
+ (persistedBinding?.provider === input.provider
+ ? persistedBinding.resumeCursor
+ : undefined);
+ const adapter = yield* registry.getByProvider(input.provider);
+ const session = yield* adapter.startSession({
+ ...input,
+ ...(effectiveResumeCursor !== undefined ? { resumeCursor: effectiveResumeCursor } : {}),
+ });
- if (session.provider !== adapter.provider) {
- return yield* toValidationError(
- "ProviderService.startSession",
- `Adapter/provider mismatch: requested '${adapter.provider}', received '${session.provider}'.`,
- );
- }
+ if (session.provider !== adapter.provider) {
+ return yield* toValidationError(
+ "ProviderService.startSession",
+ `Adapter/provider mismatch: requested '${adapter.provider}', received '${session.provider}'.`,
+ );
+ }
- yield* upsertSessionBinding(session, threadId, {
- modelSelection: input.modelSelection,
- });
- yield* analytics.record("provider.session.started", {
- provider: session.provider,
- runtimeMode: input.runtimeMode,
- hasResumeCursor: session.resumeCursor !== undefined,
- hasCwd: typeof input.cwd === "string" && input.cwd.trim().length > 0,
- hasModel:
- typeof input.modelSelection?.model === "string" &&
- input.modelSelection.model.trim().length > 0,
- });
+ yield* upsertSessionBinding(session, threadId, {
+ modelSelection: input.modelSelection,
+ });
+ yield* analytics.record("provider.session.started", {
+ provider: session.provider,
+ runtimeMode: input.runtimeMode,
+ hasResumeCursor: session.resumeCursor !== undefined,
+ hasCwd: typeof input.cwd === "string" && input.cwd.trim().length > 0,
+ hasModel:
+ typeof input.modelSelection?.model === "string" &&
+ input.modelSelection.model.trim().length > 0,
+ });
- return session;
+ return session;
+ }).pipe(
+ withMetrics({
+ counter: providerSessionsTotal,
+ attributes: providerMetricAttributes(input.provider, {
+ operation: "start",
+ }),
+ }),
+ );
},
);
@@ -357,32 +401,61 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
"Either input text or at least one attachment is required",
);
}
- const routed = yield* resolveRoutableSession({
- threadId: input.threadId,
- operation: "ProviderService.sendTurn",
- allowRecovery: true,
- });
- const turn = yield* routed.adapter.sendTurn(input);
- yield* directory.upsert({
- threadId: input.threadId,
- provider: routed.adapter.provider,
- status: "running",
- ...(turn.resumeCursor !== undefined ? { resumeCursor: turn.resumeCursor } : {}),
- runtimePayload: {
- ...(input.modelSelection !== undefined ? { modelSelection: input.modelSelection } : {}),
- activeTurnId: turn.turnId,
- lastRuntimeEvent: "provider.sendTurn",
- lastRuntimeEventAt: new Date().toISOString(),
- },
- });
- yield* analytics.record("provider.turn.sent", {
- provider: routed.adapter.provider,
- model: input.modelSelection?.model,
- interactionMode: input.interactionMode,
- attachmentCount: input.attachments.length,
- hasInput: typeof input.input === "string" && input.input.trim().length > 0,
+ yield* Effect.annotateCurrentSpan({
+ "provider.operation": "send-turn",
+ "provider.thread_id": input.threadId,
+ "provider.interaction_mode": input.interactionMode,
+ "provider.attachment_count": input.attachments.length,
});
- return turn;
+ let metricProvider = "unknown";
+ let metricModel = input.modelSelection?.model;
+ return yield* Effect.gen(function* () {
+ const routed = yield* resolveRoutableSession({
+ threadId: input.threadId,
+ operation: "ProviderService.sendTurn",
+ allowRecovery: true,
+ });
+ metricProvider = routed.adapter.provider;
+ metricModel = input.modelSelection?.model;
+ yield* Effect.annotateCurrentSpan({
+ "provider.kind": routed.adapter.provider,
+ ...(input.modelSelection?.model ? { "provider.model": input.modelSelection.model } : {}),
+ });
+ const turn = yield* routed.adapter.sendTurn(input);
+ yield* directory.upsert({
+ threadId: input.threadId,
+ provider: routed.adapter.provider,
+ status: "running",
+ ...(turn.resumeCursor !== undefined ? { resumeCursor: turn.resumeCursor } : {}),
+ runtimePayload: {
+ ...(input.modelSelection !== undefined ? { modelSelection: input.modelSelection } : {}),
+ activeTurnId: turn.turnId,
+ lastRuntimeEvent: "provider.sendTurn",
+ lastRuntimeEventAt: new Date().toISOString(),
+ },
+ });
+ yield* analytics.record("provider.turn.sent", {
+ provider: routed.adapter.provider,
+ model: input.modelSelection?.model,
+ interactionMode: input.interactionMode,
+ attachmentCount: input.attachments.length,
+ hasInput: typeof input.input === "string" && input.input.trim().length > 0,
+ });
+ return turn;
+ }).pipe(
+ withMetrics({
+ counter: providerTurnsTotal,
+ timer: providerTurnDuration,
+ attributes: () =>
+ providerTurnMetricAttributes({
+ provider: metricProvider,
+ model: metricModel,
+ extra: {
+ operation: "send",
+ },
+ }),
+ }),
+ );
});
const interruptTurn: ProviderServiceShape["interruptTurn"] = Effect.fn("interruptTurn")(
@@ -392,15 +465,33 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
schema: ProviderInterruptTurnInput,
payload: rawInput,
});
- const routed = yield* resolveRoutableSession({
- threadId: input.threadId,
- operation: "ProviderService.interruptTurn",
- allowRecovery: true,
- });
- yield* routed.adapter.interruptTurn(routed.threadId, input.turnId);
- yield* analytics.record("provider.turn.interrupted", {
- provider: routed.adapter.provider,
- });
+ let metricProvider = "unknown";
+ return yield* Effect.gen(function* () {
+ const routed = yield* resolveRoutableSession({
+ threadId: input.threadId,
+ operation: "ProviderService.interruptTurn",
+ allowRecovery: true,
+ });
+ metricProvider = routed.adapter.provider;
+ yield* Effect.annotateCurrentSpan({
+ "provider.operation": "interrupt-turn",
+ "provider.kind": routed.adapter.provider,
+ "provider.thread_id": input.threadId,
+ "provider.turn_id": input.turnId,
+ });
+ yield* routed.adapter.interruptTurn(routed.threadId, input.turnId);
+ yield* analytics.record("provider.turn.interrupted", {
+ provider: routed.adapter.provider,
+ });
+ }).pipe(
+ withMetrics({
+ counter: providerTurnsTotal,
+ outcomeAttributes: () =>
+ providerMetricAttributes(metricProvider, {
+ operation: "interrupt",
+ }),
+ }),
+ );
},
);
@@ -411,16 +502,34 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
schema: ProviderRespondToRequestInput,
payload: rawInput,
});
- const routed = yield* resolveRoutableSession({
- threadId: input.threadId,
- operation: "ProviderService.respondToRequest",
- allowRecovery: true,
- });
- yield* routed.adapter.respondToRequest(routed.threadId, input.requestId, input.decision);
- yield* analytics.record("provider.request.responded", {
- provider: routed.adapter.provider,
- decision: input.decision,
- });
+ let metricProvider = "unknown";
+ return yield* Effect.gen(function* () {
+ const routed = yield* resolveRoutableSession({
+ threadId: input.threadId,
+ operation: "ProviderService.respondToRequest",
+ allowRecovery: true,
+ });
+ metricProvider = routed.adapter.provider;
+ yield* Effect.annotateCurrentSpan({
+ "provider.operation": "respond-to-request",
+ "provider.kind": routed.adapter.provider,
+ "provider.thread_id": input.threadId,
+ "provider.request_id": input.requestId,
+ });
+ yield* routed.adapter.respondToRequest(routed.threadId, input.requestId, input.decision);
+ yield* analytics.record("provider.request.responded", {
+ provider: routed.adapter.provider,
+ decision: input.decision,
+ });
+ }).pipe(
+ withMetrics({
+ counter: providerTurnsTotal,
+ outcomeAttributes: () =>
+ providerMetricAttributes(metricProvider, {
+ operation: "approval-response",
+ }),
+ }),
+ );
},
);
@@ -432,12 +541,30 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
schema: ProviderRespondToUserInputInput,
payload: rawInput,
});
- const routed = yield* resolveRoutableSession({
- threadId: input.threadId,
- operation: "ProviderService.respondToUserInput",
- allowRecovery: true,
- });
- yield* routed.adapter.respondToUserInput(routed.threadId, input.requestId, input.answers);
+ let metricProvider = "unknown";
+ return yield* Effect.gen(function* () {
+ const routed = yield* resolveRoutableSession({
+ threadId: input.threadId,
+ operation: "ProviderService.respondToUserInput",
+ allowRecovery: true,
+ });
+ metricProvider = routed.adapter.provider;
+ yield* Effect.annotateCurrentSpan({
+ "provider.operation": "respond-to-user-input",
+ "provider.kind": routed.adapter.provider,
+ "provider.thread_id": input.threadId,
+ "provider.request_id": input.requestId,
+ });
+ yield* routed.adapter.respondToUserInput(routed.threadId, input.requestId, input.answers);
+ }).pipe(
+ withMetrics({
+ counter: providerTurnsTotal,
+ outcomeAttributes: () =>
+ providerMetricAttributes(metricProvider, {
+ operation: "user-input-response",
+ }),
+ }),
+ );
});
const stopSession: ProviderServiceShape["stopSession"] = Effect.fn("stopSession")(
@@ -447,18 +574,35 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
schema: ProviderStopSessionInput,
payload: rawInput,
});
- const routed = yield* resolveRoutableSession({
- threadId: input.threadId,
- operation: "ProviderService.stopSession",
- allowRecovery: false,
- });
- if (routed.isActive) {
- yield* routed.adapter.stopSession(routed.threadId);
- }
- yield* directory.remove(input.threadId);
- yield* analytics.record("provider.session.stopped", {
- provider: routed.adapter.provider,
- });
+ let metricProvider = "unknown";
+ return yield* Effect.gen(function* () {
+ const routed = yield* resolveRoutableSession({
+ threadId: input.threadId,
+ operation: "ProviderService.stopSession",
+ allowRecovery: false,
+ });
+ metricProvider = routed.adapter.provider;
+ yield* Effect.annotateCurrentSpan({
+ "provider.operation": "stop-session",
+ "provider.kind": routed.adapter.provider,
+ "provider.thread_id": input.threadId,
+ });
+ if (routed.isActive) {
+ yield* routed.adapter.stopSession(routed.threadId);
+ }
+ yield* directory.remove(input.threadId);
+ yield* analytics.record("provider.session.stopped", {
+ provider: routed.adapter.provider,
+ });
+ }).pipe(
+ withMetrics({
+ counter: providerSessionsTotal,
+ outcomeAttributes: () =>
+ providerMetricAttributes(metricProvider, {
+ operation: "stop",
+ }),
+ }),
+ );
},
);
@@ -524,16 +668,34 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
if (input.numTurns === 0) {
return;
}
- const routed = yield* resolveRoutableSession({
- threadId: input.threadId,
- operation: "ProviderService.rollbackConversation",
- allowRecovery: true,
- });
- yield* routed.adapter.rollbackThread(routed.threadId, input.numTurns);
- yield* analytics.record("provider.conversation.rolled_back", {
- provider: routed.adapter.provider,
- turns: input.numTurns,
- });
+ let metricProvider = "unknown";
+ return yield* Effect.gen(function* () {
+ const routed = yield* resolveRoutableSession({
+ threadId: input.threadId,
+ operation: "ProviderService.rollbackConversation",
+ allowRecovery: true,
+ });
+ metricProvider = routed.adapter.provider;
+ yield* Effect.annotateCurrentSpan({
+ "provider.operation": "rollback-conversation",
+ "provider.kind": routed.adapter.provider,
+ "provider.thread_id": input.threadId,
+ "provider.rollback_turns": input.numTurns,
+ });
+ yield* routed.adapter.rollbackThread(routed.threadId, input.numTurns);
+ yield* analytics.record("provider.conversation.rolled_back", {
+ provider: routed.adapter.provider,
+ turns: input.numTurns,
+ });
+ }).pipe(
+ withMetrics({
+ counter: providerTurnsTotal,
+ outcomeAttributes: () =>
+ providerMetricAttributes(metricProvider, {
+ operation: "rollback",
+ }),
+ }),
+ );
});
const runStopAll = Effect.fn("runStopAll")(function* () {
diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts
index 4367ca40a..0e8b909c9 100644
--- a/apps/server/src/server.test.ts
+++ b/apps/server/src/server.test.ts
@@ -139,8 +139,17 @@ const buildAppUnderTest = (options?: {
const baseDir = options?.config?.baseDir ?? tempBaseDir;
const devUrl = options?.config?.devUrl;
const derivedPaths = yield* deriveServerPaths(baseDir, devUrl);
- const config = {
+ const config: ServerConfigShape = {
logLevel: "Info",
+ traceMinLevel: "Info",
+ traceTimingEnabled: true,
+ traceBatchWindowMs: 200,
+ traceMaxBytes: 10 * 1024 * 1024,
+ traceMaxFiles: 10,
+ otlpTracesUrl: undefined,
+ otlpMetricsUrl: undefined,
+ otlpExportIntervalMs: 10_000,
+ otlpServiceName: "t3-server",
mode: "web",
port: 0,
host: "127.0.0.1",
@@ -154,7 +163,7 @@ const buildAppUnderTest = (options?: {
autoBootstrapProjectFromCwd: false,
logWebSocketEvents: false,
...options?.config,
- } satisfies ServerConfigShape;
+ };
const layerConfig = Layer.succeed(ServerConfig, config);
const appLayer = HttpRouter.serve(makeRoutesLayer, {
@@ -533,6 +542,10 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
} as const;
yield* buildAppUnderTest({
+ config: {
+ otlpTracesUrl: "http://localhost:4318/v1/traces",
+ otlpMetricsUrl: "http://localhost:4318/v1/metrics",
+ },
layers: {
keybindings: {
loadConfigState: Effect.succeed({
@@ -561,6 +574,12 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
assert.deepEqual(first.config.keybindings, []);
assert.deepEqual(first.config.issues, []);
assert.deepEqual(first.config.providers, providers);
+ assert.equal(first.config.observability.logsDirectoryPath.endsWith("/logs"), true);
+ assert.equal(first.config.observability.localTracingEnabled, true);
+ assert.equal(first.config.observability.otlpTracesUrl, "http://localhost:4318/v1/traces");
+ assert.equal(first.config.observability.otlpTracesEnabled, true);
+ assert.equal(first.config.observability.otlpMetricsUrl, "http://localhost:4318/v1/metrics");
+ assert.equal(first.config.observability.otlpMetricsEnabled, true);
assert.deepEqual(first.config.settings, DEFAULT_SERVER_SETTINGS);
}
assert.deepEqual(second, {
diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts
index 4e296e817..f630444a6 100644
--- a/apps/server/src/server.ts
+++ b/apps/server/src/server.ts
@@ -29,7 +29,6 @@ import { RoutingTextGenerationLive } from "./git/Layers/RoutingTextGeneration";
import { TerminalManagerLive } from "./terminal/Layers/Manager";
import { GitManagerLive } from "./git/Layers/GitManager";
import { KeybindingsLive } from "./keybindings";
-import { ServerLoggerLive } from "./serverLogger";
import { ServerRuntimeStartup, ServerRuntimeStartupLive } from "./serverRuntimeStartup";
import { OrchestrationReactorLive } from "./orchestration/Layers/OrchestrationReactor";
import { RuntimeReceiptBusLive } from "./orchestration/Layers/RuntimeReceiptBus";
@@ -42,6 +41,7 @@ import { ProjectFaviconResolverLive } from "./project/Layers/ProjectFaviconResol
import { WorkspaceEntriesLive } from "./workspace/Layers/WorkspaceEntries";
import { WorkspaceFileSystemLive } from "./workspace/Layers/WorkspaceFileSystem";
import { WorkspacePathsLive } from "./workspace/Layers/WorkspacePaths";
+import { ObservabilityLive } from "./observability/Layers/Observability";
const PtyAdapterLive = Layer.unwrap(
Effect.gen(function* () {
@@ -231,7 +231,7 @@ export const makeServerLayer = Layer.unwrap(
return serverApplicationLayer.pipe(
Layer.provideMerge(RuntimeServicesLive),
Layer.provideMerge(HttpServerLive),
- Layer.provide(ServerLoggerLive),
+ Layer.provide(ObservabilityLive),
Layer.provideMerge(FetchHttpClient.layer),
Layer.provideMerge(PlatformServicesLive),
);
diff --git a/apps/server/src/serverLogger.ts b/apps/server/src/serverLogger.ts
index aea53aacf..ea098dcbb 100644
--- a/apps/server/src/serverLogger.ts
+++ b/apps/server/src/serverLogger.ts
@@ -4,11 +4,8 @@ import { ServerConfig } from "./config";
export const ServerLoggerLive = Effect.gen(function* () {
const config = yield* ServerConfig;
- const { serverLogPath } = config;
-
- const fileLogger = Logger.formatSimple.pipe(Logger.toFile(serverLogPath));
const minimumLogLevelLayer = Layer.succeed(References.MinimumLogLevel, config.logLevel);
- const loggerLayer = Logger.layer([Logger.consolePretty(), fileLogger], {
+ const loggerLayer = Logger.layer([Logger.consolePretty(), Logger.tracerLogger], {
mergeWithExisting: false,
});
diff --git a/apps/server/src/serverRuntimeStartup.ts b/apps/server/src/serverRuntimeStartup.ts
index 026145eca..7c9231ac9 100644
--- a/apps/server/src/serverRuntimeStartup.ts
+++ b/apps/server/src/serverRuntimeStartup.ts
@@ -141,6 +141,8 @@ export const recordStartupHeartbeat = Effect.gen(function* () {
});
export const launchStartupHeartbeat = recordStartupHeartbeat.pipe(
+ Effect.annotateSpans({ "startup.phase": "heartbeat.record" }),
+ Effect.withSpan("server.startup.heartbeat.record"),
Effect.ignoreCause({ log: true }),
Effect.forkScoped,
Effect.asVoid,
@@ -248,7 +250,14 @@ const maybeOpenBrowser = Effect.gen(function* () {
);
});
+const runStartupPhase = <A, E, R>(phase: string, effect: Effect.Effect<A, E, R>) =>
+ effect.pipe(
+ Effect.annotateSpans({ "startup.phase": phase }),
+ Effect.withSpan(`server.startup.${phase}`),
+ );
+
const makeServerRuntimeStartup = Effect.gen(function* () {
+ const serverConfig = yield* ServerConfig;
const keybindings = yield* Keybindings;
const orchestrationReactor = yield* OrchestrationReactor;
const lifecycleEvents = yield* ServerLifecycleEvents;
@@ -262,46 +271,65 @@ const makeServerRuntimeStartup = Effect.gen(function* () {
const startup = Effect.gen(function* () {
yield* Effect.logDebug("startup phase: starting keybindings runtime");
- yield* keybindings.start.pipe(
- Effect.catch((error) =>
- Effect.logWarning("failed to start keybindings runtime", {
- path: error.configPath,
- detail: error.detail,
- cause: error.cause,
- }),
+ yield* runStartupPhase(
+ "keybindings.start",
+ keybindings.start.pipe(
+ Effect.catch((error) =>
+ Effect.logWarning("failed to start keybindings runtime", {
+ path: error.configPath,
+ detail: error.detail,
+ cause: error.cause,
+ }),
+ ),
+ Effect.forkScoped,
),
- Effect.forkScoped,
);
yield* Effect.logDebug("startup phase: starting server settings runtime");
- yield* serverSettings.start.pipe(
- Effect.catch((error) =>
- Effect.logWarning("failed to start server settings runtime", {
- path: error.settingsPath,
- detail: error.detail,
- cause: error.cause,
- }),
+ yield* runStartupPhase(
+ "settings.start",
+ serverSettings.start.pipe(
+ Effect.catch((error) =>
+ Effect.logWarning("failed to start server settings runtime", {
+ path: error.settingsPath,
+ detail: error.detail,
+ cause: error.cause,
+ }),
+ ),
+ Effect.forkScoped,
),
- Effect.forkScoped,
);
yield* Effect.logDebug("startup phase: starting orchestration reactors");
- yield* orchestrationReactor.start().pipe(Scope.provide(reactorScope));
+ yield* runStartupPhase(
+ "reactors.start",
+ orchestrationReactor.start().pipe(Scope.provide(reactorScope)),
+ );
yield* Effect.logDebug("startup phase: preparing welcome payload");
- const welcome = yield* autoBootstrapWelcome;
+ const welcome = yield* runStartupPhase("welcome.prepare", autoBootstrapWelcome);
yield* Effect.logDebug("startup phase: publishing welcome event", {
cwd: welcome.cwd,
projectName: welcome.projectName,
bootstrapProjectId: welcome.bootstrapProjectId,
bootstrapThreadId: welcome.bootstrapThreadId,
});
- yield* lifecycleEvents.publish({
- version: 1,
- type: "welcome",
- payload: welcome,
- });
- });
+ yield* runStartupPhase(
+ "welcome.publish",
+ lifecycleEvents.publish({
+ version: 1,
+ type: "welcome",
+ payload: welcome,
+ }),
+ );
+ }).pipe(
+ Effect.annotateSpans({
+ "server.mode": serverConfig.mode,
+ "server.port": serverConfig.port,
+ "server.host": serverConfig.host ?? "default",
+ }),
+ Effect.withSpan("server.startup", { kind: "server", root: true }),
+ );
yield* Effect.forkScoped(
Effect.gen(function* () {
@@ -319,18 +347,21 @@ const makeServerRuntimeStartup = Effect.gen(function* () {
yield* Effect.logDebug("Accepting commands");
yield* commandGate.signalCommandReady;
yield* Effect.logDebug("startup phase: waiting for http listener");
- yield* Deferred.await(httpListening);
+ yield* runStartupPhase("http.wait", Deferred.await(httpListening));
yield* Effect.logDebug("startup phase: publishing ready event");
- yield* lifecycleEvents.publish({
- version: 1,
- type: "ready",
- payload: { at: new Date().toISOString() },
- });
+ yield* runStartupPhase(
+ "ready.publish",
+ lifecycleEvents.publish({
+ version: 1,
+ type: "ready",
+ payload: { at: new Date().toISOString() },
+ }),
+ );
yield* Effect.logDebug("startup phase: recording startup heartbeat");
yield* launchStartupHeartbeat;
yield* Effect.logDebug("startup phase: browser open check");
- yield* maybeOpenBrowser;
+ yield* runStartupPhase("browser.open", maybeOpenBrowser);
yield* Effect.logDebug("startup phase: complete");
}),
);
diff --git a/apps/server/src/telemetry/Layers/AnalyticsService.ts b/apps/server/src/telemetry/Layers/AnalyticsService.ts
index 06ecfd023..e933576df 100644
--- a/apps/server/src/telemetry/Layers/AnalyticsService.ts
+++ b/apps/server/src/telemetry/Layers/AnalyticsService.ts
@@ -125,17 +125,19 @@ const makeAnalyticsService = Effect.gen(function* () {
}
}).pipe(Effect.catch((cause) => Effect.logError("Failed to flush telemetry", { cause })));
- const record: AnalyticsServiceShape["record"] = Effect.fnUntraced(function* (event, properties) {
- if (!telemetryConfig.enabled || !identifier) return;
-
- const enqueueResult = yield* enqueueBufferedEvent(event, properties);
- if (enqueueResult.dropped) {
- yield* Effect.logDebug("analytics buffer full; dropping oldest event", {
- size: enqueueResult.size,
- event,
- });
- }
- });
+ const record: AnalyticsServiceShape["record"] = Effect.fn("record")(
+ function* (event, properties) {
+ if (!telemetryConfig.enabled || !identifier) return;
+
+ const enqueueResult = yield* enqueueBufferedEvent(event, properties);
+ if (enqueueResult.dropped) {
+ yield* Effect.logDebug("analytics buffer full; dropping oldest event", {
+ size: enqueueResult.size,
+ event,
+ });
+ }
+ },
+ );
yield* Effect.forever(Effect.sleep(1000).pipe(Effect.flatMap(() => flush)), {
disableYield: true,
diff --git a/apps/server/src/terminal/Layers/Manager.ts b/apps/server/src/terminal/Layers/Manager.ts
index 7ef18b165..a076e7600 100644
--- a/apps/server/src/terminal/Layers/Manager.ts
+++ b/apps/server/src/terminal/Layers/Manager.ts
@@ -23,6 +23,11 @@ import {
} from "effect";
import { ServerConfig } from "../../config";
+import {
+ increment,
+ terminalRestartsTotal,
+ terminalSessionsTotal,
+} from "../../observability/Metrics";
import { runProcess } from "../../processRunner";
import {
TerminalCwdError,
@@ -1294,6 +1299,12 @@ export const makeTerminalManagerWithOptions = Effect.fn("makeTerminalManagerWith
eventType: "started" | "restarted",
) {
yield* stopProcess(session);
+ yield* Effect.annotateCurrentSpan({
+ "terminal.thread_id": session.threadId,
+ "terminal.id": session.terminalId,
+ "terminal.event_type": eventType,
+ "terminal.cwd": input.cwd,
+ });
yield* modifyManagerState((state) => {
session.status = "starting";
@@ -1314,45 +1325,49 @@ export const makeTerminalManagerWithOptions = Effect.fn("makeTerminalManagerWith
let startedShell: string | null = null;
const startResult = yield* Effect.result(
- Effect.gen(function* () {
- const shellCandidates = resolveShellCandidates(shellResolver);
- const terminalEnv = createTerminalSpawnEnv(process.env, session.runtimeEnv);
- const spawnResult = yield* trySpawn(shellCandidates, terminalEnv, session);
- ptyProcess = spawnResult.process;
- startedShell = spawnResult.shellLabel;
-
- const processPid = ptyProcess.pid;
- const unsubscribeData = ptyProcess.onData((data) => {
- if (!enqueueProcessEvent(session, processPid, { type: "output", data })) {
- return;
- }
- runFork(drainProcessEvents(session, processPid));
- });
- const unsubscribeExit = ptyProcess.onExit((event) => {
- if (!enqueueProcessEvent(session, processPid, { type: "exit", event })) {
- return;
- }
- runFork(drainProcessEvents(session, processPid));
- });
-
- yield* modifyManagerState((state) => {
- session.process = ptyProcess;
- session.pid = processPid;
- session.status = "running";
- session.updatedAt = new Date().toISOString();
- session.unsubscribeData = unsubscribeData;
- session.unsubscribeExit = unsubscribeExit;
- return [undefined, state] as const;
- });
-
- yield* publishEvent({
- type: eventType,
- threadId: session.threadId,
- terminalId: session.terminalId,
- createdAt: new Date().toISOString(),
- snapshot: snapshot(session),
- });
- }),
+ increment(terminalSessionsTotal, { lifecycle: eventType }).pipe(
+ Effect.andThen(
+ Effect.gen(function* () {
+ const shellCandidates = resolveShellCandidates(shellResolver);
+ const terminalEnv = createTerminalSpawnEnv(process.env, session.runtimeEnv);
+ const spawnResult = yield* trySpawn(shellCandidates, terminalEnv, session);
+ ptyProcess = spawnResult.process;
+ startedShell = spawnResult.shellLabel;
+
+ const processPid = ptyProcess.pid;
+ const unsubscribeData = ptyProcess.onData((data) => {
+ if (!enqueueProcessEvent(session, processPid, { type: "output", data })) {
+ return;
+ }
+ runFork(drainProcessEvents(session, processPid));
+ });
+ const unsubscribeExit = ptyProcess.onExit((event) => {
+ if (!enqueueProcessEvent(session, processPid, { type: "exit", event })) {
+ return;
+ }
+ runFork(drainProcessEvents(session, processPid));
+ });
+
+ yield* modifyManagerState((state) => {
+ session.process = ptyProcess;
+ session.pid = processPid;
+ session.status = "running";
+ session.updatedAt = new Date().toISOString();
+ session.unsubscribeData = unsubscribeData;
+ session.unsubscribeExit = unsubscribeExit;
+ return [undefined, state] as const;
+ });
+
+ yield* publishEvent({
+ type: eventType,
+ threadId: session.threadId,
+ terminalId: session.terminalId,
+ createdAt: new Date().toISOString(),
+ snapshot: snapshot(session),
+ });
+ }),
+ ),
+ ),
);
if (startResult._tag === "Success") {
@@ -1722,6 +1737,7 @@ export const makeTerminalManagerWithOptions = Effect.fn("makeTerminalManagerWith
withThreadLock(
input.threadId,
Effect.gen(function* () {
+ yield* increment(terminalRestartsTotal, { scope: "thread" });
const terminalId = input.terminalId ?? DEFAULT_TERMINAL_ID;
yield* assertValidCwd(input.cwd);
diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts
index e467ea445..de263ecb9 100644
--- a/apps/server/src/ws.ts
+++ b/apps/server/src/ws.ts
@@ -28,6 +28,11 @@ import { Open, resolveAvailableEditors } from "./open";
import { normalizeDispatchCommand } from "./orchestration/Normalizer";
import { OrchestrationEngineService } from "./orchestration/Services/OrchestrationEngine";
import { ProjectionSnapshotQuery } from "./orchestration/Services/ProjectionSnapshotQuery";
+import {
+ observeRpcEffect,
+ observeRpcStream,
+ observeRpcStreamEffect,
+} from "./observability/RpcInstrumentation";
import { ProviderRegistry } from "./provider/Services/ProviderRegistry";
import { ServerLifecycleEvents } from "./serverLifecycleEvents";
import { ServerRuntimeStartup } from "./serverRuntimeStartup";
@@ -67,85 +72,114 @@ const WsRpcLayer = WsRpcGroup.toLayer(
issues: keybindingsConfig.issues,
providers,
availableEditors: resolveAvailableEditors(),
+ observability: {
+ logsDirectoryPath: config.logsDir,
+ localTracingEnabled: true,
+ ...(config.otlpTracesUrl !== undefined ? { otlpTracesUrl: config.otlpTracesUrl } : {}),
+ otlpTracesEnabled: config.otlpTracesUrl !== undefined,
+ ...(config.otlpMetricsUrl !== undefined ? { otlpMetricsUrl: config.otlpMetricsUrl } : {}),
+ otlpMetricsEnabled: config.otlpMetricsUrl !== undefined,
+ },
settings,
};
});
return WsRpcGroup.of({
[ORCHESTRATION_WS_METHODS.getSnapshot]: (_input) =>
- projectionSnapshotQuery.getSnapshot().pipe(
- Effect.mapError(
- (cause) =>
- new OrchestrationGetSnapshotError({
- message: "Failed to load orchestration snapshot",
- cause,
- }),
+ observeRpcEffect(
+ ORCHESTRATION_WS_METHODS.getSnapshot,
+ projectionSnapshotQuery.getSnapshot().pipe(
+ Effect.mapError(
+ (cause) =>
+ new OrchestrationGetSnapshotError({
+ message: "Failed to load orchestration snapshot",
+ cause,
+ }),
+ ),
),
+ { "rpc.aggregate": "orchestration" },
),
[ORCHESTRATION_WS_METHODS.dispatchCommand]: (command) =>
- Effect.gen(function* () {
- const normalizedCommand = yield* normalizeDispatchCommand(command);
- const result = yield* startup.enqueueCommand(
- orchestrationEngine.dispatch(normalizedCommand),
- );
- if (normalizedCommand.type === "thread.archive") {
- yield* terminalManager.close({ threadId: normalizedCommand.threadId }).pipe(
- Effect.catch((error) =>
- Effect.logWarning("failed to close thread terminals after archive", {
- threadId: normalizedCommand.threadId,
- error: error.message,
- }),
- ),
+ observeRpcEffect(
+ ORCHESTRATION_WS_METHODS.dispatchCommand,
+ Effect.gen(function* () {
+ const normalizedCommand = yield* normalizeDispatchCommand(command);
+ const result = yield* startup.enqueueCommand(
+ orchestrationEngine.dispatch(normalizedCommand),
);
- }
- return result;
- }).pipe(
- Effect.mapError((cause) =>
- Schema.is(OrchestrationDispatchCommandError)(cause)
- ? cause
- : new OrchestrationDispatchCommandError({
- message: "Failed to dispatch orchestration command",
- cause,
- }),
+ if (normalizedCommand.type === "thread.archive") {
+ yield* terminalManager.close({ threadId: normalizedCommand.threadId }).pipe(
+ Effect.catch((error) =>
+ Effect.logWarning("failed to close thread terminals after archive", {
+ threadId: normalizedCommand.threadId,
+ error: error.message,
+ }),
+ ),
+ );
+ }
+ return result;
+ }).pipe(
+ Effect.mapError((cause) =>
+ Schema.is(OrchestrationDispatchCommandError)(cause)
+ ? cause
+ : new OrchestrationDispatchCommandError({
+ message: "Failed to dispatch orchestration command",
+ cause,
+ }),
+ ),
),
+ { "rpc.aggregate": "orchestration" },
),
[ORCHESTRATION_WS_METHODS.getTurnDiff]: (input) =>
- checkpointDiffQuery.getTurnDiff(input).pipe(
- Effect.mapError(
- (cause) =>
- new OrchestrationGetTurnDiffError({
- message: "Failed to load turn diff",
- cause,
- }),
+ observeRpcEffect(
+ ORCHESTRATION_WS_METHODS.getTurnDiff,
+ checkpointDiffQuery.getTurnDiff(input).pipe(
+ Effect.mapError(
+ (cause) =>
+ new OrchestrationGetTurnDiffError({
+ message: "Failed to load turn diff",
+ cause,
+ }),
+ ),
),
+ { "rpc.aggregate": "orchestration" },
),
[ORCHESTRATION_WS_METHODS.getFullThreadDiff]: (input) =>
- checkpointDiffQuery.getFullThreadDiff(input).pipe(
- Effect.mapError(
- (cause) =>
- new OrchestrationGetFullThreadDiffError({
- message: "Failed to load full thread diff",
- cause,
- }),
+ observeRpcEffect(
+ ORCHESTRATION_WS_METHODS.getFullThreadDiff,
+ checkpointDiffQuery.getFullThreadDiff(input).pipe(
+ Effect.mapError(
+ (cause) =>
+ new OrchestrationGetFullThreadDiffError({
+ message: "Failed to load full thread diff",
+ cause,
+ }),
+ ),
),
+ { "rpc.aggregate": "orchestration" },
),
[ORCHESTRATION_WS_METHODS.replayEvents]: (input) =>
- Stream.runCollect(
- orchestrationEngine.readEvents(
- clamp(input.fromSequenceExclusive, { maximum: Number.MAX_SAFE_INTEGER, minimum: 0 }),
- ),
- ).pipe(
- Effect.map((events) => Array.from(events)),
- Effect.mapError(
- (cause) =>
- new OrchestrationReplayEventsError({
- message: "Failed to replay orchestration events",
- cause,
- }),
+ observeRpcEffect(
+ ORCHESTRATION_WS_METHODS.replayEvents,
+ Stream.runCollect(
+ orchestrationEngine.readEvents(
+ clamp(input.fromSequenceExclusive, { maximum: Number.MAX_SAFE_INTEGER, minimum: 0 }),
+ ),
+ ).pipe(
+ Effect.map((events) => Array.from(events)),
+ Effect.mapError(
+ (cause) =>
+ new OrchestrationReplayEventsError({
+ message: "Failed to replay orchestration events",
+ cause,
+ }),
+ ),
),
+ { "rpc.aggregate": "orchestration" },
),
[WS_METHODS.subscribeOrchestrationDomainEvents]: (_input) =>
- Stream.unwrap(
+ observeRpcStreamEffect(
+ WS_METHODS.subscribeOrchestrationDomainEvents,
Effect.gen(function* () {
const snapshot = yield* orchestrationEngine.getReadModel();
const fromSequenceExclusive = snapshot.snapshotSequence;
@@ -200,82 +234,167 @@ const WsRpcLayer = WsRpcGroup.toLayer(
Stream.flatMap((events) => Stream.fromIterable(events)),
);
}),
+ { "rpc.aggregate": "orchestration" },
),
- [WS_METHODS.serverGetConfig]: (_input) => loadServerConfig,
+ [WS_METHODS.serverGetConfig]: (_input) =>
+ observeRpcEffect(WS_METHODS.serverGetConfig, loadServerConfig, {
+ "rpc.aggregate": "server",
+ }),
[WS_METHODS.serverRefreshProviders]: (_input) =>
- providerRegistry.refresh().pipe(Effect.map((providers) => ({ providers }))),
+ observeRpcEffect(
+ WS_METHODS.serverRefreshProviders,
+ providerRegistry.refresh().pipe(Effect.map((providers) => ({ providers }))),
+ { "rpc.aggregate": "server" },
+ ),
[WS_METHODS.serverUpsertKeybinding]: (rule) =>
- Effect.gen(function* () {
- const keybindingsConfig = yield* keybindings.upsertKeybindingRule(rule);
- return { keybindings: keybindingsConfig, issues: [] };
+ observeRpcEffect(
+ WS_METHODS.serverUpsertKeybinding,
+ Effect.gen(function* () {
+ const keybindingsConfig = yield* keybindings.upsertKeybindingRule(rule);
+ return { keybindings: keybindingsConfig, issues: [] };
+ }),
+ { "rpc.aggregate": "server" },
+ ),
+ [WS_METHODS.serverGetSettings]: (_input) =>
+ observeRpcEffect(WS_METHODS.serverGetSettings, serverSettings.getSettings, {
+ "rpc.aggregate": "server",
+ }),
+ [WS_METHODS.serverUpdateSettings]: ({ patch }) =>
+ observeRpcEffect(WS_METHODS.serverUpdateSettings, serverSettings.updateSettings(patch), {
+ "rpc.aggregate": "server",
}),
- [WS_METHODS.serverGetSettings]: (_input) => serverSettings.getSettings,
- [WS_METHODS.serverUpdateSettings]: ({ patch }) => serverSettings.updateSettings(patch),
[WS_METHODS.projectsSearchEntries]: (input) =>
- workspaceEntries.search(input).pipe(
- Effect.mapError(
- (cause) =>
- new ProjectSearchEntriesError({
- message: `Failed to search workspace entries: ${cause.detail}`,
- cause,
- }),
+ observeRpcEffect(
+ WS_METHODS.projectsSearchEntries,
+ workspaceEntries.search(input).pipe(
+ Effect.mapError(
+ (cause) =>
+ new ProjectSearchEntriesError({
+ message: `Failed to search workspace entries: ${cause.detail}`,
+ cause,
+ }),
+ ),
),
+ { "rpc.aggregate": "workspace" },
),
[WS_METHODS.projectsWriteFile]: (input) =>
- workspaceFileSystem.writeFile(input).pipe(
- Effect.mapError((cause) => {
- const message = Schema.is(WorkspacePathOutsideRootError)(cause)
- ? "Workspace file path must stay within the project root."
- : "Failed to write workspace file";
- return new ProjectWriteFileError({
- message,
- cause,
- });
- }),
+ observeRpcEffect(
+ WS_METHODS.projectsWriteFile,
+ workspaceFileSystem.writeFile(input).pipe(
+ Effect.mapError((cause) => {
+ const message = Schema.is(WorkspacePathOutsideRootError)(cause)
+ ? "Workspace file path must stay within the project root."
+ : "Failed to write workspace file";
+ return new ProjectWriteFileError({
+ message,
+ cause,
+ });
+ }),
+ ),
+ { "rpc.aggregate": "workspace" },
),
- [WS_METHODS.shellOpenInEditor]: (input) => open.openInEditor(input),
- [WS_METHODS.gitStatus]: (input) => gitManager.status(input),
- [WS_METHODS.gitPull]: (input) => git.pullCurrentBranch(input.cwd),
+ [WS_METHODS.shellOpenInEditor]: (input) =>
+ observeRpcEffect(WS_METHODS.shellOpenInEditor, open.openInEditor(input), {
+ "rpc.aggregate": "workspace",
+ }),
+ [WS_METHODS.gitStatus]: (input) =>
+ observeRpcEffect(WS_METHODS.gitStatus, gitManager.status(input), {
+ "rpc.aggregate": "git",
+ }),
+ [WS_METHODS.gitPull]: (input) =>
+ observeRpcEffect(WS_METHODS.gitPull, git.pullCurrentBranch(input.cwd), {
+ "rpc.aggregate": "git",
+ }),
[WS_METHODS.gitRunStackedAction]: (input) =>
- Stream.callback<GitActionProgressEvent, GitManagerServiceError>((queue) =>
- gitManager
- .runStackedAction(input, {
- actionId: input.actionId,
- progressReporter: {
- publish: (event) => Queue.offer(queue, event).pipe(Effect.asVoid),
- },
- })
- .pipe(
- Effect.matchCauseEffect({
- onFailure: (cause) => Queue.failCause(queue, cause),
- onSuccess: () => Queue.end(queue).pipe(Effect.asVoid),
- }),
- ),
+ observeRpcStream(
+ WS_METHODS.gitRunStackedAction,
+ Stream.callback<GitActionProgressEvent, GitManagerServiceError>((queue) =>
+ gitManager
+ .runStackedAction(input, {
+ actionId: input.actionId,
+ progressReporter: {
+ publish: (event) => Queue.offer(queue, event).pipe(Effect.asVoid),
+ },
+ })
+ .pipe(
+ Effect.matchCauseEffect({
+ onFailure: (cause) => Queue.failCause(queue, cause),
+ onSuccess: () => Queue.end(queue).pipe(Effect.asVoid),
+ }),
+ ),
+ ),
+ { "rpc.aggregate": "git" },
),
- [WS_METHODS.gitResolvePullRequest]: (input) => gitManager.resolvePullRequest(input),
+ [WS_METHODS.gitResolvePullRequest]: (input) =>
+ observeRpcEffect(WS_METHODS.gitResolvePullRequest, gitManager.resolvePullRequest(input), {
+ "rpc.aggregate": "git",
+ }),
[WS_METHODS.gitPreparePullRequestThread]: (input) =>
- gitManager.preparePullRequestThread(input),
- [WS_METHODS.gitListBranches]: (input) => git.listBranches(input),
- [WS_METHODS.gitCreateWorktree]: (input) => git.createWorktree(input),
- [WS_METHODS.gitRemoveWorktree]: (input) => git.removeWorktree(input),
- [WS_METHODS.gitCreateBranch]: (input) => git.createBranch(input),
- [WS_METHODS.gitCheckout]: (input) => Effect.scoped(git.checkoutBranch(input)),
- [WS_METHODS.gitInit]: (input) => git.initRepo(input),
- [WS_METHODS.terminalOpen]: (input) => terminalManager.open(input),
- [WS_METHODS.terminalWrite]: (input) => terminalManager.write(input),
- [WS_METHODS.terminalResize]: (input) => terminalManager.resize(input),
- [WS_METHODS.terminalClear]: (input) => terminalManager.clear(input),
- [WS_METHODS.terminalRestart]: (input) => terminalManager.restart(input),
- [WS_METHODS.terminalClose]: (input) => terminalManager.close(input),
+ observeRpcEffect(
+ WS_METHODS.gitPreparePullRequestThread,
+ gitManager.preparePullRequestThread(input),
+ { "rpc.aggregate": "git" },
+ ),
+ [WS_METHODS.gitListBranches]: (input) =>
+ observeRpcEffect(WS_METHODS.gitListBranches, git.listBranches(input), {
+ "rpc.aggregate": "git",
+ }),
+ [WS_METHODS.gitCreateWorktree]: (input) =>
+ observeRpcEffect(WS_METHODS.gitCreateWorktree, git.createWorktree(input), {
+ "rpc.aggregate": "git",
+ }),
+ [WS_METHODS.gitRemoveWorktree]: (input) =>
+ observeRpcEffect(WS_METHODS.gitRemoveWorktree, git.removeWorktree(input), {
+ "rpc.aggregate": "git",
+ }),
+ [WS_METHODS.gitCreateBranch]: (input) =>
+ observeRpcEffect(WS_METHODS.gitCreateBranch, git.createBranch(input), {
+ "rpc.aggregate": "git",
+ }),
+ [WS_METHODS.gitCheckout]: (input) =>
+ observeRpcEffect(WS_METHODS.gitCheckout, Effect.scoped(git.checkoutBranch(input)), {
+ "rpc.aggregate": "git",
+ }),
+ [WS_METHODS.gitInit]: (input) =>
+ observeRpcEffect(WS_METHODS.gitInit, git.initRepo(input), { "rpc.aggregate": "git" }),
+ [WS_METHODS.terminalOpen]: (input) =>
+ observeRpcEffect(WS_METHODS.terminalOpen, terminalManager.open(input), {
+ "rpc.aggregate": "terminal",
+ }),
+ [WS_METHODS.terminalWrite]: (input) =>
+ observeRpcEffect(WS_METHODS.terminalWrite, terminalManager.write(input), {
+ "rpc.aggregate": "terminal",
+ }),
+ [WS_METHODS.terminalResize]: (input) =>
+ observeRpcEffect(WS_METHODS.terminalResize, terminalManager.resize(input), {
+ "rpc.aggregate": "terminal",
+ }),
+ [WS_METHODS.terminalClear]: (input) =>
+ observeRpcEffect(WS_METHODS.terminalClear, terminalManager.clear(input), {
+ "rpc.aggregate": "terminal",
+ }),
+ [WS_METHODS.terminalRestart]: (input) =>
+ observeRpcEffect(WS_METHODS.terminalRestart, terminalManager.restart(input), {
+ "rpc.aggregate": "terminal",
+ }),
+ [WS_METHODS.terminalClose]: (input) =>
+ observeRpcEffect(WS_METHODS.terminalClose, terminalManager.close(input), {
+ "rpc.aggregate": "terminal",
+ }),
[WS_METHODS.subscribeTerminalEvents]: (_input) =>
- Stream.callback<TerminalEvent>((queue) =>
- Effect.acquireRelease(
- terminalManager.subscribe((event) => Queue.offer(queue, event)),
- (unsubscribe) => Effect.sync(unsubscribe),
+ observeRpcStream(
+ WS_METHODS.subscribeTerminalEvents,
+ Stream.callback<TerminalEvent>((queue) =>
+ Effect.acquireRelease(
+ terminalManager.subscribe((event) => Queue.offer(queue, event)),
+ (unsubscribe) => Effect.sync(unsubscribe),
+ ),
),
+ { "rpc.aggregate": "terminal" },
),
[WS_METHODS.subscribeServerConfig]: (_input) =>
- Stream.unwrap(
+ observeRpcStreamEffect(
+ WS_METHODS.subscribeServerConfig,
Effect.gen(function* () {
const keybindingsUpdates = keybindings.streamChanges.pipe(
Stream.map((event) => ({
@@ -310,9 +429,11 @@ const WsRpcLayer = WsRpcGroup.toLayer(
Stream.merge(keybindingsUpdates, Stream.merge(providerStatuses, settingsUpdates)),
);
}),
+ { "rpc.aggregate": "server" },
),
[WS_METHODS.subscribeServerLifecycle]: (_input) =>
- Stream.unwrap(
+ observeRpcStreamEffect(
+ WS_METHODS.subscribeServerLifecycle,
Effect.gen(function* () {
const snapshot = yield* lifecycleEvents.snapshot;
const snapshotEvents = Array.from(snapshot.events).toSorted(
@@ -323,6 +444,7 @@ const WsRpcLayer = WsRpcGroup.toLayer(
);
return Stream.concat(Stream.fromIterable(snapshotEvents), liveEvents);
}),
+ { "rpc.aggregate": "server" },
),
});
}),
@@ -330,9 +452,13 @@ const WsRpcLayer = WsRpcGroup.toLayer(
export const websocketRpcRouteLayer = Layer.unwrap(
Effect.gen(function* () {
- const rpcWebSocketHttpEffect = yield* RpcServer.toHttpEffectWebsocket(WsRpcGroup).pipe(
- Effect.provide(Layer.mergeAll(WsRpcLayer, RpcSerialization.layerJson)),
- );
+ const rpcWebSocketHttpEffect = yield* RpcServer.toHttpEffectWebsocket(WsRpcGroup, {
+ spanPrefix: "ws.rpc",
+ spanAttributes: {
+ "rpc.transport": "websocket",
+ "rpc.system": "effect-rpc",
+ },
+ }).pipe(Effect.provide(Layer.mergeAll(WsRpcLayer, RpcSerialization.layerJson)));
return HttpRouter.add(
"GET",
"/ws",
diff --git a/apps/web/src/components/ChatView.browser.tsx b/apps/web/src/components/ChatView.browser.tsx
index b0263cbf4..f9f1362d0 100644
--- a/apps/web/src/components/ChatView.browser.tsx
+++ b/apps/web/src/components/ChatView.browser.tsx
@@ -135,6 +135,12 @@ function createBaseServerConfig(): ServerConfig {
},
],
availableEditors: [],
+ observability: {
+ logsDirectoryPath: "/repo/project/.t3/logs",
+ localTracingEnabled: true,
+ otlpTracesEnabled: false,
+ otlpMetricsEnabled: false,
+ },
settings: {
...DEFAULT_SERVER_SETTINGS,
...DEFAULT_CLIENT_SETTINGS,
diff --git a/apps/web/src/components/KeybindingsToast.browser.tsx b/apps/web/src/components/KeybindingsToast.browser.tsx
index b8398e7b8..de7d84b4e 100644
--- a/apps/web/src/components/KeybindingsToast.browser.tsx
+++ b/apps/web/src/components/KeybindingsToast.browser.tsx
@@ -56,6 +56,12 @@ function createBaseServerConfig(): ServerConfig {
},
],
availableEditors: [],
+ observability: {
+ logsDirectoryPath: "/repo/project/.t3/logs",
+ localTracingEnabled: true,
+ otlpTracesEnabled: false,
+ otlpMetricsEnabled: false,
+ },
settings: {
enableAssistantStreaming: false,
defaultThreadEnvMode: "local" as const,
diff --git a/apps/web/src/components/settings/SettingsPanels.browser.tsx b/apps/web/src/components/settings/SettingsPanels.browser.tsx
new file mode 100644
index 000000000..090f6f12a
--- /dev/null
+++ b/apps/web/src/components/settings/SettingsPanels.browser.tsx
@@ -0,0 +1,91 @@
+import "../../index.css";
+
+import { DEFAULT_SERVER_SETTINGS, type NativeApi, type ServerConfig } from "@t3tools/contracts";
+import { page } from "vitest/browser";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { render } from "vitest-browser-react";
+
+import { __resetNativeApiForTests } from "../../nativeApi";
+import { AppAtomRegistryProvider } from "../../rpc/atomRegistry";
+import { resetServerStateForTests, setServerConfigSnapshot } from "../../rpc/serverState";
+import { GeneralSettingsPanel } from "./SettingsPanels";
+
+function createBaseServerConfig(): ServerConfig {
+ return {
+ cwd: "/repo/project",
+ keybindingsConfigPath: "/repo/project/.t3code-keybindings.json",
+ keybindings: [],
+ issues: [],
+ providers: [],
+ availableEditors: ["cursor"],
+ observability: {
+ logsDirectoryPath: "/repo/project/.t3/logs",
+ localTracingEnabled: true,
+ otlpTracesUrl: "http://localhost:4318/v1/traces",
+ otlpTracesEnabled: true,
+ otlpMetricsEnabled: false,
+ },
+ settings: DEFAULT_SERVER_SETTINGS,
+ };
+}
+
+describe("GeneralSettingsPanel observability", () => {
+ beforeEach(() => {
+ resetServerStateForTests();
+ __resetNativeApiForTests();
+ localStorage.clear();
+ document.body.innerHTML = "";
+ });
+
+ afterEach(() => {
+ resetServerStateForTests();
+ __resetNativeApiForTests();
+ document.body.innerHTML = "";
+ });
+
+ it("shows diagnostics inside About with a single logs-folder action", async () => {
+ setServerConfigSnapshot(createBaseServerConfig());
+
+ await render(
+ <AppAtomRegistryProvider>
+ <GeneralSettingsPanel />
+ </AppAtomRegistryProvider>,
+ );
+
+ await expect.element(page.getByText("About")).toBeInTheDocument();
+ await expect.element(page.getByText("Diagnostics")).toBeInTheDocument();
+ await expect.element(page.getByText("Open logs folder")).toBeInTheDocument();
+ await expect
+ .element(page.getByText("/repo/project/.t3/logs", { exact: true }))
+ .toBeInTheDocument();
+ await expect
+ .element(
+ page.getByText(
+ "Local trace file. OTLP exporting traces to http://localhost:4318/v1/traces.",
+ ),
+ )
+ .toBeInTheDocument();
+ });
+
+ it("opens the logs folder in the preferred editor", async () => {
+ const openInEditor = vi.fn<NativeApi["shell"]["openInEditor"]>().mockResolvedValue(undefined);
+ window.nativeApi = {
+ shell: {
+ openInEditor,
+ },
+ } as unknown as NativeApi;
+
+ setServerConfigSnapshot(createBaseServerConfig());
+
+ await render(
+ <AppAtomRegistryProvider>
+ <GeneralSettingsPanel />
+ </AppAtomRegistryProvider>,
+ );
+
+ const openLogsButton = page.getByText("Open logs folder");
+ await openLogsButton.click();
+
+ expect(openInEditor).toHaveBeenCalledWith("/repo/project/.t3/logs", "cursor");
+ });
+});
diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx
index e3e620030..d534eefaa 100644
--- a/apps/web/src/components/settings/SettingsPanels.tsx
+++ b/apps/web/src/components/settings/SettingsPanels.tsx
@@ -61,6 +61,7 @@ import { ProjectFavicon } from "../ProjectFavicon";
import {
useServerAvailableEditors,
useServerKeybindingsConfigPath,
+ useServerObservability,
useServerProviders,
} from "../../rpc/serverState";
@@ -518,8 +519,13 @@ export function GeneralSettingsPanel() {
const { theme, setTheme } = useTheme();
const settings = useSettings();
const { updateSettings } = useUpdateSettings();
- const [isOpeningKeybindings, setIsOpeningKeybindings] = useState(false);
- const [openKeybindingsError, setOpenKeybindingsError] = useState<string | null>(null);
+ const [openingPathByTarget, setOpeningPathByTarget] = useState({
+ keybindings: false,
+ logsDirectory: false,
+ });
+ const [openPathErrorByTarget, setOpenPathErrorByTarget] = useState<
+ Partial<Record<"keybindings" | "logsDirectory", string | null>>
+ >({});
const [openProviderDetails, setOpenProviderDetails] = useState<Record<ProviderKind, boolean>>({
codex: Boolean(
settings.providers.codex.binaryPath !== DEFAULT_UNIFIED_SETTINGS.providers.codex.binaryPath ||
@@ -561,8 +567,21 @@ export function GeneralSettingsPanel() {
const keybindingsConfigPath = useServerKeybindingsConfigPath();
const availableEditors = useServerAvailableEditors();
+ const observability = useServerObservability();
const serverProviders = useServerProviders();
const codexHomePath = settings.providers.codex.homePath;
+ const logsDirectoryPath = observability?.logsDirectoryPath ?? null;
+ const diagnosticsDescription = (() => {
+ const exports: string[] = [];
+ if (observability?.otlpTracesEnabled && observability.otlpTracesUrl) {
+ exports.push(`traces to ${observability.otlpTracesUrl}`);
+ }
+ if (observability?.otlpMetricsEnabled && observability.otlpMetricsUrl) {
+ exports.push(`metrics to ${observability.otlpMetricsUrl}`);
+ }
+ const mode = observability?.localTracingEnabled ? "Local trace file" : "Terminal logs only";
+ return exports.length > 0 ? `${mode}. OTLP exporting ${exports.join(" and ")}.` : `${mode}.`;
+ })();
const textGenerationModelSelection = resolveAppModelSelectionState(settings, serverProviders);
const textGenProvider = textGenerationModelSelection.provider;
@@ -579,27 +598,49 @@ export function GeneralSettingsPanel() {
DEFAULT_UNIFIED_SETTINGS.textGenerationModelSelection ?? null,
);
+ const openInPreferredEditor = useCallback(
+ (target: "keybindings" | "logsDirectory", path: string | null, failureMessage: string) => {
+ if (!path) return;
+ setOpenPathErrorByTarget((existing) => ({ ...existing, [target]: null }));
+ setOpeningPathByTarget((existing) => ({ ...existing, [target]: true }));
+
+ const editor = resolveAndPersistPreferredEditor(availableEditors ?? []);
+ if (!editor) {
+ setOpenPathErrorByTarget((existing) => ({
+ ...existing,
+ [target]: "No available editors found.",
+ }));
+ setOpeningPathByTarget((existing) => ({ ...existing, [target]: false }));
+ return;
+ }
+
+ void ensureNativeApi()
+ .shell.openInEditor(path, editor)
+ .catch((error) => {
+ setOpenPathErrorByTarget((existing) => ({
+ ...existing,
+ [target]: error instanceof Error ? error.message : failureMessage,
+ }));
+ })
+ .finally(() => {
+ setOpeningPathByTarget((existing) => ({ ...existing, [target]: false }));
+ });
+ },
+ [availableEditors],
+ );
+
const openKeybindingsFile = useCallback(() => {
- if (!keybindingsConfigPath) return;
- setOpenKeybindingsError(null);
- setIsOpeningKeybindings(true);
- const editor = resolveAndPersistPreferredEditor(availableEditors ?? []);
- if (!editor) {
- setOpenKeybindingsError("No available editors found.");
- setIsOpeningKeybindings(false);
- return;
- }
- void ensureNativeApi()
- .shell.openInEditor(keybindingsConfigPath, editor)
- .catch((error) => {
- setOpenKeybindingsError(
- error instanceof Error ? error.message : "Unable to open keybindings file.",
- );
- })
- .finally(() => {
- setIsOpeningKeybindings(false);
- });
- }, [availableEditors, keybindingsConfigPath]);
+ openInPreferredEditor("keybindings", keybindingsConfigPath, "Unable to open keybindings file.");
+ }, [keybindingsConfigPath, openInPreferredEditor]);
+
+ const openLogsDirectory = useCallback(() => {
+ openInPreferredEditor("logsDirectory", logsDirectoryPath, "Unable to open logs folder.");
+ }, [logsDirectoryPath, openInPreferredEditor]);
+
+ const openKeybindingsError = openPathErrorByTarget.keybindings ?? null;
+ const openDiagnosticsError = openPathErrorByTarget.logsDirectory ?? null;
+ const isOpeningKeybindings = openingPathByTarget.keybindings;
+ const isOpeningLogsDirectory = openingPathByTarget.logsDirectory;
const addCustomModel = useCallback(
(provider: ProviderKind) => {
@@ -1408,6 +1449,30 @@ export function GeneralSettingsPanel() {
description="Current version of the application."
/>
)}
+ <SettingsRow
+ title="Diagnostics"
+ description={diagnosticsDescription}
+ status={
+ <>
+ <span className="block break-all font-mono text-[11px] text-foreground">
+ {logsDirectoryPath ?? "Resolving logs directory..."}
+ </span>
+ {openDiagnosticsError ? (
+ <span className="mt-1 block text-destructive">{openDiagnosticsError}</span>
+ ) : null}
+ </>
+ }
+ control={
+ <Button
+ size="xs"
+ variant="outline"
+ disabled={!logsDirectoryPath || isOpeningLogsDirectory}
+ onClick={openLogsDirectory}
+ >
+ {isOpeningLogsDirectory ? "Opening..." : "Open logs folder"}
+ </Button>
+ }
+ />
</SettingsSection>
</SettingsPageContainer>
);
diff --git a/apps/web/src/rpc/serverState.test.ts b/apps/web/src/rpc/serverState.test.ts
index 8da5846fe..721ce25fb 100644
--- a/apps/web/src/rpc/serverState.test.ts
+++ b/apps/web/src/rpc/serverState.test.ts
@@ -57,6 +57,12 @@ const baseServerConfig: ServerConfig = {
issues: [],
providers: defaultProviders,
availableEditors: ["cursor"],
+ observability: {
+ logsDirectoryPath: "/tmp/workspace/.config/logs",
+ localTracingEnabled: true,
+ otlpTracesEnabled: false,
+ otlpMetricsEnabled: false,
+ },
settings: DEFAULT_SERVER_SETTINGS,
};
diff --git a/apps/web/src/rpc/serverState.ts b/apps/web/src/rpc/serverState.ts
index 2a6939406..3e8dbab28 100644
--- a/apps/web/src/rpc/serverState.ts
+++ b/apps/web/src/rpc/serverState.ts
@@ -50,6 +50,7 @@ const selectAvailableEditors = (config: ServerConfig | null): ReadonlyArray<Edit
const selectKeybindings = (config: ServerConfig | null) => config?.keybindings ?? EMPTY_KEYBINDINGS;
const selectKeybindingsConfigPath = (config: ServerConfig | null) =>
config?.keybindingsConfigPath ?? null;
+const selectObservability = (config: ServerConfig | null) => config?.observability ?? null;
const selectProviders = (config: ServerConfig | null) =>
config?.providers ?? EMPTY_SERVER_PROVIDERS;
const selectSettings = (config: ServerConfig | null): ServerSettings =>
@@ -281,6 +282,10 @@ export function useServerKeybindingsConfigPath(): string | null {
return useAtomValue(serverConfigAtom, selectKeybindingsConfigPath);
}
+export function useServerObservability(): ServerConfig["observability"] | null {
+ return useAtomValue(serverConfigAtom, selectObservability);
+}
+
export function useServerWelcomeSubscription(
listener: (payload: ServerLifecycleWelcomePayload) => void,
): void {
diff --git a/apps/web/src/wsNativeApi.test.ts b/apps/web/src/wsNativeApi.test.ts
index c9bcc7ffd..47f78f967 100644
--- a/apps/web/src/wsNativeApi.test.ts
+++ b/apps/web/src/wsNativeApi.test.ts
@@ -159,6 +159,12 @@ const baseServerConfig: ServerConfig = {
issues: [],
providers: defaultProviders,
availableEditors: ["cursor"],
+ observability: {
+ logsDirectoryPath: "/tmp/workspace/.config/logs",
+ localTracingEnabled: true,
+ otlpTracesEnabled: false,
+ otlpMetricsEnabled: false,
+ },
settings: DEFAULT_SERVER_SETTINGS,
};
diff --git a/docs/observability.md b/docs/observability.md
new file mode 100644
index 000000000..dde109357
--- /dev/null
+++ b/docs/observability.md
@@ -0,0 +1,519 @@
+# Observability
+
+T3 Code has one server-side observability model:
+
+- pretty logs go to stdout for humans
+- completed spans go to a local NDJSON trace file
+- traces and metrics can also be exported over OTLP to a real backend like Grafana LGTM
+
+The local trace file is the persisted source of truth. There is no separate persisted server log file anymore.
+
+## Where To Find Things
+
+### Logs
+
+Logs are human-facing only:
+
+- destination: stdout
+- format: `Logger.consolePretty()`
+- persistence: none
+
+If you want a log message to show up in the trace file, emit it inside an active span with `Effect.log...`. `Logger.tracerLogger` will attach it as a span event.
+
+### Traces
+
+Completed spans are written as NDJSON records to `serverTracePath` (by default, `~/.t3/userdata/logs/server.trace.ndjson`).
+
+Important fields in each record:
+
+- `name`: span name
+- `traceId`, `spanId`, `parentSpanId`: correlation
+- `durationMs`: elapsed time
+- `attributes`: structured context
+- `events`: embedded logs and custom events
+- `exit`: `Success`, `Failure`, or `Interrupted`
+
+The schema lives in `apps/server/src/observability/TraceRecord.ts`.
+
+### Metrics
+
+Metrics are not written to a local file.
+
+- local persistence: none
+- remote export: OTLP only, when configured
+- current definitions: `apps/server/src/observability/Metrics.ts`
+
+If OTLP is not configured, metrics still exist in-process, but you will not have a local artifact to inspect.
+
+### Related Artifacts
+
+Provider event NDJSON files still exist for provider runtime streams. Those are separate from the main server trace file.
+
+## Run The Server In Instrumented Mode
+
+There are two useful modes:
+
+- local-only: stdout + local `server.trace.ndjson`
+- full local observability: stdout + local trace file + OTLP export to Grafana/Tempo/Prometheus
+
+The local trace file is always on. OTLP export is opt-in.
+
+### Option 1: Local Traces Only
+
+You do not need any extra env vars. Just run the app normally and inspect `server.trace.ndjson`.
+
+Examples:
+
+```bash
+npx t3
+```
+
+```bash
+bun dev
+```
+
+```bash
+bun dev:desktop
+```
+
+### Option 2: Run With A Local LGTM Stack
+
+#### 1. Start Grafana LGTM
+
+```bash
+docker run --name lgtm \
+ -p 3000:3000 \
+ -p 4317:4317 \
+ -p 4318:4318 \
+ --rm -ti \
+ grafana/otel-lgtm
+```
+
+Then open `http://localhost:3000`.
+
+Default Grafana login:
+
+- username: `admin`
+- password: `admin`
+
+#### 2. Export OTLP env vars
+
+```bash
+export T3CODE_OTLP_TRACES_URL=http://localhost:4318/v1/traces
+export T3CODE_OTLP_METRICS_URL=http://localhost:4318/v1/metrics
+export T3CODE_OTLP_SERVICE_NAME=t3-local
+```
+
+Optional:
+
+```bash
+export T3CODE_TRACE_MIN_LEVEL=Info
+export T3CODE_TRACE_TIMING_ENABLED=true
+```
+
+#### 3. Launch the app from that same shell
+
+CLI:
+
+```bash
+npx t3
+```
+
+Monorepo web/server dev:
+
+```bash
+bun dev
+```
+
+Monorepo desktop dev:
+
+```bash
+bun dev:desktop
+```
+
+Packaged desktop app:
+
+Launch the actual app executable from the same shell so the desktop app and embedded backend inherit `T3CODE_OTLP_*`.
+
+macOS app bundle example:
+
+```bash
+T3CODE_OTLP_TRACES_URL=http://localhost:4318/v1/traces \
+T3CODE_OTLP_METRICS_URL=http://localhost:4318/v1/metrics \
+T3CODE_OTLP_SERVICE_NAME=t3-desktop \
+"/Applications/T3 Code.app/Contents/MacOS/T3 Code"
+```
+
+Direct binary example:
+
+```bash
+T3CODE_OTLP_TRACES_URL=http://localhost:4318/v1/traces \
+T3CODE_OTLP_METRICS_URL=http://localhost:4318/v1/metrics \
+T3CODE_OTLP_SERVICE_NAME=t3-desktop \
+./path/to/your/desktop-app-binary
+```
+
+Do not rely on launching from Finder, Spotlight, the dock, or the Start menu after setting shell env vars. Those launches usually will not pick them up.
+
+#### 4. Fully restart after changing env
+
+The backend reads observability config at process start. If you change OTLP env vars, stop the app completely and start it again.
+
+## How To Use Traces And Metrics To Debug The Server
+
+### Start With The Local Trace File
+
+The trace file is the fastest way to inspect raw span data.
+
+Tail it:
+
+```bash
+tail -f "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
+```
+
+In monorepo dev, use:
+
+```bash
+tail -f ./dev/logs/server.trace.ndjson
+```
+
+Show failed spans:
+
+```bash
+jq -c 'select(.exit._tag != "Success") | {
+ name,
+ durationMs,
+ exit,
+ attributes
+}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
+```
+
+Show slow spans:
+
+```bash
+jq -c 'select(.durationMs > 1000) | {
+ name,
+ durationMs,
+ traceId,
+ spanId
+}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
+```
+
+Inspect embedded log events:
+
+```bash
+jq -c 'select(any(.events[]?; .attributes["effect.logLevel"] != null)) | {
+ name,
+ durationMs,
+ events: [
+ .events[]
+ | select(.attributes["effect.logLevel"] != null)
+ | {
+ message: .name,
+ level: .attributes["effect.logLevel"]
+ }
+ ]
+}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
+```
+
+Follow one trace:
+
+```bash
+jq -r 'select(.traceId == "TRACE_ID_HERE") | [
+ .name,
+ .spanId,
+ (.parentSpanId // "-"),
+ .durationMs
+] | @tsv' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
+```
+
+Filter orchestration commands:
+
+```bash
+jq -c 'select(.attributes["orchestration.command_type"] != null) | {
+ name,
+ durationMs,
+ commandType: .attributes["orchestration.command_type"],
+ aggregateKind: .attributes["orchestration.aggregate_kind"]
+}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
+```
+
+Filter git activity:
+
+```bash
+jq -c 'select(.attributes["git.operation"] != null) | {
+ name,
+ durationMs,
+ operation: .attributes["git.operation"],
+ cwd: .attributes["git.cwd"],
+ hookEvents: [
+ .events[]
+ | select(.name == "git.hook.started" or .name == "git.hook.finished")
+ ]
+}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
+```
+
+### Use Tempo When You Need A Real Trace Viewer
+
+Tempo is better than raw NDJSON when you want to:
+
+- search across many traces
+- inspect parent/child relationships visually
+- compare many slow traces
+- drill into one failing request without hand-joining by `traceId`
+
+Recommended flow in Grafana:
+
+1. Open `Explore`.
+2. Pick the `Tempo` data source.
+3. Set the time range to something recent like `Last 15 minutes`.
+4. Start broad. Do not begin with a very narrow query.
+5. Look for spans from your configured service name, then narrow by span name or attributes.
+
+Good first searches:
+
+- service name such as `t3-local`, `t3-dev`, or `t3-desktop`
+- span names like `sql.execute`, `git.runCommand`, `provider.sendTurn`
+- orchestration spans with attributes like `orchestration.command_type`
+
+Once you know traces are arriving, narrower TraceQL queries like `name = "sql.execute"` become useful.
+
+### Use Metrics To See Systemic Problems
+
+Traces are best for one request. Metrics are best for trends.
+
+Good metric families to watch:
+
+- `t3_rpc_request_duration`
+- `t3_orchestration_command_duration`
+- `t3_orchestration_command_ack_duration`
+- `t3_provider_turn_duration`
+- `t3_git_command_duration`
+- `t3_db_query_duration`
+
+Counters tell you volume and failure rate:
+
+- `t3_rpc_requests_total`
+- `t3_orchestration_commands_total`
+- `t3_provider_turns_total`
+- `t3_git_commands_total`
+- `t3_db_queries_total`
+
+Use metrics when the question is:
+
+- "is this always slow?"
+- "did this get worse after a change?"
+- "which command type is failing most often?"
+
+Use traces when the question is:
+
+- "what happened in this specific request?"
+- "which child span caused this one slow interaction?"
+- "what logs were emitted inside the failing flow?"
+
+### What The New Ack Metric Means
+
+`t3_orchestration_command_ack_duration` measures:
+
+- start: command dispatch enters the orchestration engine
+- end: the first committed domain event for that command is published by the server
+
+That is a server-side acknowledgment metric. It does not measure:
+
+- websocket transit to the browser
+- client receipt
+- React render time
+
+If you need those later, add client-side instrumentation or a dedicated server fanout metric.
+
+## Common Workflows
+
+### "Why did this request fail?"
+
+1. Start with the local NDJSON file.
+2. Find spans where `exit._tag != "Success"`.
+3. Group by `traceId`.
+4. Inspect sibling spans and span events.
+5. If needed, move to Tempo for the full trace tree.
+
+### "Why is the UI feeling slow?"
+
+1. Search for slow top-level spans in the trace file or Tempo.
+2. Check child spans for sqlite, git, provider, or terminal work.
+3. Look at the matching duration metrics to see whether the slowness is systemic.
+
+### "Did this command take too long to acknowledge?"
+
+1. Check `t3_orchestration_command_ack_duration` by `commandType`.
+2. If it is high, inspect the corresponding orchestration trace.
+3. Look at child spans for projection, sqlite, provider, or git work.
+
+### "Are git hooks causing latency?"
+
+1. Filter `git.operation` spans.
+2. Inspect `git.hook.started` and `git.hook.finished` events.
+3. Compare hook timing to the enclosing git span duration.
+
+### "Why do I have spans locally but nothing in Grafana?"
+
+Usually one of these is true:
+
+- `T3CODE_OTLP_TRACES_URL` was not set
+- the app was launched from a different environment than the one where you exported the vars
+- the app was not fully restarted after changing env
+- Grafana is looking at the wrong time range or service name
+
+If the local NDJSON file is updating, local tracing is working. The problem is almost always OTLP export configuration or process startup.
+
+## How To Think About Adding Tracing To Future Code
+
+### Prefer Boundaries Over Tiny Helpers
+
+Good span boundaries:
+
+- RPC methods
+- orchestration command handling
+- provider adapter calls
+- external process calls
+- persistence writes
+- queue handoffs
+
+Avoid tracing every tiny helper. Most helpers should inherit the active span rather than create a new one.
+
+### Reuse `Effect.fn(...)` Where It Already Exists
+
+The codebase already uses `Effect.fn("name")` heavily. That should usually be your first tracing boundary.
+
+For ad hoc work:
+
+```ts
+import { Effect } from "effect";
+
+const runThing = Effect.gen(function* () {
+ yield* Effect.annotateCurrentSpan({
+ "thing.id": "abc123",
+ "thing.kind": "example",
+ });
+
+ yield* Effect.logInfo("starting thing");
+ return yield* doWork();
+}).pipe(Effect.withSpan("thing.run"));
+```
+
+### Put High-Cardinality Detail On Spans
+
+Use span annotations for IDs, paths, and other detailed context:
+
+```ts
+yield *
+ Effect.annotateCurrentSpan({
+ "provider.thread_id": input.threadId,
+ "provider.request_id": input.requestId,
+ "git.cwd": input.cwd,
+ });
+```
+
+### Keep Metric Labels Low Cardinality
+
+Good metric labels:
+
+- operation kind
+- method name
+- provider kind
+- aggregate kind
+- outcome
+
+Bad metric labels:
+
+- raw thread IDs
+- command IDs
+- file paths
+- cwd
+- full prompts
+- full model strings when a normalized family label would do
+
+Detailed context belongs on spans, not metrics.
+
+### Use Logs As Span Events
+
+Logs inside a span become part of the trace story:
+
+```ts
+yield * Effect.logInfo("starting provider turn");
+yield * Effect.logDebug("waiting for approval response");
+```
+
+Those messages show up as span events because `Logger.tracerLogger` is installed.
+
+### Use The Pipeable Metrics API
+
+`withMetrics(...)` is the default way to attach a counter and timer to an effect:
+
+```ts
+import { someCounter, someDuration, withMetrics } from "../observability/Metrics.ts";
+
+const program = doWork().pipe(
+ withMetrics({
+ counter: someCounter,
+ timer: someDuration,
+ attributes: {
+ operation: "work",
+ },
+ }),
+);
+```
+
+## Detailed API Reference
+
+### Runtime Wiring
+
+The server observability layer is assembled in `apps/server/src/observability/Layers/Observability.ts`.
+
+It provides:
+
+- pretty stdout logger
+- `Logger.tracerLogger`
+- local NDJSON tracer
+- optional OTLP trace exporter
+- optional OTLP metrics exporter
+- Effect trace-level and timing refs
+
+### Env Vars
+
+Local trace file:
+
+- `T3CODE_TRACE_FILE`: override trace file path
+- `T3CODE_TRACE_MAX_BYTES`: per-file rotation size, default `10485760`
+- `T3CODE_TRACE_MAX_FILES`: rotated file count, default `10`
+- `T3CODE_TRACE_BATCH_WINDOW_MS`: flush window, default `200`
+- `T3CODE_TRACE_MIN_LEVEL`: minimum trace level, default `Info`
+- `T3CODE_TRACE_TIMING_ENABLED`: enable timing metadata, default `true`
+
+OTLP export:
+
+- `T3CODE_OTLP_TRACES_URL`: OTLP trace endpoint
+- `T3CODE_OTLP_METRICS_URL`: OTLP metric endpoint
+- `T3CODE_OTLP_EXPORT_INTERVAL_MS`: export interval, default `10000`
+- `T3CODE_OTLP_SERVICE_NAME`: service name, default `t3-server`
+
+If the OTLP URLs are unset, local tracing still works and metrics stay in-process only.
+
+### What Is Instrumented Today
+
+Current high-value span and metric boundaries include:
+
+- Effect RPC websocket request spans from `effect/rpc`
+- RPC request metrics in `apps/server/src/observability/RpcInstrumentation.ts`
+- startup phases
+- orchestration command processing
+- orchestration command acknowledgment latency
+- provider session and turn operations
+- git command execution and git hook events
+- terminal session lifecycle
+- sqlite query execution
+
+### Current Constraints
+
+- logs outside spans are not persisted
+- metrics are not snapshotted locally
+- the old `serverLogPath` still exists in config for compatibility, but the trace file is the persisted artifact that matters
diff --git a/packages/contracts/src/server.ts b/packages/contracts/src/server.ts
index d4038deb3..776a0a89e 100644
--- a/packages/contracts/src/server.ts
+++ b/packages/contracts/src/server.ts
@@ -72,6 +72,16 @@ export type ServerProvider = typeof ServerProvider.Type;
export const ServerProviders = Schema.Array(ServerProvider);
export type ServerProviders = typeof ServerProviders.Type;
+export const ServerObservability = Schema.Struct({
+ logsDirectoryPath: TrimmedNonEmptyString,
+ localTracingEnabled: Schema.Boolean,
+ otlpTracesUrl: Schema.optional(TrimmedNonEmptyString),
+ otlpTracesEnabled: Schema.Boolean,
+ otlpMetricsUrl: Schema.optional(TrimmedNonEmptyString),
+ otlpMetricsEnabled: Schema.Boolean,
+});
+export type ServerObservability = typeof ServerObservability.Type;
+
export const ServerConfig = Schema.Struct({
cwd: TrimmedNonEmptyString,
keybindingsConfigPath: TrimmedNonEmptyString,
@@ -79,6 +89,7 @@ export const ServerConfig = Schema.Struct({
issues: ServerConfigIssues,
providers: ServerProviders,
availableEditors: Schema.Array(EditorId),
+ observability: ServerObservability,
settings: ServerSettings,
});
export type ServerConfig = typeof ServerConfig.Type;
diff --git a/turbo.json b/turbo.json
index 9359376f5..47d709138 100644
--- a/turbo.json
+++ b/turbo.json
@@ -11,7 +11,17 @@
"T3CODE_NO_BROWSER",
"T3CODE_HOME",
"T3CODE_AUTH_TOKEN",
- "T3CODE_DESKTOP_WS_URL"
+ "T3CODE_DESKTOP_WS_URL",
+ "T3CODE_TRACE_MIN_LEVEL",
+ "T3CODE_TRACE_TIMING_ENABLED",
+ "T3CODE_TRACE_FILE",
+ "T3CODE_TRACE_MAX_BYTES",
+ "T3CODE_TRACE_MAX_FILES",
+ "T3CODE_TRACE_BATCH_WINDOW_MS",
+ "T3CODE_OTLP_TRACES_URL",
+ "T3CODE_OTLP_METRICS_URL",
+ "T3CODE_OTLP_EXPORT_INTERVAL_MS",
+ "T3CODE_OTLP_SERVICE_NAME"
],
"tasks": {
"build": {