Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
feat(server): record runtime mode per turn and on mode changes (#5593)
Theo Browne7963cc70f3c9
diff --git a/apps/server/integration/providerService.integration.test.ts b/apps/server/integration/providerService.integration.test.tsindex e703af4b1..c57d289f9 100644--- a/apps/server/integration/providerService.integration.test.ts+++ b/apps/server/integration/providerService.integration.test.ts@@ -8,6 +8,7 @@ import * as FileSystem from "effect/FileSystem";import * as Layer from "effect/Layer";import * as Path from "effect/Path";import * as Queue from "effect/Queue";+import * as Ref from "effect/Ref";import * as Stream from "effect/Stream";import { ProviderAdapterRegistry } from "../src/provider/Services/ProviderAdapterRegistry.ts";@@ -54,35 +55,58 @@ interface IntegrationFixture {readonly layer: Layer.Layer<ProviderService, unknown, never>;}-const makeIntegrationFixture = Effect.gen(function* () {- const cwd = yield* makeWorkspaceDirectory;- const harness = yield* makeTestProviderAdapterHarness();-- const registry = makeAdapterRegistryMock({- [ProviderDriverKind.make("codex")]: harness.adapter,- });+interface RecordedAnalyticsEvent {+ readonly event: string;+ readonly properties: Readonly<Record<string, unknown>> | undefined;+}- const directoryLayer = ProviderSessionDirectoryLive.pipe(- Layer.provide(ProviderSessionRuntime.layer),+/**+ * Analytics layer that keeps captured events in memory so tests can assert on+ * telemetry payloads. `AnalyticsService.layerTest` discards them.+ */+const makeRecordingAnalytics = Effect.gen(function* () {+ const recorded = yield* Ref.make<ReadonlyArray<RecordedAnalyticsEvent>>([]);+ const layer = Layer.succeed(+ AnalyticsService,+ AnalyticsService.of({+ record: (event, properties) =>+ Ref.update(recorded, (current) => [...current, { event, properties }]),+ flush: Effect.void,+ }),);-- const shared = Layer.mergeAll(- directoryLayer,- Layer.succeed(ProviderAdapterRegistry, registry),- ServerSettingsService.layerTest(DEFAULT_SERVER_SETTINGS),- AnalyticsService.layerTest,- Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers),- ).pipe(Layer.provide(SqlitePersistenceMemory));-- const layer = makeProviderServiceLive().pipe(Layer.provide(shared));-- return {- cwd,- harness,- layer,- } satisfies IntegrationFixture;+ return { layer, get: Ref.get(recorded) } as const;});+const makeIntegrationFixture = (options?: { readonly analytics?: Layer.Layer<AnalyticsService> }) =>+ Effect.gen(function* () {+ const cwd = yield* makeWorkspaceDirectory;+ const harness = yield* makeTestProviderAdapterHarness();++ const registry = makeAdapterRegistryMock({+ [ProviderDriverKind.make("codex")]: harness.adapter,+ });++ const directoryLayer = ProviderSessionDirectoryLive.pipe(+ Layer.provide(ProviderSessionRuntime.layer),+ );++ const shared = Layer.mergeAll(+ directoryLayer,+ Layer.succeed(ProviderAdapterRegistry, registry),+ ServerSettingsService.layerTest(DEFAULT_SERVER_SETTINGS),+ options?.analytics ?? AnalyticsService.layerTest,+ Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers),+ ).pipe(Layer.provide(SqlitePersistenceMemory));++ const layer = makeProviderServiceLive().pipe(Layer.provide(shared));++ return {+ cwd,+ harness,+ layer,+ } satisfies IntegrationFixture;+ });+const collectEventsDuring = <A, E, R>(stream: Stream.Stream<ProviderRuntimeEvent>,count: number,@@ -126,7 +150,7 @@ const runTurn = (input: {it.live("replays typed runtime fixture events", () =>Effect.gen(function* () {- const fixture = yield* makeIntegrationFixture;+ const fixture = yield* makeIntegrationFixture();yield* Effect.gen(function* () {const provider = yield* ProviderService;@@ -161,7 +185,7 @@ it.live("replays typed runtime fixture events", () =>it.live("replays file-changing fixture turn events", () =>Effect.gen(function* () {- const fixture = yield* makeIntegrationFixture;+ const fixture = yield* makeIntegrationFixture();const { join } = yield* Path.Path;const { writeFileString } = yield* FileSystem.FileSystem;@@ -198,7 +222,7 @@ it.live("replays file-changing fixture turn events", () =>it.live("runs multi-turn tool/approval flow", () =>Effect.gen(function* () {- const fixture = yield* makeIntegrationFixture;+ const fixture = yield* makeIntegrationFixture();const { join } = yield* Path.Path;const { writeFileString } = yield* FileSystem.FileSystem;@@ -250,7 +274,7 @@ it.live("runs multi-turn tool/approval flow", () =>it.live("rolls back provider conversation state only", () =>Effect.gen(function* () {- const fixture = yield* makeIntegrationFixture;+ const fixture = yield* makeIntegrationFixture();const { join } = yield* Path.Path;const { writeFileString, readFileString } = yield* FileSystem.FileSystem;@@ -302,3 +326,59 @@ it.live("rolls back provider conversation state only", () =>}).pipe(Effect.provide(fixture.layer));}).pipe(Effect.provide(NodeServices.layer)),);++it.live("reports runtime mode per turn and on mode transitions", () =>+ Effect.gen(function* () {+ const analytics = yield* makeRecordingAnalytics;+ const fixture = yield* makeIntegrationFixture({ analytics: analytics.layer });+ const threadId = ThreadId.make("thread-integration-runtime-mode");++ yield* Effect.gen(function* () {+ const provider = yield* ProviderService;+ const startSession = (runtimeMode: "approval-required" | "full-access") =>+ provider.startSession(threadId, {+ threadId,+ provider: ProviderDriverKind.make("codex"),+ providerInstanceId: codexInstanceId,+ cwd: fixture.cwd,+ runtimeMode,+ });++ yield* startSession("approval-required");+ yield* runTurn({+ provider,+ harness: fixture.harness,+ threadId,+ userText: "supervised turn",+ response: { events: codexTurnTextFixture },+ });++ // Toggling the mode restarts the session, which is the only place the+ // transition is observable.+ yield* startSession("full-access");+ yield* runTurn({+ provider,+ harness: fixture.harness,+ threadId,+ userText: "full access turn",+ response: { events: codexTurnTextFixture },+ });++ const recorded = yield* analytics.get;++ assert.deepEqual(+ recorded+ .filter((entry) => entry.event === "provider.turn.sent")+ .map((entry) => entry.properties?.runtimeMode),+ ["approval-required", "full-access"],+ );++ assert.deepEqual(+ recorded+ .filter((entry) => entry.event === "provider.runtime_mode.changed")+ .map((entry) => [entry.properties?.from, entry.properties?.to]),+ [["approval-required", "full-access"]],+ );+ }).pipe(Effect.provide(fixture.layer));+ }).pipe(Effect.provide(NodeServices.layer)),+);diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.tsindex ecf26a914..d0acc1039 100644--- a/apps/server/src/provider/Layers/ProviderService.ts+++ b/apps/server/src/provider/Layers/ProviderService.ts@@ -459,6 +459,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (adapter,instanceId,threadId: input.threadId,+ runtimeMode: binding.runtimeMode,isActive: true,} as const;}@@ -468,6 +469,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (adapter,instanceId,threadId: input.threadId,+ runtimeMode: binding.runtimeMode,isActive: false,} as const;}@@ -480,6 +482,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (adapter: recovered.adapter,instanceId,threadId: input.threadId,+ runtimeMode: recovered.session.runtimeMode,isActive: true,} as const;});@@ -629,6 +632,19 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (input.modelSelection.model.trim().length > 0,});+ // Changing runtime mode restarts the session, so the transition is only+ // observable here, by diffing against the mode the previous session for+ // this thread was bound to. Recording it separately is what makes the+ // "started supervised, switched to full access" funnel answerable.+ const previousRuntimeMode = persistedBinding?.runtimeMode;+ if (previousRuntimeMode !== undefined && previousRuntimeMode !== input.runtimeMode) {+ yield* analytics.record("provider.runtime_mode.changed", {+ provider: sessionWithInstance.provider,+ from: previousRuntimeMode,+ to: input.runtimeMode,+ });+ }+return sessionWithInstance;}).pipe(withMetrics({@@ -703,6 +719,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (provider: routed.adapter.provider,model: input.modelSelection?.model,interactionMode: input.interactionMode,+ // Session-start events alone skew runtime mode toward users who toggle+ // often, since every toggle restarts the session. Recording it per turn+ // gives a usage-weighted view and lets it cross with interactionMode.+ runtimeMode: routed.runtimeMode,attachmentCount: input.attachments.length,hasInput: typeof input.input === "string" && input.input.trim().length > 0,});