Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

feat(server): record runtime mode per turn and on mode changes (#5593)

Theo Browne7963cc70f3c9
diff --git a/apps/server/integration/providerService.integration.test.ts b/apps/server/integration/providerService.integration.test.ts
index e703af4b1..c57d289f9 100644
--- a/apps/server/integration/providerService.integration.test.ts
+++ b/apps/server/integration/providerService.integration.test.ts
@@ -8,6 +8,7 @@ import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Path from "effect/Path";
import * as Queue from "effect/Queue";
+import * as Ref from "effect/Ref";
import * as Stream from "effect/Stream";
import { ProviderAdapterRegistry } from "../src/provider/Services/ProviderAdapterRegistry.ts";
@@ -54,35 +55,58 @@ interface IntegrationFixture {
readonly layer: Layer.Layer<ProviderService, unknown, never>;
}
-const makeIntegrationFixture = Effect.gen(function* () {
- const cwd = yield* makeWorkspaceDirectory;
- const harness = yield* makeTestProviderAdapterHarness();
-
- const registry = makeAdapterRegistryMock({
- [ProviderDriverKind.make("codex")]: harness.adapter,
- });
+interface RecordedAnalyticsEvent {
+ readonly event: string;
+ readonly properties: Readonly<Record<string, unknown>> | undefined;
+}
- const directoryLayer = ProviderSessionDirectoryLive.pipe(
- Layer.provide(ProviderSessionRuntime.layer),
+/**
+ * Analytics layer that keeps captured events in memory so tests can assert on
+ * telemetry payloads. `AnalyticsService.layerTest` discards them.
+ */
+const makeRecordingAnalytics = Effect.gen(function* () {
+ const recorded = yield* Ref.make<ReadonlyArray<RecordedAnalyticsEvent>>([]);
+ const layer = Layer.succeed(
+ AnalyticsService,
+ AnalyticsService.of({
+ record: (event, properties) =>
+ Ref.update(recorded, (current) => [...current, { event, properties }]),
+ flush: Effect.void,
+ }),
);
-
- const shared = Layer.mergeAll(
- directoryLayer,
- Layer.succeed(ProviderAdapterRegistry, registry),
- ServerSettingsService.layerTest(DEFAULT_SERVER_SETTINGS),
- AnalyticsService.layerTest,
- Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers),
- ).pipe(Layer.provide(SqlitePersistenceMemory));
-
- const layer = makeProviderServiceLive().pipe(Layer.provide(shared));
-
- return {
- cwd,
- harness,
- layer,
- } satisfies IntegrationFixture;
+ return { layer, get: Ref.get(recorded) } as const;
});
+const makeIntegrationFixture = (options?: { readonly analytics?: Layer.Layer<AnalyticsService> }) =>
+ Effect.gen(function* () {
+ const cwd = yield* makeWorkspaceDirectory;
+ const harness = yield* makeTestProviderAdapterHarness();
+
+ const registry = makeAdapterRegistryMock({
+ [ProviderDriverKind.make("codex")]: harness.adapter,
+ });
+
+ const directoryLayer = ProviderSessionDirectoryLive.pipe(
+ Layer.provide(ProviderSessionRuntime.layer),
+ );
+
+ const shared = Layer.mergeAll(
+ directoryLayer,
+ Layer.succeed(ProviderAdapterRegistry, registry),
+ ServerSettingsService.layerTest(DEFAULT_SERVER_SETTINGS),
+ options?.analytics ?? AnalyticsService.layerTest,
+ Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers),
+ ).pipe(Layer.provide(SqlitePersistenceMemory));
+
+ const layer = makeProviderServiceLive().pipe(Layer.provide(shared));
+
+ return {
+ cwd,
+ harness,
+ layer,
+ } satisfies IntegrationFixture;
+ });
+
const collectEventsDuring = <A, E, R>(
stream: Stream.Stream<ProviderRuntimeEvent>,
count: number,
@@ -126,7 +150,7 @@ const runTurn = (input: {
it.live("replays typed runtime fixture events", () =>
Effect.gen(function* () {
- const fixture = yield* makeIntegrationFixture;
+ const fixture = yield* makeIntegrationFixture();
yield* Effect.gen(function* () {
const provider = yield* ProviderService;
@@ -161,7 +185,7 @@ it.live("replays typed runtime fixture events", () =>
it.live("replays file-changing fixture turn events", () =>
Effect.gen(function* () {
- const fixture = yield* makeIntegrationFixture;
+ const fixture = yield* makeIntegrationFixture();
const { join } = yield* Path.Path;
const { writeFileString } = yield* FileSystem.FileSystem;
@@ -198,7 +222,7 @@ it.live("replays file-changing fixture turn events", () =>
it.live("runs multi-turn tool/approval flow", () =>
Effect.gen(function* () {
- const fixture = yield* makeIntegrationFixture;
+ const fixture = yield* makeIntegrationFixture();
const { join } = yield* Path.Path;
const { writeFileString } = yield* FileSystem.FileSystem;
@@ -250,7 +274,7 @@ it.live("runs multi-turn tool/approval flow", () =>
it.live("rolls back provider conversation state only", () =>
Effect.gen(function* () {
- const fixture = yield* makeIntegrationFixture;
+ const fixture = yield* makeIntegrationFixture();
const { join } = yield* Path.Path;
const { writeFileString, readFileString } = yield* FileSystem.FileSystem;
@@ -302,3 +326,59 @@ it.live("rolls back provider conversation state only", () =>
}).pipe(Effect.provide(fixture.layer));
}).pipe(Effect.provide(NodeServices.layer)),
);
+
+it.live("reports runtime mode per turn and on mode transitions", () =>
+ Effect.gen(function* () {
+ const analytics = yield* makeRecordingAnalytics;
+ const fixture = yield* makeIntegrationFixture({ analytics: analytics.layer });
+ const threadId = ThreadId.make("thread-integration-runtime-mode");
+
+ yield* Effect.gen(function* () {
+ const provider = yield* ProviderService;
+ const startSession = (runtimeMode: "approval-required" | "full-access") =>
+ provider.startSession(threadId, {
+ threadId,
+ provider: ProviderDriverKind.make("codex"),
+ providerInstanceId: codexInstanceId,
+ cwd: fixture.cwd,
+ runtimeMode,
+ });
+
+ yield* startSession("approval-required");
+ yield* runTurn({
+ provider,
+ harness: fixture.harness,
+ threadId,
+ userText: "supervised turn",
+ response: { events: codexTurnTextFixture },
+ });
+
+ // Toggling the mode restarts the session, which is the only place the
+ // transition is observable.
+ yield* startSession("full-access");
+ yield* runTurn({
+ provider,
+ harness: fixture.harness,
+ threadId,
+ userText: "full access turn",
+ response: { events: codexTurnTextFixture },
+ });
+
+ const recorded = yield* analytics.get;
+
+ assert.deepEqual(
+ recorded
+ .filter((entry) => entry.event === "provider.turn.sent")
+ .map((entry) => entry.properties?.runtimeMode),
+ ["approval-required", "full-access"],
+ );
+
+ assert.deepEqual(
+ recorded
+ .filter((entry) => entry.event === "provider.runtime_mode.changed")
+ .map((entry) => [entry.properties?.from, entry.properties?.to]),
+ [["approval-required", "full-access"]],
+ );
+ }).pipe(Effect.provide(fixture.layer));
+ }).pipe(Effect.provide(NodeServices.layer)),
+);
diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts
index ecf26a914..d0acc1039 100644
--- a/apps/server/src/provider/Layers/ProviderService.ts
+++ b/apps/server/src/provider/Layers/ProviderService.ts
@@ -459,6 +459,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
adapter,
instanceId,
threadId: input.threadId,
+ runtimeMode: binding.runtimeMode,
isActive: true,
} as const;
}
@@ -468,6 +469,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
adapter,
instanceId,
threadId: input.threadId,
+ runtimeMode: binding.runtimeMode,
isActive: false,
} as const;
}
@@ -480,6 +482,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
adapter: recovered.adapter,
instanceId,
threadId: input.threadId,
+ runtimeMode: recovered.session.runtimeMode,
isActive: true,
} as const;
});
@@ -629,6 +632,19 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
input.modelSelection.model.trim().length > 0,
});
+ // Changing runtime mode restarts the session, so the transition is only
+ // observable here, by diffing against the mode the previous session for
+ // this thread was bound to. Recording it separately is what makes the
+ // "started supervised, switched to full access" funnel answerable.
+ const previousRuntimeMode = persistedBinding?.runtimeMode;
+ if (previousRuntimeMode !== undefined && previousRuntimeMode !== input.runtimeMode) {
+ yield* analytics.record("provider.runtime_mode.changed", {
+ provider: sessionWithInstance.provider,
+ from: previousRuntimeMode,
+ to: input.runtimeMode,
+ });
+ }
+
return sessionWithInstance;
}).pipe(
withMetrics({
@@ -703,6 +719,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
provider: routed.adapter.provider,
model: input.modelSelection?.model,
interactionMode: input.interactionMode,
+ // Session-start events alone skew runtime mode toward users who toggle
+ // often, since every toggle restarts the session. Recording it per turn
+ // gives a usage-weighted view and lets it cross with interactionMode.
+ runtimeMode: routed.runtimeMode,
attachmentCount: input.attachments.length,
hasInput: typeof input.input === "string" && input.input.trim().length > 0,
});