TypeScript
perf(mobile): faster thread sync, connects, and reconnects on bad wifi (#28487)
Bilal Bakr2917f2846558
diff --git a/apps/mobile/src/connection/runtime.ts b/apps/mobile/src/connection/runtime.tsindex 94ee7d5ebeb..2f88de1432a 100644--- a/apps/mobile/src/connection/runtime.ts+++ b/apps/mobile/src/connection/runtime.ts@@ -36,7 +36,11 @@ type ConnectionLayerSource =const layerProvidedClientConnection = layerSnapshotLoader.pipe(Layer.provideMerge(- Connection.layerWithOptions({ usageLimitSources: true, usageLimitsCommand: true }),+ Connection.layerWithOptions({+ usageLimitSources: true,+ usageLimitsCommand: true,+ providerCatalogsOnDemand: true,+ }),),Layer.provideMerge(Layer.mergeAll(diff --git a/apps/mobile/src/state/server.ts b/apps/mobile/src/state/server.tsindex 8de9354d385..b6659a2e982 100644--- a/apps/mobile/src/state/server.ts+++ b/apps/mobile/src/state/server.ts@@ -13,6 +13,7 @@ export const serverEnvironment = createServerEnvironmentAtoms(connectionAtomRuntinitialConfigValueAtom: environmentSession.initialConfigValueAtom,usageLimitSources: true,usageLimitsCommand: true,+ providerCatalogsOnDemand: true,});export const describeOutdatedServer = createOutdatedServerDescriptionCommand(connectionAtomRuntime);export const updateOutdatedServer = createOutdatedServerUpdateCommand(connectionAtomRuntime);diff --git a/apps/server/src/terminal/OutputProtocol.test.ts b/apps/server/src/terminal/OutputProtocol.test.tsindex a804a6610b7..9750f418152 100644--- a/apps/server/src/terminal/OutputProtocol.test.ts+++ b/apps/server/src/terminal/OutputProtocol.test.ts@@ -1,5 +1,6 @@import { assert, describe, it } from "@effect/vitest";import {+ ORCHESTRATION_V2_WS_METHODS,type TerminalAttachStreamEvent,TerminalAttachStreamEvent as TerminalAttachStreamEventSchema,WS_METHODS,@@ -13,7 +14,7 @@ import * as Stream from "effect/Stream";import * as TestClock from "effect/testing/TestClock";import { Rpc, RpcGroup, RpcMessage, RpcSerialization, RpcServer } from "effect/rpc";-import { terminalOutputStream, withTerminalOutputWindow } from "./OutputProtocol.ts";+import { terminalOutputStream, withOutputWindow } from "./OutputProtocol.ts";describe("terminal output window", () => {it.effect.each([@@ -21,6 +22,9 @@ describe("terminal output window", () => {{ tag: WS_METHODS.terminalObserve, size: 1, limit: 8 },{ tag: WS_METHODS.terminalAttach, size: 64 * 1024, limit: 1 },{ tag: WS_METHODS.terminalObserve, size: 64 * 1024, limit: 1 },+ { tag: WS_METHODS.subscribeServerConfig, size: 1, limit: 8 },+ { tag: ORCHESTRATION_V2_WS_METHODS.subscribeShell, size: 1, limit: 8 },+ { tag: ORCHESTRATION_V2_WS_METHODS.subscribeThread, size: 64 * 1024, limit: 8 },{ tag: WS_METHODS.subscribeTerminalMetadata, size: 1, limit: 1 },])("limits $tag with $size-byte values to $limit pending chunks", ({ tag, size, limit }) =>Effect.gen(function* () {@@ -48,7 +52,7 @@ describe("terminal output window", () => {);yield* RpcServer.make(group).pipe(Effect.provide(group.toLayerHandler(tag, () => Stream.fromQueue(output))),- Effect.provideService(RpcServer.Protocol, withTerminalOutputWindow(protocol)),+ Effect.provideService(RpcServer.Protocol, withOutputWindow(protocol)),Effect.forkScoped,);const write = yield* Deferred.await(receive);@@ -191,7 +195,7 @@ describe("terminal output stream", () => {),),),- Effect.provideService(RpcServer.Protocol, withTerminalOutputWindow(protocol)),+ Effect.provideService(RpcServer.Protocol, withOutputWindow(protocol)),Effect.forkScoped,);const write = yield* Deferred.await(receive);diff --git a/apps/server/src/terminal/OutputProtocol.ts b/apps/server/src/terminal/OutputProtocol.tsindex e52785d6985..11339fc2bfa 100644--- a/apps/server/src/terminal/OutputProtocol.ts+++ b/apps/server/src/terminal/OutputProtocol.ts@@ -1,4 +1,9 @@-import { type TerminalAttachStreamEvent, type TerminalEvent, WS_METHODS } from "@t3tools/contracts";+import {+ ORCHESTRATION_V2_WS_METHODS,+ type TerminalAttachStreamEvent,+ type TerminalEvent,+ WS_METHODS,+} from "@t3tools/contracts";import type * as Arr from "effect/Array";import * as Cause from "effect/Cause";import * as Effect from "effect/Effect";@@ -11,31 +16,62 @@ const MAX_PENDING_BYTES = 64 * 1024;const isFull = (sizes: number[]) =>sizes.length >= MAX_PENDING_CHUNKS ||sizes.reduce((total, size) => total + size, 0) >= MAX_PENDING_BYTES;+// Terminal output is small chunks, so its window also caps bytes. The other streams+// window by count only: sizing a multi-MB thread snapshot would serialize it twice.+const BYTE_WINDOWED_STREAMS: ReadonlySet<string> = new Set([+ WS_METHODS.terminalAttach,+ WS_METHODS.terminalObserve,+]);+// One ACK round trip per chunk stalls sync on high-latency links.+const COUNT_WINDOWED_STREAMS: ReadonlySet<string> = new Set([+ WS_METHODS.subscribeServerConfig,+ ORCHESTRATION_V2_WS_METHODS.subscribeShell,+ ORCHESTRATION_V2_WS_METHODS.subscribeThread,+]);-export function withTerminalOutputWindow(+interface Window {+ readonly bytes: boolean;+ readonly sizes: number[];+}++export function withOutputWindow(protocol: RpcServer.Protocol["Service"],): RpcServer.Protocol["Service"] {- const windows = new Map<string, number[]>();+ // RpcServer drops a disconnected client before its streams exit, so their Exit never+ // reaches send; departed clients are pruned when a window opens instead.+ const clients = new Map<number, Map<string, Window>>();let receive: Parameters<RpcServer.Protocol["Service"]["run"]>[0];+ const windowFor = (clientId: number, requestId: string) => clients.get(clientId)?.get(requestId);+ const openWindow = Effect.fnUntraced(function* (+ clientId: number,+ requestId: string,+ bytes: boolean,+ ) {+ const connected = yield* protocol.clientIds;+ for (const id of clients.keys()) if (!connected.has(id)) clients.delete(id);+ let windows = clients.get(clientId);+ if (!windows) clients.set(clientId, (windows = new Map()));+ if (!windows.has(requestId)) windows.set(requestId, { bytes, sizes: [] });+ });return {...protocol,run: (write) => {receive = write;return protocol.run((clientId, message) =>Effect.suspend(() => {- if (- message._tag === "Request" &&- (message.tag === WS_METHODS.terminalAttach ||- message.tag === WS_METHODS.terminalObserve)- ) {- const key = `${clientId}:${message.id}`;- if (!windows.has(key)) windows.set(key, []);+ if (message._tag === "Request") {+ const bytes = BYTE_WINDOWED_STREAMS.has(message.tag);+ if (bytes || COUNT_WINDOWED_STREAMS.has(message.tag)) {+ return openWindow(clientId, String(message.id), bytes).pipe(+ Effect.andThen(write(clientId, message)),+ );+ }} else if (message._tag === "Ack") {- const window = windows.get(`${clientId}:${message.requestId}`);+ const window = windowFor(clientId, String(message.requestId));if (window) {- const wasFull = isFull(window);- window.shift();- if (!wasFull || isFull(window)) return Effect.void;+ const wasFull = isFull(window.sizes);+ window.sizes.shift();+ if (!wasFull || isFull(window.sizes)) return Effect.void;}}return write(clientId, message);@@ -46,14 +82,15 @@ export function withTerminalOutputWindow(Effect.suspend(() => {const send = protocol.send(clientId, response, transferables);if (response._tag === "Exit") {- windows.delete(`${clientId}:${response.requestId}`);+ clients.get(clientId)?.delete(String(response.requestId));} else if (response._tag === "Chunk") {- const window = windows.get(`${clientId}:${response.requestId}`);+ const window = windowFor(clientId, String(response.requestId));if (window) {- // @effect-diagnostics-next-line globalJsonInEffect:off- const size = Buffer.byteLength(JSON.stringify(response));- window.push(size);- if (!isFull(window)) {+ window.sizes.push(+ // @effect-diagnostics-next-line globalJsonInEffect:off+ window.bytes ? Buffer.byteLength(JSON.stringify(response)) : 0,+ );+ if (!isFull(window.sizes)) {return send.pipe(Effect.andThen(() =>receive(clientId, {diff --git a/apps/server/src/ws.test.ts b/apps/server/src/ws.test.tsindex ca53e5e3943..bf2298427e4 100644--- a/apps/server/src/ws.test.ts+++ b/apps/server/src/ws.test.ts@@ -116,6 +116,7 @@ import * as WorkspacePaths from "./workspace/WorkspacePaths.ts";import * as ServerSecretStore from "./auth/ServerSecretStore.ts";import * as ServerConfigModule from "./config.ts";import * as Keybindings from "./keybindings.ts";+import { USAGE_LIMITS_COMMAND } from "@t3tools/shared/usageLimits";import * as ServerSettings from "./serverSettings.ts";import * as ManagedPolicy from "./policy/ManagedPolicy.ts";@@ -1024,13 +1025,21 @@ it.effect("on-demand clients get provider config without workspace snapshots", (auth: { status: "authenticated" },checkedAt: "2026-10-25T00:00:00.000Z",models: [],- slashCommands: [{ name: "machine-command" }],- skills: [],+ slashCommands: [{ name: "machine-command", description: "A machine command" }],+ skills: [+ {+ name: "machine-skill",+ description: "Machine skill",+ path: "/skills/machine-skill",+ enabled: true,+ },+ ],workspaceSnapshots: [snapshotFor("/a", "2026-10-25T00:00:00.000Z"),snapshotFor("/b", "2026-10-25T00:00:00.000Z"),],};+ let scansCwd = true;const providerChanges = yield* Queue.unbounded<ReadonlyArray<ServerProvider>>();const pulled = yield* Queue.unbounded<void>();const responses = yield* Queue.unbounded<unknown>();@@ -1047,6 +1056,20 @@ it.effect("on-demand clients get provider config without workspace snapshots", (),refreshWorkspaceSnapshot: () => Effect.succeed([provider]),}),+ Layer.mock(ProviderInstanceRegistry.ProviderInstanceRegistry)({+ listInstances: Effect.sync(() => [+ (scansCwd+ ? { instanceId, snapshotForCwd: () => Effect.die("unused") }+ : { instanceId }) as never,+ ]),+ getInstance: () =>+ Effect.sync(+ () =>+ (scansCwd+ ? { instanceId, snapshotForCwd: () => Effect.die("unused") }+ : { instanceId }) as never,+ ),+ }),Layer.mock(UsageLimitSources.UsageLimitSources)({current: Effect.succeed([]),streamChanges: Stream.make([]),@@ -1116,12 +1139,17 @@ it.effect("on-demand clients get provider config without workspace snapshots", (_tag: "Request",id: configRequest,tag: WS_METHODS.subscribeServerConfig,- payload: { workspaceSnapshotsOnDemand: true },+ payload: { workspaceSnapshotsOnDemand: true, providerCatalogsOnDemand: true },headers: Headers.empty,});const snapshot = (yield* Queue.take(responses)) as ServerConfigStreamEvent;if (snapshot.type !== "snapshot") return yield* Effect.die("Missing initial snapshot");assert.isUndefined(snapshot.config.providers[0]?.workspaceSnapshots);+ // The cwd snapshot the client fetches carries the catalog, so only command names are sent.+ assert.deepEqual(snapshot.config.providers[0]?.slashCommands, [{ name: "machine-command" }]);+ assert.deepEqual(snapshot.config.providers[0]?.skills, [+ { name: "machine-skill", path: "/skills/machine-skill", enabled: true },+ ]);yield* server.write(0, { _tag: "Ack", requestId: configRequest });// A rescan that only touches a snapshot sends nothing; the next real change does.@@ -1130,13 +1158,34 @@ it.effect("on-demand clients get provider config without workspace snapshots", (]);yield* Queue.take(pulled);yield* TestClock.adjust("1 second");- yield* Queue.offer(providerChanges, [{ ...provider, status: "warning" }]);+ // Late /usage-limits discovery still reaches the client, so it refetches its cwd snapshot.+ yield* Queue.offer(providerChanges, [+ {+ ...provider,+ status: "warning",+ slashCommands: [...provider.slashCommands, USAGE_LIMITS_COMMAND],+ },+ ]);yield* Queue.take(pulled);yield* TestClock.adjust("1 second");const update = (yield* Queue.take(responses)) as ServerConfigStreamEvent;if (update.type !== "providerStatuses") return yield* Effect.die("Expected provider update");assert.equal(update.payload.providers[0]?.status, "warning");assert.isUndefined(update.payload.providers[0]?.workspaceSnapshots);+ assert.deepEqual(update.payload.providers[0]?.slashCommands, [+ { name: "machine-command" },+ { name: USAGE_LIMITS_COMMAND.name },+ ]);+ yield* server.write(0, { _tag: "Ack", requestId: configRequest });+ // A replacement that cannot scan a cwd sends its full machine catalog.+ scansCwd = false;+ yield* Queue.offer(providerChanges, [{ ...provider, status: "warning" }]);+ yield* Queue.take(pulled);+ yield* TestClock.adjust("1 second");+ const replaced = (yield* Queue.take(responses)) as ServerConfigStreamEvent;+ if (replaced.type !== "providerStatuses") return yield* Effect.die("Expected provider update");+ assert.deepEqual(replaced.payload.providers[0]?.slashCommands, provider.slashCommands);+ assert.deepEqual(replaced.payload.providers[0]?.skills, provider.skills);yield* server.write(0, {_tag: "Request",@@ -1152,6 +1201,32 @@ it.effect("on-demand clients get provider config without workspace snapshots", (refreshed.providers.map((entry) => [entry.workspaceSnapshots, entry.slashCommands]),[[[snapshotFor("/b", "2026-10-25T00:00:00.000Z")], []]],);+ // A failed scan still gives a client without the machine catalog its commands.+ scansCwd = true;+ yield* server.write(0, {+ _tag: "Request",+ id: RpcMessage.RequestId("refresh-failed-scan"),+ tag: WS_METHODS.serverRefreshProviders,+ payload: {+ instanceId,+ cwd: "/c",+ workspaceSnapshotsOnDemand: true,+ providerCatalogsOnDemand: true,+ },+ headers: Headers.empty,+ });+ const fallback = (yield* Queue.take(responses)) as {+ readonly providers: ReadonlyArray<ServerProvider>;+ };+ assert.deepEqual(fallback.providers[0]?.workspaceSnapshots, [+ {+ cwd: "/c",+ checkedAt: provider.checkedAt,+ slashCommands: provider.slashCommands,+ slashCommandsPending: true,+ skills: provider.skills,+ },+ ]);yield* server.write(0, { _tag: "Interrupt", requestId: configRequest, interruptors: [] });}).pipe(Effect.provide(diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.tsindex 9816f6c6008..e3c7fdaa474 100644--- a/apps/server/src/ws.ts+++ b/apps/server/src/ws.ts@@ -188,7 +188,7 @@ import { invalidateSettingsSecrets } from "./sourceControl/sourceControlHostSecrimport * as DatabaseStorage from "./persistence/DatabaseStorage.ts";import * as StorageCleanup from "./storageCleanup.ts";import * as TerminalManager from "./terminal/Manager.ts";-import { terminalOutputStream, withTerminalOutputWindow } from "./terminal/OutputProtocol.ts";+import { terminalOutputStream, withOutputWindow } from "./terminal/OutputProtocol.ts";import * as ServerBrowser from "./preview/ServerBrowser.ts";import * as DeviceService from "./device/DeviceService.ts";import { remoteSshDeviceHosts } from "./device/localSshDeviceHost.ts";@@ -426,8 +426,44 @@ export const resolveOpenDiscoveryForConfig = <T, D, E1, R1, E2, R2, E3, R3, E4,},{ concurrency: "unbounded" },).pipe(Effect.map(({ editorConfig, ...endpoints }) => ({ ...editorConfig, ...endpoints })));-function withoutWorkspaceSnapshots(providers: ReadonlyArray<ServerProvider>): ServerProvider[] {- return providers.map(({ workspaceSnapshots: _workspaceSnapshots, ...provider }) => provider);+// A failed scan leaves no cwd snapshot, so a client without the machine catalog gets it+// as a pending snapshot, which it shows while it retries the scan.+function workspaceSnapshotOrCatalog(+ provider: ServerProvider,+ cwd: string,+ catalogOnDemand: boolean,+) {+ const snapshots = (provider.workspaceSnapshots ?? []).filter((snapshot) => snapshot.cwd === cwd);+ if (snapshots.length > 0 || !catalogOnDemand) return snapshots;+ return [+ {+ cwd,+ checkedAt: provider.checkedAt,+ slashCommands: provider.slashCommands,+ slashCommandsPending: true,+ skills: provider.skills,+ },+ ];+}++// On-demand catalog clients read full entries from the cwd snapshot they fetch. Names still+// change the config when the machine catalog does, so the client refetches, and skill names+// and labels keep skill links working where no snapshot is fetched.+function withoutWorkspaceSnapshots(+ providers: ReadonlyArray<ServerProvider>,+ catalogsPerCwd: ReadonlySet<string> = new Set(),+): ServerProvider[] {+ return providers.map(({ workspaceSnapshots: _workspaceSnapshots, ...provider }) =>+ catalogsPerCwd.has(provider.instanceId)+ ? {+ ...provider,+ slashCommands: provider.slashCommands.map(({ name }) => ({ name })),+ skills: provider.skills.map(+ ({ description: _d, shortDescription: _s, ...skill }) => skill,+ ),+ }+ : provider,+ );}function unexpectedCompatibilityError(error: never): never {@@ -1860,6 +1896,7 @@ export const layerWsRpc = (readonly usageLimitsCommand: boolean;readonly workspaceSnapshotsOnDemand: boolean;readonly deferNetworkDiscovery: boolean;+ readonly catalogsPerCwd?: ReadonlySet<string>;}) {const keybindingsConfig = yield* keybindings.loadConfigState;const currentProviders = yield* providerRegistry.getProviders;@@ -1867,7 +1904,7 @@ export const layerWsRpc = (? withUsageLimitsCommands(currentProviders, yield* usageLimitSources.current): currentProviders;const providers = options.workspaceSnapshotsOnDemand- ? withoutWorkspaceSnapshots(commandProviders)+ ? withoutWorkspaceSnapshots(commandProviders, options.catalogsPerCwd): commandProviders;const settings = ServerSettings.redactServerSettingsForClient(yield* serverSettings.getSettings,@@ -2498,6 +2535,9 @@ export const layerWsRpc = (if (instanceId === undefined || cwd === undefined) {return { providers: withoutWorkspaceSnapshots(commandProviders) };}+ const catalogOnDemand =+ input.providerCatalogsOnDemand === true &&+ (yield* providerInstances.getInstance(instanceId))?.snapshotForCwd !== undefined;return {providers: commandProviders.filter((provider) => provider.instanceId === instanceId)@@ -2507,9 +2547,7 @@ export const layerWsRpc = (models: [],slashCommands: [],skills: [],- workspaceSnapshots: (provider.workspaceSnapshots ?? []).filter(- (snapshot) => snapshot.cwd === cwd,- ),+ workspaceSnapshots: workspaceSnapshotOrCatalog(provider, cwd, catalogOnDemand),})),};}),@@ -3234,7 +3272,22 @@ export const layerWsRpc = (Effect.gen(function* () {const usageLimitsCommand = input.usageLimitsCommand === true;const workspaceSnapshotsOnDemand = input.workspaceSnapshotsOnDemand === true;- // Buffer changes through discovery and the initial snapshot's RPC ACK wait.+ // Only providers that can scan a cwd answer the client's on-demand catalog fetch.+ // Read per update, since a replaced instance can change that.+ const currentCatalogsPerCwd =+ workspaceSnapshotsOnDemand && input.providerCatalogsOnDemand === true+ ? providerInstances.listInstances.pipe(+ Effect.map(+ (instances) =>+ new Set(+ instances+ .filter((instance) => instance.snapshotForCwd !== undefined)+ .map((instance) => instance.instanceId),+ ),+ ),+ )+ : Effect.succeed(new Set<string>());+ // Buffer changes through discovery and the initial snapshot's send.const keybindingsChanges = yield* keybindings.subscribeChanges;const settingsChanges = yield* serverSettings.subscribeChanges;const remoteTargets = yield* Effect.forkScoped(remoteOpenTargets.resolveTargets(), {@@ -3250,6 +3303,7 @@ export const layerWsRpc = (usageLimitsCommand,workspaceSnapshotsOnDemand,deferNetworkDiscovery: true,+ catalogsPerCwd: yield* currentCatalogsPerCwd,});const keybindingsUpdates = keybindingsChanges.pipe(Stream.map((event) => ({@@ -3272,16 +3326,19 @@ export const layerWsRpc = (usageLimitsCommand ? sameUsageLimitCommandCoverage : () => true,),),- (providers, sources) => {- const commandProviders = usageLimitsCommand- ? withUsageLimitsCommands(providers, sources)- : providers;- // Stripped before the de-dupe below, so snapshot-only changes send nothing.- return workspaceSnapshotsOnDemand- ? withoutWorkspaceSnapshots(commandProviders)- : commandProviders;- },+ (providers, sources) =>+ usageLimitsCommand ? withUsageLimitsCommands(providers, sources) : providers,).pipe(+ // Stripped before the de-dupe below, so snapshot-only changes send nothing.+ Stream.mapEffect((commandProviders) =>+ workspaceSnapshotsOnDemand+ ? currentCatalogsPerCwd.pipe(+ Effect.map((catalogsPerCwd) =>+ withoutWorkspaceSnapshots(commandProviders, catalogsPerCwd),+ ),+ )+ : Effect.succeed(commandProviders),+ ),// Compare initial paired values with the snapshot; blindly dropping them loses changes between snapshot and subscription.(updates) => Stream.concat(rpcInitialItems([config.providers]), updates),Stream.changesWith(@@ -3480,7 +3537,7 @@ export const layer = Layer.unwrap(const rpcWebSocketHttpEffect = yield* Effect.gen(function* () {const { protocol, httpEffect } = yield* RpcServer.makeProtocolWithHttpEffectWebsocket;yield* RpcServer.make(ServerWsRpcGroup, WS_RPC_SERVER_OPTIONS).pipe(- Effect.provideService(RpcServer.Protocol, withTerminalOutputWindow(protocol)),+ Effect.provideService(RpcServer.Protocol, withOutputWindow(protocol)),Effect.provide(Layer.merge(RpcAuthorization.layer(session.scopes, (method, payload) => {diff --git a/packages/client-runtime/src/connection/supervisor.test.ts b/packages/client-runtime/src/connection/supervisor.test.tsindex b4fc6955366..f284c48e827 100644--- a/packages/client-runtime/src/connection/supervisor.test.ts+++ b/packages/client-runtime/src/connection/supervisor.test.ts@@ -571,7 +571,7 @@ describe("EnvironmentSupervisor", () => {);expect(Option.isNone(yield* SubscriptionRef.get(supervisor.prepared))).toBe(true);- yield* TestClock.adjust("3 seconds");+ yield* TestClock.adjust("250 millis");yield* awaitState(supervisor.state,(state) => state.phase === "connected" && state.generation === 2,@@ -794,6 +794,26 @@ describe("EnvironmentSupervisor", () => {}).pipe(Effect.provide(TestClock.layer())),);+ it.effect("replaces a silent remote socket without the wait after a long resume", () =>+ Effect.gen(function* () {+ const harness = yield* makeSilentSocketHarness();+ const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {+ initiallyDesired: true,+ }).pipe(Effect.provide(harness.dependencies));++ yield* awaitState(supervisor.state, (state) => state.phase === "connected");+ yield* harness.wake("application-active-long");+ yield* TestClock.adjust("499 millis");+ expect(yield* Ref.get(harness.sessionCount)).toBe(1);++ yield* TestClock.adjust("1 millis");+ yield* awaitState(+ supervisor.state,+ (state) => state.phase === "connected" && state.generation === 2,+ );+ }).pipe(Effect.provide(TestClock.layer())),+ );+it.effect("replaces a silent socket even while the network reports offline", () =>Effect.gen(function* () {const harness = yield* makeSilentSocketHarness();@@ -869,11 +889,11 @@ describe("EnvironmentSupervisor", () => {yield* awaitState(supervisor.state, (state) => state.phase === "connected");yield* harness.wake("application-active-probe");yield* TestClock.adjust("1 second");- yield* Ref.update(frames, (count) => count + 1);- yield* TestClock.adjust("1 second");yield* harness.wake("network-changed");- // The running check sees the frame at 2.5 s, so a fresh one starts then.- yield* TestClock.adjust("2999 millis");+ yield* TestClock.adjust("500 millis");+ yield* Ref.update(frames, (count) => count + 1);+ // The running check sees the frame at 1.6 s, so a fresh one starts then.+ yield* TestClock.adjust("2599 millis");expect(yield* Ref.get(harness.sessionCount)).toBe(1);yield* TestClock.adjust("1 millis");@@ -967,22 +987,49 @@ describe("EnvironmentSupervisor", () => {}).pipe(Effect.provide(TestClock.layer())),);- it.effect("keeps waiting for the probe when a frame arrives during the host check", () =>+ it.effect.each(["application-active-probe", "application-active-long"] as const)(+ "keeps waiting for the probe when a frame arrives during the host check (%s)",+ (wakeup) =>+ Effect.gen(function* () {+ const frames = yield* Ref.make(0);+ const harness = yield* makeHarness({+ prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),+ initialConfig: () => Effect.succeed(PROBE_CAPABLE_CONFIG),+ probe: (attempt) => (attempt === 1 ? Effect.sleep("2 seconds") : Effect.void),+ inboundFrames: Ref.get(frames),+ checkHost: () => Ref.update(frames, (count) => count + 1).pipe(Effect.as("answered")),+ });+ const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, {+ initiallyDesired: true,+ }).pipe(Effect.provide(harness.dependencies));++ yield* awaitState(supervisor.state, (state) => state.phase === "connected");+ yield* harness.wake(wakeup);+ yield* TestClock.adjust("5 seconds");++ expect(yield* Ref.get(harness.sessionCount)).toBe(1);+ expect((yield* SubscriptionRef.get(supervisor.state)).generation).toBe(1);+ }).pipe(Effect.provide(TestClock.layer())),+ );++ it.effect("keeps a long-resume socket that delivers a frame during a slow host check", () =>Effect.gen(function* () {const frames = yield* Ref.make(0);const harness = yield* makeHarness({prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),initialConfig: () => Effect.succeed(PROBE_CAPABLE_CONFIG),- probe: (attempt) => (attempt === 1 ? Effect.sleep("2 seconds") : Effect.void),+ probe: (attempt) => (attempt === 1 ? Effect.sleep("2500 millis") : Effect.void),inboundFrames: Ref.get(frames),- checkHost: () => Ref.update(frames, (count) => count + 1).pipe(Effect.as("answered")),+ checkHost: () => Effect.sleep("2 seconds").pipe(Effect.as("answered")),});- const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, {+ const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {initiallyDesired: true,}).pipe(Effect.provide(harness.dependencies));yield* awaitState(supervisor.state, (state) => state.phase === "connected");- yield* harness.wake("application-active-probe");+ yield* harness.wake("application-active-long");+ yield* TestClock.adjust("700 millis");+ yield* Ref.update(frames, (count) => count + 1);yield* TestClock.adjust("5 seconds");expect(yield* Ref.get(harness.sessionCount)).toBe(1);@@ -1049,7 +1096,7 @@ describe("EnvironmentSupervisor", () => {yield* awaitState(supervisor.state, (state) => state.phase === "connected");yield* harness.wake("application-active-long");yield* harness.setNetworkStatus("offline");- yield* TestClock.adjust("2999 millis");+ yield* TestClock.adjust("1499 millis");expect(yield* Ref.get(harness.sessionCount)).toBe(1);yield* TestClock.adjust("1 millis");diff --git a/packages/client-runtime/src/connection/supervisor.ts b/packages/client-runtime/src/connection/supervisor.tsindex 5f279edf568..8f753660581 100644--- a/packages/client-runtime/src/connection/supervisor.ts+++ b/packages/client-runtime/src/connection/supervisor.ts@@ -42,6 +42,7 @@ import {const RETRY_BASE_DELAY_MS = 1_000;const RETRY_MAX_DELAY_MS = 300_000;+const SESSION_LOSS_RETRY_DELAY_MS = 250;// A cold server with a large database can take longer than 15 seconds to// answer, and that is not a reason to drop a working connection.const CONNECTION_ESTABLISHMENT_TIMEOUT = "15 seconds";@@ -49,13 +50,17 @@ const establishmentTimeout = Duration.fromInputUnsafe(CONNECTION_ESTABLISHMENT_Tconst synchronizationTimeout = Duration.fromInputUnsafe("75 seconds");const CONNECTION_PROBE_TIMEOUT = "15 seconds";const QUICK_CONNECTION_PROBE_TIMEOUT = "3 seconds";+// After a long suspension pooled connections are often dead too, so the host+// check can hang with the socket. A replacement is cheaper than the wait.+const LONG_RESUME_PROBE_TIMEOUT = "1500 millis";// A probe still unanswered this long checks the host with a fresh request. If// the host answers while the socket stays silent, the socket is dead.const SILENT_SOCKET_CHECK_AFTER = "500 millis";const SILENT_SOCKET_RECHECK_EVERY = "1 second";+const FRAME_CHECK_EVERY = "100 millis";// Off the LAN one lost packet can stall a live socket for a second, so a remote-// socket must stay silent this long. It must end before-// QUICK_CONNECTION_PROBE_TIMEOUT so a long resume replaces a silent socket+// socket must stay silent this long, except after a long resume. It must end+// before QUICK_CONNECTION_PROBE_TIMEOUT so a resume replaces a silent socket// instead of timing out.const REMOTE_SILENT_SOCKET_AFTER = "2500 millis";const BACKOFF_RESET_AFTER_MS = 30_000;@@ -668,7 +673,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (// answer reconnects as quickly as a missing one, even on a remote// route where a reconnect takes longer than the wait it replaces.if (next.reason === "application-active-long") {- return QUICK_CONNECTION_PROBE_TIMEOUT;+ return LONG_RESUME_PROBE_TIMEOUT;}if (next.reason !== "application-active-probe" && next.reason !== "network-changed") {return undefined;@@ -693,6 +698,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (const awaitSilentSocket = Effect.fn("EnvironmentSupervisor.awaitSilentSocket")(function* (lease: ConnectedLease,routeKind: ConnectionRouteKind | undefined,+ longResume: boolean,) {const startedAt = yield* Clock.monotonicTimeNanos;yield* Effect.sleep(SILENT_SOCKET_CHECK_AFTER);@@ -707,17 +713,26 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (const socketSilent = lease.session.inboundFrames.pipe(Effect.map((frames) => frames === framesBefore),);- // A radio still waking up or a stale pooled connection can fail one check.- while ((yield* driver.checkHost(lease.prepared)) !== "answered") {- if (!(yield* socketSilent)) return "alive" as const;- yield* Effect.sleep(SILENT_SOCKET_RECHECK_EVERY);- }- if (!answersQuickly(routeKind)) {- const silentFor = (yield* Clock.monotonicTimeNanos) - startedAt;- const remaining = Duration.toNanosUnsafe(REMOTE_SILENT_SOCKET_AFTER) - silentFor;- if (remaining > 0n) yield* Effect.sleep(Duration.nanos(remaining));- }- return (yield* socketSilent) ? ("silent" as const) : ("alive" as const);+ // The session only counts frames, so a host check slower than the probe+ // deadline would otherwise hide a frame until after the socket is replaced.+ const frameArrives = Effect.sleep(FRAME_CHECK_EVERY).pipe(+ Effect.andThen(socketSilent),+ Effect.repeat({ while: (silent) => silent }),+ Effect.as("alive" as const),+ );+ const hostAnswersWhileSilent = Effect.gen(function* () {+ // A radio still waking up or a stale pooled connection can fail one check.+ while ((yield* driver.checkHost(lease.prepared)) !== "answered") {+ yield* Effect.sleep(SILENT_SOCKET_RECHECK_EVERY);+ }+ if (!answersQuickly(routeKind) && !longResume) {+ const silentFor = (yield* Clock.monotonicTimeNanos) - startedAt;+ const remaining = Duration.toNanosUnsafe(REMOTE_SILENT_SOCKET_AFTER) - silentFor;+ if (remaining > 0n) yield* Effect.sleep(Duration.nanos(remaining));+ }+ return (yield* socketSilent) ? ("silent" as const) : ("alive" as const);+ });+ return yield* Effect.race(hostAnswersWhileSilent, frameArrives);});// Returning to the app, an explicit retry, and the network reporting offline@@ -764,7 +779,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (let recheck = false;let longResume = isLongResume(next);const startSilentSocketCheck = Effect.gen(function* () {- silentSocket = yield* Effect.forkChild(awaitSilentSocket(lease, routeKind));+ silentSocket = yield* Effect.forkChild(awaitSilentSocket(lease, routeKind, longResume));recheck = false;});// The web and desktop foreground probe allows a busy host 15 seconds,@@ -784,7 +799,8 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ({ discard: true },),);- let deadline = (yield* Clock.monotonicTimeNanos) + Duration.toNanosUnsafe(probeTimeout);+ let longResumeStartedAt = yield* Clock.monotonicTimeNanos;+ let deadline = longResumeStartedAt + Duration.toNanosUnsafe(probeTimeout);for (;;) {const remaining = deadline - (yield* Clock.monotonicTimeNanos);const probeEvent = yield* Effect.raceAllFirst([@@ -805,6 +821,12 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (]);if (probeEvent._tag === "SocketAlive") {silentSocket = undefined;+ // A frame shows the socket still works, so its probe gets the usual resume time.+ if (longResume) {+ const aliveDeadline =+ longResumeStartedAt + Duration.toNanosUnsafe(QUICK_CONNECTION_PROBE_TIMEOUT);+ if (aliveDeadline > deadline) deadline = aliveDeadline;+ }if (recheck) yield* startSilentSocketCheck;continue;}@@ -839,8 +861,11 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (}const signalTimeout = probeTimeoutFor(probeEvent.signal, routeKind);if (signalTimeout !== undefined) {+ if (isLongResume(probeEvent.signal) && !longResume) {+ longResume = true;+ longResumeStartedAt = yield* Clock.monotonicTimeNanos;+ }yield* armSilentSocketCheck(probeEvent.signal);- longResume ||= isLongResume(probeEvent.signal);const signalDeadline =(yield* Clock.monotonicTimeNanos) + Duration.toNanosUnsafe(signalTimeout);if (signalDeadline < deadline) deadline = signalDeadline;@@ -1155,8 +1180,13 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (continue;}+ // The first loss of a live session retries almost at once; the ladder+ // still counts it, so a host that keeps dropping sessions backs off.+ const firstLoss = outcome.established && failureCount === 0;failureCount += 1;- const delayMs = retryDelayMs(failureCount - 1, yield* Random.next);+ const delayMs = firstLoss+ ? Math.round(SESSION_LOSS_RETRY_DELAY_MS * (0.5 + (yield* Random.next) / 2))+ : retryDelayMs(failureCount - 1, yield* Random.next);pendingRetry = Option.map(attemptSpan, (previousAttempt) => ({previousAttempt,failureCount,diff --git a/packages/client-runtime/src/rpc/session.test.ts b/packages/client-runtime/src/rpc/session.test.tsindex 9e63f8485ad..8f4bee5dc16 100644--- a/packages/client-runtime/src/rpc/session.test.ts+++ b/packages/client-runtime/src/rpc/session.test.ts@@ -404,6 +404,38 @@ describe("RpcSessionFactory", () => {}),);+ it.effect("ends a stream at once when its socket closes", () =>+ Effect.gen(function* () {+ const { factory, sockets } = yield* makeFactory();+ const session = yield* factory.connect(PREPARED);+ const readyFiber = yield* Effect.forkChild(session.ready);+ const socket = yield* awaitSocket(sockets);+ socket.open();+ yield* completeInitialConfig(socket);+ yield* Fiber.join(readyFiber);+ // Like the supervisor: the closed session interrupts its streams, and a+ // stream interrupt must not wait for a socket that is gone.+ const ended = yield* Effect.forkChild(+ Effect.raceFirst(+ Effect.flip(session.closed),+ session.client[WS_METHODS.subscribeServerLifecycle]({}).pipe(+ Stream.runDrain,+ Effect.andThen(Effect.never),+ ),+ ),+ );+ yield* awaitRequest(socket, 1);+ socket.close(1006);+ yield* TestClock.adjust("999 millis");++ expect(ended.pollUnsafe()).toBeDefined();+ // A request made after the close still fails instead of waiting forever.+ const lateProbe = yield* Effect.forkChild(session.probe);+ yield* TestClock.adjust("999 millis");+ expect(lateProbe.pollUnsafe()?._tag).toBe("Failure");+ }).pipe(Effect.scoped, Effect.provide(TestClock.layer())),+ );+it.effect("replays current config and broadcasts updates to every subscriber", () =>Effect.scoped(Effect.gen(function* () {diff --git a/packages/client-runtime/src/rpc/session.ts b/packages/client-runtime/src/rpc/session.tsindex ddf591b1391..3ea4b73bf6b 100644--- a/packages/client-runtime/src/rpc/session.ts+++ b/packages/client-runtime/src/rpc/session.ts@@ -66,6 +66,7 @@ export interface RpcSessionOptions {readonly environmentThemes?: boolean;readonly usageLimitSources?: boolean;readonly usageLimitsCommand?: boolean;+ readonly providerCatalogsOnDemand?: boolean;}export class RpcSessionFactory extends Context.Service<@@ -175,6 +176,7 @@ export const make = Effect.fn("RpcSessionFactory.make")(function* (...(options.environmentThemes === true ? { environmentThemes: true } : {}),...(options.usageLimitSources === true ? { usageLimitSources: true } : {}),...(options.usageLimitsCommand === true ? { usageLimitsCommand: true } : {}),+ ...(options.providerCatalogsOnDemand === true ? { providerCatalogsOnDemand: true } : {}),};// Config state merges the snapshots it fetches back in, so no subscriber needs them broadcast.const onDemandSnapshots = (input: ServerConfigSubscriptionInput) => ({@@ -247,7 +249,18 @@ export const make = Effect.fn("RpcSessionFactory.make")(function* (retryTransientErrors: false,retryPolicy: Schedule.recurs(0),onPingTimeout: Ref.set(pingTimedOut, true),- }),+ }).pipe(+ // Teardown sends an interrupt per open stream, and a write to a closed+ // socket waits out the client's 1s interrupt timeout before reconnecting.+ // Requests still go through, so they fail with the protocol error.+ Effect.map((protocol) => ({+ ...protocol,+ send: (clientId, request, transferables) =>+ request._tag !== "Request" && Deferred.isDoneUnsafe(disconnected)+ ? Effect.void+ : protocol.send(clientId, request, transferables),+ })),+ ),).pipe(Layer.provide(Layer.mergeAll(diff --git a/packages/client-runtime/src/state/server.ts b/packages/client-runtime/src/state/server.tsindex 04a2d663215..f5c9d8f45b3 100644--- a/packages/client-runtime/src/state/server.ts+++ b/packages/client-runtime/src/state/server.ts@@ -365,6 +365,7 @@ export interface ServerConfigSubscriptionOptions {readonly environmentThemes?: boolean;readonly usageLimitSources?: boolean;readonly usageLimitsCommand?: boolean;+ readonly providerCatalogsOnDemand?: boolean;}// Configs are JSON-shaped. Missing and undefined object fields persist identically.@@ -486,6 +487,7 @@ export const makeEnvironmentServerConfigState = Effect.fn("EnvironmentServerConf...(subscription.environmentThemes === true ? { environmentThemes: true } : {}),...(subscription.usageLimitSources === true ? { usageLimitSources: true } : {}),...(subscription.usageLimitsCommand === true ? { usageLimitsCommand: true } : {}),+ ...(subscription.providerCatalogsOnDemand === true ? { providerCatalogsOnDemand: true } : {}),}).pipe(Stream.runForEach((event) =>Effect.gen(function* () {@@ -755,6 +757,7 @@ export function createServerEnvironmentAtoms<R, E>(readonly environmentThemes?: boolean;readonly usageLimitSources?: boolean;readonly usageLimitsCommand?: boolean;+ readonly providerCatalogsOnDemand?: boolean;},) {const configScheduler = createAtomCommandScheduler();@@ -770,6 +773,7 @@ export function createServerEnvironmentAtoms<R, E>(...(options.environmentThemes === true ? { environmentThemes: true } : {}),...(options.usageLimitSources === true ? { usageLimitSources: true } : {}),...(options.usageLimitsCommand === true ? { usageLimitsCommand: true } : {}),+ ...(options.providerCatalogsOnDemand === true ? { providerCatalogsOnDemand: true } : {}),}),).pipe(@@ -1401,6 +1405,7 @@ export function createServerEnvironmentAtoms<R, E>(...input,workspaceSnapshotsOnDemand: true,...(options.usageLimitsCommand === true ? { usageLimitsCommand: true } : {}),+ ...(options.providerCatalogsOnDemand === true ? { providerCatalogsOnDemand: true } : {}),}),onSuccess: ({ environmentId, input }, registry, result) =>Effect.sync(() => {diff --git a/packages/client-runtime/src/state/shell-sync.test.ts b/packages/client-runtime/src/state/shell-sync.test.tsindex c74ecd82fe8..032ede63714 100644--- a/packages/client-runtime/src/state/shell-sync.test.ts+++ b/packages/client-runtime/src/state/shell-sync.test.ts@@ -56,7 +56,10 @@ const LIVE_SHELL_SNAPSHOT: OrchestrationV2ShellSnapshot = {function session(client: WsRpcProtocolClient): RpcSession.RpcSession {return {client,- initialConfig: Effect.succeed({ shellResumeCompletionMarker: true } as never),+ initialConfig: Effect.succeed({+ shellResumeCompletionMarker: true,+ environment: { serverVersion: "1.0.0" },+ } as never),subscribeServerConfig: (input = {}) => client.subscribeServerConfig(input),ready: Effect.void,probe: Effect.void,@@ -817,12 +820,12 @@ describe("environment shell synchronization", () => {it.effect("opts in to shell omissions and counts authoritative snapshots and removals", () =>Effect.gen(function* () {const events = yield* Queue.unbounded<OrchestrationV2ShellStreamItem>();- const subscribeInput = yield* Deferred.make<unknown>();+ const subscribeInputs = yield* Queue.unbounded<unknown>();const loadOptions = yield* Ref.make<ReadonlyArray<unknown>>([]);const client = {[ORCHESTRATION_V2_WS_METHODS.subscribeShell]: (input: unknown) =>Stream.unwrap(- Deferred.succeed(subscribeInput, input).pipe(Effect.as(Stream.fromQueue(events))),+ Queue.offer(subscribeInputs, input).pipe(Effect.as(Stream.fromQueue(events))),),} as unknown as WsRpcProtocolClient;const familySession: RpcSession.RpcSession = {@@ -831,12 +834,14 @@ describe("environment shell synchronization", () => {shellResumeCompletionMarker: true,shellThreadFamilies: true,shellOlderSettled: true,+ environment: { serverVersion: "1.0.0", databaseMigration: 70 },} as never),};+ const activeSession = yield* SubscriptionRef.make(Option.some(familySession));const supervisor = EnvironmentSupervisor.EnvironmentSupervisor.of({target: TARGET,state: yield* SubscriptionRef.make(AVAILABLE_CONNECTION_STATE),- session: yield* SubscriptionRef.make(Option.some(familySession)),+ session: activeSession,prepared: yield* SubscriptionRef.make(Option.some(PREPARED)),connect: Effect.void,disconnect: Effect.void,@@ -871,7 +876,7 @@ describe("environment shell synchronization", () => {SubscriptionRef.changes(shellState).pipe(Stream.filter(predicate), Stream.runHead);// Both transports must omit, or omitted children would come back through the socket.- expect(yield* Deferred.await(subscribeInput)).toMatchObject({+ expect(yield* Queue.take(subscribeInputs)).toMatchObject({afterSequence: 1,omitIdleSubagents: true,omitOlderSettled: true,@@ -907,6 +912,31 @@ describe("environment shell synchronization", () => {yield* stateWhere((state) => state.generation > loaded.generation),);expect(reset.removedAt.size).toBe(0);++ // A reconnect resumes the server-loaded shell from its cursor, with no second HTTP load.+ yield* SubscriptionRef.set(activeSession, Option.some({ ...familySession }));+ expect(yield* Queue.take(subscribeInputs)).toMatchObject({+ afterSequence: 5,+ omitIdleSubagents: true,+ omitOlderSettled: true,+ });+ expect(yield* Ref.get(loadOptions)).toHaveLength(1);++ // A migrated server may have rewritten rows without events, so it reloads.+ yield* SubscriptionRef.set(+ activeSession,+ Option.some({+ ...familySession,+ initialConfig: familySession.initialConfig.pipe(+ Effect.map((config) => ({+ ...config,+ environment: { ...config.environment, databaseMigration: 71 },+ })),+ ),+ }),+ );+ yield* Queue.take(subscribeInputs);+ expect(yield* Ref.get(loadOptions)).toHaveLength(2);}),);@@ -1076,7 +1106,17 @@ describe("environment shell synchronization", () => {);yield* Deferred.await(firstFillStarted);- yield* SubscriptionRef.set(activeSession, Option.some(session(client)));+ // New omissions make the next session reload instead of resuming.+ yield* SubscriptionRef.set(+ activeSession,+ Option.some({+ ...session(client),+ initialConfig: Effect.succeed({+ shellThreadFamilies: true,+ environment: { serverVersion: "1.0.0" },+ } as never),+ }),+ );yield* SubscriptionRef.changes(shellState).pipe(Stream.filter((state) => Option.isSome(state.snapshot) && state.snapshot.value.snapshotSequence === 2,diff --git a/packages/client-runtime/src/state/shell.ts b/packages/client-runtime/src/state/shell.tsindex 442e99b6f29..d75ea5ff64a 100644--- a/packages/client-runtime/src/state/shell.ts+++ b/packages/client-runtime/src/state/shell.ts@@ -26,7 +26,7 @@ import * as ConnectionWakeups from "../connection/wakeups.ts";import { safeErrorLogAttributes } from "../errors/safeLog.ts";import * as Persistence from "../platform/persistence.ts";import { runCachePersistence } from "./cachePersistence.ts";-import { subscribeDynamic } from "../rpc/client.ts";+import { subscribeDynamicWithSession } from "../rpc/client.ts";import type { RpcSession } from "../rpc/session.ts";import * as ShellSnapshotLoader from "./shellSnapshotHttp.ts";import type { DeferredShellSnapshot } from "./shellPullRequests.ts";@@ -82,6 +82,16 @@ function withoutDeferredPullRequests(value: DeferredShellSnapshot): Orchestratioconst SHELL_SYNCHRONIZATION_ERROR_MESSAGE = "Could not synchronize environment data.";+function groupBySession<A>(entries: ReadonlyArray<readonly [RpcSession, A]>) {+ const groups: Array<[RpcSession, Array<A>]> = [];+ for (const [session, value] of entries) {+ const last = groups.at(-1);+ if (last?.[0] === session) last[1].push(value);+ else groups.push([session, [value]]);+ }+ return groups;+}+export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")(function* () {const supervisor = yield* EnvironmentSupervisor.EnvironmentSupervisor;const cache = yield* Persistence.EnvironmentCacheStore;@@ -106,8 +116,11 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")status: shellStatusForSnapshot(cachedSnapshot),});const awaitingCompletion = yield* Ref.make(false);- const lastAuthoritativeSession = yield* Ref.make<RpcSession | null>(null);- const activeSubscriptionSession = yield* Ref.make<RpcSession | null>(null);+ // The omissions and server build of the subscription whose snapshot the+ // in-memory shell last came from; null while it holds only cached data.+ const authoritativeOmission = yield* Ref.make<string | null>(null);+ // Items buffered from a previous session keep that session's key.+ const sessionOmissions = new WeakMap<RpcSession, string | null>();const latestLiveSnapshot = yield* Ref.make<Option.Option<OrchestrationV2ShellSnapshot>>(Option.none(),);@@ -208,12 +221,13 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")const applyItems = Effect.fn("EnvironmentShellState.applyItems")(function* (items: ReadonlyArray<OrchestrationV2ShellStreamItem>,+ omission: string | null,pendingLinkIds?: ReadonlySet<ThreadId>,) {const initial = yield* SubscriptionRef.get(state);let waiting = yield* Ref.get(awaitingCompletion);let next = initial;- let receivedSnapshot = false;+ let receivedAuthoritativeSnapshot = false;let deltas: Array<ShellStreamEvent> = [];const flushDeltas = () => {if (deltas.length > 0 && Option.isSome(next.snapshot)) {@@ -267,7 +281,7 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")? undefined: { resolvedRepositoryIdentityRoots: item.resolvedRepositoryIdentityRoots },);- receivedSnapshot = true;+ if (authoritative) receivedAuthoritativeSnapshot = true;next = {snapshot: Option.some(nextSnapshot),status: waiting ? "synchronizing" : "live",@@ -288,11 +302,8 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")yield* Ref.set(latestLiveSnapshot, next.snapshot);}yield* SubscriptionRef.set(state, next);- if (receivedSnapshot) {- const session = yield* Ref.get(activeSubscriptionSession);- if (session !== null) {- yield* Ref.set(lastAuthoritativeSession, session);- }+ if (receivedAuthoritativeSnapshot && omission !== null) {+ yield* Ref.set(authoritativeOmission, omission);}if (next.snapshot !== initial.snapshot && Option.isSome(next.snapshot)) {yield* Queue.offer(persistence, next.snapshot.value);@@ -374,25 +385,42 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")yield* setSynchronizing;yield* Effect.forkScoped(- subscribeDynamic(+ subscribeDynamicWithSession(ORCHESTRATION_V2_WS_METHODS.subscribeShell,Effect.fn("EnvironmentShellState.makeSubscribeInput")(function* (session) {- yield* Ref.set(activeSubscriptionSession, session);- const { omitIdleSubagents, omitOlderSettled } = yield* session.initialConfig.pipe(- Effect.map((config) => ({- omitIdleSubagents: config.shellThreadFamilies === true,- omitOlderSettled:- config.shellOlderSettled === true && config.shellThreadFamilies === true,- })),- Effect.orElseSucceed(() => ({ omitIdleSubagents: false, omitOlderSettled: false })),- );+ const { omitIdleSubagents, omitOlderSettled, omissionKey } =+ yield* session.initialConfig.pipe(+ Effect.map((config) => {+ const omitIdleSubagents = config.shellThreadFamilies === true;+ const omitOlderSettled = config.shellOlderSettled === true && omitIdleSubagents;+ // Migrations rewrite projections without events, so an upgraded server reloads.+ const { serverVersion, databaseMigration } = config.environment;+ return {+ omitIdleSubagents,+ omitOlderSettled,+ omissionKey: JSON.stringify([+ omitIdleSubagents,+ omitOlderSettled,+ serverVersion,+ databaseMigration ?? null,+ ]),+ };+ }),+ Effect.orElseSucceed(() => ({+ omitIdleSubagents: false,+ omitOlderSettled: false,+ omissionKey: null,+ })),+ );+ sessionOmissions.set(session, omissionKey);yield* Ref.set(awaitingCompletion, true);yield* setSynchronizing;- // Foreground resubscriptions on the same live session can resume from- // the in-memory cursor. A new session reloads the authoritative HTTP- // snapshot so a valid cursor cannot preserve incomplete cached data.- const hasAuthoritativeSnapshot = (yield* Ref.get(lastAuthoritativeSession)) === session;+ // A shell from a server snapshot resumes from its cursor, on this session or a+ // new one. Cached data, other omissions, or another server build reload the+ // authoritative HTTP snapshot so a valid cursor cannot preserve stale data.+ const hasAuthoritativeSnapshot =+ omissionKey !== null && (yield* Ref.get(authoritativeOmission)) === omissionKey;let canResume = hasAuthoritativeSnapshot;let current = yield* SubscriptionRef.get(state);if (!hasAuthoritativeSnapshot || Option.isNone(current.snapshot)) {@@ -427,7 +455,7 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")? []: httpSnapshot.value.threads.map((thread) => thread.id),);- yield* applyItems([{ kind: "snapshot", snapshot }], pendingLinkIds);+ yield* applyItems([{ kind: "snapshot", snapshot }], omissionKey, pendingLinkIds);canResume = true;current = yield* SubscriptionRef.get(state);if (Option.isSome(current.snapshot)) {@@ -454,7 +482,15 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")retryExpectedFailureAfter: "250 millis",resubscribe: foregroundResubscriptions,},- ).pipe(Stream.runForEachArray((items) => applyItems(items))),+ ).pipe(+ Stream.runForEachArray((entries) =>+ Effect.forEach(+ groupBySession(entries),+ ([session, items]) => applyItems(items, sessionOmissions.get(session) ?? null),+ { discard: true },+ ),+ ),+ ),);yield* SubscriptionRef.changes(supervisor.state).pipe(Stream.runForEach((connectionState) => {diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.tsindex c014114cc06..3106462d357 100644--- a/packages/contracts/src/rpc.ts+++ b/packages/contracts/src/rpc.ts@@ -655,6 +655,8 @@ export const WsServerRefreshProvidersRpc = Rpc.make(WS_METHODS.serverRefreshProvworkspaceSnapshotsOnDemand: Schema.optional(Schema.Boolean),/** As on `subscribeServerConfig`: add /usage-limits to the replied snapshot. */usageLimitsCommand: Schema.optional(Schema.Boolean),+ /** As on `subscribeServerConfig`: a failed cwd scan replies with the machine catalog as a pending snapshot. */+ providerCatalogsOnDemand: Schema.optional(Schema.Boolean),}),success: ServerProviderUpdatedPayload,error: Schema.Union([EnvironmentAuthorizationError, ProviderSetupError]),@@ -1893,6 +1895,8 @@ export const WsSubscribeServerConfigRpc = Rpc.make(WS_METHODS.subscribeServerConusageLimitsCommand: Schema.optional(Schema.Boolean),/** Omit per-cwd provider snapshots; the client fetches its cwd through `server.refreshProviders`. */workspaceSnapshotsOnDemand: Schema.optional(Schema.Boolean),+ /** With on-demand snapshots, also omit machine slash commands and skills a cwd snapshot replaces. */+ providerCatalogsOnDemand: Schema.optional(Schema.Boolean),}),success: ServerConfigStreamEvent,error: Schema.Union([KeybindingsConfigError, ServerSettingsError, EnvironmentAuthorizationError]),