Skip to content
TypeScript

perf(mobile): faster thread sync, connects, and reconnects on bad wifi (#28487)

Bilal Bakr2917f2846558
diff --git a/apps/mobile/src/connection/runtime.ts b/apps/mobile/src/connection/runtime.ts
index 94ee7d5ebeb..2f88de1432a 100644
--- a/apps/mobile/src/connection/runtime.ts
+++ b/apps/mobile/src/connection/runtime.ts
@@ -36,7 +36,11 @@ type ConnectionLayerSource =
const layerProvidedClientConnection = layerSnapshotLoader.pipe(
Layer.provideMerge(
- Connection.layerWithOptions({ usageLimitSources: true, usageLimitsCommand: true }),
+ Connection.layerWithOptions({
+ usageLimitSources: true,
+ usageLimitsCommand: true,
+ providerCatalogsOnDemand: true,
+ }),
),
Layer.provideMerge(
Layer.mergeAll(
diff --git a/apps/mobile/src/state/server.ts b/apps/mobile/src/state/server.ts
index 8de9354d385..b6659a2e982 100644
--- a/apps/mobile/src/state/server.ts
+++ b/apps/mobile/src/state/server.ts
@@ -13,6 +13,7 @@ export const serverEnvironment = createServerEnvironmentAtoms(connectionAtomRunt
initialConfigValueAtom: environmentSession.initialConfigValueAtom,
usageLimitSources: true,
usageLimitsCommand: true,
+ providerCatalogsOnDemand: true,
});
export const describeOutdatedServer = createOutdatedServerDescriptionCommand(connectionAtomRuntime);
export const updateOutdatedServer = createOutdatedServerUpdateCommand(connectionAtomRuntime);
diff --git a/apps/server/src/terminal/OutputProtocol.test.ts b/apps/server/src/terminal/OutputProtocol.test.ts
index a804a6610b7..9750f418152 100644
--- a/apps/server/src/terminal/OutputProtocol.test.ts
+++ b/apps/server/src/terminal/OutputProtocol.test.ts
@@ -1,5 +1,6 @@
import { assert, describe, it } from "@effect/vitest";
import {
+ ORCHESTRATION_V2_WS_METHODS,
type TerminalAttachStreamEvent,
TerminalAttachStreamEvent as TerminalAttachStreamEventSchema,
WS_METHODS,
@@ -13,7 +14,7 @@ import * as Stream from "effect/Stream";
import * as TestClock from "effect/testing/TestClock";
import { Rpc, RpcGroup, RpcMessage, RpcSerialization, RpcServer } from "effect/rpc";
-import { terminalOutputStream, withTerminalOutputWindow } from "./OutputProtocol.ts";
+import { terminalOutputStream, withOutputWindow } from "./OutputProtocol.ts";
describe("terminal output window", () => {
it.effect.each([
@@ -21,6 +22,9 @@ describe("terminal output window", () => {
{ tag: WS_METHODS.terminalObserve, size: 1, limit: 8 },
{ tag: WS_METHODS.terminalAttach, size: 64 * 1024, limit: 1 },
{ tag: WS_METHODS.terminalObserve, size: 64 * 1024, limit: 1 },
+ { tag: WS_METHODS.subscribeServerConfig, size: 1, limit: 8 },
+ { tag: ORCHESTRATION_V2_WS_METHODS.subscribeShell, size: 1, limit: 8 },
+ { tag: ORCHESTRATION_V2_WS_METHODS.subscribeThread, size: 64 * 1024, limit: 8 },
{ tag: WS_METHODS.subscribeTerminalMetadata, size: 1, limit: 1 },
])("limits $tag with $size-byte values to $limit pending chunks", ({ tag, size, limit }) =>
Effect.gen(function* () {
@@ -48,7 +52,7 @@ describe("terminal output window", () => {
);
yield* RpcServer.make(group).pipe(
Effect.provide(group.toLayerHandler(tag, () => Stream.fromQueue(output))),
- Effect.provideService(RpcServer.Protocol, withTerminalOutputWindow(protocol)),
+ Effect.provideService(RpcServer.Protocol, withOutputWindow(protocol)),
Effect.forkScoped,
);
const write = yield* Deferred.await(receive);
@@ -191,7 +195,7 @@ describe("terminal output stream", () => {
),
),
),
- Effect.provideService(RpcServer.Protocol, withTerminalOutputWindow(protocol)),
+ Effect.provideService(RpcServer.Protocol, withOutputWindow(protocol)),
Effect.forkScoped,
);
const write = yield* Deferred.await(receive);
diff --git a/apps/server/src/terminal/OutputProtocol.ts b/apps/server/src/terminal/OutputProtocol.ts
index e52785d6985..11339fc2bfa 100644
--- a/apps/server/src/terminal/OutputProtocol.ts
+++ b/apps/server/src/terminal/OutputProtocol.ts
@@ -1,4 +1,9 @@
-import { type TerminalAttachStreamEvent, type TerminalEvent, WS_METHODS } from "@t3tools/contracts";
+import {
+ ORCHESTRATION_V2_WS_METHODS,
+ type TerminalAttachStreamEvent,
+ type TerminalEvent,
+ WS_METHODS,
+} from "@t3tools/contracts";
import type * as Arr from "effect/Array";
import * as Cause from "effect/Cause";
import * as Effect from "effect/Effect";
@@ -11,31 +16,62 @@ const MAX_PENDING_BYTES = 64 * 1024;
const isFull = (sizes: number[]) =>
sizes.length >= MAX_PENDING_CHUNKS ||
sizes.reduce((total, size) => total + size, 0) >= MAX_PENDING_BYTES;
+// Terminal output is small chunks, so its window also caps bytes. The other streams
+// window by count only: sizing a multi-MB thread snapshot would serialize it twice.
+const BYTE_WINDOWED_STREAMS: ReadonlySet<string> = new Set([
+ WS_METHODS.terminalAttach,
+ WS_METHODS.terminalObserve,
+]);
+// One ACK round trip per chunk stalls sync on high-latency links.
+const COUNT_WINDOWED_STREAMS: ReadonlySet<string> = new Set([
+ WS_METHODS.subscribeServerConfig,
+ ORCHESTRATION_V2_WS_METHODS.subscribeShell,
+ ORCHESTRATION_V2_WS_METHODS.subscribeThread,
+]);
-export function withTerminalOutputWindow(
+interface Window {
+ readonly bytes: boolean;
+ readonly sizes: number[];
+}
+
+export function withOutputWindow(
protocol: RpcServer.Protocol["Service"],
): RpcServer.Protocol["Service"] {
- const windows = new Map<string, number[]>();
+ // RpcServer drops a disconnected client before its streams exit, so their Exit never
+ // reaches send; departed clients are pruned when a window opens instead.
+ const clients = new Map<number, Map<string, Window>>();
let receive: Parameters<RpcServer.Protocol["Service"]["run"]>[0];
+ const windowFor = (clientId: number, requestId: string) => clients.get(clientId)?.get(requestId);
+ const openWindow = Effect.fnUntraced(function* (
+ clientId: number,
+ requestId: string,
+ bytes: boolean,
+ ) {
+ const connected = yield* protocol.clientIds;
+ for (const id of clients.keys()) if (!connected.has(id)) clients.delete(id);
+ let windows = clients.get(clientId);
+ if (!windows) clients.set(clientId, (windows = new Map()));
+ if (!windows.has(requestId)) windows.set(requestId, { bytes, sizes: [] });
+ });
return {
...protocol,
run: (write) => {
receive = write;
return protocol.run((clientId, message) =>
Effect.suspend(() => {
- if (
- message._tag === "Request" &&
- (message.tag === WS_METHODS.terminalAttach ||
- message.tag === WS_METHODS.terminalObserve)
- ) {
- const key = `${clientId}:${message.id}`;
- if (!windows.has(key)) windows.set(key, []);
+ if (message._tag === "Request") {
+ const bytes = BYTE_WINDOWED_STREAMS.has(message.tag);
+ if (bytes || COUNT_WINDOWED_STREAMS.has(message.tag)) {
+ return openWindow(clientId, String(message.id), bytes).pipe(
+ Effect.andThen(write(clientId, message)),
+ );
+ }
} else if (message._tag === "Ack") {
- const window = windows.get(`${clientId}:${message.requestId}`);
+ const window = windowFor(clientId, String(message.requestId));
if (window) {
- const wasFull = isFull(window);
- window.shift();
- if (!wasFull || isFull(window)) return Effect.void;
+ const wasFull = isFull(window.sizes);
+ window.sizes.shift();
+ if (!wasFull || isFull(window.sizes)) return Effect.void;
}
}
return write(clientId, message);
@@ -46,14 +82,15 @@ export function withTerminalOutputWindow(
Effect.suspend(() => {
const send = protocol.send(clientId, response, transferables);
if (response._tag === "Exit") {
- windows.delete(`${clientId}:${response.requestId}`);
+ clients.get(clientId)?.delete(String(response.requestId));
} else if (response._tag === "Chunk") {
- const window = windows.get(`${clientId}:${response.requestId}`);
+ const window = windowFor(clientId, String(response.requestId));
if (window) {
- // @effect-diagnostics-next-line globalJsonInEffect:off
- const size = Buffer.byteLength(JSON.stringify(response));
- window.push(size);
- if (!isFull(window)) {
+ window.sizes.push(
+ // @effect-diagnostics-next-line globalJsonInEffect:off
+ window.bytes ? Buffer.byteLength(JSON.stringify(response)) : 0,
+ );
+ if (!isFull(window.sizes)) {
return send.pipe(
Effect.andThen(() =>
receive(clientId, {
diff --git a/apps/server/src/ws.test.ts b/apps/server/src/ws.test.ts
index ca53e5e3943..bf2298427e4 100644
--- a/apps/server/src/ws.test.ts
+++ b/apps/server/src/ws.test.ts
@@ -116,6 +116,7 @@ import * as WorkspacePaths from "./workspace/WorkspacePaths.ts";
import * as ServerSecretStore from "./auth/ServerSecretStore.ts";
import * as ServerConfigModule from "./config.ts";
import * as Keybindings from "./keybindings.ts";
+import { USAGE_LIMITS_COMMAND } from "@t3tools/shared/usageLimits";
import * as ServerSettings from "./serverSettings.ts";
import * as ManagedPolicy from "./policy/ManagedPolicy.ts";
@@ -1024,13 +1025,21 @@ it.effect("on-demand clients get provider config without workspace snapshots", (
auth: { status: "authenticated" },
checkedAt: "2026-10-25T00:00:00.000Z",
models: [],
- slashCommands: [{ name: "machine-command" }],
- skills: [],
+ slashCommands: [{ name: "machine-command", description: "A machine command" }],
+ skills: [
+ {
+ name: "machine-skill",
+ description: "Machine skill",
+ path: "/skills/machine-skill",
+ enabled: true,
+ },
+ ],
workspaceSnapshots: [
snapshotFor("/a", "2026-10-25T00:00:00.000Z"),
snapshotFor("/b", "2026-10-25T00:00:00.000Z"),
],
};
+ let scansCwd = true;
const providerChanges = yield* Queue.unbounded<ReadonlyArray<ServerProvider>>();
const pulled = yield* Queue.unbounded<void>();
const responses = yield* Queue.unbounded<unknown>();
@@ -1047,6 +1056,20 @@ it.effect("on-demand clients get provider config without workspace snapshots", (
),
refreshWorkspaceSnapshot: () => Effect.succeed([provider]),
}),
+ Layer.mock(ProviderInstanceRegistry.ProviderInstanceRegistry)({
+ listInstances: Effect.sync(() => [
+ (scansCwd
+ ? { instanceId, snapshotForCwd: () => Effect.die("unused") }
+ : { instanceId }) as never,
+ ]),
+ getInstance: () =>
+ Effect.sync(
+ () =>
+ (scansCwd
+ ? { instanceId, snapshotForCwd: () => Effect.die("unused") }
+ : { instanceId }) as never,
+ ),
+ }),
Layer.mock(UsageLimitSources.UsageLimitSources)({
current: Effect.succeed([]),
streamChanges: Stream.make([]),
@@ -1116,12 +1139,17 @@ it.effect("on-demand clients get provider config without workspace snapshots", (
_tag: "Request",
id: configRequest,
tag: WS_METHODS.subscribeServerConfig,
- payload: { workspaceSnapshotsOnDemand: true },
+ payload: { workspaceSnapshotsOnDemand: true, providerCatalogsOnDemand: true },
headers: Headers.empty,
});
const snapshot = (yield* Queue.take(responses)) as ServerConfigStreamEvent;
if (snapshot.type !== "snapshot") return yield* Effect.die("Missing initial snapshot");
assert.isUndefined(snapshot.config.providers[0]?.workspaceSnapshots);
+ // The cwd snapshot the client fetches carries the catalog, so only command names are sent.
+ assert.deepEqual(snapshot.config.providers[0]?.slashCommands, [{ name: "machine-command" }]);
+ assert.deepEqual(snapshot.config.providers[0]?.skills, [
+ { name: "machine-skill", path: "/skills/machine-skill", enabled: true },
+ ]);
yield* server.write(0, { _tag: "Ack", requestId: configRequest });
// A rescan that only touches a snapshot sends nothing; the next real change does.
@@ -1130,13 +1158,34 @@ it.effect("on-demand clients get provider config without workspace snapshots", (
]);
yield* Queue.take(pulled);
yield* TestClock.adjust("1 second");
- yield* Queue.offer(providerChanges, [{ ...provider, status: "warning" }]);
+ // Late /usage-limits discovery still reaches the client, so it refetches its cwd snapshot.
+ yield* Queue.offer(providerChanges, [
+ {
+ ...provider,
+ status: "warning",
+ slashCommands: [...provider.slashCommands, USAGE_LIMITS_COMMAND],
+ },
+ ]);
yield* Queue.take(pulled);
yield* TestClock.adjust("1 second");
const update = (yield* Queue.take(responses)) as ServerConfigStreamEvent;
if (update.type !== "providerStatuses") return yield* Effect.die("Expected provider update");
assert.equal(update.payload.providers[0]?.status, "warning");
assert.isUndefined(update.payload.providers[0]?.workspaceSnapshots);
+ assert.deepEqual(update.payload.providers[0]?.slashCommands, [
+ { name: "machine-command" },
+ { name: USAGE_LIMITS_COMMAND.name },
+ ]);
+ yield* server.write(0, { _tag: "Ack", requestId: configRequest });
+ // A replacement that cannot scan a cwd sends its full machine catalog.
+ scansCwd = false;
+ yield* Queue.offer(providerChanges, [{ ...provider, status: "warning" }]);
+ yield* Queue.take(pulled);
+ yield* TestClock.adjust("1 second");
+ const replaced = (yield* Queue.take(responses)) as ServerConfigStreamEvent;
+ if (replaced.type !== "providerStatuses") return yield* Effect.die("Expected provider update");
+ assert.deepEqual(replaced.payload.providers[0]?.slashCommands, provider.slashCommands);
+ assert.deepEqual(replaced.payload.providers[0]?.skills, provider.skills);
yield* server.write(0, {
_tag: "Request",
@@ -1152,6 +1201,32 @@ it.effect("on-demand clients get provider config without workspace snapshots", (
refreshed.providers.map((entry) => [entry.workspaceSnapshots, entry.slashCommands]),
[[[snapshotFor("/b", "2026-10-25T00:00:00.000Z")], []]],
);
+ // A failed scan still gives a client without the machine catalog its commands.
+ scansCwd = true;
+ yield* server.write(0, {
+ _tag: "Request",
+ id: RpcMessage.RequestId("refresh-failed-scan"),
+ tag: WS_METHODS.serverRefreshProviders,
+ payload: {
+ instanceId,
+ cwd: "/c",
+ workspaceSnapshotsOnDemand: true,
+ providerCatalogsOnDemand: true,
+ },
+ headers: Headers.empty,
+ });
+ const fallback = (yield* Queue.take(responses)) as {
+ readonly providers: ReadonlyArray<ServerProvider>;
+ };
+ assert.deepEqual(fallback.providers[0]?.workspaceSnapshots, [
+ {
+ cwd: "/c",
+ checkedAt: provider.checkedAt,
+ slashCommands: provider.slashCommands,
+ slashCommandsPending: true,
+ skills: provider.skills,
+ },
+ ]);
yield* server.write(0, { _tag: "Interrupt", requestId: configRequest, interruptors: [] });
}).pipe(
Effect.provide(
diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts
index 9816f6c6008..e3c7fdaa474 100644
--- a/apps/server/src/ws.ts
+++ b/apps/server/src/ws.ts
@@ -188,7 +188,7 @@ import { invalidateSettingsSecrets } from "./sourceControl/sourceControlHostSecr
import * as DatabaseStorage from "./persistence/DatabaseStorage.ts";
import * as StorageCleanup from "./storageCleanup.ts";
import * as TerminalManager from "./terminal/Manager.ts";
-import { terminalOutputStream, withTerminalOutputWindow } from "./terminal/OutputProtocol.ts";
+import { terminalOutputStream, withOutputWindow } from "./terminal/OutputProtocol.ts";
import * as ServerBrowser from "./preview/ServerBrowser.ts";
import * as DeviceService from "./device/DeviceService.ts";
import { remoteSshDeviceHosts } from "./device/localSshDeviceHost.ts";
@@ -426,8 +426,44 @@ export const resolveOpenDiscoveryForConfig = <T, D, E1, R1, E2, R2, E3, R3, E4,
},
{ concurrency: "unbounded" },
).pipe(Effect.map(({ editorConfig, ...endpoints }) => ({ ...editorConfig, ...endpoints })));
-function withoutWorkspaceSnapshots(providers: ReadonlyArray<ServerProvider>): ServerProvider[] {
- return providers.map(({ workspaceSnapshots: _workspaceSnapshots, ...provider }) => provider);
+// A failed scan leaves no cwd snapshot, so a client without the machine catalog gets it
+// as a pending snapshot, which it shows while it retries the scan.
+function workspaceSnapshotOrCatalog(
+ provider: ServerProvider,
+ cwd: string,
+ catalogOnDemand: boolean,
+) {
+ const snapshots = (provider.workspaceSnapshots ?? []).filter((snapshot) => snapshot.cwd === cwd);
+ if (snapshots.length > 0 || !catalogOnDemand) return snapshots;
+ return [
+ {
+ cwd,
+ checkedAt: provider.checkedAt,
+ slashCommands: provider.slashCommands,
+ slashCommandsPending: true,
+ skills: provider.skills,
+ },
+ ];
+}
+
+// On-demand catalog clients read full entries from the cwd snapshot they fetch. Names still
+// change the config when the machine catalog does, so the client refetches, and skill names
+// and labels keep skill links working where no snapshot is fetched.
+function withoutWorkspaceSnapshots(
+ providers: ReadonlyArray<ServerProvider>,
+ catalogsPerCwd: ReadonlySet<string> = new Set(),
+): ServerProvider[] {
+ return providers.map(({ workspaceSnapshots: _workspaceSnapshots, ...provider }) =>
+ catalogsPerCwd.has(provider.instanceId)
+ ? {
+ ...provider,
+ slashCommands: provider.slashCommands.map(({ name }) => ({ name })),
+ skills: provider.skills.map(
+ ({ description: _d, shortDescription: _s, ...skill }) => skill,
+ ),
+ }
+ : provider,
+ );
}
function unexpectedCompatibilityError(error: never): never {
@@ -1860,6 +1896,7 @@ export const layerWsRpc = (
readonly usageLimitsCommand: boolean;
readonly workspaceSnapshotsOnDemand: boolean;
readonly deferNetworkDiscovery: boolean;
+ readonly catalogsPerCwd?: ReadonlySet<string>;
}) {
const keybindingsConfig = yield* keybindings.loadConfigState;
const currentProviders = yield* providerRegistry.getProviders;
@@ -1867,7 +1904,7 @@ export const layerWsRpc = (
? withUsageLimitsCommands(currentProviders, yield* usageLimitSources.current)
: currentProviders;
const providers = options.workspaceSnapshotsOnDemand
- ? withoutWorkspaceSnapshots(commandProviders)
+ ? withoutWorkspaceSnapshots(commandProviders, options.catalogsPerCwd)
: commandProviders;
const settings = ServerSettings.redactServerSettingsForClient(
yield* serverSettings.getSettings,
@@ -2498,6 +2535,9 @@ export const layerWsRpc = (
if (instanceId === undefined || cwd === undefined) {
return { providers: withoutWorkspaceSnapshots(commandProviders) };
}
+ const catalogOnDemand =
+ input.providerCatalogsOnDemand === true &&
+ (yield* providerInstances.getInstance(instanceId))?.snapshotForCwd !== undefined;
return {
providers: commandProviders
.filter((provider) => provider.instanceId === instanceId)
@@ -2507,9 +2547,7 @@ export const layerWsRpc = (
models: [],
slashCommands: [],
skills: [],
- workspaceSnapshots: (provider.workspaceSnapshots ?? []).filter(
- (snapshot) => snapshot.cwd === cwd,
- ),
+ workspaceSnapshots: workspaceSnapshotOrCatalog(provider, cwd, catalogOnDemand),
})),
};
}),
@@ -3234,7 +3272,22 @@ export const layerWsRpc = (
Effect.gen(function* () {
const usageLimitsCommand = input.usageLimitsCommand === true;
const workspaceSnapshotsOnDemand = input.workspaceSnapshotsOnDemand === true;
- // Buffer changes through discovery and the initial snapshot's RPC ACK wait.
+ // Only providers that can scan a cwd answer the client's on-demand catalog fetch.
+ // Read per update, since a replaced instance can change that.
+ const currentCatalogsPerCwd =
+ workspaceSnapshotsOnDemand && input.providerCatalogsOnDemand === true
+ ? providerInstances.listInstances.pipe(
+ Effect.map(
+ (instances) =>
+ new Set(
+ instances
+ .filter((instance) => instance.snapshotForCwd !== undefined)
+ .map((instance) => instance.instanceId),
+ ),
+ ),
+ )
+ : Effect.succeed(new Set<string>());
+ // Buffer changes through discovery and the initial snapshot's send.
const keybindingsChanges = yield* keybindings.subscribeChanges;
const settingsChanges = yield* serverSettings.subscribeChanges;
const remoteTargets = yield* Effect.forkScoped(remoteOpenTargets.resolveTargets(), {
@@ -3250,6 +3303,7 @@ export const layerWsRpc = (
usageLimitsCommand,
workspaceSnapshotsOnDemand,
deferNetworkDiscovery: true,
+ catalogsPerCwd: yield* currentCatalogsPerCwd,
});
const keybindingsUpdates = keybindingsChanges.pipe(
Stream.map((event) => ({
@@ -3272,16 +3326,19 @@ export const layerWsRpc = (
usageLimitsCommand ? sameUsageLimitCommandCoverage : () => true,
),
),
- (providers, sources) => {
- const commandProviders = usageLimitsCommand
- ? withUsageLimitsCommands(providers, sources)
- : providers;
- // Stripped before the de-dupe below, so snapshot-only changes send nothing.
- return workspaceSnapshotsOnDemand
- ? withoutWorkspaceSnapshots(commandProviders)
- : commandProviders;
- },
+ (providers, sources) =>
+ usageLimitsCommand ? withUsageLimitsCommands(providers, sources) : providers,
).pipe(
+ // Stripped before the de-dupe below, so snapshot-only changes send nothing.
+ Stream.mapEffect((commandProviders) =>
+ workspaceSnapshotsOnDemand
+ ? currentCatalogsPerCwd.pipe(
+ Effect.map((catalogsPerCwd) =>
+ withoutWorkspaceSnapshots(commandProviders, catalogsPerCwd),
+ ),
+ )
+ : Effect.succeed(commandProviders),
+ ),
// Compare initial paired values with the snapshot; blindly dropping them loses changes between snapshot and subscription.
(updates) => Stream.concat(rpcInitialItems([config.providers]), updates),
Stream.changesWith(
@@ -3480,7 +3537,7 @@ export const layer = Layer.unwrap(
const rpcWebSocketHttpEffect = yield* Effect.gen(function* () {
const { protocol, httpEffect } = yield* RpcServer.makeProtocolWithHttpEffectWebsocket;
yield* RpcServer.make(ServerWsRpcGroup, WS_RPC_SERVER_OPTIONS).pipe(
- Effect.provideService(RpcServer.Protocol, withTerminalOutputWindow(protocol)),
+ Effect.provideService(RpcServer.Protocol, withOutputWindow(protocol)),
Effect.provide(
Layer.merge(
RpcAuthorization.layer(session.scopes, (method, payload) => {
diff --git a/packages/client-runtime/src/connection/supervisor.test.ts b/packages/client-runtime/src/connection/supervisor.test.ts
index b4fc6955366..f284c48e827 100644
--- a/packages/client-runtime/src/connection/supervisor.test.ts
+++ b/packages/client-runtime/src/connection/supervisor.test.ts
@@ -571,7 +571,7 @@ describe("EnvironmentSupervisor", () => {
);
expect(Option.isNone(yield* SubscriptionRef.get(supervisor.prepared))).toBe(true);
- yield* TestClock.adjust("3 seconds");
+ yield* TestClock.adjust("250 millis");
yield* awaitState(
supervisor.state,
(state) => state.phase === "connected" && state.generation === 2,
@@ -794,6 +794,26 @@ describe("EnvironmentSupervisor", () => {
}).pipe(Effect.provide(TestClock.layer())),
);
+ it.effect("replaces a silent remote socket without the wait after a long resume", () =>
+ Effect.gen(function* () {
+ const harness = yield* makeSilentSocketHarness();
+ const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
+ initiallyDesired: true,
+ }).pipe(Effect.provide(harness.dependencies));
+
+ yield* awaitState(supervisor.state, (state) => state.phase === "connected");
+ yield* harness.wake("application-active-long");
+ yield* TestClock.adjust("499 millis");
+ expect(yield* Ref.get(harness.sessionCount)).toBe(1);
+
+ yield* TestClock.adjust("1 millis");
+ yield* awaitState(
+ supervisor.state,
+ (state) => state.phase === "connected" && state.generation === 2,
+ );
+ }).pipe(Effect.provide(TestClock.layer())),
+ );
+
it.effect("replaces a silent socket even while the network reports offline", () =>
Effect.gen(function* () {
const harness = yield* makeSilentSocketHarness();
@@ -869,11 +889,11 @@ describe("EnvironmentSupervisor", () => {
yield* awaitState(supervisor.state, (state) => state.phase === "connected");
yield* harness.wake("application-active-probe");
yield* TestClock.adjust("1 second");
- yield* Ref.update(frames, (count) => count + 1);
- yield* TestClock.adjust("1 second");
yield* harness.wake("network-changed");
- // The running check sees the frame at 2.5 s, so a fresh one starts then.
- yield* TestClock.adjust("2999 millis");
+ yield* TestClock.adjust("500 millis");
+ yield* Ref.update(frames, (count) => count + 1);
+ // The running check sees the frame at 1.6 s, so a fresh one starts then.
+ yield* TestClock.adjust("2599 millis");
expect(yield* Ref.get(harness.sessionCount)).toBe(1);
yield* TestClock.adjust("1 millis");
@@ -967,22 +987,49 @@ describe("EnvironmentSupervisor", () => {
}).pipe(Effect.provide(TestClock.layer())),
);
- it.effect("keeps waiting for the probe when a frame arrives during the host check", () =>
+ it.effect.each(["application-active-probe", "application-active-long"] as const)(
+ "keeps waiting for the probe when a frame arrives during the host check (%s)",
+ (wakeup) =>
+ Effect.gen(function* () {
+ const frames = yield* Ref.make(0);
+ const harness = yield* makeHarness({
+ prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
+ initialConfig: () => Effect.succeed(PROBE_CAPABLE_CONFIG),
+ probe: (attempt) => (attempt === 1 ? Effect.sleep("2 seconds") : Effect.void),
+ inboundFrames: Ref.get(frames),
+ checkHost: () => Ref.update(frames, (count) => count + 1).pipe(Effect.as("answered")),
+ });
+ const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, {
+ initiallyDesired: true,
+ }).pipe(Effect.provide(harness.dependencies));
+
+ yield* awaitState(supervisor.state, (state) => state.phase === "connected");
+ yield* harness.wake(wakeup);
+ yield* TestClock.adjust("5 seconds");
+
+ expect(yield* Ref.get(harness.sessionCount)).toBe(1);
+ expect((yield* SubscriptionRef.get(supervisor.state)).generation).toBe(1);
+ }).pipe(Effect.provide(TestClock.layer())),
+ );
+
+ it.effect("keeps a long-resume socket that delivers a frame during a slow host check", () =>
Effect.gen(function* () {
const frames = yield* Ref.make(0);
const harness = yield* makeHarness({
prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
initialConfig: () => Effect.succeed(PROBE_CAPABLE_CONFIG),
- probe: (attempt) => (attempt === 1 ? Effect.sleep("2 seconds") : Effect.void),
+ probe: (attempt) => (attempt === 1 ? Effect.sleep("2500 millis") : Effect.void),
inboundFrames: Ref.get(frames),
- checkHost: () => Ref.update(frames, (count) => count + 1).pipe(Effect.as("answered")),
+ checkHost: () => Effect.sleep("2 seconds").pipe(Effect.as("answered")),
});
- const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, {
+ const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
initiallyDesired: true,
}).pipe(Effect.provide(harness.dependencies));
yield* awaitState(supervisor.state, (state) => state.phase === "connected");
- yield* harness.wake("application-active-probe");
+ yield* harness.wake("application-active-long");
+ yield* TestClock.adjust("700 millis");
+ yield* Ref.update(frames, (count) => count + 1);
yield* TestClock.adjust("5 seconds");
expect(yield* Ref.get(harness.sessionCount)).toBe(1);
@@ -1049,7 +1096,7 @@ describe("EnvironmentSupervisor", () => {
yield* awaitState(supervisor.state, (state) => state.phase === "connected");
yield* harness.wake("application-active-long");
yield* harness.setNetworkStatus("offline");
- yield* TestClock.adjust("2999 millis");
+ yield* TestClock.adjust("1499 millis");
expect(yield* Ref.get(harness.sessionCount)).toBe(1);
yield* TestClock.adjust("1 millis");
diff --git a/packages/client-runtime/src/connection/supervisor.ts b/packages/client-runtime/src/connection/supervisor.ts
index 5f279edf568..8f753660581 100644
--- a/packages/client-runtime/src/connection/supervisor.ts
+++ b/packages/client-runtime/src/connection/supervisor.ts
@@ -42,6 +42,7 @@ import {
const RETRY_BASE_DELAY_MS = 1_000;
const RETRY_MAX_DELAY_MS = 300_000;
+const SESSION_LOSS_RETRY_DELAY_MS = 250;
// A cold server with a large database can take longer than 15 seconds to
// answer, and that is not a reason to drop a working connection.
const CONNECTION_ESTABLISHMENT_TIMEOUT = "15 seconds";
@@ -49,13 +50,17 @@ const establishmentTimeout = Duration.fromInputUnsafe(CONNECTION_ESTABLISHMENT_T
const synchronizationTimeout = Duration.fromInputUnsafe("75 seconds");
const CONNECTION_PROBE_TIMEOUT = "15 seconds";
const QUICK_CONNECTION_PROBE_TIMEOUT = "3 seconds";
+// After a long suspension pooled connections are often dead too, so the host
+// check can hang with the socket. A replacement is cheaper than the wait.
+const LONG_RESUME_PROBE_TIMEOUT = "1500 millis";
// A probe still unanswered this long checks the host with a fresh request. If
// the host answers while the socket stays silent, the socket is dead.
const SILENT_SOCKET_CHECK_AFTER = "500 millis";
const SILENT_SOCKET_RECHECK_EVERY = "1 second";
+const FRAME_CHECK_EVERY = "100 millis";
// Off the LAN one lost packet can stall a live socket for a second, so a remote
-// socket must stay silent this long. It must end before
-// QUICK_CONNECTION_PROBE_TIMEOUT so a long resume replaces a silent socket
+// socket must stay silent this long, except after a long resume. It must end
+// before QUICK_CONNECTION_PROBE_TIMEOUT so a resume replaces a silent socket
// instead of timing out.
const REMOTE_SILENT_SOCKET_AFTER = "2500 millis";
const BACKOFF_RESET_AFTER_MS = 30_000;
@@ -668,7 +673,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
// answer reconnects as quickly as a missing one, even on a remote
// route where a reconnect takes longer than the wait it replaces.
if (next.reason === "application-active-long") {
- return QUICK_CONNECTION_PROBE_TIMEOUT;
+ return LONG_RESUME_PROBE_TIMEOUT;
}
if (next.reason !== "application-active-probe" && next.reason !== "network-changed") {
return undefined;
@@ -693,6 +698,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
const awaitSilentSocket = Effect.fn("EnvironmentSupervisor.awaitSilentSocket")(function* (
lease: ConnectedLease,
routeKind: ConnectionRouteKind | undefined,
+ longResume: boolean,
) {
const startedAt = yield* Clock.monotonicTimeNanos;
yield* Effect.sleep(SILENT_SOCKET_CHECK_AFTER);
@@ -707,17 +713,26 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
const socketSilent = lease.session.inboundFrames.pipe(
Effect.map((frames) => frames === framesBefore),
);
- // A radio still waking up or a stale pooled connection can fail one check.
- while ((yield* driver.checkHost(lease.prepared)) !== "answered") {
- if (!(yield* socketSilent)) return "alive" as const;
- yield* Effect.sleep(SILENT_SOCKET_RECHECK_EVERY);
- }
- if (!answersQuickly(routeKind)) {
- const silentFor = (yield* Clock.monotonicTimeNanos) - startedAt;
- const remaining = Duration.toNanosUnsafe(REMOTE_SILENT_SOCKET_AFTER) - silentFor;
- if (remaining > 0n) yield* Effect.sleep(Duration.nanos(remaining));
- }
- return (yield* socketSilent) ? ("silent" as const) : ("alive" as const);
+ // The session only counts frames, so a host check slower than the probe
+ // deadline would otherwise hide a frame until after the socket is replaced.
+ const frameArrives = Effect.sleep(FRAME_CHECK_EVERY).pipe(
+ Effect.andThen(socketSilent),
+ Effect.repeat({ while: (silent) => silent }),
+ Effect.as("alive" as const),
+ );
+ const hostAnswersWhileSilent = Effect.gen(function* () {
+ // A radio still waking up or a stale pooled connection can fail one check.
+ while ((yield* driver.checkHost(lease.prepared)) !== "answered") {
+ yield* Effect.sleep(SILENT_SOCKET_RECHECK_EVERY);
+ }
+ if (!answersQuickly(routeKind) && !longResume) {
+ const silentFor = (yield* Clock.monotonicTimeNanos) - startedAt;
+ const remaining = Duration.toNanosUnsafe(REMOTE_SILENT_SOCKET_AFTER) - silentFor;
+ if (remaining > 0n) yield* Effect.sleep(Duration.nanos(remaining));
+ }
+ return (yield* socketSilent) ? ("silent" as const) : ("alive" as const);
+ });
+ return yield* Effect.race(hostAnswersWhileSilent, frameArrives);
});
// Returning to the app, an explicit retry, and the network reporting offline
@@ -764,7 +779,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
let recheck = false;
let longResume = isLongResume(next);
const startSilentSocketCheck = Effect.gen(function* () {
- silentSocket = yield* Effect.forkChild(awaitSilentSocket(lease, routeKind));
+ silentSocket = yield* Effect.forkChild(awaitSilentSocket(lease, routeKind, longResume));
recheck = false;
});
// The web and desktop foreground probe allows a busy host 15 seconds,
@@ -784,7 +799,8 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
{ discard: true },
),
);
- let deadline = (yield* Clock.monotonicTimeNanos) + Duration.toNanosUnsafe(probeTimeout);
+ let longResumeStartedAt = yield* Clock.monotonicTimeNanos;
+ let deadline = longResumeStartedAt + Duration.toNanosUnsafe(probeTimeout);
for (;;) {
const remaining = deadline - (yield* Clock.monotonicTimeNanos);
const probeEvent = yield* Effect.raceAllFirst([
@@ -805,6 +821,12 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
]);
if (probeEvent._tag === "SocketAlive") {
silentSocket = undefined;
+ // A frame shows the socket still works, so its probe gets the usual resume time.
+ if (longResume) {
+ const aliveDeadline =
+ longResumeStartedAt + Duration.toNanosUnsafe(QUICK_CONNECTION_PROBE_TIMEOUT);
+ if (aliveDeadline > deadline) deadline = aliveDeadline;
+ }
if (recheck) yield* startSilentSocketCheck;
continue;
}
@@ -839,8 +861,11 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
}
const signalTimeout = probeTimeoutFor(probeEvent.signal, routeKind);
if (signalTimeout !== undefined) {
+ if (isLongResume(probeEvent.signal) && !longResume) {
+ longResume = true;
+ longResumeStartedAt = yield* Clock.monotonicTimeNanos;
+ }
yield* armSilentSocketCheck(probeEvent.signal);
- longResume ||= isLongResume(probeEvent.signal);
const signalDeadline =
(yield* Clock.monotonicTimeNanos) + Duration.toNanosUnsafe(signalTimeout);
if (signalDeadline < deadline) deadline = signalDeadline;
@@ -1155,8 +1180,13 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
continue;
}
+ // The first loss of a live session retries almost at once; the ladder
+ // still counts it, so a host that keeps dropping sessions backs off.
+ const firstLoss = outcome.established && failureCount === 0;
failureCount += 1;
- const delayMs = retryDelayMs(failureCount - 1, yield* Random.next);
+ const delayMs = firstLoss
+ ? Math.round(SESSION_LOSS_RETRY_DELAY_MS * (0.5 + (yield* Random.next) / 2))
+ : retryDelayMs(failureCount - 1, yield* Random.next);
pendingRetry = Option.map(attemptSpan, (previousAttempt) => ({
previousAttempt,
failureCount,
diff --git a/packages/client-runtime/src/rpc/session.test.ts b/packages/client-runtime/src/rpc/session.test.ts
index 9e63f8485ad..8f4bee5dc16 100644
--- a/packages/client-runtime/src/rpc/session.test.ts
+++ b/packages/client-runtime/src/rpc/session.test.ts
@@ -404,6 +404,38 @@ describe("RpcSessionFactory", () => {
}),
);
+ it.effect("ends a stream at once when its socket closes", () =>
+ Effect.gen(function* () {
+ const { factory, sockets } = yield* makeFactory();
+ const session = yield* factory.connect(PREPARED);
+ const readyFiber = yield* Effect.forkChild(session.ready);
+ const socket = yield* awaitSocket(sockets);
+ socket.open();
+ yield* completeInitialConfig(socket);
+ yield* Fiber.join(readyFiber);
+ // Like the supervisor: the closed session interrupts its streams, and a
+ // stream interrupt must not wait for a socket that is gone.
+ const ended = yield* Effect.forkChild(
+ Effect.raceFirst(
+ Effect.flip(session.closed),
+ session.client[WS_METHODS.subscribeServerLifecycle]({}).pipe(
+ Stream.runDrain,
+ Effect.andThen(Effect.never),
+ ),
+ ),
+ );
+ yield* awaitRequest(socket, 1);
+ socket.close(1006);
+ yield* TestClock.adjust("999 millis");
+
+ expect(ended.pollUnsafe()).toBeDefined();
+ // A request made after the close still fails instead of waiting forever.
+ const lateProbe = yield* Effect.forkChild(session.probe);
+ yield* TestClock.adjust("999 millis");
+ expect(lateProbe.pollUnsafe()?._tag).toBe("Failure");
+ }).pipe(Effect.scoped, Effect.provide(TestClock.layer())),
+ );
+
it.effect("replays current config and broadcasts updates to every subscriber", () =>
Effect.scoped(
Effect.gen(function* () {
diff --git a/packages/client-runtime/src/rpc/session.ts b/packages/client-runtime/src/rpc/session.ts
index ddf591b1391..3ea4b73bf6b 100644
--- a/packages/client-runtime/src/rpc/session.ts
+++ b/packages/client-runtime/src/rpc/session.ts
@@ -66,6 +66,7 @@ export interface RpcSessionOptions {
readonly environmentThemes?: boolean;
readonly usageLimitSources?: boolean;
readonly usageLimitsCommand?: boolean;
+ readonly providerCatalogsOnDemand?: boolean;
}
export class RpcSessionFactory extends Context.Service<
@@ -175,6 +176,7 @@ export const make = Effect.fn("RpcSessionFactory.make")(function* (
...(options.environmentThemes === true ? { environmentThemes: true } : {}),
...(options.usageLimitSources === true ? { usageLimitSources: true } : {}),
...(options.usageLimitsCommand === true ? { usageLimitsCommand: true } : {}),
+ ...(options.providerCatalogsOnDemand === true ? { providerCatalogsOnDemand: true } : {}),
};
// Config state merges the snapshots it fetches back in, so no subscriber needs them broadcast.
const onDemandSnapshots = (input: ServerConfigSubscriptionInput) => ({
@@ -247,7 +249,18 @@ export const make = Effect.fn("RpcSessionFactory.make")(function* (
retryTransientErrors: false,
retryPolicy: Schedule.recurs(0),
onPingTimeout: Ref.set(pingTimedOut, true),
- }),
+ }).pipe(
+ // Teardown sends an interrupt per open stream, and a write to a closed
+ // socket waits out the client's 1s interrupt timeout before reconnecting.
+ // Requests still go through, so they fail with the protocol error.
+ Effect.map((protocol) => ({
+ ...protocol,
+ send: (clientId, request, transferables) =>
+ request._tag !== "Request" && Deferred.isDoneUnsafe(disconnected)
+ ? Effect.void
+ : protocol.send(clientId, request, transferables),
+ })),
+ ),
).pipe(
Layer.provide(
Layer.mergeAll(
diff --git a/packages/client-runtime/src/state/server.ts b/packages/client-runtime/src/state/server.ts
index 04a2d663215..f5c9d8f45b3 100644
--- a/packages/client-runtime/src/state/server.ts
+++ b/packages/client-runtime/src/state/server.ts
@@ -365,6 +365,7 @@ export interface ServerConfigSubscriptionOptions {
readonly environmentThemes?: boolean;
readonly usageLimitSources?: boolean;
readonly usageLimitsCommand?: boolean;
+ readonly providerCatalogsOnDemand?: boolean;
}
// Configs are JSON-shaped. Missing and undefined object fields persist identically.
@@ -486,6 +487,7 @@ export const makeEnvironmentServerConfigState = Effect.fn("EnvironmentServerConf
...(subscription.environmentThemes === true ? { environmentThemes: true } : {}),
...(subscription.usageLimitSources === true ? { usageLimitSources: true } : {}),
...(subscription.usageLimitsCommand === true ? { usageLimitsCommand: true } : {}),
+ ...(subscription.providerCatalogsOnDemand === true ? { providerCatalogsOnDemand: true } : {}),
}).pipe(
Stream.runForEach((event) =>
Effect.gen(function* () {
@@ -755,6 +757,7 @@ export function createServerEnvironmentAtoms<R, E>(
readonly environmentThemes?: boolean;
readonly usageLimitSources?: boolean;
readonly usageLimitsCommand?: boolean;
+ readonly providerCatalogsOnDemand?: boolean;
},
) {
const configScheduler = createAtomCommandScheduler();
@@ -770,6 +773,7 @@ export function createServerEnvironmentAtoms<R, E>(
...(options.environmentThemes === true ? { environmentThemes: true } : {}),
...(options.usageLimitSources === true ? { usageLimitSources: true } : {}),
...(options.usageLimitsCommand === true ? { usageLimitsCommand: true } : {}),
+ ...(options.providerCatalogsOnDemand === true ? { providerCatalogsOnDemand: true } : {}),
}),
)
.pipe(
@@ -1401,6 +1405,7 @@ export function createServerEnvironmentAtoms<R, E>(
...input,
workspaceSnapshotsOnDemand: true,
...(options.usageLimitsCommand === true ? { usageLimitsCommand: true } : {}),
+ ...(options.providerCatalogsOnDemand === true ? { providerCatalogsOnDemand: true } : {}),
}),
onSuccess: ({ environmentId, input }, registry, result) =>
Effect.sync(() => {
diff --git a/packages/client-runtime/src/state/shell-sync.test.ts b/packages/client-runtime/src/state/shell-sync.test.ts
index c74ecd82fe8..032ede63714 100644
--- a/packages/client-runtime/src/state/shell-sync.test.ts
+++ b/packages/client-runtime/src/state/shell-sync.test.ts
@@ -56,7 +56,10 @@ const LIVE_SHELL_SNAPSHOT: OrchestrationV2ShellSnapshot = {
function session(client: WsRpcProtocolClient): RpcSession.RpcSession {
return {
client,
- initialConfig: Effect.succeed({ shellResumeCompletionMarker: true } as never),
+ initialConfig: Effect.succeed({
+ shellResumeCompletionMarker: true,
+ environment: { serverVersion: "1.0.0" },
+ } as never),
subscribeServerConfig: (input = {}) => client.subscribeServerConfig(input),
ready: Effect.void,
probe: Effect.void,
@@ -817,12 +820,12 @@ describe("environment shell synchronization", () => {
it.effect("opts in to shell omissions and counts authoritative snapshots and removals", () =>
Effect.gen(function* () {
const events = yield* Queue.unbounded<OrchestrationV2ShellStreamItem>();
- const subscribeInput = yield* Deferred.make<unknown>();
+ const subscribeInputs = yield* Queue.unbounded<unknown>();
const loadOptions = yield* Ref.make<ReadonlyArray<unknown>>([]);
const client = {
[ORCHESTRATION_V2_WS_METHODS.subscribeShell]: (input: unknown) =>
Stream.unwrap(
- Deferred.succeed(subscribeInput, input).pipe(Effect.as(Stream.fromQueue(events))),
+ Queue.offer(subscribeInputs, input).pipe(Effect.as(Stream.fromQueue(events))),
),
} as unknown as WsRpcProtocolClient;
const familySession: RpcSession.RpcSession = {
@@ -831,12 +834,14 @@ describe("environment shell synchronization", () => {
shellResumeCompletionMarker: true,
shellThreadFamilies: true,
shellOlderSettled: true,
+ environment: { serverVersion: "1.0.0", databaseMigration: 70 },
} as never),
};
+ const activeSession = yield* SubscriptionRef.make(Option.some(familySession));
const supervisor = EnvironmentSupervisor.EnvironmentSupervisor.of({
target: TARGET,
state: yield* SubscriptionRef.make(AVAILABLE_CONNECTION_STATE),
- session: yield* SubscriptionRef.make(Option.some(familySession)),
+ session: activeSession,
prepared: yield* SubscriptionRef.make(Option.some(PREPARED)),
connect: Effect.void,
disconnect: Effect.void,
@@ -871,7 +876,7 @@ describe("environment shell synchronization", () => {
SubscriptionRef.changes(shellState).pipe(Stream.filter(predicate), Stream.runHead);
// Both transports must omit, or omitted children would come back through the socket.
- expect(yield* Deferred.await(subscribeInput)).toMatchObject({
+ expect(yield* Queue.take(subscribeInputs)).toMatchObject({
afterSequence: 1,
omitIdleSubagents: true,
omitOlderSettled: true,
@@ -907,6 +912,31 @@ describe("environment shell synchronization", () => {
yield* stateWhere((state) => state.generation > loaded.generation),
);
expect(reset.removedAt.size).toBe(0);
+
+ // A reconnect resumes the server-loaded shell from its cursor, with no second HTTP load.
+ yield* SubscriptionRef.set(activeSession, Option.some({ ...familySession }));
+ expect(yield* Queue.take(subscribeInputs)).toMatchObject({
+ afterSequence: 5,
+ omitIdleSubagents: true,
+ omitOlderSettled: true,
+ });
+ expect(yield* Ref.get(loadOptions)).toHaveLength(1);
+
+ // A migrated server may have rewritten rows without events, so it reloads.
+ yield* SubscriptionRef.set(
+ activeSession,
+ Option.some({
+ ...familySession,
+ initialConfig: familySession.initialConfig.pipe(
+ Effect.map((config) => ({
+ ...config,
+ environment: { ...config.environment, databaseMigration: 71 },
+ })),
+ ),
+ }),
+ );
+ yield* Queue.take(subscribeInputs);
+ expect(yield* Ref.get(loadOptions)).toHaveLength(2);
}),
);
@@ -1076,7 +1106,17 @@ describe("environment shell synchronization", () => {
);
yield* Deferred.await(firstFillStarted);
- yield* SubscriptionRef.set(activeSession, Option.some(session(client)));
+ // New omissions make the next session reload instead of resuming.
+ yield* SubscriptionRef.set(
+ activeSession,
+ Option.some({
+ ...session(client),
+ initialConfig: Effect.succeed({
+ shellThreadFamilies: true,
+ environment: { serverVersion: "1.0.0" },
+ } as never),
+ }),
+ );
yield* SubscriptionRef.changes(shellState).pipe(
Stream.filter(
(state) => Option.isSome(state.snapshot) && state.snapshot.value.snapshotSequence === 2,
diff --git a/packages/client-runtime/src/state/shell.ts b/packages/client-runtime/src/state/shell.ts
index 442e99b6f29..d75ea5ff64a 100644
--- a/packages/client-runtime/src/state/shell.ts
+++ b/packages/client-runtime/src/state/shell.ts
@@ -26,7 +26,7 @@ import * as ConnectionWakeups from "../connection/wakeups.ts";
import { safeErrorLogAttributes } from "../errors/safeLog.ts";
import * as Persistence from "../platform/persistence.ts";
import { runCachePersistence } from "./cachePersistence.ts";
-import { subscribeDynamic } from "../rpc/client.ts";
+import { subscribeDynamicWithSession } from "../rpc/client.ts";
import type { RpcSession } from "../rpc/session.ts";
import * as ShellSnapshotLoader from "./shellSnapshotHttp.ts";
import type { DeferredShellSnapshot } from "./shellPullRequests.ts";
@@ -82,6 +82,16 @@ function withoutDeferredPullRequests(value: DeferredShellSnapshot): Orchestratio
const SHELL_SYNCHRONIZATION_ERROR_MESSAGE = "Could not synchronize environment data.";
+function groupBySession<A>(entries: ReadonlyArray<readonly [RpcSession, A]>) {
+ const groups: Array<[RpcSession, Array<A>]> = [];
+ for (const [session, value] of entries) {
+ const last = groups.at(-1);
+ if (last?.[0] === session) last[1].push(value);
+ else groups.push([session, [value]]);
+ }
+ return groups;
+}
+
export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")(function* () {
const supervisor = yield* EnvironmentSupervisor.EnvironmentSupervisor;
const cache = yield* Persistence.EnvironmentCacheStore;
@@ -106,8 +116,11 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")
status: shellStatusForSnapshot(cachedSnapshot),
});
const awaitingCompletion = yield* Ref.make(false);
- const lastAuthoritativeSession = yield* Ref.make<RpcSession | null>(null);
- const activeSubscriptionSession = yield* Ref.make<RpcSession | null>(null);
+ // The omissions and server build of the subscription whose snapshot the
+ // in-memory shell last came from; null while it holds only cached data.
+ const authoritativeOmission = yield* Ref.make<string | null>(null);
+ // Items buffered from a previous session keep that session's key.
+ const sessionOmissions = new WeakMap<RpcSession, string | null>();
const latestLiveSnapshot = yield* Ref.make<Option.Option<OrchestrationV2ShellSnapshot>>(
Option.none(),
);
@@ -208,12 +221,13 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")
const applyItems = Effect.fn("EnvironmentShellState.applyItems")(function* (
items: ReadonlyArray<OrchestrationV2ShellStreamItem>,
+ omission: string | null,
pendingLinkIds?: ReadonlySet<ThreadId>,
) {
const initial = yield* SubscriptionRef.get(state);
let waiting = yield* Ref.get(awaitingCompletion);
let next = initial;
- let receivedSnapshot = false;
+ let receivedAuthoritativeSnapshot = false;
let deltas: Array<ShellStreamEvent> = [];
const flushDeltas = () => {
if (deltas.length > 0 && Option.isSome(next.snapshot)) {
@@ -267,7 +281,7 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")
? undefined
: { resolvedRepositoryIdentityRoots: item.resolvedRepositoryIdentityRoots },
);
- receivedSnapshot = true;
+ if (authoritative) receivedAuthoritativeSnapshot = true;
next = {
snapshot: Option.some(nextSnapshot),
status: waiting ? "synchronizing" : "live",
@@ -288,11 +302,8 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")
yield* Ref.set(latestLiveSnapshot, next.snapshot);
}
yield* SubscriptionRef.set(state, next);
- if (receivedSnapshot) {
- const session = yield* Ref.get(activeSubscriptionSession);
- if (session !== null) {
- yield* Ref.set(lastAuthoritativeSession, session);
- }
+ if (receivedAuthoritativeSnapshot && omission !== null) {
+ yield* Ref.set(authoritativeOmission, omission);
}
if (next.snapshot !== initial.snapshot && Option.isSome(next.snapshot)) {
yield* Queue.offer(persistence, next.snapshot.value);
@@ -374,25 +385,42 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")
yield* setSynchronizing;
yield* Effect.forkScoped(
- subscribeDynamic(
+ subscribeDynamicWithSession(
ORCHESTRATION_V2_WS_METHODS.subscribeShell,
Effect.fn("EnvironmentShellState.makeSubscribeInput")(function* (session) {
- yield* Ref.set(activeSubscriptionSession, session);
- const { omitIdleSubagents, omitOlderSettled } = yield* session.initialConfig.pipe(
- Effect.map((config) => ({
- omitIdleSubagents: config.shellThreadFamilies === true,
- omitOlderSettled:
- config.shellOlderSettled === true && config.shellThreadFamilies === true,
- })),
- Effect.orElseSucceed(() => ({ omitIdleSubagents: false, omitOlderSettled: false })),
- );
+ const { omitIdleSubagents, omitOlderSettled, omissionKey } =
+ yield* session.initialConfig.pipe(
+ Effect.map((config) => {
+ const omitIdleSubagents = config.shellThreadFamilies === true;
+ const omitOlderSettled = config.shellOlderSettled === true && omitIdleSubagents;
+ // Migrations rewrite projections without events, so an upgraded server reloads.
+ const { serverVersion, databaseMigration } = config.environment;
+ return {
+ omitIdleSubagents,
+ omitOlderSettled,
+ omissionKey: JSON.stringify([
+ omitIdleSubagents,
+ omitOlderSettled,
+ serverVersion,
+ databaseMigration ?? null,
+ ]),
+ };
+ }),
+ Effect.orElseSucceed(() => ({
+ omitIdleSubagents: false,
+ omitOlderSettled: false,
+ omissionKey: null,
+ })),
+ );
+ sessionOmissions.set(session, omissionKey);
yield* Ref.set(awaitingCompletion, true);
yield* setSynchronizing;
- // Foreground resubscriptions on the same live session can resume from
- // the in-memory cursor. A new session reloads the authoritative HTTP
- // snapshot so a valid cursor cannot preserve incomplete cached data.
- const hasAuthoritativeSnapshot = (yield* Ref.get(lastAuthoritativeSession)) === session;
+ // A shell from a server snapshot resumes from its cursor, on this session or a
+ // new one. Cached data, other omissions, or another server build reload the
+ // authoritative HTTP snapshot so a valid cursor cannot preserve stale data.
+ const hasAuthoritativeSnapshot =
+ omissionKey !== null && (yield* Ref.get(authoritativeOmission)) === omissionKey;
let canResume = hasAuthoritativeSnapshot;
let current = yield* SubscriptionRef.get(state);
if (!hasAuthoritativeSnapshot || Option.isNone(current.snapshot)) {
@@ -427,7 +455,7 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")
? []
: httpSnapshot.value.threads.map((thread) => thread.id),
);
- yield* applyItems([{ kind: "snapshot", snapshot }], pendingLinkIds);
+ yield* applyItems([{ kind: "snapshot", snapshot }], omissionKey, pendingLinkIds);
canResume = true;
current = yield* SubscriptionRef.get(state);
if (Option.isSome(current.snapshot)) {
@@ -454,7 +482,15 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")
retryExpectedFailureAfter: "250 millis",
resubscribe: foregroundResubscriptions,
},
- ).pipe(Stream.runForEachArray((items) => applyItems(items))),
+ ).pipe(
+ Stream.runForEachArray((entries) =>
+ Effect.forEach(
+ groupBySession(entries),
+ ([session, items]) => applyItems(items, sessionOmissions.get(session) ?? null),
+ { discard: true },
+ ),
+ ),
+ ),
);
yield* SubscriptionRef.changes(supervisor.state).pipe(
Stream.runForEach((connectionState) => {
diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts
index c014114cc06..3106462d357 100644
--- a/packages/contracts/src/rpc.ts
+++ b/packages/contracts/src/rpc.ts
@@ -655,6 +655,8 @@ export const WsServerRefreshProvidersRpc = Rpc.make(WS_METHODS.serverRefreshProv
workspaceSnapshotsOnDemand: Schema.optional(Schema.Boolean),
/** As on `subscribeServerConfig`: add /usage-limits to the replied snapshot. */
usageLimitsCommand: Schema.optional(Schema.Boolean),
+ /** As on `subscribeServerConfig`: a failed cwd scan replies with the machine catalog as a pending snapshot. */
+ providerCatalogsOnDemand: Schema.optional(Schema.Boolean),
}),
success: ServerProviderUpdatedPayload,
error: Schema.Union([EnvironmentAuthorizationError, ProviderSetupError]),
@@ -1893,6 +1895,8 @@ export const WsSubscribeServerConfigRpc = Rpc.make(WS_METHODS.subscribeServerCon
usageLimitsCommand: Schema.optional(Schema.Boolean),
/** Omit per-cwd provider snapshots; the client fetches its cwd through `server.refreshProviders`. */
workspaceSnapshotsOnDemand: Schema.optional(Schema.Boolean),
+ /** With on-demand snapshots, also omit machine slash commands and skills a cwd snapshot replaces. */
+ providerCatalogsOnDemand: Schema.optional(Schema.Boolean),
}),
success: ServerConfigStreamEvent,
error: Schema.Union([KeybindingsConfigError, ServerSettingsError, EnvironmentAuthorizationError]),