Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

fix(server): stop Windows terminal polling from spiking CPU (#211)

Leoacd2b6ef90c8
diff --git a/apps/server/src/diagnostics/ProcessDiagnostics.test.ts b/apps/server/src/diagnostics/ProcessDiagnostics.test.ts
index 382793716..3a3d8da26 100644
--- a/apps/server/src/diagnostics/ProcessDiagnostics.test.ts
+++ b/apps/server/src/diagnostics/ProcessDiagnostics.test.ts
@@ -19,7 +19,7 @@ function makeNativeSnapshot(
processes: ResourceMonitorSnapshotEvent["processes"],
): ResourceMonitorSnapshotEvent {
return {
- version: 2,
+ version: 3,
type: "snapshot",
sequence: 1,
sampledAtUnixMs: DateTime.toEpochMillis(DateTime.makeUnsafe("2026-05-05T10:00:00.000Z")),
diff --git a/apps/server/src/resourceTelemetry/Model.test.ts b/apps/server/src/resourceTelemetry/Model.test.ts
index 94690e396..6f759ac97 100644
--- a/apps/server/src/resourceTelemetry/Model.test.ts
+++ b/apps/server/src/resourceTelemetry/Model.test.ts
@@ -39,7 +39,7 @@ function nativeSnapshot(
sequence = 1,
): ResourceMonitorSnapshotEvent {
return {
- version: 2,
+ version: 3,
type: "snapshot",
sequence,
sampledAtUnixMs,
diff --git a/apps/server/src/resourceTelemetry/NativeTelemetryClient.ts b/apps/server/src/resourceTelemetry/NativeTelemetryClient.ts
index e864c0228..24c0f5448 100644
--- a/apps/server/src/resourceTelemetry/NativeTelemetryClient.ts
+++ b/apps/server/src/resourceTelemetry/NativeTelemetryClient.ts
@@ -5,6 +5,7 @@ import type {
ResourceMonitorEvent,
ResourceMonitorExternalProcess,
ResourceMonitorHelloEvent,
+ ResourceMonitorProcessTableEntry,
ResourceMonitorSnapshotEvent,
ResourceTelemetrySourceStatus,
} from "@t3tools/contracts";
@@ -44,6 +45,7 @@ const BATTERY_SAMPLE_INTERVAL_MS = 5_000;
const CONSTRAINED_SAMPLE_INTERVAL_MS = 15_000;
const HANDSHAKE_TIMEOUT = Duration.seconds(5);
const SAMPLE_REQUEST_TIMEOUT = Duration.seconds(5);
+const PROCESS_TABLE_REQUEST_TIMEOUT = Duration.seconds(5);
const HISTORY_REQUEST_TIMEOUT = Duration.seconds(15);
const INITIAL_RESTART_DELAY = Duration.millis(500);
const MAX_RESTART_DELAY = Duration.seconds(10);
@@ -76,7 +78,7 @@ export class NativeTelemetryHandshakeTimedOut extends Schema.TaggedErrorClass<Na
export class NativeTelemetryRequestTimedOut extends Schema.TaggedErrorClass<NativeTelemetryRequestTimedOut>()(
"NativeTelemetryRequestTimedOut",
{
- operation: Schema.Literals(["readHistory", "sampleNow"]),
+ operation: Schema.Literals(["processTable", "readHistory", "sampleNow"]),
timeoutMs: Schema.Number,
},
) {
@@ -192,6 +194,10 @@ export class NativeTelemetryClient extends Context.Service<
snapshot: HostPowerSnapshot,
) => Effect.Effect<void, NativeTelemetryClientError>;
readonly sampleNow: Effect.Effect<NativeTelemetrySnapshot, NativeTelemetryClientError>;
+ readonly processTable: Effect.Effect<
+ ReadonlyArray<ResourceMonitorProcessTableEntry>,
+ NativeTelemetryClientError
+ >;
readonly retry: Effect.Effect<boolean>;
readonly health: Effect.Effect<NativeTelemetryClientHealth>;
readonly subscribeHealth: Effect.Effect<
@@ -386,6 +392,12 @@ export const make = Effect.fn("resourceTelemetry.nativeTelemetryClient.make")(fu
const pendingSamples = yield* Ref.make(
new Map<string, Deferred.Deferred<NativeTelemetrySnapshot, NativeTelemetryClientError>>(),
);
+ const pendingProcessTables = yield* Ref.make(
+ new Map<
+ string,
+ Deferred.Deferred<ReadonlyArray<ResourceMonitorProcessTableEntry>, NativeTelemetryClientError>
+ >(),
+ );
const pendingHistories = yield* Ref.make(new Map<string, PendingHistoryRequest>());
const snapshots = yield* PubSub.sliding<NativeTelemetrySnapshot>(8);
const healthChanges = yield* PubSub.sliding<NativeTelemetryClientHealth>(4);
@@ -403,10 +415,14 @@ export const make = Effect.fn("resourceTelemetry.nativeTelemetryClient.make")(fu
const failPending = (error: NativeTelemetryClientError) =>
Effect.gen(function* () {
const samples = yield* Ref.getAndSet(pendingSamples, new Map());
+ const processTables = yield* Ref.getAndSet(pendingProcessTables, new Map());
const histories = yield* Ref.getAndSet(pendingHistories, new Map());
yield* Effect.forEach(samples.values(), (deferred) => Deferred.fail(deferred, error), {
discard: true,
});
+ yield* Effect.forEach(processTables.values(), (deferred) => Deferred.fail(deferred, error), {
+ discard: true,
+ });
yield* Effect.forEach(
histories.values(),
(request) => Deferred.fail(request.deferred, error),
@@ -482,6 +498,21 @@ export const make = Effect.fn("resourceTelemetry.nativeTelemetryClient.make")(fu
}
}
});
+ case "processTable":
+ return Ref.modify(pendingProcessTables, (pending) => {
+ const next = new Map(pending);
+ const deferred = next.get(event.requestId);
+ next.delete(event.requestId);
+ return [Option.fromUndefinedOr(deferred), next] as const;
+ }).pipe(
+ Effect.flatMap(
+ Option.match({
+ onNone: () => Effect.void,
+ onSome: (deferred) => Deferred.succeed(deferred, event.processes),
+ }),
+ ),
+ Effect.asVoid,
+ );
case "historyChunk":
return Effect.gen(function* () {
const latestSnapshot = event.snapshots.at(-1);
@@ -934,6 +965,60 @@ export const make = Effect.fn("resourceTelemetry.nativeTelemetryClient.make")(fu
);
});
+ const processTable: NativeTelemetryClient["Service"]["processTable"] = Effect.gen(function* () {
+ const current = yield* Ref.get(state);
+ if (!canCommandNativeTelemetrySidecar(current.status, Option.isSome(current.handle))) {
+ return yield* new NativeTelemetryUnavailable({
+ reason: Option.getOrElse(current.lastError, () => "sidecar is not running"),
+ });
+ }
+
+ const requestId = yield* crypto.randomUUIDv4.pipe(
+ Effect.mapError(
+ (cause) => new NativeTelemetryCommandFailed({ operation: "createRequestId", cause }),
+ ),
+ );
+ const deferred = yield* Deferred.make<
+ ReadonlyArray<ResourceMonitorProcessTableEntry>,
+ NativeTelemetryClientError
+ >();
+ yield* Ref.update(pendingProcessTables, (pending) => {
+ const next = new Map(pending);
+ next.set(requestId, deferred);
+ return next;
+ });
+ return yield* writeCommand(Option.getOrThrow(current.handle), {
+ version: RESOURCE_MONITOR_PROTOCOL_VERSION,
+ type: "processTable",
+ requestId,
+ }).pipe(
+ Effect.andThen(
+ Deferred.await(deferred).pipe(
+ Effect.timeoutOption(PROCESS_TABLE_REQUEST_TIMEOUT),
+ Effect.flatMap(
+ Option.match({
+ onNone: () =>
+ Effect.fail(
+ new NativeTelemetryRequestTimedOut({
+ operation: "processTable",
+ timeoutMs: Duration.toMillis(PROCESS_TABLE_REQUEST_TIMEOUT),
+ }),
+ ),
+ onSome: Effect.succeed,
+ }),
+ ),
+ ),
+ ),
+ Effect.ensuring(
+ Ref.update(pendingProcessTables, (pending) => {
+ const next = new Map(pending);
+ next.delete(requestId);
+ return next;
+ }),
+ ),
+ );
+ });
+
const health = currentHealth;
return NativeTelemetryClient.of({
@@ -955,6 +1040,7 @@ export const make = Effect.fn("resourceTelemetry.nativeTelemetryClient.make")(fu
setExternalProcesses,
setHostPowerState,
sampleNow,
+ processTable,
retry: Ref.get(state).pipe(
Effect.flatMap((current) =>
!canRequestNativeTelemetryRetry(current.status, Option.isSome(current.handle))
@@ -1008,6 +1094,11 @@ export const layerTest = (
reason: "No resource monitor sample was configured for this test.",
}),
),
+ processTable: Effect.fail(
+ new NativeTelemetryUnavailable({
+ reason: "No resource monitor process table was configured for this test.",
+ }),
+ ),
retry: Effect.succeed(false),
health,
subscribeHealth:
diff --git a/apps/server/src/resourceTelemetry/ResourceTelemetry.test.ts b/apps/server/src/resourceTelemetry/ResourceTelemetry.test.ts
index a96423607..9c3710783 100644
--- a/apps/server/src/resourceTelemetry/ResourceTelemetry.test.ts
+++ b/apps/server/src/resourceTelemetry/ResourceTelemetry.test.ts
@@ -82,7 +82,7 @@ function nativeSnapshot(input: {
}),
];
return {
- version: 2,
+ version: 3,
type: "snapshot",
sequence: input.sequence,
sampledAtUnixMs: input.sampledAtUnixMs,
@@ -497,7 +497,7 @@ describe("ResourceTelemetry", () => {
const nativeHealth = yield* Ref.make<NativeTelemetryClient.NativeTelemetryClientHealth>({
status: "healthy",
hello: Option.some({
- version: 2,
+ version: 3,
type: "hello",
sidecarVersion: "0.1.0",
sidecarPid: 9_000,
diff --git a/apps/server/src/resourceTelemetry/ResourceTelemetryHistory.test.ts b/apps/server/src/resourceTelemetry/ResourceTelemetryHistory.test.ts
index 879c83d86..fff4588c8 100644
--- a/apps/server/src/resourceTelemetry/ResourceTelemetryHistory.test.ts
+++ b/apps/server/src/resourceTelemetry/ResourceTelemetryHistory.test.ts
@@ -64,7 +64,7 @@ function snapshot(
}),
];
return {
- version: 2,
+ version: 3,
type: "snapshot",
sequence,
sampledAtUnixMs,
diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts
index bc4b55885..06935733b 100644
--- a/apps/server/src/server.ts
+++ b/apps/server/src/server.ts
@@ -338,6 +338,7 @@ const PortScannerLayerLive = PortScanner.layer.pipe(Layer.provide(ProcessRunner.
const TerminalLayerLive = TerminalManager.layer.pipe(
Layer.provide(PtyAdapterLive),
Layer.provide(PortScannerLayerLive),
+ Layer.provide(NativeTelemetryLayerLive),
);
const PreviewLayerLive = Layer.empty.pipe(
diff --git a/apps/server/src/terminal/Manager.test.ts b/apps/server/src/terminal/Manager.test.ts
index 47d91e451..fc9e69fda 100644
--- a/apps/server/src/terminal/Manager.test.ts
+++ b/apps/server/src/terminal/Manager.test.ts
@@ -10,6 +10,7 @@ import {
} from "@t3tools/contracts";
import { HostProcessPlatform } from "@t3tools/shared/hostProcess";
import * as Data from "effect/Data";
+import * as Clock from "effect/Clock";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Encoding from "effect/Encoding";
@@ -210,6 +211,10 @@ interface CreateManagerOptions {
readonly childCommand: string | null;
readonly processIds: ReadonlyArray<number>;
}>;
+ processTable?: Effect.Effect<
+ ReadonlyArray<{ readonly pid: number; readonly ppid: number; readonly name: string }>,
+ never
+ >;
subprocessPollIntervalMs?: number;
processKillGraceMs?: number;
maxRetainedInactiveSessions?: number;
@@ -248,6 +253,7 @@ const createManager = (
...(options.subprocessInspector !== undefined
? { subprocessInspector: options.subprocessInspector }
: {}),
+ ...(options.processTable !== undefined ? { processTable: options.processTable } : {}),
...(options.subprocessPollIntervalMs !== undefined
? { subprocessPollIntervalMs: options.subprocessPollIntervalMs }
: {}),
@@ -1073,6 +1079,92 @@ it.layer(
}),
);
+ it("calculates snapshot failure backoff and success reset delays", () => {
+ assert.equal(TerminalManager.subprocessSnapshotPollDelayMs(1_000, 0), 1_000);
+ assert.equal(TerminalManager.subprocessSnapshotPollDelayMs(1_000, 1), 2_000);
+ assert.equal(TerminalManager.subprocessSnapshotPollDelayMs(1_000, 2), 4_000);
+ assert.equal(TerminalManager.subprocessSnapshotPollDelayMs(1_000, 30), 60_000);
+ });
+
+ it.effect("uses process snapshots from the resource monitor", () =>
+ Effect.gen(function* () {
+ let snapshotCalls = 0;
+ const { manager, getEvents } = yield* createManager(5, {
+ subprocessPollIntervalMs: 20,
+ processTable: Effect.sync(() => {
+ snapshotCalls += 1;
+ return [{ pid: 100, ppid: 9000, name: "ping.exe" }];
+ }),
+ }).pipe(Effect.provide(withHostPlatform("win32")));
+
+ yield* manager.open(openInput());
+ yield* waitFor(
+ Effect.map(getEvents, (events) =>
+ events.some(
+ (event) =>
+ event.type === "activity" && event.hasRunningSubprocess && event.label === "ping",
+ ),
+ ),
+ "1200 millis",
+ );
+ expect(snapshotCalls).toBeGreaterThan(0);
+ }),
+ );
+
+ it.effect("backs off the spawned fallback when the resource monitor snapshot fails", () =>
+ Effect.gen(function* () {
+ const fallbackCalls: Array<number> = [];
+ const processRunner: ProcessRunner.ProcessRunner["Service"] = {
+ run: () =>
+ Clock.currentTimeMillis.pipe(
+ Effect.map((now) => {
+ fallbackCalls.push(now);
+ return {
+ stdout: " 100 9000 vim",
+ stderr: "",
+ code: ChildProcessSpawner.ExitCode(0),
+ timedOut: false,
+ stdoutTruncated: false,
+ stderrInvalidUtf8: false,
+ stdoutInvalidUtf8: false,
+ stderrTruncated: false,
+ };
+ }),
+ ),
+ };
+
+ const { manager, getEvents } = yield* createManager(5, {
+ subprocessPollIntervalMs: 20,
+ processTable: Effect.fail("sidecar unavailable").pipe(
+ Effect.mapError((cause) => cause as never),
+ ),
+ }).pipe(
+ Effect.provideService(ProcessRunner.ProcessRunner, processRunner),
+ Effect.provide(withHostPlatform("linux")),
+ );
+
+ yield* manager.open(openInput());
+ yield* waitFor(
+ Effect.map(getEvents, (events) =>
+ events.some(
+ (event) =>
+ event.type === "activity" &&
+ event.hasRunningSubprocess === true &&
+ event.label === "vim",
+ ),
+ ),
+ "1200 millis",
+ );
+
+ yield* waitFor(
+ Effect.sync(() => fallbackCalls.length >= 4),
+ "2000 millis",
+ );
+ const spanMs = fallbackCalls[3]! - fallbackCalls[0]!;
+ expect(spanMs).toBeGreaterThan(150);
+ }),
+ );
+
it.effect("caps persisted history to configured line limit", () =>
Effect.gen(function* () {
const { manager, ptyAdapter } = yield* createManager(3);
diff --git a/apps/server/src/terminal/Manager.ts b/apps/server/src/terminal/Manager.ts
index ecb67b00b..ee3e9d7e1 100644
--- a/apps/server/src/terminal/Manager.ts
+++ b/apps/server/src/terminal/Manager.ts
@@ -26,6 +26,7 @@ import {
type TerminalMetadataStreamEvent,
type TerminalOpenInput,
type TerminalResizeInput,
+ type ResourceMonitorProcessTableEntry,
type TerminalRestartInput,
type TerminalSessionSnapshot,
type TerminalSessionStatus,
@@ -59,6 +60,7 @@ import {
} from "../observability/Metrics.ts";
import * as ProcessRunner from "../processRunner.ts";
import * as PortScanner from "../preview/PortScanner.ts";
+import * as NativeTelemetryClient from "../resourceTelemetry/NativeTelemetryClient.ts";
import * as PtyAdapter from "./PtyAdapter.ts";
export {
@@ -77,6 +79,7 @@ export {
const DEFAULT_HISTORY_LINE_LIMIT = 5_000;
const DEFAULT_PERSIST_DEBOUNCE_MS = 40;
const DEFAULT_SUBPROCESS_POLL_INTERVAL_MS = 1_000;
+const MAX_SUBPROCESS_POLL_INTERVAL_MS = 60_000;
const DEFAULT_PROCESS_KILL_GRACE_MS = 1_000;
const DEFAULT_MAX_RETAINED_INACTIVE_SESSIONS = 128;
const DEFAULT_OPEN_COLS = 120;
@@ -89,7 +92,7 @@ class TerminalSubprocessCheckError extends Schema.TaggedErrorClass<TerminalSubpr
"TerminalSubprocessCheckError",
{
cause: Schema.optional(Schema.Defect()),
- command: Schema.Literals(["powershell", "ps"]),
+ command: Schema.Literals(["powershell", "ps", "resource-monitor"]),
exitCode: Schema.optional(Schema.NullOr(Schema.Number)),
timedOut: Schema.optional(Schema.Boolean),
stdoutTruncated: Schema.optional(Schema.Boolean),
@@ -643,15 +646,22 @@ function parsePosixProcessTable(stdout: string): TerminalProcessTableSnapshot {
return { childrenByParent, commandById };
}
-function parseWindowsProcessTable(stdout: string): TerminalProcessTableSnapshot {
+export function subprocessSnapshotPollDelayMs(
+ pollIntervalMs: number,
+ failureCount: number,
+): number {
+ return Math.min(pollIntervalMs * 2 ** failureCount, MAX_SUBPROCESS_POLL_INTERVAL_MS);
+}
+
+function processTableSnapshotFromProcesses(
+ processes: ReadonlyArray<ResourceMonitorProcessTableEntry>,
+): TerminalProcessTableSnapshot {
const childrenByParent = new Map<number, number[]>();
const commandById = new Map<number, string>();
- for (const line of stdout.split(/\r?\n/g)) {
- const [pidRaw, parentPidRaw, nameRaw] = line.trim().split("|", 3);
- const pid = Number(pidRaw);
- const parentPid = Number(parentPidRaw);
+ for (const process of processes) {
+ const { pid, ppid: parentPid, name } = process;
if (!Number.isInteger(pid) || !Number.isInteger(parentPid)) continue;
- commandById.set(pid, nameRaw?.trim() ?? "");
+ commandById.set(pid, name.trim());
const children = childrenByParent.get(parentPid) ?? [];
children.push(pid);
childrenByParent.set(parentPid, children);
@@ -746,14 +756,11 @@ const windowsProcessTableSnapshot = Effect.fn("terminal.windowsProcessTableSnaps
TerminalSubprocessCheckError,
ProcessRunner.ProcessRunner
> {
+ const processRunner = yield* ProcessRunner.ProcessRunner;
const command =
'Get-CimInstance Win32_Process -ErrorAction Stop | ForEach-Object { Write-Output "$($_.ProcessId)|$($_.ParentProcessId)|$($_.Name)" }';
- const processRunner = yield* ProcessRunner.ProcessRunner;
const result = yield* processRunner
.run({
- // powershell.exe is a real executable — never spawn it through cmd.exe
- // shell mode, which would re-tokenize the `-Command` payload (pipes,
- // semicolons) before PowerShell ever sees it.
command: "powershell.exe",
args: ["-NoProfile", "-NonInteractive", "-Command", command],
timeout: "1500 millis",
@@ -763,16 +770,10 @@ const windowsProcessTableSnapshot = Effect.fn("terminal.windowsProcessTableSnaps
})
.pipe(
Effect.mapError(
- (cause) =>
- new TerminalSubprocessCheckError({
- cause,
- command: "powershell",
- }),
+ (cause) => new TerminalSubprocessCheckError({ cause, command: "powershell" }),
),
);
if (result.code !== 0 || result.timedOut || result.stdoutTruncated) {
- // Not authoritative: an empty or partial table would mark every terminal
- // idle and clear its registered process ids. Failing skips the tick.
return yield* new TerminalSubprocessCheckError({
command: "powershell",
exitCode: result.code,
@@ -780,7 +781,15 @@ const windowsProcessTableSnapshot = Effect.fn("terminal.windowsProcessTableSnaps
stdoutTruncated: result.stdoutTruncated,
});
}
- return parseWindowsProcessTable(result.stdout);
+ const processes = result.stdout.split(/\r?\n/g).flatMap((line) => {
+ const [pidRaw, ppidRaw, name = ""] = line.trim().split("|", 3);
+ const pid = Number(pidRaw);
+ const ppid = Number(ppidRaw);
+ return Number.isInteger(pid) && pid > 0 && Number.isInteger(ppid)
+ ? [{ pid, ppid, name }]
+ : [];
+ });
+ return processTableSnapshotFromProcesses(processes);
},
);
@@ -1115,6 +1124,10 @@ interface TerminalManagerOptions {
shellResolver?: () => string;
env?: NodeJS.ProcessEnv;
subprocessInspector?: TerminalSubprocessInspector;
+ processTable?: Effect.Effect<
+ ReadonlyArray<ResourceMonitorProcessTableEntry>,
+ TerminalSubprocessCheckError
+ >;
subprocessPollIntervalMs?: number;
processKillGraceMs?: number;
maxRetainedInactiveSessions?: number;
@@ -1133,9 +1146,15 @@ export const make = Effect.fn("TerminalManager.make")(function* () {
const { terminalLogsDir } = yield* ServerConfig.ServerConfig;
const ptyAdapter = yield* PtyAdapter.PtyAdapter;
const portDiscovery = yield* PortScanner.PortDiscovery;
+ const nativeTelemetry = yield* NativeTelemetryClient.NativeTelemetryClient;
return yield* makeWithOptions({
logsDir: terminalLogsDir,
ptyAdapter,
+ processTable: nativeTelemetry.processTable.pipe(
+ Effect.mapError(
+ (cause) => new TerminalSubprocessCheckError({ cause, command: "resource-monitor" }),
+ ),
+ ),
registerTerminalProcesses: portDiscovery.registerTerminalProcesses,
unregisterTerminal: portDiscovery.unregisterTerminal,
});
@@ -1162,23 +1181,60 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func
// One process-table snapshot per poll tick, shared across every terminal.
// Per-terminal `pgrep`/`ps` calls multiply spawn load by terminal count and
// can exhaust the PID space on hosts with many sessions (#6332).
- const fetchProcessTableSnapshot = (
+ const fallbackProcessTableSnapshot = (
platform === "win32"
? windowsProcessTableSnapshot()
: posixProcessTableSnapshot(yield* resolvePosixPsCommand())
).pipe(Effect.provideService(ProcessRunner.ProcessRunner, processRunner));
+ const fetchProcessTableSnapshot: Effect.Effect<
+ {
+ readonly snapshot: TerminalProcessTableSnapshot;
+ /**
+ * False when the sidecar snapshot failed and this table came from the
+ * spawned fallback. The data is still applied, but the tick counts as
+ * a failure so polling backs off instead of hot-looping the fallback.
+ */
+ readonly snapshotSucceeded: boolean;
+ },
+ TerminalSubprocessCheckError
+ > = options.processTable
+ ? options.processTable.pipe(
+ Effect.map((entries) => ({
+ snapshot: processTableSnapshotFromProcesses(entries),
+ snapshotSucceeded: true,
+ })),
+ Effect.catch(() =>
+ fallbackProcessTableSnapshot.pipe(
+ Effect.map((snapshot) => ({ snapshot, snapshotSucceeded: false })),
+ ),
+ ),
+ )
+ : fallbackProcessTableSnapshot.pipe(
+ Effect.map((snapshot) => ({ snapshot, snapshotSucceeded: true })),
+ );
const customSubprocessInspector = options.subprocessInspector;
const acquireSubprocessInspector: Effect.Effect<
- TerminalSubprocessInspector,
+ {
+ readonly inspector: TerminalSubprocessInspector;
+ readonly snapshotSucceeded: boolean;
+ },
TerminalSubprocessCheckError
> =
customSubprocessInspector !== undefined
- ? Effect.succeed(customSubprocessInspector)
+ ? Effect.succeed({ inspector: customSubprocessInspector, snapshotSucceeded: true })
: Effect.map(
fetchProcessTableSnapshot,
- (snapshot): TerminalSubprocessInspector =>
- (terminalPid) =>
+ ({
+ snapshot,
+ snapshotSucceeded,
+ }): {
+ readonly inspector: TerminalSubprocessInspector;
+ readonly snapshotSucceeded: boolean;
+ } => ({
+ inspector: (terminalPid) =>
Effect.succeed(deriveSubprocessInspectResult(snapshot, terminalPid, platform)),
+ snapshotSucceeded,
+ }),
);
const subprocessPollIntervalMs =
options.subprocessPollIntervalMs ?? DEFAULT_SUBPROCESS_POLL_INTERVAL_MS;
@@ -2008,7 +2064,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func
);
if (runningSessions.length === 0) {
- return;
+ return true;
}
const inspectorOption = yield* acquireSubprocessInspector.pipe(
@@ -2016,15 +2072,22 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func
Effect.catch((reason) =>
Effect.logWarning("failed to snapshot processes for terminal subprocess polling", {
reason,
- }).pipe(Effect.as(Option.none<TerminalSubprocessInspector>())),
+ }).pipe(
+ Effect.as(
+ Option.none<{
+ readonly inspector: TerminalSubprocessInspector;
+ readonly snapshotSucceeded: boolean;
+ }>(),
+ ),
+ ),
),
);
if (Option.isNone(inspectorOption)) {
- return;
+ return false;
}
- const subprocessInspector = inspectorOption.value;
+ const { inspector: subprocessInspector, snapshotSucceeded } = inspectorOption.value;
const checkSubprocessActivity = Effect.fn("terminal.checkSubprocessActivity")(function* (
session: TerminalSessionState & { pid: number },
@@ -2093,6 +2156,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func
concurrency: "unbounded",
discard: true,
});
+ return snapshotSucceeded;
});
const hasRunningSessions = readManagerState.pipe(
@@ -2101,14 +2165,26 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func
),
);
+ let subprocessSnapshotFailureCount = 0;
yield* Effect.forever(
hasRunningSessions.pipe(
Effect.flatMap((active) =>
active
? pollSubprocessActivity().pipe(
- Effect.flatMap(() => Effect.sleep(subprocessPollIntervalMs)),
+ Effect.flatMap((snapshotSucceeded) => {
+ subprocessSnapshotFailureCount = snapshotSucceeded
+ ? 0
+ : Math.min(subprocessSnapshotFailureCount + 1, 30);
+ const delayMs = subprocessSnapshotPollDelayMs(
+ subprocessPollIntervalMs,
+ subprocessSnapshotFailureCount,
+ );
+ return Effect.sleep(delayMs);
+ }),
)
- : Effect.sleep(subprocessPollIntervalMs),
+ : Effect.sync(() => {
+ subprocessSnapshotFailureCount = 0;
+ }).pipe(Effect.flatMap(() => Effect.sleep(subprocessPollIntervalMs))),
),
),
).pipe(Effect.forkIn(workerScope));
diff --git a/docs/internals/resource-telemetry.md b/docs/internals/resource-telemetry.md
index 51a30cc84..44162bfab 100644
--- a/docs/internals/resource-telemetry.md
+++ b/docs/internals/resource-telemetry.md
@@ -7,7 +7,10 @@ Status: implemented
## Purpose
Resource telemetry replaces recurring `ps`, PowerShell, `ioreg`, and `pmset`
-subprocess probes with two persistent, direct data sources:
+subprocess probes with two persistent, direct data sources. Terminal subprocess
+polling reads one shared `processTable` snapshot from the same sidecar, then
+falls back to `ps` or PowerShell with exponential backoff if the sidecar is
+down.
1. a standalone Rust resource-monitor executable that reads process counters
through operating-system APIs via `sysinfo`;
diff --git a/native/resource-monitor/src/main.rs b/native/resource-monitor/src/main.rs
index 0e5dd6630..8b8dbcfc1 100644
--- a/native/resource-monitor/src/main.rs
+++ b/native/resource-monitor/src/main.rs
@@ -8,7 +8,7 @@ use sysinfo::{
MINIMUM_CPU_UPDATE_INTERVAL, Pid, ProcessRefreshKind, ProcessesToUpdate, System, UpdateKind,
};
-const PROTOCOL_VERSION: u32 = 2;
+const PROTOCOL_VERSION: u32 = 3;
const MIN_SAMPLE_INTERVAL_MS: u64 = 250;
const MAX_SAMPLE_INTERVAL_MS: u64 = 60_000;
const PROCESS_START_TIME_PRECISION_MS: u64 = 1_000;
@@ -66,6 +66,10 @@ enum Command {
version: u32,
request_id: String,
},
+ ProcessTable {
+ version: u32,
+ request_id: String,
+ },
ReadHistory {
version: u32,
request_id: String,
@@ -84,6 +88,7 @@ impl Command {
| Self::SetSampleInterval { version, .. }
| Self::SetStreaming { version, .. }
| Self::SampleNow { version, .. }
+ | Self::ProcessTable { version, .. }
| Self::ReadHistory { version, .. }
| Self::Shutdown { version } => *version,
}
@@ -146,6 +151,24 @@ struct ProcessSample {
io_semantics: IoSemantics,
}
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct ProcessTableEntry {
+ pid: u32,
+ ppid: u32,
+ name: String,
+}
+
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct ProcessTableEvent<'a> {
+ version: u32,
+ #[serde(rename = "type")]
+ event_type: &'static str,
+ request_id: &'a str,
+ processes: Vec<ProcessTableEntry>,
+}
+
impl ProcessSample {
fn estimated_history_bytes(&self) -> usize {
std::mem::size_of::<Self>()
@@ -342,6 +365,41 @@ impl Collector {
self.cpu_baseline_refreshed_at = Some(Instant::now());
}
+ fn process_table(&self) -> Vec<ProcessTableEntry> {
+ // Use a dedicated System so this refresh cannot reset the CPU
+ // baseline tracked by self.system for snapshots.
+ let mut process_table_system = System::new();
+ process_table_system.refresh_processes_specifics(
+ ProcessesToUpdate::All,
+ true,
+ ProcessRefreshKind::nothing().without_tasks(),
+ );
+ let mut processes = process_table_system
+ .processes()
+ .iter()
+ .filter_map(|(pid, process)| {
+ let pid = pid.as_u32();
+ // Pid 0 is the kernel idle process on some platforms. The
+ // processTable contract requires positive pids, and one zero
+ // would fail the whole event decode on the server, so drop it
+ // here. It can never be a terminal descendant.
+ if pid == 0 {
+ return None;
+ }
+ Some(ProcessTableEntry {
+ pid,
+ ppid: process.parent().map(Pid::as_u32).unwrap_or(0),
+ name: truncate_utf8(
+ process.name().to_string_lossy().into_owned(),
+ MAX_PROCESS_NAME_BYTES,
+ ),
+ })
+ })
+ .collect::<Vec<_>>();
+ processes.sort_by_key(|process| process.pid);
+ processes
+ }
+
fn sample(&mut self, config: &CollectorConfig, request_id: Option<String>) -> SnapshotEvent {
if let Some(delay) =
remaining_cpu_measurement_delay(self.cpu_baseline_refreshed_at.take(), Instant::now())
@@ -806,6 +864,15 @@ fn main() -> io::Result<()> {
)?;
}
}
+ Command::ProcessTable { request_id, .. } => {
+ let event = ProcessTableEvent {
+ version: PROTOCOL_VERSION,
+ event_type: "processTable",
+ request_id: &request_id,
+ processes: collector.process_table(),
+ };
+ write_event(&mut writer, &event)?;
+ }
Command::ReadHistory {
request_id,
window_ms,
@@ -883,7 +950,7 @@ mod tests {
#[test]
fn decodes_protocol_commands() {
let configure = serde_json::from_str::<Command>(
- r#"{"version":2,"type":"configure","rootPid":42,"sampleIntervalMs":1000,"externalProcesses":[{"pid":7}]}"#,
+ r#"{"version":3,"type":"configure","rootPid":42,"sampleIntervalMs":1000,"externalProcesses":[{"pid":7}]}"#,
)
.expect("configure command");
@@ -903,7 +970,7 @@ mod tests {
}
let read_history = serde_json::from_str::<Command>(
- r#"{"version":2,"type":"readHistory","requestId":"history-1","windowMs":60000}"#,
+ r#"{"version":3,"type":"readHistory","requestId":"history-1","windowMs":60000}"#,
)
.expect("read history command");
assert!(matches!(
@@ -914,6 +981,15 @@ mod tests {
..
} if request_id == "history-1"
));
+
+ let process_table = serde_json::from_str::<Command>(
+ r#"{"version":3,"type":"processTable","requestId":"processes-1"}"#,
+ )
+ .expect("process table command");
+ assert!(matches!(
+ process_table,
+ Command::ProcessTable { request_id, .. } if request_id == "processes-1"
+ ));
}
#[test]
diff --git a/packages/contracts/src/resourceTelemetry.ts b/packages/contracts/src/resourceTelemetry.ts
index 2c1b2ffec..2cc816925 100644
--- a/packages/contracts/src/resourceTelemetry.ts
+++ b/packages/contracts/src/resourceTelemetry.ts
@@ -3,7 +3,7 @@ import * as Schema from "effect/Schema";
import { NonNegativeInt, PositiveInt, TrimmedNonEmptyString } from "./baseSchemas.ts";
import { HostPowerSnapshot } from "./background.ts";
-export const RESOURCE_MONITOR_PROTOCOL_VERSION = 2 as const;
+export const RESOURCE_MONITOR_PROTOCOL_VERSION = 3 as const;
export const ResourceTelemetryIoSemantics = Schema.Literals([
"storage",
@@ -101,6 +101,13 @@ export const ResourceMonitorSampleNowCommand = Schema.Struct({
});
export type ResourceMonitorSampleNowCommand = typeof ResourceMonitorSampleNowCommand.Type;
+export const ResourceMonitorProcessTableCommand = Schema.Struct({
+ version: Schema.Literal(RESOURCE_MONITOR_PROTOCOL_VERSION),
+ type: Schema.Literal("processTable"),
+ requestId: TrimmedNonEmptyString,
+});
+export type ResourceMonitorProcessTableCommand = typeof ResourceMonitorProcessTableCommand.Type;
+
export const ResourceMonitorSetSampleIntervalCommand = Schema.Struct({
version: Schema.Literal(RESOURCE_MONITOR_PROTOCOL_VERSION),
type: Schema.Literal("setSampleInterval"),
@@ -136,6 +143,7 @@ export const ResourceMonitorCommand = Schema.Union([
ResourceMonitorSetSampleIntervalCommand,
ResourceMonitorSetStreamingCommand,
ResourceMonitorSampleNowCommand,
+ ResourceMonitorProcessTableCommand,
ResourceMonitorReadHistoryCommand,
ResourceMonitorShutdownCommand,
]);
@@ -167,6 +175,21 @@ export const ResourceMonitorSnapshotEvent = Schema.Struct({
});
export type ResourceMonitorSnapshotEvent = typeof ResourceMonitorSnapshotEvent.Type;
+export const ResourceMonitorProcessTableEntry = Schema.Struct({
+ pid: PositiveInt,
+ ppid: NonNegativeInt,
+ name: Schema.String,
+});
+export type ResourceMonitorProcessTableEntry = typeof ResourceMonitorProcessTableEntry.Type;
+
+export const ResourceMonitorProcessTableEvent = Schema.Struct({
+ version: Schema.Literal(RESOURCE_MONITOR_PROTOCOL_VERSION),
+ type: Schema.Literal("processTable"),
+ requestId: TrimmedNonEmptyString,
+ processes: Schema.Array(ResourceMonitorProcessTableEntry),
+});
+export type ResourceMonitorProcessTableEvent = typeof ResourceMonitorProcessTableEvent.Type;
+
export const ResourceMonitorHistoryChunkEvent = Schema.Struct({
version: Schema.Literal(RESOURCE_MONITOR_PROTOCOL_VERSION),
type: Schema.Literal("historyChunk"),
@@ -188,6 +211,7 @@ export type ResourceMonitorErrorEvent = typeof ResourceMonitorErrorEvent.Type;
export const ResourceMonitorEvent = Schema.Union([
ResourceMonitorHelloEvent,
ResourceMonitorSnapshotEvent,
+ ResourceMonitorProcessTableEvent,
ResourceMonitorHistoryChunkEvent,
ResourceMonitorErrorEvent,
]);