Skip to content
TypeScript

fix(server): SSH device host no longer reconnects on a single slow health probe (#30159)

gyurixb9d7239dcb42
diff --git a/apps/server/src/device/SshDeviceHost.test.ts b/apps/server/src/device/SshDeviceHost.test.ts
index 80b669732eb..6600425331c 100644
--- a/apps/server/src/device/SshDeviceHost.test.ts
+++ b/apps/server/src/device/SshDeviceHost.test.ts
@@ -11,6 +11,7 @@ import * as Stream from "effect/Stream";
import * as HttpClient from "effect/http/HttpClient";
import * as HttpClientResponse from "effect/http/HttpClientResponse";
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
+import * as TestClock from "effect/testing/TestClock";
import * as ServerConfig from "../config.ts";
import * as DeviceHost from "./DeviceHost.ts";
import * as SshDeviceHost from "./SshDeviceHost.ts";
@@ -309,3 +310,79 @@ it.effect("lists Android devices without probing the host again once it reports
yield* host.stop;
}).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
);
+
+it.effect(
+ "keeps the SSH forward through slow health probes and reconnects once the hub stays down",
+ () =>
+ Effect.gen(function* () {
+ const fs = yield* FileSystem.FileSystem;
+ const home = yield* fs.makeTempDirectoryScoped();
+ let forwardSpawns = 0;
+ let hubAnswers = true;
+ const spawner = ChildProcessSpawner.make((command) =>
+ Effect.gen(function* () {
+ if (command._tag !== "StandardCommand") return yield* Effect.die("Unexpected command");
+ const forwarding = command.args.includes("-L");
+ // A fresh forward reaches a working hub again.
+ if (forwarding && ++forwardSpawns > 1) hubAnswers = true;
+ const output = JSON.stringify({
+ nodePath: "/node",
+ platforms: [{ platform: "ios", available: true }],
+ hubPort: 1234,
+ helpers: { serveSimAxSettings: null, serveSimCli: null },
+ });
+ return ChildProcessSpawner.makeHandle({
+ pid: ChildProcessSpawner.ProcessId(123),
+ stdout: Stream.make(new TextEncoder().encode(output)),
+ stderr: Stream.empty,
+ all: Stream.empty,
+ exitCode: forwarding ? Effect.never : Effect.succeed(ChildProcessSpawner.ExitCode(0)),
+ isRunning: Effect.succeed(forwarding),
+ kill: () => Effect.void,
+ stdin: Sink.drain,
+ getInputFd: () => Sink.drain,
+ getOutputFd: () => Stream.empty,
+ unref: Effect.succeed(Effect.void),
+ });
+ }),
+ );
+ const host = yield* SshDeviceHost.make({
+ kind: "ssh",
+ id: "test",
+ label: "Test",
+ target: "test.example",
+ }).pipe(
+ Effect.provide(Layer.mergeAll(ServerConfig.layerTest(home, home), Net.layer)),
+ Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
+ Effect.provideService(
+ HttpClient.HttpClient,
+ HttpClient.make((request) =>
+ hubAnswers
+ ? Effect.succeed(HttpClientResponse.fromWeb(request, new Response("ok")))
+ : Effect.never,
+ ),
+ ),
+ );
+ yield* host.ensureReady(() => Effect.void);
+ expect(forwardSpawns).toBe(1);
+
+ // A loaded host can miss a probe or two without being gone; the forward stays up.
+ hubAnswers = false;
+ yield* TestClock.adjust("30 seconds");
+ hubAnswers = true;
+ yield* TestClock.adjust("2 minutes");
+ expect(forwardSpawns).toBe(1);
+
+ // A hub that keeps missing probes is still replaced.
+ hubAnswers = false;
+ // Recovery reset the miss count, so two new misses do not reconnect yet.
+ yield* TestClock.adjust("30 seconds");
+ expect(forwardSpawns).toBe(1);
+ yield* TestClock.adjust("30 seconds");
+ expect(forwardSpawns).toBe(2);
+ yield* host.stop;
+ }).pipe(
+ Effect.scoped,
+ Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer, TestClock.layer())),
+ ),
+);
diff --git a/apps/server/src/device/SshDeviceHost.ts b/apps/server/src/device/SshDeviceHost.ts
index 789b700308b..ef051cb9075 100644
--- a/apps/server/src/device/SshDeviceHost.ts
+++ b/apps/server/src/device/SshDeviceHost.ts
@@ -27,6 +27,8 @@ import * as DeviceHost from "./DeviceHost.ts";
import { makeDeviceHubOperations } from "./DeviceHubOperations.ts";
import { quoteRemoteArg, remoteDeviceEnvironment, remoteDeviceScript } from "./sshDeviceScript.ts";
+const UNHEALTHY_PROBE_LIMIT = 3;
+
const Probe = Schema.Struct({
nodePath: Schema.String,
tools: Schema.optional(DeviceToolVersions),
@@ -365,7 +367,10 @@ export const make = Effect.fn("SshDeviceHost.make")(function* (
: next;
yield* onStatus("ready");
// Reconnect also repairs helpers that died while SSH itself stayed connected.
+ // A loaded host can miss a probe or two while still working, and reconnecting
+ // then drops the forward in use, so only consecutive misses count.
const unhealthy = Effect.gen(function* () {
+ let misses = 0;
while (true) {
yield* Effect.sleep("10 seconds");
const alive = yield* http.get(`${next.hub.origin}/readyz`).pipe(
@@ -380,7 +385,8 @@ export const make = Effect.fn("SshDeviceHost.make")(function* (
Effect.orElseSucceed(() => false),
)
: true;
- if (!alive || !daemonAlive) return;
+ misses = alive && daemonAlive ? 0 : misses + 1;
+ if (misses >= UNHEALTHY_PROBE_LIMIT) return;
}
});
yield* Effect.gen(function* () {