An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(ssh): prevent orphaned multiplexed connections
Andras Bacsai57d879263d2f
diff --git a/app/Console/Kernel.php b/app/Console/Kernel.phpindex 3ec59adb3e..665553fcb3 100644--- a/app/Console/Kernel.php+++ b/app/Console/Kernel.php@@ -8,6 +8,7 @@ use App\Jobs\CheckHelperImageJob;use App\Jobs\CheckTraefikVersionJob;use App\Jobs\CleanupInstanceStuffsJob;use App\Jobs\CleanupOrphanedPreviewContainersJob;+use App\Jobs\CleanupStaleMultiplexedConnections;use App\Jobs\PullChangelog;use App\Jobs\PullTemplatesFromCDN;use App\Jobs\RegenerateSslCertJob;@@ -40,7 +41,7 @@ class Kernel extends ConsoleKernel$this->instanceTimezone = config('app.timezone');}- // $this->scheduleInstance->job(new CleanupStaleMultiplexedConnections)->hourly();+ $this->scheduleInstance->job(new CleanupStaleMultiplexedConnections)->hourly()->onOneServer();$this->scheduleInstance->command('cleanup:redis --clear-locks')->daily();$this->scheduleInstance->command('sanctum:prune-expired --hours=1')->hourly()->onOneServer();$this->scheduleInstance->job(new ApiTokenExpirationWarningJob)->hourly()->onOneServer();diff --git a/app/Helpers/SshMultiplexingHelper.php b/app/Helpers/SshMultiplexingHelper.phpindex 4629df571d..78084a1577 100644--- a/app/Helpers/SshMultiplexingHelper.php+++ b/app/Helpers/SshMultiplexingHelper.php@@ -4,6 +4,7 @@ namespace App\Helpers;use App\Models\PrivateKey;use App\Models\Server;+use Illuminate\Contracts\Cache\LockTimeoutException;use Illuminate\Support\Facades\Cache;use Illuminate\Support\Facades\Hash;use Illuminate\Support\Facades\Log;@@ -30,37 +31,99 @@ class SshMultiplexingHelperreturn false;}+ // Fast path: a usable master already exists, no need to lock.+ if (self::connectionIsReusable($server)) {+ return true;+ }++ // Slow path: establishing or refreshing the master. Serialize per server+ // so concurrent workers do not each spawn their own master process,+ // leaving orphaned non-master ssh connections that ControlPersist never reaps.+ try {+ return Cache::lock(+ self::connectionLockKey($server),+ config('constants.ssh.mux_lock_ttl')+ )->block(config('constants.ssh.mux_lock_timeout'), function () use ($server) {+ // Double-checked: another worker may have established the master+ // while we were waiting for the lock.+ if (self::connectionIsReusable($server)) {+ return true;+ }++ // A master exists but is stale or expired: close and re-establish.+ if (self::masterConnectionExists($server)) {+ return self::refreshMultiplexedConnection($server);+ }++ return self::establishNewMultiplexedConnection($server);+ });+ } catch (LockTimeoutException) {+ Log::warning('SSH multiplexing lock timeout, falling back to non-multiplexed connection', [+ 'server' => $server->name ?? $server->ip,+ ]);++ return false;+ } catch (\Throwable $e) {+ Log::warning('SSH multiplexing lock unavailable, falling back to non-multiplexed connection', [+ 'server' => $server->name ?? $server->ip,+ 'error' => $e->getMessage(),+ ]);++ return false;+ }+ }++ /**+ * Per-server, per-host lock key for serializing master establishment.+ *+ * The mux socket is a host-local unix socket, so the lock is scoped to the+ * current Coolify host: workers on the same host share a master and must+ * serialize, while workers on other hosts manage their own masters and must+ * not block on each other.+ */+ private static function connectionLockKey(Server $server): string+ {+ return 'ssh_mux_lock_'.(gethostname() ?: 'unknown').'_'.$server->uuid;+ }++ /**+ * Check whether a multiplexed master connection currently exists for the server.+ */+ private static function masterConnectionExists(Server $server): bool+ {$sshConfig = self::serverSshConfiguration($server);$muxSocket = $sshConfig['muxFilename'];- // Check if connection exists$checkCommand = "ssh -O check -o ControlPath=$muxSocket ";if (data_get($server, 'settings.is_cloudflare_tunnel')) {$checkCommand .= '-o ProxyCommand="cloudflared access ssh --hostname %h" ';}$checkCommand .= self::escapedUserAtHost($server);- $process = Process::run($checkCommand);- if ($process->exitCode() !== 0) {- return self::establishNewMultiplexedConnection($server);+ return Process::run($checkCommand)->exitCode() === 0;+ }++ /**+ * Determine whether the existing master connection can be reused as-is+ * (it exists, has not exceeded its max age, and passes the health check).+ */+ private static function connectionIsReusable(Server $server): bool+ {+ if (! self::masterConnectionExists($server)) {+ return false;}- // Connection exists, ensure we have metadata for age tracking+ // Existing connection but no metadata, store current time as fallback.if (self::getConnectionAge($server) === null) {- // Existing connection but no metadata, store current time as fallbackself::storeConnectionMetadata($server);}- // Connection exists, check if it needs refresh due to ageif (self::isConnectionExpired($server)) {- return self::refreshMultiplexedConnection($server);+ return false;}- // Perform health check if enabled- if (config('constants.ssh.mux_health_check_enabled')) {- if (! self::isConnectionHealthy($server)) {- return self::refreshMultiplexedConnection($server);- }+ if (config('constants.ssh.mux_health_check_enabled') && ! self::isConnectionHealthy($server)) {+ return false;}return true;@@ -75,7 +138,10 @@ class SshMultiplexingHelper$serverInterval = config('constants.ssh.server_interval');$muxPersistTime = config('constants.ssh.mux_persist_time');- $establishCommand = "ssh -fNM -o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";+ // No -M: it forces master mode and overrides ControlMaster=auto. When a+ // socket already exists -M leaves an orphaned non-master ssh -fN process+ // that ControlPersist never reaps. ControlMaster=auto reuses instead.+ $establishCommand = "ssh -fN -o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";if (data_get($server, 'settings.is_cloudflare_tunnel')) {$establishCommand .= ' -o ProxyCommand="cloudflared access ssh --hostname %h" ';@@ -176,6 +242,10 @@ class SshMultiplexingHelper$ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";}} catch (\Exception $e) {+ Log::warning('SSH multiplexing failed, falling back to non-multiplexed connection', [+ 'server' => $server->name ?? $server->ip,+ 'error' => $e->getMessage(),+ ]);// Continue without multiplexing}}diff --git a/app/Jobs/CleanupStaleMultiplexedConnections.php b/app/Jobs/CleanupStaleMultiplexedConnections.phpindex 6d49bee4b0..0d3029c668 100644--- a/app/Jobs/CleanupStaleMultiplexedConnections.php+++ b/app/Jobs/CleanupStaleMultiplexedConnections.php@@ -9,6 +9,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;use Illuminate\Foundation\Bus\Dispatchable;use Illuminate\Queue\InteractsWithQueue;use Illuminate\Queue\SerializesModels;+use Illuminate\Support\Facades\Log;use Illuminate\Support\Facades\Process;use Illuminate\Support\Facades\Storage;@@ -20,6 +21,132 @@ class CleanupStaleMultiplexedConnections implements ShouldQueue{$this->cleanupStaleConnections();$this->cleanupNonExistentServerConnections();+ $this->cleanupOrphanedSshProcesses();+ $this->cleanupOrphanedCloudflaredProcesses();+ }++ /**+ * Kill backgrounded ssh master processes that lost the ControlPath socket+ * race. Such processes are not masters, so ControlPersist never reaps them+ * and they leak memory until the container restarts. A legitimate master+ * always owns its socket file; an orphan has none.+ *+ * Processes younger than the minimum age are skipped: a freshly forked+ * master creates its socket a few milliseconds after starting, so a young+ * process with no socket may simply be mid-establish rather than orphaned.+ */+ private function cleanupOrphanedSshProcesses(): void+ {+ $muxDir = storage_path('app/ssh/mux');+ $minAge = (int) config('constants.ssh.mux_orphan_min_age');++ foreach ($this->listProcesses() as $process) {+ // Backgrounded ssh master: current `ssh -fN` or legacy `ssh -fNM`.+ if (! preg_match('#(^|/)ssh -fN#', $process['args'])) {+ continue;+ }++ // Only ever touch ssh processes pointing at Coolify's mux directory.+ if (! preg_match('#ControlPath=('.preg_quote($muxDir, '#').'/\S+)#', $process['args'], $pathMatch)) {+ continue;+ }++ if ($process['etimes'] >= $minAge && ! file_exists($pathMatch[1])) {+ $this->reapOrphan('ssh', $process);+ }+ }+ }++ /**+ * Kill orphaned `cloudflared access ssh` proxy processes. Each is spawned+ * as the SSH ProxyCommand transport for a Cloudflare Tunnel server and must+ * die with its parent ssh. When that ssh is killed or orphaned (e.g. a lost+ * mux master), the cloudflared process can leak and accumulate. A legitimate+ * proxy always has a live ssh parent; one without is safe to reap.+ *+ * Processes younger than the minimum age are skipped so a proxy whose parent+ * ssh is still starting up, or a transient `ssh -O check` proxy mid-exit, is+ * never mistaken for an orphan.+ */+ private function cleanupOrphanedCloudflaredProcesses(): void+ {+ $minAge = (int) config('constants.ssh.mux_orphan_min_age');+ $processes = $this->listProcesses();++ $sshPids = [];+ foreach ($processes as $process) {+ // The ssh binary itself, not `cloudflared access ssh` (space before ssh).+ if (preg_match('#(^|/)ssh\s#', $process['args'])) {+ $sshPids[$process['pid']] = true;+ }+ }++ foreach ($processes as $process) {+ // `cloudflared access ssh`, never the `cloudflared tunnel` daemon.+ if (! str_contains($process['args'], 'cloudflared access ssh')) {+ continue;+ }++ // Orphaned when no live ssh process is its parent.+ if ($process['etimes'] >= $minAge && ! isset($sshPids[$process['ppid']])) {+ $this->reapOrphan('cloudflared', $process);+ }+ }+ }++ /**+ * Reap a detected orphan process. When orphan reaping is disabled (the+ * default), the orphan is only logged — a dry-run mode that lets operators+ * verify what would be killed before enabling it for real.+ *+ * @param array{pid: string, ppid: string, etimes: int, args: string} $process+ */+ private function reapOrphan(string $kind, array $process): void+ {+ if (! config('constants.ssh.mux_orphan_reap_enabled')) {+ Log::info("Orphaned {$kind} process detected (dry-run, not killed)", [+ 'pid' => $process['pid'],+ 'etimes' => $process['etimes'],+ 'command' => $process['args'],+ ]);++ return;+ }++ Process::run('kill '.escapeshellarg($process['pid']));+ Log::info("Killed orphaned {$kind} process", [+ 'pid' => $process['pid'],+ 'etimes' => $process['etimes'],+ 'command' => $process['args'],+ ]);+ }++ /**+ * Snapshot of running processes.+ *+ * @return list<array{pid: string, ppid: string, etimes: int, args: string}>+ */+ private function listProcesses(): array+ {+ $ps = Process::run('ps -ww -eo pid=,ppid=,etimes=,args=');+ if ($ps->exitCode() !== 0) {+ return [];+ }++ $processes = [];+ foreach (explode("\n", trim($ps->output())) as $line) {+ if (! preg_match('/^\s*(\d+)\s+(\d+)\s+(\d+)\s+(.*)$/', $line, $matches)) {+ continue;+ }+ $processes[] = [+ 'pid' => $matches[1],+ 'ppid' => $matches[2],+ 'etimes' => (int) $matches[3],+ 'args' => $matches[4],+ ];+ }++ return $processes;}private function cleanupStaleConnections()@@ -31,7 +158,7 @@ class CleanupStaleMultiplexedConnections implements ShouldQueue$server = Server::where('uuid', $serverUuid)->first();if (! $server) {- $this->removeMultiplexFile($muxFile);+ $this->removeMultiplexFile($muxFile, 'server_not_found');continue;}@@ -41,14 +168,14 @@ class CleanupStaleMultiplexedConnections implements ShouldQueue$checkProcess = Process::run($checkCommand);if ($checkProcess->exitCode() !== 0) {- $this->removeMultiplexFile($muxFile);+ $this->removeMultiplexFile($muxFile, 'connection_check_failed');} else {$muxContent = Storage::disk('ssh-mux')->get($muxFile);$establishedAt = Carbon::parse(substr($muxContent, 37));$expirationTime = $establishedAt->addSeconds(config('constants.ssh.mux_persist_time'));if (Carbon::now()->isAfter($expirationTime)) {- $this->removeMultiplexFile($muxFile);+ $this->removeMultiplexFile($muxFile, 'expired');}}}@@ -62,7 +189,7 @@ class CleanupStaleMultiplexedConnections implements ShouldQueueforeach ($muxFiles as $muxFile) {$serverUuid = $this->extractServerUuidFromMuxFile($muxFile);if (! in_array($serverUuid, $existingServerUuids)) {- $this->removeMultiplexFile($muxFile);+ $this->removeMultiplexFile($muxFile, 'server_does_not_exist');}}}@@ -72,11 +199,30 @@ class CleanupStaleMultiplexedConnections implements ShouldQueuereturn substr($muxFile, 4);}- private function removeMultiplexFile($muxFile)+ /**+ * Close and delete a stale mux socket file. When orphan reaping is disabled+ * (the default), the file is only logged — a dry-run mode that lets operators+ * verify what would be removed before enabling it for real.+ */+ private function removeMultiplexFile(string $muxFile, string $reason): void{+ if (! config('constants.ssh.mux_orphan_reap_enabled')) {+ Log::info('Stale mux file detected (dry-run, not removed)', [+ 'file' => $muxFile,+ 'reason' => $reason,+ ]);++ return;+ }+$muxSocket = "/var/www/html/storage/app/ssh/mux/{$muxFile}";$closeCommand = "ssh -O exit -o ControlPath={$muxSocket} localhost 2>/dev/null";Process::run($closeCommand);Storage::disk('ssh-mux')->delete($muxFile);++ Log::info('Removed stale mux file', [+ 'file' => $muxFile,+ 'reason' => $reason,+ ]);}}diff --git a/config/constants.php b/config/constants.phpindex 10db1085cc..7721ff2134 100644--- a/config/constants.php+++ b/config/constants.php@@ -70,6 +70,10 @@ return ['mux_health_check_enabled' => env('SSH_MUX_HEALTH_CHECK_ENABLED', true),'mux_health_check_timeout' => env('SSH_MUX_HEALTH_CHECK_TIMEOUT', 5),'mux_max_age' => env('SSH_MUX_MAX_AGE', 1800), // 30 minutes+ 'mux_lock_ttl' => env('SSH_MUX_LOCK_TTL', 30), // lock auto-release, seconds+ 'mux_lock_timeout' => env('SSH_MUX_LOCK_TIMEOUT', 10), // max wait for lock, seconds+ 'mux_orphan_min_age' => env('SSH_MUX_ORPHAN_MIN_AGE', 600), // min process age before reaping orphans, seconds+ 'mux_orphan_reap_enabled' => env('SSH_MUX_ORPHAN_REAP_ENABLED', false), // false = dry-run, only log orphans'connection_timeout' => 10,'server_interval' => 20,'command_timeout' => 3600,diff --git a/tests/Feature/SshMultiplexingLockTest.php b/tests/Feature/SshMultiplexingLockTest.phpnew file mode 100644index 0000000000..8d56f52351--- /dev/null+++ b/tests/Feature/SshMultiplexingLockTest.php@@ -0,0 +1,219 @@+<?php++use App\Helpers\SshMultiplexingHelper;+use App\Jobs\CleanupStaleMultiplexedConnections;+use App\Models\PrivateKey;+use App\Models\Server;+use App\Models\User;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Facades\Cache;+use Illuminate\Support\Facades\File;+use Illuminate\Support\Facades\Process;+use Illuminate\Support\Facades\Storage;++/**+ * Tests for the per-server lock that prevents concurrent workers from each+ * spawning their own SSH master, which leaves orphaned non-master ssh+ * connections that ControlPersist never reaps (memory leak).+ */+uses(RefreshDatabase::class);++function makeMuxServer(): Server+{+ $user = User::factory()->create();+ $team = $user->teams()->first();++ $privateKey = PrivateKey::create([+ 'name' => 'mux-test-key-'.uniqid(),+ 'private_key' => "-----BEGIN OPENSSH PRIVATE KEY-----\n".+ "b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW\n".+ "QyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevAAAAJi/QySHv0Mk\n".+ "hwAAAAtzc2gtZWQyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevA\n".+ "AAAECBQw4jg1WRT2IGHMncCiZhURCts2s24HoDS0thHnnRKVuGmoeGq/pojrsyP1pszcNV\n".+ "uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==\n".+ '-----END OPENSSH PRIVATE KEY-----',+ 'team_id' => $team->id,+ ]);++ return Server::factory()->create([+ 'team_id' => $team->id,+ 'private_key_id' => $privateKey->id,+ ]);+}++it('establishes a master with ssh -fN and never the orphan-prone ssh -fNM', function () {+ config(['constants.ssh.mux_enabled' => true]);+ $server = makeMuxServer();++ Process::fake([+ '*-O check*' => Process::result(exitCode: 1), // no existing master+ '*-fN *' => Process::result(exitCode: 0), // establish succeeds+ ]);++ expect(SshMultiplexingHelper::ensureMultiplexedConnection($server))->toBeTrue();++ Process::assertRan(fn ($process) => str_contains($process->command, 'ssh -fN ')+ && ! str_contains($process->command, 'ssh -fNM'));+});++it('reuses an existing healthy master without spawning a new one', function () {+ config([+ 'constants.ssh.mux_enabled' => true,+ 'constants.ssh.mux_health_check_enabled' => true,+ ]);+ $server = makeMuxServer();++ Process::fake([+ '*-O check*' => Process::result(exitCode: 0),+ '*health_check_ok*' => Process::result(output: 'health_check_ok', exitCode: 0),+ ]);++ expect(SshMultiplexingHelper::ensureMultiplexedConnection($server))->toBeTrue();++ Process::assertNotRan(fn ($process) => str_contains($process->command, 'ssh -fN'));+});++it('does not spawn a master when the per-server lock is already held', function () {+ config([+ 'constants.ssh.mux_enabled' => true,+ 'constants.ssh.mux_lock_timeout' => 0,+ ]);+ $server = makeMuxServer();++ Process::fake([+ '*-O check*' => Process::result(exitCode: 1), // forces the slow path+ ]);++ // Simulate another worker on the same host holding the lock for this server.+ $lockKey = 'ssh_mux_lock_'.(gethostname() ?: 'unknown').'_'.$server->uuid;+ $held = Cache::lock($lockKey, 30);+ expect($held->get())->toBeTrue();++ expect(SshMultiplexingHelper::ensureMultiplexedConnection($server))->toBeFalse();++ Process::assertNotRan(fn ($process) => str_contains($process->command, 'ssh -fN '));++ $held->release();+});++it('returns false and runs no ssh when multiplexing is disabled', function () {+ config(['constants.ssh.mux_enabled' => false]);+ $server = makeMuxServer();++ Process::fake();++ expect(SshMultiplexingHelper::ensureMultiplexedConnection($server))->toBeFalse();++ Process::assertNothingRan();+});++it('kills only old orphaned ssh masters whose control socket no longer exists', function () {+ config(['constants.ssh.mux_orphan_reap_enabled' => true]);+ $muxDir = storage_path('app/ssh/mux');+ File::ensureDirectoryExists($muxDir);++ $liveSocket = $muxDir.'/mux_live_'.uniqid();+ $orphanSocket = $muxDir.'/mux_orphan_'.uniqid();+ $youngSocket = $muxDir.'/mux_young_'.uniqid();+ File::put($liveSocket, 'x'); // live master owns its socket file; the others do not++ // Columns: pid ppid etimes args+ Process::fake([+ 'ps*' => Process::result(output: "111 1 5000 ssh -fN -o ControlMaster=auto -o ControlPath={$liveSocket} [email protected]\n".+ "222 1 5000 ssh -fN -o ControlMaster=auto -o ControlPath={$orphanSocket} [email protected]\n".+ "333 1 30 ssh -fN -o ControlMaster=auto -o ControlPath={$youngSocket} [email protected]\n"),+ 'kill*' => Process::result(exitCode: 0),+ ]);++ $job = new CleanupStaleMultiplexedConnections;+ $method = new ReflectionMethod($job, 'cleanupOrphanedSshProcesses');+ $method->setAccessible(true);+ $method->invoke($job);++ // Old orphan: killed.+ Process::assertRan(fn ($process) => str_contains($process->command, 'kill') && str_contains($process->command, '222'));+ // Live master (socket exists): spared.+ Process::assertNotRan(fn ($process) => str_contains($process->command, 'kill') && str_contains($process->command, '111'));+ // Young process (may be mid-establish): spared despite missing socket.+ Process::assertNotRan(fn ($process) => str_contains($process->command, 'kill') && str_contains($process->command, '333'));++ File::delete($liveSocket);+});++it('kills only old orphaned cloudflared proxies whose parent ssh is gone', function () {+ config(['constants.ssh.mux_orphan_reap_enabled' => true]);+ // pid 100 = live ssh master; 200 = its legit child; 300 = old orphan;+ // 400 = young orphan (parent ssh may still be starting up).+ Process::fake([+ 'ps*' => Process::result(output: "100 1 5000 ssh -fN -o ControlMaster=auto [email protected]\n".+ "200 100 5000 cloudflared access ssh --hostname host.example.com\n".+ "300 2176 5000 cloudflared access ssh --hostname host.example.com\n".+ "400 2176 30 cloudflared access ssh --hostname host.example.com\n".+ "2176 1 9000 /usr/bin/some-supervisor\n"),+ 'kill*' => Process::result(exitCode: 0),+ ]);++ $job = new CleanupStaleMultiplexedConnections;+ $method = new ReflectionMethod($job, 'cleanupOrphanedCloudflaredProcesses');+ $method->setAccessible(true);+ $method->invoke($job);++ // Old orphan (parent not ssh): killed.+ Process::assertRan(fn ($process) => str_contains($process->command, 'kill') && str_contains($process->command, '300'));+ // Legit proxy (parent ssh alive): spared.+ Process::assertNotRan(fn ($process) => str_contains($process->command, 'kill') && str_contains($process->command, '200'));+ // Young orphan: spared.+ Process::assertNotRan(fn ($process) => str_contains($process->command, 'kill') && str_contains($process->command, '400'));+});++it('dry-run mode logs orphans but kills nothing when reaping is disabled', function () {+ config(['constants.ssh.mux_orphan_reap_enabled' => false]);+ $muxDir = storage_path('app/ssh/mux');+ File::ensureDirectoryExists($muxDir);++ $orphanSocket = $muxDir.'/mux_orphan_'.uniqid(); // no file: a real old orphan++ Process::fake([+ 'ps*' => Process::result(output: "222 1 5000 ssh -fN -o ControlMaster=auto -o ControlPath={$orphanSocket} [email protected]\n"),+ 'kill*' => Process::result(exitCode: 0),+ ]);++ $job = new CleanupStaleMultiplexedConnections;+ $method = new ReflectionMethod($job, 'cleanupOrphanedSshProcesses');+ $method->setAccessible(true);+ $method->invoke($job);++ // Orphan detected, but dry-run: nothing killed.+ Process::assertNotRan(fn ($process) => str_contains($process->command, 'kill'));+});++it('removes mux files for non-existent servers when reaping is enabled', function () {+ config(['constants.ssh.mux_orphan_reap_enabled' => true]);+ Storage::fake('ssh-mux');+ $file = 'mux_ghost'.uniqid();+ Storage::disk('ssh-mux')->put($file, 'x');+ Process::fake(); // the `ssh -O exit` close command++ $job = new CleanupStaleMultiplexedConnections;+ $method = new ReflectionMethod($job, 'cleanupNonExistentServerConnections');+ $method->setAccessible(true);+ $method->invoke($job);++ expect(Storage::disk('ssh-mux')->exists($file))->toBeFalse();+});++it('keeps mux files for non-existent servers in dry-run mode', function () {+ config(['constants.ssh.mux_orphan_reap_enabled' => false]);+ Storage::fake('ssh-mux');+ $file = 'mux_ghost'.uniqid();+ Storage::disk('ssh-mux')->put($file, 'x');+ Process::fake();++ $job = new CleanupStaleMultiplexedConnections;+ $method = new ReflectionMethod($job, 'cleanupNonExistentServerConnections');+ $method->setAccessible(true);+ $method->invoke($job);++ expect(Storage::disk('ssh-mux')->exists($file))->toBeTrue();+ Process::assertNothingRan();+});