An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
chore(ssh): remove stale mux cleanup job
Andras Bacsai43884823c6d1
diff --git a/app/Console/Kernel.php b/app/Console/Kernel.phpindex 494e6d7b67..e971058365 100644--- a/app/Console/Kernel.php+++ b/app/Console/Kernel.php@@ -8,7 +8,6 @@ use App\Jobs\CheckHelperImageJob;use App\Jobs\CheckTraefikVersionJob;use App\Jobs\CleanupInstanceStuffsJob;use App\Jobs\CleanupOrphanedPreviewContainersJob;-use App\Jobs\CleanupStaleMultiplexedConnections;use App\Jobs\PullChangelog;use App\Jobs\PullTemplatesFromCDN;use App\Jobs\RegenerateSslCertJob;@@ -41,9 +40,6 @@ class Kernel extends ConsoleKernel$this->instanceTimezone = config('app.timezone');}- $this->scheduleInstance->job(new CleanupStaleMultiplexedConnections, crons_queue())- ->hourly()- ->onOneServer();$this->scheduleInstance->command('cleanup:redis --clear-locks')->daily();$this->scheduleInstance->command('sanctum:prune-expired --hours=1')->hourly()->onOneServer();$this->scheduleInstance->job(new ApiTokenExpirationWarningJob)->hourly()->onOneServer();diff --git a/app/Jobs/CleanupStaleMultiplexedConnections.php b/app/Jobs/CleanupStaleMultiplexedConnections.phpdeleted file mode 100644index 0a10fa4209..0000000000--- a/app/Jobs/CleanupStaleMultiplexedConnections.php+++ /dev/null@@ -1,295 +0,0 @@-<?php--namespace App\Jobs;--use App\Models\Server;-use Carbon\Carbon;-use Illuminate\Bus\Queueable;-use Illuminate\Contracts\Queue\ShouldQueue;-use Illuminate\Foundation\Bus\Dispatchable;-use Illuminate\Queue\InteractsWithQueue;-use Illuminate\Queue\SerializesModels;-use Illuminate\Support\Facades\Log;-use Illuminate\Support\Facades\Process;-use Illuminate\Support\Facades\Storage;--class CleanupStaleMultiplexedConnections implements ShouldQueue-{- use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;-- public function handle()- {- $this->cleanupStaleConnections();- $this->cleanupNonExistentServerConnections();- $this->cleanupDuplicateSshProcesses();- $this->cleanupOrphanedSshProcesses();- $this->cleanupOrphanedCloudflaredProcesses();- }-- /**- * Once two background ssh masters share the same ControlPath, OpenSSH's- * control socket state is no longer trustworthy: `ssh -O check` may report- * one PID while the socket lifecycle is tied to another. Reset the whole- * duplicate group rather than trying to choose an owner.- */- private function cleanupDuplicateSshProcesses(): void- {- $muxDir = storage_path('app/ssh/mux');- $groups = [];-- foreach ($this->listProcesses() as $process) {- $controlPath = $this->extractControlPath($process['args']);- if (! is_string($controlPath) || ! str_starts_with($controlPath, $muxDir.'/')) {- continue;- }-- $groups[$controlPath][] = $process;- }-- foreach ($groups as $controlPath => $processes) {- if (count($processes) < 2) {- continue;- }-- $this->resetDuplicateGroup($controlPath, $processes);- }- }-- /**- * Kill backgrounded ssh master processes that lost the ControlPath socket- * race. Such processes are not masters, so ControlPersist never reaps them- * and they leak memory until the container restarts. A legitimate master- * always owns its socket file; an orphan has none.- *- * Processes younger than the minimum age are skipped: a freshly forked- * master creates its socket a few milliseconds after starting, so a young- * process with no socket may simply be mid-establish rather than orphaned.- */- private function cleanupOrphanedSshProcesses(): void- {- $muxDir = storage_path('app/ssh/mux');- $minAge = (int) config('constants.ssh.mux_orphan_min_age');-- foreach ($this->listProcesses() as $process) {- // Only ever touch ssh processes pointing at Coolify's mux directory.- $controlPath = $this->extractControlPath($process['args']);- if (! is_string($controlPath) || ! str_starts_with($controlPath, $muxDir.'/')) {- continue;- }-- if ($process['etimes'] >= $minAge && ! file_exists($controlPath)) {- $this->reapOrphan('ssh', $process);- }- }- }-- /**- * Kill orphaned `cloudflared access ssh` proxy processes. Each is spawned- * as the SSH ProxyCommand transport for a Cloudflare Tunnel server and must- * die with its parent ssh. When that ssh is killed or orphaned (e.g. a lost- * mux master), the cloudflared process can leak and accumulate. A legitimate- * proxy always has a live ssh parent; one without is safe to reap.- *- * Processes younger than the minimum age are skipped so a proxy whose parent- * ssh is still starting up, or a transient `ssh -O check` proxy mid-exit, is- * never mistaken for an orphan.- */- private function cleanupOrphanedCloudflaredProcesses(): void- {- $minAge = (int) config('constants.ssh.mux_orphan_min_age');- $processes = $this->listProcesses();-- $sshPids = [];- foreach ($processes as $process) {- // The ssh binary itself, not `cloudflared access ssh` (space before ssh).- if (preg_match('#(^|/)ssh\s#', $process['args'])) {- $sshPids[$process['pid']] = true;- }- }-- foreach ($processes as $process) {- // `cloudflared access ssh`, never the `cloudflared tunnel` daemon.- if (! str_contains($process['args'], 'cloudflared access ssh')) {- continue;- }-- // Orphaned when no live ssh process is its parent.- if ($process['etimes'] >= $minAge && ! isset($sshPids[$process['ppid']])) {- $this->reapOrphan('cloudflared', $process);- }- }- }-- /**- * Reap a detected orphan process. When orphan reaping is disabled (the- * default), the orphan is only logged — a dry-run mode that lets operators- * verify what would be killed before enabling it for real.- *- * @param array{pid: string, ppid: string, etimes: int, args: string} $process- */- private function reapOrphan(string $kind, array $process): void- {- if (! config('constants.ssh.mux_orphan_reap_enabled')) {- Log::info("Orphaned {$kind} process detected (dry-run, not killed)", [- 'pid' => $process['pid'],- 'etimes' => $process['etimes'],- 'command' => $process['args'],- ]);-- return;- }-- Process::run('kill '.escapeshellarg($process['pid']));- Log::info("Killed orphaned {$kind} process", [- 'pid' => $process['pid'],- 'etimes' => $process['etimes'],- 'command' => $process['args'],- ]);- }-- /**- * Snapshot of running processes.- *- * @return list<array{pid: string, ppid: string, etimes: int, args: string}>- */- private function listProcesses(): array- {- $ps = Process::run('ps -ww -eo pid=,ppid=,etimes=,args=');- if ($ps->exitCode() !== 0) {- return [];- }-- $processes = [];- foreach (explode("\n", trim($ps->output())) as $line) {- if (! preg_match('/^\s*(\d+)\s+(\d+)\s+(\d+)\s+(.*)$/', $line, $matches)) {- continue;- }- $processes[] = [- 'pid' => $matches[1],- 'ppid' => $matches[2],- 'etimes' => (int) $matches[3],- 'args' => $matches[4],- ];- }-- return $processes;- }-- /**- * @param list<array{pid: string, ppid: string, etimes: int, args: string}> $processes- */- private function resetDuplicateGroup(string $controlPath, array $processes): void- {- if (! config('constants.ssh.mux_orphan_reap_enabled')) {- Log::info('Duplicate ssh mux processes detected (dry-run, not killed)', [- 'control_path' => $controlPath,- 'pids' => array_column($processes, 'pid'),- ]);-- return;- }-- foreach ($processes as $process) {- Process::run('kill '.escapeshellarg($process['pid']));- }-- if (file_exists($controlPath)) {- @unlink($controlPath);- }-- Log::info('Reset duplicate ssh mux processes', [- 'control_path' => $controlPath,- 'pids' => array_column($processes, 'pid'),- ]);- }-- private function extractControlPath(string $args): ?string- {- if (! preg_match('/(?:^|\s)-o\s+ControlPath=(?:"([^"]+)"|\'([^\']+)\'|(\S+))/', $args, $matches)) {- if (preg_match('/^ssh:\s+(\S+)\s+\[mux\]$/', $args, $matches)) {- return $matches[1];- }-- return null;- }-- return $matches[1] ?: ($matches[2] ?: $matches[3]);- }-- private function cleanupStaleConnections()- {- $muxFiles = Storage::disk('ssh-mux')->files();-- foreach ($muxFiles as $muxFile) {- $serverUuid = $this->extractServerUuidFromMuxFile($muxFile);- $server = Server::where('uuid', $serverUuid)->first();-- if (! $server) {- $this->removeMultiplexFile($muxFile, 'server_not_found');-- continue;- }-- $muxSocket = "/var/www/html/storage/app/ssh/mux/{$muxFile}";- $checkCommand = "ssh -O check -o ControlPath={$muxSocket} {$server->user}@{$server->ip} 2>/dev/null";- $checkProcess = Process::run($checkCommand);-- if ($checkProcess->exitCode() !== 0) {- $this->removeMultiplexFile($muxFile, 'connection_check_failed');- } else {- $muxContent = Storage::disk('ssh-mux')->get($muxFile);- $establishedAt = Carbon::parse(substr($muxContent, 37));- $expirationTime = $establishedAt->addSeconds(config('constants.ssh.mux_persist_time'));-- if (Carbon::now()->isAfter($expirationTime)) {- $this->removeMultiplexFile($muxFile, 'expired');- }- }- }- }-- private function cleanupNonExistentServerConnections()- {- $muxFiles = Storage::disk('ssh-mux')->files();- $existingServerUuids = Server::pluck('uuid')->toArray();-- foreach ($muxFiles as $muxFile) {- $serverUuid = $this->extractServerUuidFromMuxFile($muxFile);- if (! in_array($serverUuid, $existingServerUuids)) {- $this->removeMultiplexFile($muxFile, 'server_does_not_exist');- }- }- }-- private function extractServerUuidFromMuxFile($muxFile)- {- return substr($muxFile, 4);- }-- /**- * Close and delete a stale mux socket file. When orphan reaping is disabled- * (the default), the file is only logged — a dry-run mode that lets operators- * verify what would be removed before enabling it for real.- */- private function removeMultiplexFile(string $muxFile, string $reason): void- {- if (! config('constants.ssh.mux_orphan_reap_enabled')) {- Log::info('Stale mux file detected (dry-run, not removed)', [- 'file' => $muxFile,- 'reason' => $reason,- ]);-- return;- }-- $muxSocket = "/var/www/html/storage/app/ssh/mux/{$muxFile}";- $closeCommand = "ssh -O exit -o ControlPath={$muxSocket} localhost 2>/dev/null";- Process::run($closeCommand);- Storage::disk('ssh-mux')->delete($muxFile);-- Log::info('Removed stale mux file', [- 'file' => $muxFile,- 'reason' => $reason,- ]);- }-}diff --git a/config/constants.php b/config/constants.phpindex 0a0227ea6c..dfff17542d 100644--- a/config/constants.php+++ b/config/constants.php@@ -67,13 +67,6 @@ return ['ssh' => ['mux_enabled' => env('MUX_ENABLED', env('SSH_MUX_ENABLED', true)),'mux_persist_time' => env('SSH_MUX_PERSIST_TIME', 3600),- 'mux_health_check_enabled' => env('SSH_MUX_HEALTH_CHECK_ENABLED', true),- 'mux_health_check_timeout' => env('SSH_MUX_HEALTH_CHECK_TIMEOUT', 5),- 'mux_max_age' => env('SSH_MUX_MAX_AGE', 1800), // 30 minutes- 'mux_lock_ttl' => env('SSH_MUX_LOCK_TTL', 30), // lock auto-release, seconds- 'mux_lock_timeout' => env('SSH_MUX_LOCK_TIMEOUT', 10), // max wait for lock, seconds- 'mux_orphan_min_age' => env('SSH_MUX_ORPHAN_MIN_AGE', 600), // min process age before reaping orphans, seconds- 'mux_orphan_reap_enabled' => env('SSH_MUX_ORPHAN_REAP_ENABLED', false), // false = dry-run, only log orphans'connection_timeout' => 10,'server_interval' => 20,'command_timeout' => 3600,diff --git a/tests/Feature/ScheduleOnOneServerTest.php b/tests/Feature/ScheduleOnOneServerTest.phpindex b110171077..10758738cc 100644--- a/tests/Feature/ScheduleOnOneServerTest.php+++ b/tests/Feature/ScheduleOnOneServerTest.php@@ -1,10 +1,8 @@<?php-use App\Jobs\CleanupStaleMultiplexedConnections;use App\Models\InstanceSettings;use Illuminate\Console\Scheduling\Schedule;use Illuminate\Foundation\Testing\RefreshDatabase;-use Illuminate\Support\Facades\Queue;uses(RefreshDatabase::class);@@ -23,23 +21,6 @@ it('schedules RegenerateSslCertJob with onOneServer to prevent multi-server doubexpect($event->onOneServer)->toBeTrue();});-it('schedules CleanupStaleMultiplexedConnections on the crons queue', function () {- config(['constants.coolify.self_hosted' => false]);- Queue::fake();-- $schedule = app(Schedule::class);-- $event = collect($schedule->events())->first(- fn ($e) => str_contains((string) $e->description, CleanupStaleMultiplexedConnections::class)- );-- expect($event)->not->toBeNull();-- $event->run(app());-- Queue::assertPushedOn('crons', CleanupStaleMultiplexedConnections::class);-});-it('schedules every production job with onOneServer', function () {$schedule = app(Schedule::class);diff --git a/tests/Feature/SshMultiplexingLockTest.php b/tests/Feature/SshMultiplexingLockTest.phpindex fee4ec6328..e97be1f502 100644--- a/tests/Feature/SshMultiplexingLockTest.php+++ b/tests/Feature/SshMultiplexingLockTest.php@@ -1,12 +1,10 @@<?phpuse App\Helpers\SshMultiplexingHelper;-use App\Jobs\CleanupStaleMultiplexedConnections;use App\Models\PrivateKey;use App\Models\Server;use App\Models\User;use Illuminate\Foundation\Testing\RefreshDatabase;-use Illuminate\Support\Facades\File;use Illuminate\Support\Facades\Process;use Illuminate\Support\Facades\Storage;@@ -116,177 +114,3 @@ it('returns false and runs no process when multiplexing is globally disabled', fProcess::assertNothingRan();});--it('kills only old orphaned ssh masters whose control socket no longer exists', function () {- config(['constants.ssh.mux_orphan_reap_enabled' => true]);- $muxDir = storage_path('app/ssh/mux');- File::ensureDirectoryExists($muxDir);-- $liveSocket = $muxDir.'/mux_live_'.uniqid();- $orphanSocket = $muxDir.'/mux_orphan_'.uniqid();- $youngSocket = $muxDir.'/mux_young_'.uniqid();- File::put($liveSocket, 'x');-- Process::fake([- 'ps*' => Process::result(output: "111 1 5000 ssh -fN -o ControlMaster=auto -o ControlPath={$liveSocket} [email protected]\n".- "222 1 5000 ssh -fN -o ControlMaster=auto -o ControlPath={$orphanSocket} [email protected]\n".- "333 1 30 ssh -fN -o ControlMaster=auto -o ControlPath={$youngSocket} [email protected]\n"),- 'kill*' => Process::result(exitCode: 0),- ]);-- $job = new CleanupStaleMultiplexedConnections;- $method = new ReflectionMethod($job, 'cleanupOrphanedSshProcesses');- $method->setAccessible(true);- $method->invoke($job);-- Process::assertRan(fn ($process) => str_contains($process->command, 'kill') && str_contains($process->command, '222'));- Process::assertNotRan(fn ($process) => str_contains($process->command, 'kill') && str_contains($process->command, '111'));- Process::assertNotRan(fn ($process) => str_contains($process->command, 'kill') && str_contains($process->command, '333'));-- File::delete($liveSocket);-});--it('kills old orphaned native openssh mux masters whose control socket no longer exists', function () {- config(['constants.ssh.mux_orphan_reap_enabled' => true]);- $muxDir = storage_path('app/ssh/mux');- File::ensureDirectoryExists($muxDir);-- $liveSocket = $muxDir.'/mux_native_live_'.uniqid();- $orphanSocket = $muxDir.'/mux_native_orphan_'.uniqid();- File::put($liveSocket, 'x');-- Process::fake([- 'ps*' => Process::result(output: "111 1 5000 ssh: {$liveSocket} [mux]\n".- "222 1 5000 ssh: {$orphanSocket} [mux]\n"),- 'kill*' => Process::result(exitCode: 0),- ]);-- $job = new CleanupStaleMultiplexedConnections;- $method = new ReflectionMethod($job, 'cleanupOrphanedSshProcesses');- $method->setAccessible(true);- $method->invoke($job);-- Process::assertRan(fn ($process) => str_contains($process->command, 'kill') && str_contains($process->command, '222'));- Process::assertNotRan(fn ($process) => str_contains($process->command, 'kill') && str_contains($process->command, '111'));-- File::delete($liveSocket);-});--it('kills only old orphaned cloudflared proxies whose parent ssh is gone', function () {- config(['constants.ssh.mux_orphan_reap_enabled' => true]);-- Process::fake([- 'ps*' => Process::result(output: "100 1 5000 ssh -fN -o ControlMaster=auto [email protected]\n".- "200 100 5000 cloudflared access ssh --hostname host.example.com\n".- "300 2176 5000 cloudflared access ssh --hostname host.example.com\n".- "400 2176 30 cloudflared access ssh --hostname host.example.com\n".- "2176 1 9000 /usr/bin/some-supervisor\n"),- 'kill*' => Process::result(exitCode: 0),- ]);-- $job = new CleanupStaleMultiplexedConnections;- $method = new ReflectionMethod($job, 'cleanupOrphanedCloudflaredProcesses');- $method->setAccessible(true);- $method->invoke($job);-- Process::assertRan(fn ($process) => str_contains($process->command, 'kill') && str_contains($process->command, '300'));- Process::assertNotRan(fn ($process) => str_contains($process->command, 'kill') && str_contains($process->command, '200'));- Process::assertNotRan(fn ($process) => str_contains($process->command, 'kill') && str_contains($process->command, '400'));-});--it('dry-run mode logs orphans but kills nothing when reaping is disabled', function () {- config(['constants.ssh.mux_orphan_reap_enabled' => false]);- $muxDir = storage_path('app/ssh/mux');- File::ensureDirectoryExists($muxDir);-- $orphanSocket = $muxDir.'/mux_orphan_'.uniqid();-- Process::fake([- 'ps*' => Process::result(output: "222 1 5000 ssh -fN -o ControlMaster=auto -o ControlPath={$orphanSocket} [email protected]\n"),- 'kill*' => Process::result(exitCode: 0),- ]);-- $job = new CleanupStaleMultiplexedConnections;- $method = new ReflectionMethod($job, 'cleanupOrphanedSshProcesses');- $method->setAccessible(true);- $method->invoke($job);-- Process::assertNotRan(fn ($process) => str_contains($process->command, 'kill'));-});--it('resets duplicate ssh mux process groups atomically when reaping is enabled', function () {- config(['constants.ssh.mux_orphan_reap_enabled' => true]);- $muxDir = storage_path('app/ssh/mux');- File::ensureDirectoryExists($muxDir);- $controlPath = $muxDir.'/mux_duplicate_'.uniqid();- File::put($controlPath, 'socket');-- Process::fake([- 'ps*' => Process::result(output: "111 1 5000 ssh -fN -o ControlMaster=auto -o ControlPath={$controlPath} [email protected]\n".- "222 1 5000 ssh -fN -o ControlMaster=auto -o ControlPath={$controlPath} [email protected]\n"),- 'kill*' => Process::result(exitCode: 0),- ]);-- $job = new CleanupStaleMultiplexedConnections;- $method = new ReflectionMethod($job, 'cleanupDuplicateSshProcesses');- $method->setAccessible(true);- $method->invoke($job);-- Process::assertRan(fn ($process) => str_contains($process->command, 'kill') && str_contains($process->command, '111'));- Process::assertRan(fn ($process) => str_contains($process->command, 'kill') && str_contains($process->command, '222'));- expect(file_exists($controlPath))->toBeFalse();-});--it('resets duplicate native openssh mux process groups atomically when reaping is enabled', function () {- config(['constants.ssh.mux_orphan_reap_enabled' => true]);- $muxDir = storage_path('app/ssh/mux');- File::ensureDirectoryExists($muxDir);- $controlPath = $muxDir.'/mux_native_duplicate_'.uniqid();- File::put($controlPath, 'socket');-- Process::fake([- 'ps*' => Process::result(output: "111 1 5000 ssh: {$controlPath} [mux]\n".- "222 1 5000 ssh: {$controlPath} [mux]\n"),- 'kill*' => Process::result(exitCode: 0),- ]);-- $job = new CleanupStaleMultiplexedConnections;- $method = new ReflectionMethod($job, 'cleanupDuplicateSshProcesses');- $method->setAccessible(true);- $method->invoke($job);-- Process::assertRan(fn ($process) => str_contains($process->command, 'kill') && str_contains($process->command, '111'));- Process::assertRan(fn ($process) => str_contains($process->command, 'kill') && str_contains($process->command, '222'));- expect(file_exists($controlPath))->toBeFalse();-});--it('removes mux files for non-existent servers when reaping is enabled', function () {- config(['constants.ssh.mux_orphan_reap_enabled' => true]);- Storage::fake('ssh-mux');- $file = 'mux_ghost'.uniqid();- Storage::disk('ssh-mux')->put($file, 'x');- Process::fake();-- $job = new CleanupStaleMultiplexedConnections;- $method = new ReflectionMethod($job, 'cleanupNonExistentServerConnections');- $method->setAccessible(true);- $method->invoke($job);-- expect(Storage::disk('ssh-mux')->exists($file))->toBeFalse();-});--it('keeps mux files for non-existent servers in dry-run mode', function () {- config(['constants.ssh.mux_orphan_reap_enabled' => false]);- Storage::fake('ssh-mux');- $file = 'mux_ghost'.uniqid();- Storage::disk('ssh-mux')->put($file, 'x');- Process::fake();-- $job = new CleanupStaleMultiplexedConnections;- $method = new ReflectionMethod($job, 'cleanupNonExistentServerConnections');- $method->setAccessible(true);- $method->invoke($job);-- expect(Storage::disk('ssh-mux')->exists($file))->toBeTrue();- Process::assertNothingRan();-});