An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(backups): forward per-backup timeout to the ssh timeout wrapper
Himanshu Ahuja419106696a44
diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.phpindex 104a84a1b8..d432a3eeb3 100644--- a/app/Jobs/DatabaseBackupJob.php+++ b/app/Jobs/DatabaseBackupJob.php@@ -785,7 +785,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue$commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc alias set{$resolveOptions} temporary {$escapedEndpoint} {$escapedKey} {$escapedSecret}";$commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc cp {$escapedBackupLocation} {$escapedS3Destination}";- instant_remote_process($commands, $this->server, true, false, null, disableMultiplexing: true);+ instant_remote_process($commands, $this->server, true, false, $this->timeout, disableMultiplexing: true);$this->s3_uploaded = true;} catch (Throwable $e) {diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.phpindex 8d1a30a62a..982dda5511 100644--- a/bootstrap/helpers/remoteProcess.php+++ b/bootstrap/helpers/remoteProcess.php@@ -177,7 +177,7 @@ function instant_remote_process(Collection|array $command, Server $server, boolreturn SshRetryHandler::retry(function () use ($server, $command_string, $effectiveTimeout, $disableMultiplexing) {- $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string, $disableMultiplexing);+ $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string, $disableMultiplexing, (int) $effectiveTimeout);$process = Process::timeout($effectiveTimeout)->run($sshCommand);$output = trim($process->output());diff --git a/config/constants.php b/config/constants.phpindex aa1b5c36c0..fc15a6899a 100644--- a/config/constants.php+++ b/config/constants.php@@ -77,7 +77,7 @@ return ['mux_orphan_reap_enabled' => env('SSH_MUX_ORPHAN_REAP_ENABLED', false), // false = dry-run, only log orphans'connection_timeout' => 10,'server_interval' => 20,- 'command_timeout' => 3600,+ 'command_timeout' => env('SSH_COMMAND_TIMEOUT', 3600),'max_retries' => env('SSH_MAX_RETRIES', 3),'retry_base_delay' => env('SSH_RETRY_BASE_DELAY', 2), // seconds'retry_max_delay' => env('SSH_RETRY_MAX_DELAY', 30), // secondsdiff --git a/tests/Feature/RemoteProcessTimeoutTest.php b/tests/Feature/RemoteProcessTimeoutTest.phpnew file mode 100644index 0000000000..9bc8e12ae4--- /dev/null+++ b/tests/Feature/RemoteProcessTimeoutTest.php@@ -0,0 +1,103 @@+<?php++use App\Helpers\SshMultiplexingHelper;+use App\Models\PrivateKey;+use App\Models\Server;+use App\Models\User;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Facades\Process;+use Illuminate\Support\Facades\Storage;++/**+ * Tests that per-call SSH command timeouts (e.g. a scheduled backup's configured+ * timeout) reach the shell-level `timeout N ssh` wrapper instead of being capped+ * by the global constants.ssh.command_timeout default.+ *+ * @see https://github.com/coollabsio/coolify DatabaseBackupJob/VolumeBackupJob pass+ * a per-backup timeout to instant_remote_process()+ */+uses(RefreshDatabase::class);++function makeTimeoutTestServer(): Server+{+ $user = User::factory()->create();+ $team = $user->teams()->first();++ $privateKeyContent = '-----BEGIN OPENSSH PRIVATE KEY-----+'.+ 'b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW+'.+ 'QyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevAAAAJi/QySHv0Mk+'.+ 'hwAAAAtzc2gtZWQyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevA+'.+ 'AAAECBQw4jg1WRT2IGHMncCiZhURCts2s24HoDS0thHnnRKVuGmoeGq/pojrsyP1pszcNV+'.+ 'uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==+'.+ '-----END OPENSSH PRIVATE KEY-----';++ $privateKey = PrivateKey::create([+ 'name' => 'timeout-test-key-'.uniqid(),+ 'private_key' => $privateKeyContent,+ 'team_id' => $team->id,+ ]);++ Storage::fake('ssh-keys');+ Storage::disk('ssh-keys')->put("ssh_key@{$privateKey->uuid}", $privateKeyContent);++ $server = Server::factory()->create([+ 'team_id' => $team->id,+ 'private_key_id' => $privateKey->id,+ ]);++ Storage::disk('ssh-keys')->put("ssh_key@{$server->privateKey->uuid}", $server->privateKey->private_key);++ return $server;+}++it('wraps ssh commands with an explicitly passed command timeout', function () {+ config(['constants.ssh.mux_enabled' => false]);+ $server = makeTimeoutTestServer();++ $command = SshMultiplexingHelper::generateSshCommand($server, 'echo ok', commandTimeout: 7200);++ expect($command)->toStartWith('timeout 7200 ssh ');+});++it('wraps ssh commands with the configured default timeout when none is passed', function () {+ config([+ 'constants.ssh.mux_enabled' => false,+ 'constants.ssh.command_timeout' => 1234,+ ]);+ $server = makeTimeoutTestServer();++ $command = SshMultiplexingHelper::generateSshCommand($server, 'echo ok');++ expect($command)->toStartWith('timeout 1234 ssh ');+});++it('forwards the per-call timeout of instant_remote_process to the ssh timeout wrapper', function () {+ config(['constants.ssh.mux_enabled' => false]);+ $server = makeTimeoutTestServer();++ Process::fake();++ instant_remote_process(['echo ok'], $server, timeout: 7200, disableMultiplexing: true);++ Process::assertRan(fn ($process) => str_starts_with($process->command, 'timeout 7200 ssh '));+});++it('uses the configured default timeout in instant_remote_process when no timeout is passed', function () {+ config([+ 'constants.ssh.mux_enabled' => false,+ 'constants.ssh.command_timeout' => 1234,+ ]);+ $server = makeTimeoutTestServer();++ Process::fake();++ instant_remote_process(['echo ok'], $server, disableMultiplexing: true);++ Process::assertRan(fn ($process) => str_starts_with($process->command, 'timeout 1234 ssh '));+});