Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(backups): forward per-backup timeout to the ssh timeout wrapper

Himanshu Ahuja419106696a44
diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php
index 104a84a1b8..d432a3eeb3 100644
--- a/app/Jobs/DatabaseBackupJob.php
+++ b/app/Jobs/DatabaseBackupJob.php
@@ -785,7 +785,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc alias set{$resolveOptions} temporary {$escapedEndpoint} {$escapedKey} {$escapedSecret}";
$commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc cp {$escapedBackupLocation} {$escapedS3Destination}";
- instant_remote_process($commands, $this->server, true, false, null, disableMultiplexing: true);
+ instant_remote_process($commands, $this->server, true, false, $this->timeout, disableMultiplexing: true);
$this->s3_uploaded = true;
} catch (Throwable $e) {
diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php
index 8d1a30a62a..982dda5511 100644
--- a/bootstrap/helpers/remoteProcess.php
+++ b/bootstrap/helpers/remoteProcess.php
@@ -177,7 +177,7 @@ function instant_remote_process(Collection|array $command, Server $server, bool
return SshRetryHandler::retry(
function () use ($server, $command_string, $effectiveTimeout, $disableMultiplexing) {
- $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string, $disableMultiplexing);
+ $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string, $disableMultiplexing, (int) $effectiveTimeout);
$process = Process::timeout($effectiveTimeout)->run($sshCommand);
$output = trim($process->output());
diff --git a/config/constants.php b/config/constants.php
index aa1b5c36c0..fc15a6899a 100644
--- a/config/constants.php
+++ b/config/constants.php
@@ -77,7 +77,7 @@ return [
'mux_orphan_reap_enabled' => env('SSH_MUX_ORPHAN_REAP_ENABLED', false), // false = dry-run, only log orphans
'connection_timeout' => 10,
'server_interval' => 20,
- 'command_timeout' => 3600,
+ 'command_timeout' => env('SSH_COMMAND_TIMEOUT', 3600),
'max_retries' => env('SSH_MAX_RETRIES', 3),
'retry_base_delay' => env('SSH_RETRY_BASE_DELAY', 2), // seconds
'retry_max_delay' => env('SSH_RETRY_MAX_DELAY', 30), // seconds
diff --git a/tests/Feature/RemoteProcessTimeoutTest.php b/tests/Feature/RemoteProcessTimeoutTest.php
new file mode 100644
index 0000000000..9bc8e12ae4
--- /dev/null
+++ b/tests/Feature/RemoteProcessTimeoutTest.php
@@ -0,0 +1,103 @@
+<?php
+
+use App\Helpers\SshMultiplexingHelper;
+use App\Models\PrivateKey;
+use App\Models\Server;
+use App\Models\User;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Storage;
+
+/**
+ * Tests that per-call SSH command timeouts (e.g. a scheduled backup's configured
+ * timeout) reach the shell-level `timeout N ssh` wrapper instead of being capped
+ * by the global constants.ssh.command_timeout default.
+ *
+ * @see https://github.com/coollabsio/coolify DatabaseBackupJob/VolumeBackupJob pass
+ * a per-backup timeout to instant_remote_process()
+ */
+uses(RefreshDatabase::class);
+
+function makeTimeoutTestServer(): Server
+{
+ $user = User::factory()->create();
+ $team = $user->teams()->first();
+
+ $privateKeyContent = '-----BEGIN OPENSSH PRIVATE KEY-----
+'.
+ 'b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+'.
+ 'QyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevAAAAJi/QySHv0Mk
+'.
+ 'hwAAAAtzc2gtZWQyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevA
+'.
+ 'AAAECBQw4jg1WRT2IGHMncCiZhURCts2s24HoDS0thHnnRKVuGmoeGq/pojrsyP1pszcNV
+'.
+ 'uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==
+'.
+ '-----END OPENSSH PRIVATE KEY-----';
+
+ $privateKey = PrivateKey::create([
+ 'name' => 'timeout-test-key-'.uniqid(),
+ 'private_key' => $privateKeyContent,
+ 'team_id' => $team->id,
+ ]);
+
+ Storage::fake('ssh-keys');
+ Storage::disk('ssh-keys')->put("ssh_key@{$privateKey->uuid}", $privateKeyContent);
+
+ $server = Server::factory()->create([
+ 'team_id' => $team->id,
+ 'private_key_id' => $privateKey->id,
+ ]);
+
+ Storage::disk('ssh-keys')->put("ssh_key@{$server->privateKey->uuid}", $server->privateKey->private_key);
+
+ return $server;
+}
+
+it('wraps ssh commands with an explicitly passed command timeout', function () {
+ config(['constants.ssh.mux_enabled' => false]);
+ $server = makeTimeoutTestServer();
+
+ $command = SshMultiplexingHelper::generateSshCommand($server, 'echo ok', commandTimeout: 7200);
+
+ expect($command)->toStartWith('timeout 7200 ssh ');
+});
+
+it('wraps ssh commands with the configured default timeout when none is passed', function () {
+ config([
+ 'constants.ssh.mux_enabled' => false,
+ 'constants.ssh.command_timeout' => 1234,
+ ]);
+ $server = makeTimeoutTestServer();
+
+ $command = SshMultiplexingHelper::generateSshCommand($server, 'echo ok');
+
+ expect($command)->toStartWith('timeout 1234 ssh ');
+});
+
+it('forwards the per-call timeout of instant_remote_process to the ssh timeout wrapper', function () {
+ config(['constants.ssh.mux_enabled' => false]);
+ $server = makeTimeoutTestServer();
+
+ Process::fake();
+
+ instant_remote_process(['echo ok'], $server, timeout: 7200, disableMultiplexing: true);
+
+ Process::assertRan(fn ($process) => str_starts_with($process->command, 'timeout 7200 ssh '));
+});
+
+it('uses the configured default timeout in instant_remote_process when no timeout is passed', function () {
+ config([
+ 'constants.ssh.mux_enabled' => false,
+ 'constants.ssh.command_timeout' => 1234,
+ ]);
+ $server = makeTimeoutTestServer();
+
+ Process::fake();
+
+ instant_remote_process(['echo ok'], $server, disableMultiplexing: true);
+
+ Process::assertRan(fn ($process) => str_starts_with($process->command, 'timeout 1234 ssh '));
+});