An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(ssh): keep the command and its secrets out of timeout errors
Andras Bacsai3e6f6bfdad83
diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.phpindex 8f4eb84be6..d17c4188ac 100644--- a/bootstrap/helpers/remoteProcess.php+++ b/bootstrap/helpers/remoteProcess.php@@ -11,7 +11,9 @@ use App\Models\PrivateKey;use App\Models\Server;use App\Support\RemoteProcessCommand;use Carbon\Carbon;+use Illuminate\Contracts\Process\ProcessResult;use Illuminate\Database\Eloquent\Model;+use Illuminate\Process\Exceptions\ProcessTimedOutException;use Illuminate\Support\Collection;use Illuminate\Support\Facades\Auth;use Illuminate\Support\Facades\Process;@@ -137,6 +139,20 @@ function instant_scp_from_server(string $remoteSource, string $localDest, Server);}+/**+ * Runs an SSH command. The timeout error of the process contains the complete command line, and with it+ * the secrets of the remote script (database passwords, S3 keys), so it is replaced with a generic error.+ * Exit code 124 (as from `timeout`) marks it as a remote command failure, so the batch is not run again.+ */+function runSshProcess(string $sshCommand, int $timeout, ?string $input = null): ProcessResult+{+ try {+ return Process::timeout($timeout)->input($input)->run($sshCommand);+ } catch (ProcessTimedOutException) {+ throw new RuntimeException("The command on the server timed out after {$timeout} seconds.", 124);+ }+}+/*** Write content to a file on a server through SSH stdin. The content never becomes part of a command* line, so it is not limited by the 128 KiB per-argument limit of Linux. A non-root SSH user writes@@ -150,7 +166,7 @@ function instant_remote_write_file(Server $server, string $path, string $contentSshRetryHandler::retry(function () use ($server, $remoteCommand, $content, $timeout) {$sshCommand = SshMultiplexingHelper::generateSshStdinCommand($server, $remoteCommand, commandTimeout: $timeout);- $process = Process::timeout($timeout)->input($content)->run($sshCommand);+ $process = runSshProcess($sshCommand, $timeout, $content);if ($process->exitCode() !== 0) {excludeCertainErrors($process->errorOutput(), $process->exitCode());@@ -176,7 +192,7 @@ function instant_remote_process_with_timeout(Collection|array $command, Server $return SshRetryHandler::retry(function () use ($server, $command_string) {$sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string);- $process = Process::timeout(30)->run($sshCommand);+ $process = runSshProcess($sshCommand, 30);$output = trim($process->output());$exitCode = $process->exitCode();@@ -212,7 +228,7 @@ function instant_remote_process(Collection|array $command, Server $server, boolreturn SshRetryHandler::retry(function () use ($server, $command_string, $effectiveTimeout, $disableMultiplexing) {$sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string, $disableMultiplexing, (int) $effectiveTimeout);- $process = Process::timeout($effectiveTimeout)->run($sshCommand);+ $process = runSshProcess($sshCommand, (int) $effectiveTimeout);$output = trim($process->output());$exitCode = $process->exitCode();diff --git a/tests/Feature/BackupTimeoutSecretRedactionTest.php b/tests/Feature/BackupTimeoutSecretRedactionTest.phpnew file mode 100644index 0000000000..9cb4ec23f8--- /dev/null+++ b/tests/Feature/BackupTimeoutSecretRedactionTest.php@@ -0,0 +1,115 @@+<?php++use App\Jobs\DatabaseBackupJob;+use App\Models\Environment;+use App\Models\InstanceSettings;+use App\Models\PrivateKey;+use App\Models\Project;+use App\Models\ScheduledDatabaseBackup;+use App\Models\Server;+use App\Models\StandaloneDocker;+use App\Models\StandaloneMysql;+use App\Models\Team;+use App\Notifications\Database\BackupFailed;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Process\Exceptions\ProcessTimedOutException;+use Illuminate\Process\ProcessResult;+use Illuminate\Support\Facades\Notification;+use Illuminate\Support\Facades\Process;+use Illuminate\Support\Facades\Storage;+use Symfony\Component\Process\Exception\ProcessTimedOutException as SymfonyTimeoutException;+use Symfony\Component\Process\Process as SymfonyProcess;++uses(RefreshDatabase::class);++beforeEach(function () {+ Server::flushIdentityMap();+ Storage::fake('ssh-keys');+ Storage::fake('ssh-mux');+ config(['constants.ssh.mux_enabled' => false]);+});++/**+ * The exception Process::run() throws when a command runs longer than its timeout. Symfony puts the+ * complete command line into the message.+ */+function backupRedactionTimeoutException(string $command, int $timeout): ProcessTimedOutException+{+ $process = SymfonyProcess::fromShellCommandline($command);+ $process->setTimeout($timeout);++ return new ProcessTimedOutException(new SymfonyTimeoutException($process, SymfonyTimeoutException::TYPE_GENERAL), new ProcessResult($process));+}++function backupRedactionServer(Team $team): Server+{+ $server = Server::factory()->create(['team_id' => $team->id]);+ $server->update(['private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id]);+ $server->settings()->update(['is_reachable' => true, 'is_usable' => true]);++ return $server->fresh();+}++it('does not put the command into the error when a remote command times out', function () {+ config(['constants.ssh.max_retries' => 3, 'constants.ssh.retry_base_delay' => 0]);+ $server = backupRedactionServer(Team::factory()->create());+ $attempts = 0;+ Process::fake(function ($process) use (&$attempts) {+ $attempts++;++ throw backupRedactionTimeoutException($process->command, 7200);+ });++ try {+ instant_remote_process(["docker exec mysql mysqldump -u root -p'TopSecretPass' app > /tmp/x"], $server, timeout: 7200, disableMultiplexing: true);+ $this->fail('The timeout was not reported.');+ } catch (Throwable $exception) {+ expect($exception->getMessage())->toBe('The command on the server timed out after 7200 seconds.')+ ->and($exception->getPrevious())->toBeNull();+ }++ expect($attempts)->toBe(1);+});++it('does not put database credentials into the backup execution or the failure notification when the dump times out', function () {+ InstanceSettings::forceCreate(['id' => 0]);+ Notification::fake();+ $team = Team::factory()->create();+ $server = backupRedactionServer($team);+ $team->emailNotificationSettings->update(['smtp_enabled' => true, 'backup_failure_email_notifications' => true]);+ $project = Project::factory()->create(['team_id' => $team->id]);+ $environment = Environment::factory()->create(['project_id' => $project->id]);+ $database = create_standalone_mysql($environment->id, StandaloneDocker::where('server_id', $server->id)->firstOrFail(), [+ 'mysql_root_password' => 'TopSecretPass',+ 'mysql_database' => 'app',+ 'status' => 'running:healthy',+ ]);+ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => '0 0 * * *',+ 'save_s3' => false,+ 'database_type' => StandaloneMysql::class,+ 'database_id' => $database->id,+ 'team_id' => $team->id,+ 'databases_to_backup' => 'app',+ 'timeout' => 600,+ ]);+ Process::fake(function ($process) {+ if (str_contains($process->command, 'mysqldump')) {+ throw backupRedactionTimeoutException($process->command, 600);+ }++ return Process::result(output: '');+ });++ try {+ (new DatabaseBackupJob($backup))->handle();+ } catch (Throwable) {+ // The job reports the failure after it records the execution.+ }++ $execution = $backup->executions()->latest('id')->firstOrFail();+ expect($execution->status)->toBe('failed')+ ->and($execution->message)->toBe('The command on the server timed out after 600 seconds.');+ Notification::assertSentTo($team, BackupFailed::class, fn (BackupFailed $notification) => ! str_contains((string) $notification->output, 'TopSecretPass')+ && str_contains((string) $notification->output, 'timed out after 600 seconds'));+});