Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(ssh): keep the command and its secrets out of timeout errors

Andras Bacsai3e6f6bfdad83
diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php
index 8f4eb84be6..d17c4188ac 100644
--- a/bootstrap/helpers/remoteProcess.php
+++ b/bootstrap/helpers/remoteProcess.php
@@ -11,7 +11,9 @@ use App\Models\PrivateKey;
use App\Models\Server;
use App\Support\RemoteProcessCommand;
use Carbon\Carbon;
+use Illuminate\Contracts\Process\ProcessResult;
use Illuminate\Database\Eloquent\Model;
+use Illuminate\Process\Exceptions\ProcessTimedOutException;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Auth;
use Illuminate\Support\Facades\Process;
@@ -137,6 +139,20 @@ function instant_scp_from_server(string $remoteSource, string $localDest, Server
);
}
+/**
+ * Runs an SSH command. The timeout error of the process contains the complete command line, and with it
+ * the secrets of the remote script (database passwords, S3 keys), so it is replaced with a generic error.
+ * Exit code 124 (as from `timeout`) marks it as a remote command failure, so the batch is not run again.
+ */
+function runSshProcess(string $sshCommand, int $timeout, ?string $input = null): ProcessResult
+{
+ try {
+ return Process::timeout($timeout)->input($input)->run($sshCommand);
+ } catch (ProcessTimedOutException) {
+ throw new RuntimeException("The command on the server timed out after {$timeout} seconds.", 124);
+ }
+}
+
/**
* Write content to a file on a server through SSH stdin. The content never becomes part of a command
* line, so it is not limited by the 128 KiB per-argument limit of Linux. A non-root SSH user writes
@@ -150,7 +166,7 @@ function instant_remote_write_file(Server $server, string $path, string $content
SshRetryHandler::retry(
function () use ($server, $remoteCommand, $content, $timeout) {
$sshCommand = SshMultiplexingHelper::generateSshStdinCommand($server, $remoteCommand, commandTimeout: $timeout);
- $process = Process::timeout($timeout)->input($content)->run($sshCommand);
+ $process = runSshProcess($sshCommand, $timeout, $content);
if ($process->exitCode() !== 0) {
excludeCertainErrors($process->errorOutput(), $process->exitCode());
@@ -176,7 +192,7 @@ function instant_remote_process_with_timeout(Collection|array $command, Server $
return SshRetryHandler::retry(
function () use ($server, $command_string) {
$sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string);
- $process = Process::timeout(30)->run($sshCommand);
+ $process = runSshProcess($sshCommand, 30);
$output = trim($process->output());
$exitCode = $process->exitCode();
@@ -212,7 +228,7 @@ function instant_remote_process(Collection|array $command, Server $server, bool
return SshRetryHandler::retry(
function () use ($server, $command_string, $effectiveTimeout, $disableMultiplexing) {
$sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string, $disableMultiplexing, (int) $effectiveTimeout);
- $process = Process::timeout($effectiveTimeout)->run($sshCommand);
+ $process = runSshProcess($sshCommand, (int) $effectiveTimeout);
$output = trim($process->output());
$exitCode = $process->exitCode();
diff --git a/tests/Feature/BackupTimeoutSecretRedactionTest.php b/tests/Feature/BackupTimeoutSecretRedactionTest.php
new file mode 100644
index 0000000000..9cb4ec23f8
--- /dev/null
+++ b/tests/Feature/BackupTimeoutSecretRedactionTest.php
@@ -0,0 +1,115 @@
+<?php
+
+use App\Jobs\DatabaseBackupJob;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\ScheduledDatabaseBackup;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\StandaloneMysql;
+use App\Models\Team;
+use App\Notifications\Database\BackupFailed;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Process\Exceptions\ProcessTimedOutException;
+use Illuminate\Process\ProcessResult;
+use Illuminate\Support\Facades\Notification;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Storage;
+use Symfony\Component\Process\Exception\ProcessTimedOutException as SymfonyTimeoutException;
+use Symfony\Component\Process\Process as SymfonyProcess;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ Server::flushIdentityMap();
+ Storage::fake('ssh-keys');
+ Storage::fake('ssh-mux');
+ config(['constants.ssh.mux_enabled' => false]);
+});
+
+/**
+ * The exception Process::run() throws when a command runs longer than its timeout. Symfony puts the
+ * complete command line into the message.
+ */
+function backupRedactionTimeoutException(string $command, int $timeout): ProcessTimedOutException
+{
+ $process = SymfonyProcess::fromShellCommandline($command);
+ $process->setTimeout($timeout);
+
+ return new ProcessTimedOutException(new SymfonyTimeoutException($process, SymfonyTimeoutException::TYPE_GENERAL), new ProcessResult($process));
+}
+
+function backupRedactionServer(Team $team): Server
+{
+ $server = Server::factory()->create(['team_id' => $team->id]);
+ $server->update(['private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id]);
+ $server->settings()->update(['is_reachable' => true, 'is_usable' => true]);
+
+ return $server->fresh();
+}
+
+it('does not put the command into the error when a remote command times out', function () {
+ config(['constants.ssh.max_retries' => 3, 'constants.ssh.retry_base_delay' => 0]);
+ $server = backupRedactionServer(Team::factory()->create());
+ $attempts = 0;
+ Process::fake(function ($process) use (&$attempts) {
+ $attempts++;
+
+ throw backupRedactionTimeoutException($process->command, 7200);
+ });
+
+ try {
+ instant_remote_process(["docker exec mysql mysqldump -u root -p'TopSecretPass' app > /tmp/x"], $server, timeout: 7200, disableMultiplexing: true);
+ $this->fail('The timeout was not reported.');
+ } catch (Throwable $exception) {
+ expect($exception->getMessage())->toBe('The command on the server timed out after 7200 seconds.')
+ ->and($exception->getPrevious())->toBeNull();
+ }
+
+ expect($attempts)->toBe(1);
+});
+
+it('does not put database credentials into the backup execution or the failure notification when the dump times out', function () {
+ InstanceSettings::forceCreate(['id' => 0]);
+ Notification::fake();
+ $team = Team::factory()->create();
+ $server = backupRedactionServer($team);
+ $team->emailNotificationSettings->update(['smtp_enabled' => true, 'backup_failure_email_notifications' => true]);
+ $project = Project::factory()->create(['team_id' => $team->id]);
+ $environment = Environment::factory()->create(['project_id' => $project->id]);
+ $database = create_standalone_mysql($environment->id, StandaloneDocker::where('server_id', $server->id)->firstOrFail(), [
+ 'mysql_root_password' => 'TopSecretPass',
+ 'mysql_database' => 'app',
+ 'status' => 'running:healthy',
+ ]);
+ $backup = ScheduledDatabaseBackup::create([
+ 'frequency' => '0 0 * * *',
+ 'save_s3' => false,
+ 'database_type' => StandaloneMysql::class,
+ 'database_id' => $database->id,
+ 'team_id' => $team->id,
+ 'databases_to_backup' => 'app',
+ 'timeout' => 600,
+ ]);
+ Process::fake(function ($process) {
+ if (str_contains($process->command, 'mysqldump')) {
+ throw backupRedactionTimeoutException($process->command, 600);
+ }
+
+ return Process::result(output: '');
+ });
+
+ try {
+ (new DatabaseBackupJob($backup))->handle();
+ } catch (Throwable) {
+ // The job reports the failure after it records the execution.
+ }
+
+ $execution = $backup->executions()->latest('id')->firstOrFail();
+ expect($execution->status)->toBe('failed')
+ ->and($execution->message)->toBe('The command on the server timed out after 600 seconds.');
+ Notification::assertSentTo($team, BackupFailed::class, fn (BackupFailed $notification) => ! str_contains((string) $notification->output, 'TopSecretPass')
+ && str_contains((string) $notification->output, 'timed out after 600 seconds'));
+});