An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(deployments): keep secrets out of failed command logs
Andras Bacsai1e207292b6cc
diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.phpindex db704f1dc4..994d9ad657 100644--- a/app/Jobs/ApplicationDeploymentJob.php+++ b/app/Jobs/ApplicationDeploymentJob.php@@ -769,6 +769,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue$this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->docker_compose_base64}' | base64 -d | tee {$this->workdir}{$this->docker_compose_location} > /dev/null"),'hidden' => true,+ 'skip_command_log' => true,]);// Modify Dockerfiles for ARGs and build secrets@@ -1151,6 +1152,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue],["echo '{$this->docker_compose_base64}' | base64 -d | tee $composeFileName > /dev/null",+ 'skip_command_log' => true,],["echo '{$readme}' > $mainDir/README.md",@@ -1774,12 +1776,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue);if (isDev()) {- $this->execute_remote_command(- [- executeInDocker($this->deployment_uuid, "cat $this->workdir/.env"),- 'hidden' => true,- ]- );+ $this->logEnvironmentFileKeys("{$this->workdir}/.env", $environment_variables);}// Write .env file to configuration directory@@ -1802,6 +1799,20 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue}}+ /**+ * Development aid: log which variables an env file contains, without their values.+ *+ * @param Collection<int, string> $environmentVariables Lines in KEY=VALUE format.+ */+ private function logEnvironmentFileKeys(string $path, Collection $environmentVariables): void+ {+ $keys = $environmentVariables+ ->map(fn (string $line): string => explode('=', $line, 2)[0])+ ->implode(', ');++ $this->application_deployment_queue->addLogEntry("[DEBUG] Variable names in {$path}: {$keys}", hidden: true);+ }+private function generate_buildtime_environment_variables(){if (isDev()) {@@ -2067,38 +2078,60 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue}/**- * Build-time names go into shell and Docker build commands, so they must be valid. Runtime-only- * variables only go into the .env file: existing ones with names that new variables can no longer- * use (such as my-var) keep working, unless the name would break a .env line.+ * Build-time names go into shell and Docker build commands, so they must be valid when the+ * deployment builds an image. Runtime-only variables only go into the .env file: existing ones+ * with names that new variables can no longer use (such as my-var) keep working, unless the+ * name would break a .env line. Deployments without a build step (Docker image) treat+ * build-time variables the same way, because no build receives them.*/private function validateDeploymentEnvironmentVariableKeys(): void{$environmentVariables = $this->pull_request_id === 0- ? $this->application->environment_variables()->get(['key', 'is_buildtime'])- : $this->application->environment_variables_preview()->get(['key', 'is_buildtime']);+ ? $this->application->environment_variables()->get(['key', 'is_buildtime', 'is_runtime'])+ : $this->application->environment_variables_preview()->get(['key', 'is_buildtime', 'is_runtime']);+ $passesBuildtimeVariables = $this->deploymentPassesBuildtimeVariables();foreach ($environmentVariables as $environmentVariable) {$key = (string) $environmentVariable->key;$isEnvFileSafe = $key !== '' && strpbrk($key, "=\n\r\0") === false;- if ($environmentVariable->is_buildtime || ! $isEnvFileSafe) {+ if (($environmentVariable->is_buildtime && $passesBuildtimeVariables) || ! $isEnvFileSafe) {$this->validatedBuildtimeEnvironmentVariableKey($key, 'the deployment environment');continue;}if (! ValidationPatterns::isValidEnvironmentVariableKey($key)) {- $this->logLegacyRuntimeEnvironmentVariableKey($key);+ $this->logLegacyEnvironmentVariableKey($key, (bool) $environmentVariable->is_buildtime, (bool) $environmentVariable->is_runtime);}}}- private function logLegacyRuntimeEnvironmentVariableKey(string $key): void+ /**+ * Only Docker image deployments never build an image. Other deployments (also restarts,+ * rollbacks, and previews) can build and pass build-time variables to the build.+ */+ private function deploymentPassesBuildtimeVariables(): bool+ {+ return $this->application->build_pack !== 'dockerimage';+ }++ private function logLegacyEnvironmentVariableKey(string $key, bool $isBuildtime, bool $isRuntime): void{$suggestedKey = (string) preg_replace('/[^A-Za-z0-9_]/', '_', $key);if (preg_match('/\A[0-9]/', $suggestedKey) === 1) {$suggestedKey = '_'.$suggestedKey;}- $this->application_deployment_queue->addLogEntry('⚠️ Runtime variable '.ValidationPatterns::displayShellEnvironmentVariableKey($key).' uses a name that new variables cannot use. It is still passed to the container, but shell scripts cannot read it.', 'stderr');+ $displayKey = ValidationPatterns::displayShellEnvironmentVariableKey($key);++ if ($isBuildtime) {+ $this->application_deployment_queue->addLogEntry("⚠️ Build-time variable {$displayKey} uses a name that new variables cannot use. This deployment does not build an image, so it is not used as a build-time variable. Rename it before you use it in a build.", 'stderr');+ }+ if ($isRuntime) {+ $this->application_deployment_queue->addLogEntry("⚠️ Runtime variable {$displayKey} uses a name that new variables cannot use. It is still passed to the container, but shell scripts cannot read it.", 'stderr');+ }+ if (! $isBuildtime && ! $isRuntime) {+ $this->application_deployment_queue->addLogEntry("⚠️ Variable {$displayKey} uses a name that new variables cannot use. This deployment does not use it.", 'stderr');+ }$this->application_deployment_queue->addLogEntry(' Suggested name: '.ValidationPatterns::displayShellEnvironmentVariableKey($suggestedKey), type: 'info');}@@ -2159,10 +2192,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue]);if (isDev()) {- $this->execute_remote_command([- executeInDocker($this->deployment_uuid, 'cat '.self::BUILD_TIME_ENV_PATH),- 'hidden' => true,- ]);+ $this->logEnvironmentFileKeys(self::BUILD_TIME_ENV_PATH, $environment_variables);}} elseif (in_array($this->build_pack, ['dockercompose', 'dockerfile', 'railpack'], true)) {$this->application_deployment_queue->addLogEntry('Creating empty build-time .env file in /artifacts (no build-time variables defined).', hidden: true);@@ -2527,6 +2557,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue[$runCommand,'hidden' => true,+ 'skip_command_log' => filled($env_flags),],['command' => executeInDocker($this->deployment_uuid, "mkdir -p {$this->basedir}"),@@ -3325,7 +3356,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue$this->application_deployment_queue->addLogEntry('Generating Railpack build plan.');$this->execute_remote_command(- [executeInDocker($this->deployment_uuid, $prepare_command), 'hidden' => true],+ [executeInDocker($this->deployment_uuid, $prepare_command), 'hidden' => true, 'skip_command_log' => true],[executeInDocker($this->deployment_uuid, 'cat /artifacts/railpack-plan.json'),'hidden' => true,@@ -3813,7 +3844,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");$this->docker_compose = Yaml::dump($docker_compose, 10);$this->docker_compose_base64 = base64_encode($this->docker_compose);- $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->docker_compose_base64}' | base64 -d | tee {$this->workdir}/docker-compose.yaml > /dev/null"), 'hidden' => true]);+ $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->docker_compose_base64}' | base64 -d | tee {$this->workdir}/docker-compose.yaml > /dev/null"), 'hidden' => true, 'skip_command_log' => true]);}private function generate_local_persistent_volumes()@@ -4044,7 +4075,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");}if ($this->application->build_pack === 'nixpacks') {$this->nixpacks_plan = base64_encode($this->nixpacks_plan);- $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->nixpacks_plan}' | base64 -d | tee ".self::NIXPACKS_PLAN_PATH.' > /dev/null'), 'hidden' => true]);+ $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->nixpacks_plan}' | base64 -d | tee ".self::NIXPACKS_PLAN_PATH.' > /dev/null'), 'hidden' => true, 'skip_command_log' => true]);if ($this->force_rebuild) {$this->execute_remote_command([executeInDocker($this->deployment_uuid, 'nixpacks build -c '.self::NIXPACKS_PLAN_PATH." --no-cache --no-error-without-start -n {$this->build_image_name} {$this->workdir} -o {$this->workdir}"),@@ -4230,7 +4261,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");} else {if ($this->application->build_pack === 'nixpacks') {$this->nixpacks_plan = base64_encode($this->nixpacks_plan);- $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->nixpacks_plan}' | base64 -d | tee ".self::NIXPACKS_PLAN_PATH.' > /dev/null'), 'hidden' => true]);+ $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->nixpacks_plan}' | base64 -d | tee ".self::NIXPACKS_PLAN_PATH.' > /dev/null'), 'hidden' => true, 'skip_command_log' => true]);if ($this->force_rebuild) {$this->execute_remote_command([executeInDocker($this->deployment_uuid, 'nixpacks build -c '.self::NIXPACKS_PLAN_PATH." --no-cache --no-error-without-start -n {$this->production_image_name} {$this->workdir} -o {$this->workdir}"),@@ -4579,8 +4610,8 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");private function generate_docker_env_flags_for_secrets(){- // Only generate env flags if build secrets are enabled- if (! $this->application->settings->use_build_secrets) {+ // Only generate env flags if build secrets are enabled and the deployment builds an image+ if (! $this->application->settings->use_build_secrets || ! $this->deploymentPassesBuildtimeVariables()) {return '';}diff --git a/app/Traits/ExecuteRemoteCommand.php b/app/Traits/ExecuteRemoteCommand.phpindex 0d8dfdc29f..5a4540ea3a 100644--- a/app/Traits/ExecuteRemoteCommand.php+++ b/app/Traits/ExecuteRemoteCommand.php@@ -16,6 +16,8 @@ trait ExecuteRemoteCommand{use SshRetryable;+ private const SENSITIVE_COMMAND_PLACEHOLDER = '[command hidden because it contains sensitive data]';+public ?string $save = null;public static int $batch_counter = 0;@@ -184,7 +186,7 @@ trait ExecuteRemoteCommand$new_log_entry = ['command' => $skip_command_log || $command_hidden ? null : $this->redact_sensitive_info($command),- 'output' => $this->redact_sensitive_info($log_output),+ 'output' => $this->redact_sensitive_info($skip_command_log ? $this->redactSensitiveCommandPayloads((string) $log_output, (string) $command) : $log_output),'type' => $customType ?? ($type === 'err' ? 'stderr' : 'stdout'),'timestamp' => Carbon::now('UTC'),'hidden' => $hidden,@@ -241,12 +243,54 @@ trait ExecuteRemoteCommandif (empty($error)) {$error = $process_result->output() ?: 'Command failed with no error output';}- $redactedCommand = $this->redact_sensitive_info($command);- throw new DeploymentException("Command execution failed (exit code {$process_result->exitCode()}): {$redactedCommand}\nError: {$error}");+ throw new DeploymentException($this->commandFailureMessage((string) $command, (int) $process_result->exitCode(), (string) $error, $skip_command_log));}}}+ /**+ * Commands marked with skip_command_log embed secrets (for example base64 encoded .env files or+ * private keys), so their text must never reach a log line or an exception message.+ */+ private function commandFailureMessage(string $command, int $exitCode, string $error, bool $isSensitiveCommand): string+ {+ if ($isSensitiveCommand) {+ $commandText = self::SENSITIVE_COMMAND_PLACEHOLDER;+ $error = $this->redactSensitiveCommandPayloads($error, $command);+ } else {+ $commandText = $this->redact_sensitive_info($command);+ }++ $error = $this->redact_sensitive_info($error);++ return "Command execution failed (exit code {$exitCode}): {$commandText}\nError: {$error}";+ }++ /**+ * Removes the encoded payloads of a sensitive command from output that could echo the command.+ */+ private function redactSensitiveCommandPayloads(string $text, string $command): string+ {+ if ($text === '' || preg_match_all('~[A-Za-z0-9+/]{16,}={0,2}~', $command, $matches) === false) {+ return $text;+ }++ $payloads = collect($matches[0])+ ->unique()+ ->filter(function (string $candidate): bool {+ $decoded = base64_decode($candidate, true);++ return $decoded !== false+ && mb_check_encoding($decoded, 'UTF-8')+ && preg_match('/[^\P{C}\t\n\r]/u', $decoded) !== 1;+ })+ ->sortByDesc(fn (string $payload): int => strlen($payload))+ ->values()+ ->all();++ return $payloads === [] ? $text : str_replace($payloads, REDACTED, $text);+ }+private function saveCommandOutput(string $output, bool $append): void{if (! $this->save) {diff --git a/tests/Feature/ApplicationDeploymentControlVarFilteringTest.php b/tests/Feature/ApplicationDeploymentControlVarFilteringTest.phpindex 1aa668e116..5f61f78c47 100644--- a/tests/Feature/ApplicationDeploymentControlVarFilteringTest.php+++ b/tests/Feature/ApplicationDeploymentControlVarFilteringTest.php@@ -1030,6 +1030,94 @@ it('rejects existing variable names that would break the .env file or build comm'build-time name with a hyphen' => ['my-var', true],]);+it('warns instead of failing for invalid build-time names when the deployment does not build an image', function (bool $isRuntime) {+ [$application, $server] = makeDeploymentControlVarFixture(['build_pack' => 'dockerimage']);+ $environmentVariable = createApplicationEnvironmentVariable($application, [+ 'key' => 'SAFE_KEY',+ 'value' => 'secret',+ 'is_buildtime' => true,+ 'is_runtime' => $isRuntime,+ ]);+ DB::table('environment_variables')->where('id', $environmentVariable->id)->update(['key' => 'my-var']);++ [$job, $reflection] = makeControlVarFilteringJob($application->fresh(), $server);+ invokeDeploymentJobMethod($job, $reflection, 'validateDeploymentEnvironmentVariableKeys');++ expect(collect($job->recordedLogEntries)->implode("\n"))+ ->toContain('my-var')+ ->toContain('Suggested name: my_var')+ ->not->toContain('Invalid environment variable name');+ expect($job->recordedCommands)->toBeEmpty();+})->with([+ 'build-time only' => [false],+ 'build-time and runtime' => [true],+]);++it('warns instead of failing for invalid build-time preview names of Docker image deployments', function () {+ [$application, $server] = makeDeploymentControlVarFixture(['build_pack' => 'dockerimage']);+ $environmentVariable = createApplicationEnvironmentVariable($application, [+ 'key' => 'SAFE_KEY',+ 'value' => 'secret',+ 'is_buildtime' => true,+ 'is_preview' => true,+ ]);+ DB::table('environment_variables')->where('id', $environmentVariable->id)->update(['key' => 'my-var']);++ [$job, $reflection] = makeControlVarFilteringJob($application->fresh(), $server, ['pull_request_id' => 7]);+ invokeDeploymentJobMethod($job, $reflection, 'validateDeploymentEnvironmentVariableKeys');++ expect(collect($job->recordedLogEntries)->implode("\n"))+ ->toContain('Suggested name: my_var')+ ->not->toContain('Invalid environment variable name');+});++it('still rejects Docker image variable names that would break the .env file', function () {+ [$application, $server] = makeDeploymentControlVarFixture(['build_pack' => 'dockerimage']);+ $environmentVariable = createApplicationEnvironmentVariable($application, [+ 'key' => 'SAFE_KEY',+ 'value' => 'secret',+ 'is_buildtime' => true,+ ]);+ DB::table('environment_variables')->where('id', $environmentVariable->id)->update(['key' => 'A=B']);++ [$job, $reflection] = makeControlVarFilteringJob($application->fresh(), $server);++ expect(fn () => invokeDeploymentJobMethod($job, $reflection, 'validateDeploymentEnvironmentVariableKeys'))+ ->toThrow(DeploymentException::class, 'Invalid environment variable name from the deployment environment');+});++it('fails with a clear message for invalid build-time names when the deployment builds an image', function (string $buildPack) {+ [$application, $server] = makeDeploymentControlVarFixture(['build_pack' => $buildPack]);+ $environmentVariable = createApplicationEnvironmentVariable($application, [+ 'key' => 'SAFE_KEY',+ 'value' => 'secret',+ 'is_buildtime' => true,+ ]);+ DB::table('environment_variables')->where('id', $environmentVariable->id)->update(['key' => 'my-var']);++ [$job, $reflection] = makeControlVarFilteringJob($application->fresh(), $server, ['build_pack' => $buildPack]);++ expect(fn () => invokeDeploymentJobMethod($job, $reflection, 'validateDeploymentEnvironmentVariableKeys'))+ ->toThrow(DeploymentException::class, 'Invalid environment variable name from the deployment environment: my-var');+ expect(collect($job->recordedLogEntries)->implode("\n"))+ ->toContain('Build-time variable names must start with a letter or underscore');+})->with(['dockerfile', 'nixpacks', 'static', 'railpack', 'dockercompose']);++it('does not pass build-time variables to the helper container of Docker image deployments', function () {+ [$application, $server] = makeDeploymentControlVarFixture(['build_pack' => 'dockerimage']);+ $application->settings()->update(['use_build_secrets' => true]);+ $environmentVariable = createApplicationEnvironmentVariable($application, [+ 'key' => 'SAFE_KEY',+ 'value' => 'harmless-build-value',+ 'is_buildtime' => true,+ ]);+ DB::table('environment_variables')->where('id', $environmentVariable->id)->update(['key' => 'my-var']);++ [$job, $reflection] = makeControlVarFilteringJob($application->fresh(), $server);++ expect(invokeDeploymentJobMethod($job, $reflection, 'generate_docker_env_flags_for_secrets'))->toBe('');+});+it('injects raw escaped remote secrets into Dockerfile args and hashes the same values', function (int $pullRequestId, bool $isPreview) {[$application, $server] = makeDeploymentControlVarFixture();diff --git a/tests/Feature/DeploymentSensitiveCommandFailureTest.php b/tests/Feature/DeploymentSensitiveCommandFailureTest.phpnew file mode 100644index 0000000000..2fa3798d2e--- /dev/null+++ b/tests/Feature/DeploymentSensitiveCommandFailureTest.php@@ -0,0 +1,449 @@+<?php++use App\Exceptions\DeploymentException;+use App\Jobs\ApplicationDeploymentJob;+use App\Models\Application;+use App\Models\ApplicationDeploymentQueue;+use App\Models\Environment;+use App\Models\EnvironmentVariable;+use App\Models\PrivateKey;+use App\Models\Project;+use App\Models\Server;+use App\Models\StandaloneDocker;+use App\Models\User;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Process\PendingProcess;+use Illuminate\Support\Collection;+use Illuminate\Support\Facades\Process;+use Illuminate\Support\Facades\Storage;++uses(RefreshDatabase::class);++const SENSITIVE_FAILURE_SECRET = 'harmless-secret-marker-7431';++class SensitiveCommandFailureDeploymentJob extends ApplicationDeploymentJob+{+ public function __construct() {}+}++beforeEach(function () {+ config()->set('constants.ssh.mux_enabled', false);+ config()->set('constants.ssh.max_retries', 1);+});++/**+ * @return array{0: SensitiveCommandFailureDeploymentJob, 1: ReflectionClass, 2: ApplicationDeploymentQueue, 3: Application}+ */+function makeSensitiveCommandFailureJob(array $environmentVariables = []): array+{+ $user = User::factory()->create();+ $team = $user->teams()->first();++ $privateKeyContent = "-----BEGIN OPENSSH PRIVATE KEY-----\n"+ ."b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW\n"+ ."QyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevAAAAJi/QySHv0Mk\n"+ ."hwAAAAtzc2gtZWQyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevA\n"+ ."AAAECBQw4jg1WRT2IGHMncCiZhURCts2s24HoDS0thHnnRKVuGmoeGq/pojrsyP1pszcNV\n"+ ."uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==\n"+ .'-----END OPENSSH PRIVATE KEY-----';++ $privateKey = PrivateKey::create([+ 'name' => 'sensitive-failure-key-'.uniqid(),+ 'private_key' => $privateKeyContent,+ 'team_id' => $team->id,+ ]);++ Storage::fake('ssh-keys');+ Storage::disk('ssh-keys')->put("ssh_key@{$privateKey->uuid}", $privateKeyContent);++ $server = Server::factory()->create([+ 'team_id' => $team->id,+ 'private_key_id' => $privateKey->id,+ 'user' => 'root',+ ]);++ $project = Project::create(['name' => 'Sensitive Failure Project', 'team_id' => $team->id]);+ $environment = Environment::where('project_id', $project->id)->firstOrFail();+ $application = Application::factory()->create([+ 'environment_id' => $environment->id,+ 'build_pack' => 'dockerfile',+ ]);+ $application->settings()->update([+ 'include_source_commit_in_build' => false,+ 'is_env_sorting_enabled' => false,+ ]);++ foreach ($environmentVariables as $attributes) {+ EnvironmentVariable::create([+ 'resourceable_type' => Application::class,+ 'resourceable_id' => $application->id,+ 'is_preview' => false,+ 'is_runtime' => true,+ 'is_buildtime' => true,+ 'is_multiline' => false,+ 'is_literal' => false,+ ...$attributes,+ ]);+ }++ $queue = ApplicationDeploymentQueue::create([+ 'deployment_uuid' => 'sensitive-failure-deployment',+ 'application_id' => $application->id,+ 'server_id' => $server->id,+ 'status' => 'in_progress',+ ]);++ $job = new SensitiveCommandFailureDeploymentJob;+ $reflection = new ReflectionClass(ApplicationDeploymentJob::class);++ foreach ([+ 'application' => $application->fresh(),+ 'application_deployment_queue' => $queue,+ 'server' => $server,+ 'mainServer' => $server,+ 'build_pack' => 'dockerfile',+ 'pull_request_id' => 0,+ 'commit' => 'HEAD',+ 'basedir' => '/artifacts/sensitive-failure',+ 'workdir' => '/artifacts/sensitive-failure',+ 'configuration_dir' => '/data/coolify/applications/sensitive-failure',+ 'deployment_uuid' => 'sensitive-failure-deployment',+ 'dockerfile_location' => '/Dockerfile',+ 'container_name' => 'sensitive-failure-app',+ 'coolify_variables' => null,+ 'dockerSecretsSupported' => false,+ 'saved_outputs' => new Collection,+ ] as $property => $value) {+ $reflectionProperty = $reflection->getProperty($property);+ $reflectionProperty->setAccessible(true);+ $reflectionProperty->setValue($job, $value);+ }++ return [$job, $reflection, $queue, $application];+}++function invokeSensitiveFailureJobMethod(object $job, ReflectionClass $reflection, string $method, mixed ...$arguments): mixed+{+ $reflectionMethod = $reflection->getMethod($method);+ $reflectionMethod->setAccessible(true);++ return $reflectionMethod->invoke($job, ...$arguments);+}++function captureDeploymentException(callable $callback): DeploymentException+{+ try {+ $callback();+ } catch (DeploymentException $exception) {+ return $exception;+ }++ throw new RuntimeException('Expected a DeploymentException.');+}++/**+ * @return list<string>+ */+function base64PayloadsInCommand(string $command): array+{+ preg_match_all('~[A-Za-z0-9+/]{16,}={0,2}~', $command, $matches);++ return array_values(array_filter(+ $matches[0],+ static fn (string $candidate): bool => str_contains((string) base64_decode($candidate, true), SENSITIVE_FAILURE_SECRET)+ ));+}++it('hides the .env write command and its base64 payload when the write fails', function () {+ [$job, $reflection, $queue] = makeSensitiveCommandFailureJob([+ ['key' => 'APP_SECRET', 'value' => SENSITIVE_FAILURE_SECRET],+ ]);++ $payloads = [];+ Process::fake(function (PendingProcess $process) use (&$payloads) {+ $payloads = array_merge($payloads, base64PayloadsInCommand($process->command));++ return Process::result(errorOutput: 'tee: /artifacts/sensitive-failure/.env: No space left on device', exitCode: 1);+ });++ $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'save_runtime_environment_variables'));++ expect($payloads)->not->toBeEmpty();+ expect($exception->getMessage())+ ->toContain('Command execution failed (exit code 1): [command hidden because it contains sensitive data]')+ ->toContain('No space left on device')+ ->not->toContain(SENSITIVE_FAILURE_SECRET)+ ->not->toContain('base64 -d');+ foreach ($payloads as $payload) {+ expect($exception->getMessage())->not->toContain($payload);+ }++ $storedLogs = (string) $queue->fresh()->logs;+ expect($storedLogs)+ ->toContain('No space left on device')+ ->not->toContain(SENSITIVE_FAILURE_SECRET);+ foreach ($payloads as $payload) {+ expect($storedLogs)->not->toContain($payload);+ }+});++it('redacts the sensitive payload when the error output echoes the command', function () {+ [$job, $reflection, $queue] = makeSensitiveCommandFailureJob([+ ['key' => 'APP_SECRET', 'value' => SENSITIVE_FAILURE_SECRET],+ ]);++ $payloads = [];+ Process::fake(function (PendingProcess $process) use (&$payloads) {+ $commandPayloads = base64PayloadsInCommand($process->command);+ $payloads = array_merge($payloads, $commandPayloads);++ return Process::result(errorOutput: "bash: line 1: echo '".($commandPayloads[0] ?? '')."': unexpected failure", exitCode: 2);+ });++ $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'save_runtime_environment_variables'));++ expect($payloads)->not->toBeEmpty();+ $storedLogs = (string) $queue->fresh()->logs;+ foreach ($payloads as $payload) {+ expect($exception->getMessage())->not->toContain($payload);+ expect($storedLogs)->not->toContain($payload);+ }+ expect($exception->getMessage())->toContain('unexpected failure');+});++it('hides the build-time env write command when the write fails', function () {+ [$job, $reflection] = makeSensitiveCommandFailureJob([+ ['key' => 'BUILD_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_runtime' => false],+ ]);++ $payloads = [];+ Process::fake(function (PendingProcess $process) use (&$payloads) {+ $payloads = array_merge($payloads, base64PayloadsInCommand($process->command));++ return Process::result(errorOutput: 'tee: /artifacts/build-time.env: No such file or directory', exitCode: 1);+ });++ $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'save_buildtime_environment_variables'));++ expect($payloads)->not->toBeEmpty();+ expect($exception->getMessage())+ ->toContain('[command hidden because it contains sensitive data]')+ ->toContain('No such file or directory')+ ->not->toContain(SENSITIVE_FAILURE_SECRET);+ foreach ($payloads as $payload) {+ expect($exception->getMessage())->not->toContain($payload);+ }+});++it('keeps the command in the error message for failed commands that are not sensitive', function () {+ [$job] = makeSensitiveCommandFailureJob();++ Process::fake(['*' => Process::result(errorOutput: 'ls: cannot access: No such file or directory', exitCode: 2)]);++ $exception = captureDeploymentException(fn () => $job->execute_remote_command([+ 'command' => 'ls /artifacts/harmless-missing-directory',+ 'hidden' => true,+ ]));++ expect($exception->getMessage())+ ->toContain('Command execution failed (exit code 2): ls /artifacts/harmless-missing-directory')+ ->toContain('No such file or directory')+ ->not->toContain('[command hidden because it contains sensitive data]');+});++it('redacts locked values from the error output of failed commands', function () {+ [$job] = makeSensitiveCommandFailureJob([+ ['key' => 'LOCKED_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_shown_once' => true],+ ]);++ Process::fake(['*' => Process::result(errorOutput: 'failed with value '.SENSITIVE_FAILURE_SECRET, exitCode: 1)]);++ $exception = captureDeploymentException(fn () => $job->execute_remote_command([+ 'command' => 'harmless-command --flag',+ ]));++ expect($exception->getMessage())+ ->toContain('harmless-command --flag')+ ->toContain('failed with value '.REDACTED)+ ->not->toContain(SENSITIVE_FAILURE_SECRET);+});++it('logs only the variable names of the .env files in development mode', function () {+ config()->set('app.env', 'local');+ [$job, $reflection, $queue] = makeSensitiveCommandFailureJob([+ ['key' => 'APP_SECRET', 'value' => SENSITIVE_FAILURE_SECRET],+ ]);++ $ranCommands = [];+ Process::fake(function (PendingProcess $process) use (&$ranCommands) {+ $ranCommands[] = $process->command;++ return Process::result(output: str_contains($process->command, 'cat ') ? 'APP_SECRET='.SENSITIVE_FAILURE_SECRET : '');+ });++ invokeSensitiveFailureJobMethod($job, $reflection, 'save_runtime_environment_variables');+ invokeSensitiveFailureJobMethod($job, $reflection, 'save_buildtime_environment_variables');++ $logEntries = collect(json_decode((string) $queue->fresh()->logs, true));+ $keyListEntries = $logEntries->filter(fn (array $entry): bool => str_contains((string) $entry['output'], '[DEBUG] Variable names in'));++ expect($keyListEntries)->toHaveCount(2)+ ->each(fn ($entry) => $entry->hidden->toBeTrue()->output->toContain('APP_SECRET'));+ expect($logEntries->pluck('output')->merge($logEntries->pluck('command'))->filter()->implode("\n"))+ ->not->toContain('APP_SECRET='.SENSITIVE_FAILURE_SECRET)+ ->not->toContain('APP_SECRET="'.SENSITIVE_FAILURE_SECRET);+ expect(collect($ranCommands)->filter(fn (string $command): bool => str_contains($command, 'cat /artifacts/sensitive-failure/.env') || str_contains($command, 'cat '.ApplicationDeploymentJob::BUILD_TIME_ENV_PATH)))+ ->toBeEmpty();+});++function setSensitiveFailureJobProperties(object $job, ReflectionClass $reflection, array $properties): void+{+ foreach ($properties as $property => $value) {+ $reflectionProperty = $reflection->getProperty($property);+ $reflectionProperty->setAccessible(true);+ $reflectionProperty->setValue($job, $value);+ }+}++it('hides the helper container command when build secrets are passed as environment flags', function () {+ [$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob([+ ['key' => 'BUILD_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_runtime' => false],+ ]);+ $application->settings()->update(['use_build_secrets' => true]);+ $server = readSensitiveFailureJobProperty($job, $reflection, 'server');+ setSensitiveFailureJobProperties($job, $reflection, [+ 'application' => $application->fresh(),+ 'destination' => StandaloneDocker::forceCreate([+ 'name' => 'sensitive-failure-network',+ 'network' => 'sensitive-failure-network',+ 'server_id' => $server->id,+ ]),+ ]);++ Process::fake(function (PendingProcess $process) {+ if (str_contains($process->command, 'docker run -d')) {+ return Process::result(errorOutput: 'docker: Error response from daemon: network not found.', exitCode: 125);+ }++ return Process::result(output: str_contains($process->command, 'echo $HOME') ? '/root' : 'NOK');+ });++ $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'prepare_builder_image'));++ expect($exception->getMessage())+ ->toContain('[command hidden because it contains sensitive data]')+ ->toContain('network not found')+ ->not->toContain(SENSITIVE_FAILURE_SECRET);+ expect((string) $queue->fresh()->logs)->not->toContain(SENSITIVE_FAILURE_SECRET);+});++it('hides the Railpack prepare command that passes build-time values', function () {+ [$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob([+ ['key' => 'BUILD_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_runtime' => false],+ ]);+ $application->update(['build_pack' => 'railpack']);+ setSensitiveFailureJobProperties($job, $reflection, [+ 'application' => $application->fresh(),+ 'build_pack' => 'railpack',+ 'dockerBuildxAvailable' => true,+ ]);++ Process::fake(function (PendingProcess $process) {+ if (str_contains($process->command, 'railpack prepare')) {+ return Process::result(errorOutput: 'railpack: failed to detect a provider', exitCode: 1);+ }++ return Process::result(output: str_contains($process->command, 'buildx version') ? 'available' : 'missing');+ });++ $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'build_railpack_image'));++ expect($exception->getMessage())+ ->toContain('[command hidden because it contains sensitive data]')+ ->toContain('failed to detect a provider')+ ->not->toContain(SENSITIVE_FAILURE_SECRET);+ expect((string) $queue->fresh()->logs)->not->toContain(SENSITIVE_FAILURE_SECRET);+});++it('hides the Nixpacks plan write command that contains build-time values', function (bool $isStatic) {+ [$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob();+ $application->update(['build_pack' => 'nixpacks']);+ $application->settings()->update(['is_static' => $isStatic]);+ setSensitiveFailureJobProperties($job, $reflection, [+ 'application' => $application->fresh(),+ 'build_pack' => 'nixpacks',+ 'build_args' => collect(),+ 'disableBuildCache' => false,+ 'force_rebuild' => false,+ 'nixpacks_plan' => json_encode(['variables' => ['BUILD_SECRET' => SENSITIVE_FAILURE_SECRET]]),+ ]);++ $payloads = [];+ Process::fake(function (PendingProcess $process) use (&$payloads) {+ $commandPayloads = base64PayloadsInCommand($process->command);+ if ($commandPayloads !== []) {+ $payloads = array_merge($payloads, $commandPayloads);++ return Process::result(errorOutput: 'tee: /artifacts/thegameplan.json: No space left on device', exitCode: 1);+ }++ return Process::result();+ });++ $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'build_image'));++ expect($payloads)->not->toBeEmpty();+ expect($exception->getMessage())+ ->toContain('[command hidden because it contains sensitive data]')+ ->not->toContain(SENSITIVE_FAILURE_SECRET);+ $storedLogs = (string) $queue->fresh()->logs;+ foreach ($payloads as $payload) {+ expect($exception->getMessage())->not->toContain($payload);+ expect($storedLogs)->not->toContain($payload);+ }+})->with([+ 'static' => [true],+ 'not static' => [false],+]);++it('hides the compose file write command because compose files can contain secrets', function () {+ [$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob();+ $server = readSensitiveFailureJobProperty($job, $reflection, 'server');+ setSensitiveFailureJobProperties($job, $reflection, [+ 'destination' => $server->standaloneDockers()->firstOrFail(),+ 'production_image_name' => 'example/app:latest',+ ]);++ $payloads = [];+ Process::fake(function (PendingProcess $process) use (&$payloads) {+ preg_match_all('~[A-Za-z0-9+/]{16,}={0,2}~', $process->command, $matches);+ $composePayloads = array_filter($matches[0], fn (string $candidate): bool => str_contains((string) base64_decode($candidate, true), 'services:'));+ if ($composePayloads !== []) {+ $payloads = array_merge($payloads, $composePayloads);++ return Process::result(errorOutput: 'tee: docker-compose.yaml: No space left on device', exitCode: 1);+ }++ return Process::result();+ });++ $exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'generate_compose_file'));++ expect($payloads)->not->toBeEmpty();+ expect($exception->getMessage())+ ->toContain('[command hidden because it contains sensitive data]')+ ->toContain('No space left on device');+ $storedLogs = (string) $queue->fresh()->logs;+ foreach ($payloads as $payload) {+ expect($exception->getMessage())->not->toContain($payload);+ expect($storedLogs)->not->toContain($payload);+ }+});++function readSensitiveFailureJobProperty(object $job, ReflectionClass $reflection, string $property): mixed+{+ $reflectionProperty = $reflection->getProperty($property);+ $reflectionProperty->setAccessible(true);++ return $reflectionProperty->getValue($job);+}