An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(deployments): prevent Git fetches from hanging (#12321)
Andras Bacsaiddeb05280e57
diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.phpindex bdc84bc6a6..d954a62f7c 100644--- a/app/Jobs/ApplicationDeploymentJob.php+++ b/app/Jobs/ApplicationDeploymentJob.php@@ -3171,11 +3171,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueueprivate function gitLsRemoteCommand(string $lsRemoteRef, ?string $identityFile = null): string{- $sshCommand = "ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";-- if ($identityFile !== null) {- $sshCommand .= " -i {$identityFile} -o IdentitiesOnly=yes";- }+ $sshCommand = gitSshCommand($this->customPort, $identityFile);return 'GIT_SSH_COMMAND="'.$sshCommand.'" git ls-remote '.escapeshellarg($this->fullRepoUrl).' '.escapeshellarg($lsRemoteRef);}diff --git a/app/Models/Application.php b/app/Models/Application.phpindex 22a8a67041..a183d412cc 100644--- a/app/Models/Application.php+++ b/app/Models/Application.php@@ -1587,7 +1587,7 @@ class Application extends BaseModel// If shallow clone is enabled and we need a specific commit,// we need to fetch that specific commit with depth=1if ($isShallowCloneEnabled) {- $git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && {$sshCommand} {$gitCommand} fetch --depth=1 origin {$escapedCommit} && {$gitCommand} -c advice.detachedHead=false checkout {$escapedCommit} >/dev/null 2>&1";+ $git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && echo 'Fetching the commit.' && {$sshCommand} {$gitCommand} fetch --depth=1 origin {$escapedCommit} && {$gitCommand} -c advice.detachedHead=false checkout {$escapedCommit} >/dev/null 2>&1";} else {$git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && {$sshCommand} {$gitCommand} -c advice.detachedHead=false checkout {$escapedCommit} >/dev/null 2>&1";}@@ -1601,10 +1601,12 @@ class Application extends BaseModel}// Add shallow submodules flag if shallow clone is enabled$submoduleFlags = $isShallowCloneEnabled ? '--depth=1' : '';- $git_clone_command = "{$git_clone_command} {$gitCommand} submodule sync && {$sshCommand} {$gitCommand} submodule update --init --recursive {$submoduleFlags}; fi";+ $git_clone_command = "{$git_clone_command} echo 'Updating submodules.' && {$gitCommand} submodule sync && {$sshCommand} {$gitCommand} submodule update --init --recursive {$submoduleFlags}; fi";}+ // `git lfs pull` opens an SSH session even without LFS files, which can hang on hosts+ // without LFS over SSH (Azure DevOps). `git lfs ls-files` is offline.if ($this->settings->is_git_lfs_enabled && ! $onlyCheckout) {- $git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && {$sshCommand} {$gitCommand} lfs pull";+ $git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && if {$gitCommand} lfs ls-files -n | grep -q .; then echo 'Downloading Git LFS objects.' && {$sshCommand} {$gitCommand} lfs pull; fi";}return $git_clone_command;@@ -1714,7 +1716,8 @@ class Application extends BaseModel$private_key = base64_encode($private_key);$gitlabPort = $oauthSource->custom_port ?? 22;$escapedCustomRepository = str_replace("'", "'\\''", $customRepository);- $base_command = "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$gitlabPort} -o Port={$gitlabPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$customSshKeyLocation} -o IdentitiesOnly=yes\" {$base_command} '{$escapedCustomRepository}'";+ $sshCommand = gitSshCommand($gitlabPort, $customSshKeyLocation);+ $base_command = "GIT_SSH_COMMAND=\"{$sshCommand}\" {$base_command} '{$escapedCustomRepository}'";$commands = $this->gitSshKeySetupCommands($deployment_uuid, $private_key, $exec_in_docker);@@ -1759,7 +1762,8 @@ class Application extends BaseModel// When used with executeInDocker (which uses bash -c '...'), we need to escape for bash context// Replace ' with '\'' to safely escape within single-quoted bash strings$escapedCustomRepository = str_replace("'", "'\\''", $customRepository);- $base_command = "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$customSshKeyLocation} -o IdentitiesOnly=yes\" {$base_command} '{$escapedCustomRepository}'";+ $sshCommand = gitSshCommand($customPort, $customSshKeyLocation);+ $base_command = "GIT_SSH_COMMAND=\"{$sshCommand}\" {$base_command} '{$escapedCustomRepository}'";$commands = $this->gitSshKeySetupCommands($deployment_uuid, $private_key, $exec_in_docker);@@ -1993,7 +1997,7 @@ class Application extends BaseModel$private_key = base64_encode($private_key);$gitlabPort = $oauthSource->custom_port ?? 22;$escapedCustomRepository = escapeshellarg($customRepository);- $gitlabSshCommand = "ssh -o ConnectTimeout=30 -p {$gitlabPort} -o Port={$gitlabPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$customSshKeyLocation} -o IdentitiesOnly=yes";+ $gitlabSshCommand = gitSshCommand($gitlabPort, $customSshKeyLocation);$gitlabGitSshCommand = "GIT_SSH_COMMAND=\"{$gitlabSshCommand}\"";$git_clone_command_base = "{$gitlabGitSshCommand} {$git_clone_command} {$escapedCustomRepository} {$escapedBaseDir}";if ($only_checkout) {@@ -2054,7 +2058,7 @@ class Application extends BaseModel}$private_key = base64_encode($private_key);$escapedCustomRepository = escapeshellarg($customRepository);- $deployKeySshCommand = "ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$customSshKeyLocation} -o IdentitiesOnly=yes";+ $deployKeySshCommand = gitSshCommand($customPort, $customSshKeyLocation);$deployKeyGitSshCommand = "GIT_SSH_COMMAND=\"{$deployKeySshCommand}\"";$git_clone_command_base = "{$deployKeyGitSshCommand} {$git_clone_command} {$escapedCustomRepository} {$escapedBaseDir}";if ($only_checkout) {@@ -2108,7 +2112,7 @@ class Application extends BaseModel$escapedCustomRepository = escapeshellarg($customRepository);$git_clone_command = "{$git_clone_command} {$escapedCustomRepository} {$escapedBaseDir}";$git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command, public: true, commit: $commit, baseDir: $baseDir, onlyCheckout: $only_checkout);- $otherSshCommand = "ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa";+ $otherSshCommand = gitSshCommand($customPort, '/root/.ssh/id_rsa', identitiesOnly: false);if ($pull_request_id !== 0) {if ($git_type === 'gitlab') {diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.phpindex 9e1f8867c5..c6d7a20a0c 100644--- a/bootstrap/helpers/applications.php+++ b/bootstrap/helpers/applications.php@@ -553,3 +553,18 @@ function clone_application(Application $source, $destination, array $overrides =return $newApplication;}++/**+ * The SSH command that git uses to reach a repository.+ * The keepalive options stop a stalled connection after about one minute; without them git waits until the deployment timeout.+ */+function gitSshCommand(int|string $port, ?string $identityFile = null, bool $identitiesOnly = true): string+{+ $command = "ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p {$port} -o Port={$port} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";++ if ($identityFile !== null) {+ $command .= " -i {$identityFile}".($identitiesOnly ? ' -o IdentitiesOnly=yes' : '');+ }++ return $command;+}diff --git a/tests/Unit/DeployKeyDedicatedPathTest.php b/tests/Unit/DeployKeyDedicatedPathTest.phpindex 40f6cecb4c..3586994bca 100644--- a/tests/Unit/DeployKeyDedicatedPathTest.php+++ b/tests/Unit/DeployKeyDedicatedPathTest.php@@ -262,8 +262,8 @@ it('uses the per-deployment deploy key for pull request fetches', function () us$result = $application->generateGitImportCommands('test-deployment-uuid', 123, 'github', false);- expectCommandListToContain($result['commands'], "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$keyPath} -o IdentitiesOnly=yes\" git fetch origin pull/123/head:pr-123-coolify");- expectCommandListNotToContain($result['commands'], 'GIT_SSH_COMMAND="ssh -o ConnectTimeout=30 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa" git fetch origin pull/123/head:pr-123-coolify');+ expectCommandListToContain($result['commands'], "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$keyPath} -o IdentitiesOnly=yes\" git fetch origin pull/123/head:pr-123-coolify");+ expectCommandListNotToContain($result['commands'], 'GIT_SSH_COMMAND="ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa" git fetch origin pull/123/head:pr-123-coolify');expectPrivateKeyMaterializationCommandsSkipLogging($result['commands']);});@@ -282,6 +282,6 @@ it('does not force a missing per-deployment key for other repository pull reques$result = $application->generateGitImportCommands('test-deployment-uuid', 123, 'github', false);- expectCommandListToContain($result['commands'], 'GIT_SSH_COMMAND="ssh -o ConnectTimeout=30 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa" git fetch origin pull/123/head:pr-123-coolify');+ expectCommandListToContain($result['commands'], 'GIT_SSH_COMMAND="ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa" git fetch origin pull/123/head:pr-123-coolify');expectCommandListNotToContain($result['commands'], $keyPath);});diff --git a/tests/Unit/GitImportStepsTest.php b/tests/Unit/GitImportStepsTest.phpnew file mode 100644index 0000000000..e429ce37b9--- /dev/null+++ b/tests/Unit/GitImportStepsTest.php@@ -0,0 +1,84 @@+<?php++use App\Models\Application;+use App\Models\ApplicationSetting;+use Symfony\Component\Process\Process;++/**+ * Runs the generated import command in bash with a stub git that records its calls.+ * The stub prints $lfsFiles for `git lfs ls-files -n`.+ *+ * @return array{0: int, 1: list<string>, 2: list<string>}+ */+function runImportCommandWithStubGit(string $lfsFiles, bool $hasSubmodules = false): array+{+ $dir = sys_get_temp_dir().'/coolify-import-'.bin2hex(random_bytes(4));+ mkdir("{$dir}/bin", 0755, true);+ file_put_contents("{$dir}/bin/git", <<<SH+ #!/bin/sh+ echo "\$*" >> {$dir}/git.log+ if [ "\$*" = "lfs ls-files -n" ]; then printf '%s' '{$lfsFiles}'; fi+ SH);+ chmod("{$dir}/bin/git", 0755);+ if ($hasSubmodules) {+ touch("{$dir}/.gitmodules");+ }++ $application = new Application;+ $application->forceFill(['uuid' => 'test-app-uuid', 'git_commit_sha' => 'abc123']);+ $settings = new ApplicationSetting;+ $settings->is_git_shallow_clone_enabled = true;+ $settings->is_git_submodules_enabled = true;+ $settings->is_git_lfs_enabled = true;+ $application->setRelation('settings', $settings);++ $command = $application->setGitImportSettings(+ deployment_uuid: 'test-uuid',+ git_clone_command: 'true',+ gitSshCommand: 'ssh -o ConnectTimeout=30 -i /root/.ssh/id_rsa',+ baseDir: $dir,+ );++ $process = Process::fromShellCommandline($command, env: ['PATH' => "{$dir}/bin:".getenv('PATH')]);+ $process->run();+ $calls = file_exists("{$dir}/git.log") ? file("{$dir}/git.log", FILE_IGNORE_NEW_LINES) : [];++ exec('rm -rf '.escapeshellarg($dir));++ return [$process->getExitCode(), $calls, explode("\n", trim($process->getOutput()))];+}++test('git lfs pull is skipped when the checkout has no LFS files', function () {+ [$exitCode, $calls, $output] = runImportCommandWithStubGit('');++ expect($exitCode)->toBe(0)+ ->and($calls)->toBe([+ 'fetch --depth=1 origin abc123',+ '-c advice.detachedHead=false checkout abc123',+ 'lfs ls-files -n',+ ])+ ->and($output)->toBe(['Fetching the commit.']);+});++test('each network step after the clone prints a log line before it runs', function () {+ [$exitCode, $calls, $output] = runImportCommandWithStubGit('assets/logo.psd', hasSubmodules: true);++ expect($exitCode)->toBe(0)+ ->and($calls)->toBe([+ 'fetch --depth=1 origin abc123',+ '-c advice.detachedHead=false checkout abc123',+ 'submodule sync',+ 'submodule update --init --recursive --depth=1',+ 'lfs ls-files -n',+ 'lfs pull',+ ])+ ->and($output)->toBe(['Fetching the commit.', 'Updating submodules.', 'Downloading Git LFS objects.']);+});++test('git ssh command stops a stalled connection instead of waiting for the deployment timeout', function () {+ expect(gitSshCommand(2222, '/root/.ssh/key'))->toBe(+ 'ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 2222 -o Port=2222 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/key -o IdentitiesOnly=yes'+ )+ ->and(gitSshCommand(22))->not->toContain(' -i ')+ ->and(gitSshCommand(22, '/root/.ssh/id_rsa', identitiesOnly: false))->toEndWith('-i /root/.ssh/id_rsa');+});diff --git a/tests/Unit/GitSubmoduleCredentialTest.php b/tests/Unit/GitSubmoduleCredentialTest.phpindex 11c28c9909..f4e8921a15 100644--- a/tests/Unit/GitSubmoduleCredentialTest.php+++ b/tests/Unit/GitSubmoduleCredentialTest.php@@ -156,7 +156,7 @@ describe('Git submodule credential propagation', function () {exec_in_docker: false,);- $sshCommand = 'ssh -o ConnectTimeout=30 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa_coolify_test-uuid -o IdentitiesOnly=yes';+ $sshCommand = 'ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa_coolify_test-uuid -o IdentitiesOnly=yes';// Commands are a list of ['command' => ..., 'hidden' => ...] entries; assert on the joined shell script$commands = collect($result['commands'])