Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(deployments): prevent Git fetches from hanging (#12321)

Andras Bacsaiddeb05280e57
diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php
index bdc84bc6a6..d954a62f7c 100644
--- a/app/Jobs/ApplicationDeploymentJob.php
+++ b/app/Jobs/ApplicationDeploymentJob.php
@@ -3171,11 +3171,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
private function gitLsRemoteCommand(string $lsRemoteRef, ?string $identityFile = null): string
{
- $sshCommand = "ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";
-
- if ($identityFile !== null) {
- $sshCommand .= " -i {$identityFile} -o IdentitiesOnly=yes";
- }
+ $sshCommand = gitSshCommand($this->customPort, $identityFile);
return 'GIT_SSH_COMMAND="'.$sshCommand.'" git ls-remote '.escapeshellarg($this->fullRepoUrl).' '.escapeshellarg($lsRemoteRef);
}
diff --git a/app/Models/Application.php b/app/Models/Application.php
index 22a8a67041..a183d412cc 100644
--- a/app/Models/Application.php
+++ b/app/Models/Application.php
@@ -1587,7 +1587,7 @@ class Application extends BaseModel
// If shallow clone is enabled and we need a specific commit,
// we need to fetch that specific commit with depth=1
if ($isShallowCloneEnabled) {
- $git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && {$sshCommand} {$gitCommand} fetch --depth=1 origin {$escapedCommit} && {$gitCommand} -c advice.detachedHead=false checkout {$escapedCommit} >/dev/null 2>&1";
+ $git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && echo 'Fetching the commit.' && {$sshCommand} {$gitCommand} fetch --depth=1 origin {$escapedCommit} && {$gitCommand} -c advice.detachedHead=false checkout {$escapedCommit} >/dev/null 2>&1";
} else {
$git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && {$sshCommand} {$gitCommand} -c advice.detachedHead=false checkout {$escapedCommit} >/dev/null 2>&1";
}
@@ -1601,10 +1601,12 @@ class Application extends BaseModel
}
// Add shallow submodules flag if shallow clone is enabled
$submoduleFlags = $isShallowCloneEnabled ? '--depth=1' : '';
- $git_clone_command = "{$git_clone_command} {$gitCommand} submodule sync && {$sshCommand} {$gitCommand} submodule update --init --recursive {$submoduleFlags}; fi";
+ $git_clone_command = "{$git_clone_command} echo 'Updating submodules.' && {$gitCommand} submodule sync && {$sshCommand} {$gitCommand} submodule update --init --recursive {$submoduleFlags}; fi";
}
+ // `git lfs pull` opens an SSH session even without LFS files, which can hang on hosts
+ // without LFS over SSH (Azure DevOps). `git lfs ls-files` is offline.
if ($this->settings->is_git_lfs_enabled && ! $onlyCheckout) {
- $git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && {$sshCommand} {$gitCommand} lfs pull";
+ $git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && if {$gitCommand} lfs ls-files -n | grep -q .; then echo 'Downloading Git LFS objects.' && {$sshCommand} {$gitCommand} lfs pull; fi";
}
return $git_clone_command;
@@ -1714,7 +1716,8 @@ class Application extends BaseModel
$private_key = base64_encode($private_key);
$gitlabPort = $oauthSource->custom_port ?? 22;
$escapedCustomRepository = str_replace("'", "'\\''", $customRepository);
- $base_command = "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$gitlabPort} -o Port={$gitlabPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$customSshKeyLocation} -o IdentitiesOnly=yes\" {$base_command} '{$escapedCustomRepository}'";
+ $sshCommand = gitSshCommand($gitlabPort, $customSshKeyLocation);
+ $base_command = "GIT_SSH_COMMAND=\"{$sshCommand}\" {$base_command} '{$escapedCustomRepository}'";
$commands = $this->gitSshKeySetupCommands($deployment_uuid, $private_key, $exec_in_docker);
@@ -1759,7 +1762,8 @@ class Application extends BaseModel
// When used with executeInDocker (which uses bash -c '...'), we need to escape for bash context
// Replace ' with '\'' to safely escape within single-quoted bash strings
$escapedCustomRepository = str_replace("'", "'\\''", $customRepository);
- $base_command = "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$customSshKeyLocation} -o IdentitiesOnly=yes\" {$base_command} '{$escapedCustomRepository}'";
+ $sshCommand = gitSshCommand($customPort, $customSshKeyLocation);
+ $base_command = "GIT_SSH_COMMAND=\"{$sshCommand}\" {$base_command} '{$escapedCustomRepository}'";
$commands = $this->gitSshKeySetupCommands($deployment_uuid, $private_key, $exec_in_docker);
@@ -1993,7 +1997,7 @@ class Application extends BaseModel
$private_key = base64_encode($private_key);
$gitlabPort = $oauthSource->custom_port ?? 22;
$escapedCustomRepository = escapeshellarg($customRepository);
- $gitlabSshCommand = "ssh -o ConnectTimeout=30 -p {$gitlabPort} -o Port={$gitlabPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$customSshKeyLocation} -o IdentitiesOnly=yes";
+ $gitlabSshCommand = gitSshCommand($gitlabPort, $customSshKeyLocation);
$gitlabGitSshCommand = "GIT_SSH_COMMAND=\"{$gitlabSshCommand}\"";
$git_clone_command_base = "{$gitlabGitSshCommand} {$git_clone_command} {$escapedCustomRepository} {$escapedBaseDir}";
if ($only_checkout) {
@@ -2054,7 +2058,7 @@ class Application extends BaseModel
}
$private_key = base64_encode($private_key);
$escapedCustomRepository = escapeshellarg($customRepository);
- $deployKeySshCommand = "ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$customSshKeyLocation} -o IdentitiesOnly=yes";
+ $deployKeySshCommand = gitSshCommand($customPort, $customSshKeyLocation);
$deployKeyGitSshCommand = "GIT_SSH_COMMAND=\"{$deployKeySshCommand}\"";
$git_clone_command_base = "{$deployKeyGitSshCommand} {$git_clone_command} {$escapedCustomRepository} {$escapedBaseDir}";
if ($only_checkout) {
@@ -2108,7 +2112,7 @@ class Application extends BaseModel
$escapedCustomRepository = escapeshellarg($customRepository);
$git_clone_command = "{$git_clone_command} {$escapedCustomRepository} {$escapedBaseDir}";
$git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command, public: true, commit: $commit, baseDir: $baseDir, onlyCheckout: $only_checkout);
- $otherSshCommand = "ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa";
+ $otherSshCommand = gitSshCommand($customPort, '/root/.ssh/id_rsa', identitiesOnly: false);
if ($pull_request_id !== 0) {
if ($git_type === 'gitlab') {
diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.php
index 9e1f8867c5..c6d7a20a0c 100644
--- a/bootstrap/helpers/applications.php
+++ b/bootstrap/helpers/applications.php
@@ -553,3 +553,18 @@ function clone_application(Application $source, $destination, array $overrides =
return $newApplication;
}
+
+/**
+ * The SSH command that git uses to reach a repository.
+ * The keepalive options stop a stalled connection after about one minute; without them git waits until the deployment timeout.
+ */
+function gitSshCommand(int|string $port, ?string $identityFile = null, bool $identitiesOnly = true): string
+{
+ $command = "ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p {$port} -o Port={$port} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";
+
+ if ($identityFile !== null) {
+ $command .= " -i {$identityFile}".($identitiesOnly ? ' -o IdentitiesOnly=yes' : '');
+ }
+
+ return $command;
+}
diff --git a/tests/Unit/DeployKeyDedicatedPathTest.php b/tests/Unit/DeployKeyDedicatedPathTest.php
index 40f6cecb4c..3586994bca 100644
--- a/tests/Unit/DeployKeyDedicatedPathTest.php
+++ b/tests/Unit/DeployKeyDedicatedPathTest.php
@@ -262,8 +262,8 @@ it('uses the per-deployment deploy key for pull request fetches', function () us
$result = $application->generateGitImportCommands('test-deployment-uuid', 123, 'github', false);
- expectCommandListToContain($result['commands'], "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$keyPath} -o IdentitiesOnly=yes\" git fetch origin pull/123/head:pr-123-coolify");
- expectCommandListNotToContain($result['commands'], 'GIT_SSH_COMMAND="ssh -o ConnectTimeout=30 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa" git fetch origin pull/123/head:pr-123-coolify');
+ expectCommandListToContain($result['commands'], "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$keyPath} -o IdentitiesOnly=yes\" git fetch origin pull/123/head:pr-123-coolify");
+ expectCommandListNotToContain($result['commands'], 'GIT_SSH_COMMAND="ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa" git fetch origin pull/123/head:pr-123-coolify');
expectPrivateKeyMaterializationCommandsSkipLogging($result['commands']);
});
@@ -282,6 +282,6 @@ it('does not force a missing per-deployment key for other repository pull reques
$result = $application->generateGitImportCommands('test-deployment-uuid', 123, 'github', false);
- expectCommandListToContain($result['commands'], 'GIT_SSH_COMMAND="ssh -o ConnectTimeout=30 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa" git fetch origin pull/123/head:pr-123-coolify');
+ expectCommandListToContain($result['commands'], 'GIT_SSH_COMMAND="ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa" git fetch origin pull/123/head:pr-123-coolify');
expectCommandListNotToContain($result['commands'], $keyPath);
});
diff --git a/tests/Unit/GitImportStepsTest.php b/tests/Unit/GitImportStepsTest.php
new file mode 100644
index 0000000000..e429ce37b9
--- /dev/null
+++ b/tests/Unit/GitImportStepsTest.php
@@ -0,0 +1,84 @@
+<?php
+
+use App\Models\Application;
+use App\Models\ApplicationSetting;
+use Symfony\Component\Process\Process;
+
+/**
+ * Runs the generated import command in bash with a stub git that records its calls.
+ * The stub prints $lfsFiles for `git lfs ls-files -n`.
+ *
+ * @return array{0: int, 1: list<string>, 2: list<string>}
+ */
+function runImportCommandWithStubGit(string $lfsFiles, bool $hasSubmodules = false): array
+{
+ $dir = sys_get_temp_dir().'/coolify-import-'.bin2hex(random_bytes(4));
+ mkdir("{$dir}/bin", 0755, true);
+ file_put_contents("{$dir}/bin/git", <<<SH
+ #!/bin/sh
+ echo "\$*" >> {$dir}/git.log
+ if [ "\$*" = "lfs ls-files -n" ]; then printf '%s' '{$lfsFiles}'; fi
+ SH);
+ chmod("{$dir}/bin/git", 0755);
+ if ($hasSubmodules) {
+ touch("{$dir}/.gitmodules");
+ }
+
+ $application = new Application;
+ $application->forceFill(['uuid' => 'test-app-uuid', 'git_commit_sha' => 'abc123']);
+ $settings = new ApplicationSetting;
+ $settings->is_git_shallow_clone_enabled = true;
+ $settings->is_git_submodules_enabled = true;
+ $settings->is_git_lfs_enabled = true;
+ $application->setRelation('settings', $settings);
+
+ $command = $application->setGitImportSettings(
+ deployment_uuid: 'test-uuid',
+ git_clone_command: 'true',
+ gitSshCommand: 'ssh -o ConnectTimeout=30 -i /root/.ssh/id_rsa',
+ baseDir: $dir,
+ );
+
+ $process = Process::fromShellCommandline($command, env: ['PATH' => "{$dir}/bin:".getenv('PATH')]);
+ $process->run();
+ $calls = file_exists("{$dir}/git.log") ? file("{$dir}/git.log", FILE_IGNORE_NEW_LINES) : [];
+
+ exec('rm -rf '.escapeshellarg($dir));
+
+ return [$process->getExitCode(), $calls, explode("\n", trim($process->getOutput()))];
+}
+
+test('git lfs pull is skipped when the checkout has no LFS files', function () {
+ [$exitCode, $calls, $output] = runImportCommandWithStubGit('');
+
+ expect($exitCode)->toBe(0)
+ ->and($calls)->toBe([
+ 'fetch --depth=1 origin abc123',
+ '-c advice.detachedHead=false checkout abc123',
+ 'lfs ls-files -n',
+ ])
+ ->and($output)->toBe(['Fetching the commit.']);
+});
+
+test('each network step after the clone prints a log line before it runs', function () {
+ [$exitCode, $calls, $output] = runImportCommandWithStubGit('assets/logo.psd', hasSubmodules: true);
+
+ expect($exitCode)->toBe(0)
+ ->and($calls)->toBe([
+ 'fetch --depth=1 origin abc123',
+ '-c advice.detachedHead=false checkout abc123',
+ 'submodule sync',
+ 'submodule update --init --recursive --depth=1',
+ 'lfs ls-files -n',
+ 'lfs pull',
+ ])
+ ->and($output)->toBe(['Fetching the commit.', 'Updating submodules.', 'Downloading Git LFS objects.']);
+});
+
+test('git ssh command stops a stalled connection instead of waiting for the deployment timeout', function () {
+ expect(gitSshCommand(2222, '/root/.ssh/key'))->toBe(
+ 'ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 2222 -o Port=2222 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/key -o IdentitiesOnly=yes'
+ )
+ ->and(gitSshCommand(22))->not->toContain(' -i ')
+ ->and(gitSshCommand(22, '/root/.ssh/id_rsa', identitiesOnly: false))->toEndWith('-i /root/.ssh/id_rsa');
+});
diff --git a/tests/Unit/GitSubmoduleCredentialTest.php b/tests/Unit/GitSubmoduleCredentialTest.php
index 11c28c9909..f4e8921a15 100644
--- a/tests/Unit/GitSubmoduleCredentialTest.php
+++ b/tests/Unit/GitSubmoduleCredentialTest.php
@@ -156,7 +156,7 @@ describe('Git submodule credential propagation', function () {
exec_in_docker: false,
);
- $sshCommand = 'ssh -o ConnectTimeout=30 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa_coolify_test-uuid -o IdentitiesOnly=yes';
+ $sshCommand = 'ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa_coolify_test-uuid -o IdentitiesOnly=yes';
// Commands are a list of ['command' => ..., 'hidden' => ...] entries; assert on the joined shell script
$commands = collect($result['commands'])