An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(deployments): retry transient GitHub token and clone failures
Andras Bacsai00678a12f0c2
diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.phpindex ec25b44b98..fe01e5b32f 100644--- a/app/Jobs/ApplicationDeploymentJob.php+++ b/app/Jobs/ApplicationDeploymentJob.php@@ -3182,13 +3182,12 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueueprivate function clone_repository(){- $importCommands = $this->generate_git_import_commands();$this->application_deployment_queue->addLogEntry("\n----------------------------------------");$this->application_deployment_queue->addLogEntry("Importing {$this->customRepository}:{$this->application->git_branch} (commit sha {$this->commit}) to {$this->basedir}.");if ($this->pull_request_id !== 0) {$this->application_deployment_queue->addLogEntry("Checking out tag pull/{$this->pull_request_id}/head.");}- $this->execute_remote_command(...$this->gitCommandDefinitions($importCommands));+ $this->import_repository();$this->create_workdir();$this->execute_remote_command([@@ -3206,6 +3205,37 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue}}+ /**+ * SSH exits with 255 when the connection drops, which can happen during a long clone. Git exits with+ * 128 for its own errors, so only 255 is retried. The interrupted clone can still run in the helper+ * container, so stop it and remove the partial checkout before cloning again.+ */+ private function import_repository(): void+ {+ $maxAttempts = config('constants.ssh.max_retries');++ for ($attempt = 1; ; $attempt++) {+ try {+ $this->execute_remote_command(...$this->gitCommandDefinitions($this->generate_git_import_commands()));++ return;+ } catch (DeploymentException $e) {+ if ($e->getCode() !== 255 || $attempt >= $maxAttempts) {+ throw $e;+ }++ $delay = $this->calculateRetryDelay($attempt - 1);+ $this->application_deployment_queue->addLogEntry('Lost the SSH connection while importing the repository. Retrying in '.$delay.'s (attempt '.($attempt + 1)."/{$maxAttempts}).", 'stderr');+ Sleep::for($delay)->seconds();+ $this->execute_remote_command([+ executeInDocker($this->deployment_uuid, 'pkill git; rm -rf '.escapeshellarg($this->basedir)),+ 'hidden' => true,+ 'ignore_errors' => true,+ ]);+ }+ }+ }+private function generate_git_import_commands(){['commands' => $commands, 'branch' => $this->branch, 'fullRepoUrl' => $this->fullRepoUrl] = $this->application->generateGitImportCommands(diff --git a/bootstrap/helpers/github.php b/bootstrap/helpers/github.phpindex 48498c0135..edcbca88f6 100644--- a/bootstrap/helpers/github.php+++ b/bootstrap/helpers/github.php@@ -4,6 +4,8 @@ use App\Models\GithubApp;use App\Models\PrivateKey;use Carbon\Carbon;use Carbon\CarbonImmutable;+use Illuminate\Http\Client\ConnectionException;+use Illuminate\Http\Client\RequestException;use Illuminate\Support\Facades\Cache;use Illuminate\Support\Facades\Http;use Illuminate\Support\Str;@@ -195,13 +197,20 @@ function generateGithubToken(GithubApp $source, string $type)return match ($type) {'jwt' => $jwt,'installation' => (function () use ($source, $jwt) {+ // GitHub sometimes answers with a 5xx page or drops the connection; a new token request is harmless.$response = Http::GitSource($source->api_url)->withHeaders(['Authorization' => "Bearer $jwt",'Accept' => 'application/vnd.github.machine-man-preview+json',- ])->post("{$source->api_url}/app/installations/{$source->installation_id}/access_tokens");+ ])->retry(+ 3,+ fn (int $attempt) => $attempt * 1000,+ fn (Throwable $exception) => $exception instanceof ConnectionException+ || ($exception instanceof RequestException && $exception->response->serverError()),+ throw: false,+ )->post("{$source->api_url}/app/installations/{$source->installation_id}/access_tokens");if (! $response->successful()) {- $error = data_get($response->json(), 'message', 'no error message found');+ $error = data_get($response->json(), 'message', "HTTP {$response->status()}");if ($error === 'Not Found') {$error = 'Repository not found. Is it moved or deleted?';}diff --git a/tests/Feature/ApplicationDeploymentImportRetryTest.php b/tests/Feature/ApplicationDeploymentImportRetryTest.phpnew file mode 100644index 0000000000..e05bb97f1c--- /dev/null+++ b/tests/Feature/ApplicationDeploymentImportRetryTest.php@@ -0,0 +1,89 @@+<?php++use App\Exceptions\DeploymentException;+use App\Jobs\ApplicationDeploymentJob;+use App\Models\Application;+use App\Models\ApplicationDeploymentQueue;+use Illuminate\Support\Sleep;++/**+ * Records remote commands and fails the repository import with the queued exit codes.+ */+class ImportRetryDeploymentJob extends ApplicationDeploymentJob+{+ public array $commands = [];++ public array $importExitCodes = [];++ public function __construct() {}++ public function execute_remote_command(...$commands): void+ {+ foreach ($commands as $command) {+ $this->commands[] = $command[0];+ if (str_contains($command[0], 'git clone') && $exitCode = array_shift($this->importExitCodes)) {+ throw new DeploymentException('Command execution failed', $exitCode);+ }+ }+ }+}++beforeEach(function () {+ Sleep::fake();+ config()->set('constants.ssh.max_retries', 3);++ $application = Mockery::mock(Application::class)->makePartial();+ $application->shouldReceive('generateGitImportCommands')->andReturn([+ 'commands' => "docker exec deploy-uuid bash -c 'git clone repo /artifacts/deploy-uuid'",+ 'branch' => 'main',+ 'fullRepoUrl' => 'repo',+ ]);++ $queue = Mockery::mock(ApplicationDeploymentQueue::class)->makePartial();+ $queue->shouldReceive('addLogEntry');++ $this->job = new ImportRetryDeploymentJob;+ foreach ([+ 'application' => $application,+ 'application_deployment_queue' => $queue,+ 'deployment_uuid' => 'deploy-uuid',+ 'basedir' => '/artifacts/deploy-uuid',+ 'pull_request_id' => 0,+ 'commit' => 'HEAD',+ ] as $property => $value) {+ (new ReflectionProperty(ApplicationDeploymentJob::class, $property))->setValue($this->job, $value);+ }+});++function importRepository(ImportRetryDeploymentJob $job): void+{+ (new ReflectionMethod(ApplicationDeploymentJob::class, 'import_repository'))->invoke($job);+}++function cloneCommandCount(ImportRetryDeploymentJob $job): int+{+ return collect($job->commands)->filter(fn (string $command) => str_contains($command, 'git clone'))->count();+}++it('cleans up and clones again when the SSH connection drops during the import', function () {+ $this->job->importExitCodes = [255];++ importRepository($this->job);++ expect(cloneCommandCount($this->job))->toBe(2)+ ->and($this->job->commands[1])->toBe(executeInDocker('deploy-uuid', "pkill git; rm -rf '/artifacts/deploy-uuid'"));+});++it('does not retry the import when git itself fails', function () {+ $this->job->importExitCodes = [128];++ expect(fn () => importRepository($this->job))->toThrow(DeploymentException::class);+ expect(cloneCommandCount($this->job))->toBe(1);+});++it('stops retrying the import after the configured SSH attempts', function () {+ $this->job->importExitCodes = [255, 255, 255];++ expect(fn () => importRepository($this->job))->toThrow(DeploymentException::class);+ expect(cloneCommandCount($this->job))->toBe(3);+});diff --git a/tests/Feature/GithubAppInstallationTokenTest.php b/tests/Feature/GithubAppInstallationTokenTest.phpindex bb83df0636..9ea349c460 100644--- a/tests/Feature/GithubAppInstallationTokenTest.php+++ b/tests/Feature/GithubAppInstallationTokenTest.php@@ -6,10 +6,13 @@ use App\Models\Team;use Illuminate\Foundation\Testing\RefreshDatabase;use Illuminate\Http\Client\Request;use Illuminate\Support\Facades\Http;+use Illuminate\Support\Sleep;uses(RefreshDatabase::class);beforeEach(function () {+ Sleep::fake();+$this->team = Team::factory()->create();openssl_pkey_export(openssl_pkey_new([@@ -36,24 +39,36 @@ beforeEach(function () {'https://api.github.com/zen' => fn () => Http::response('Keep it logically awesome.', 200, ['Date' => now()->toRfc7231String(),]),- 'https://api.github.com/app/installations/*/access_tokens' => Http::sequence()- ->push(['token' => 'first-token'], 201)- ->push(['token' => 'second-token'], 201),]);});+function fakeGithubInstallationTokenResponses(mixed $response): void+{+ Http::fake(['https://api.github.com/app/installations/*/access_tokens' => $response]);+}++function fakeTwoGithubInstallationTokens(): void+{+ fakeGithubInstallationTokenResponses(Http::sequence()+ ->push(['token' => 'first-token'], 201)+ ->push(['token' => 'second-token'], 201));+}+function githubInstallationTokenRequests(): int{return Http::recorded(fn (Request $request): bool => str_ends_with($request->url(), '/access_tokens'))->count();}it('reuses the installation token of the same GitHub App instance', function () {+ fakeTwoGithubInstallationTokens();+expect(generateGithubInstallationToken($this->githubApp))->toBe('first-token')->and(generateGithubInstallationToken($this->githubApp))->toBe('first-token')->and(githubInstallationTokenRequests())->toBe(1);});it('requests a new installation token after the in-memory token expires', function () {+ fakeTwoGithubInstallationTokens();generateGithubInstallationToken($this->githubApp);$this->travel(51)->minutes();@@ -63,6 +78,7 @@ it('requests a new installation token after the in-memory token expires', functi});it('requests a new installation token when the GitHub App installation changes', function () {+ fakeTwoGithubInstallationTokens();generateGithubInstallationToken($this->githubApp);$this->githubApp->installation_id = 11111;@@ -72,8 +88,34 @@ it('requests a new installation token when the GitHub App installation changes',});it('does not share the installation token between GitHub App instances', function () {+ fakeTwoGithubInstallationTokens();generateGithubInstallationToken($this->githubApp);expect(generateGithubInstallationToken(GithubApp::find($this->githubApp->id)))->toBe('second-token')->and(githubInstallationTokenRequests())->toBe(2);});++it('retries the installation token request when GitHub returns a server error', function () {+ fakeGithubInstallationTokenResponses(Http::sequence()+ ->push('<html>Bad Gateway</html>', 502)+ ->push(['token' => 'retried-token'], 201));++ expect(generateGithubInstallationToken($this->githubApp))->toBe('retried-token')+ ->and(githubInstallationTokenRequests())->toBe(2);+});++it('does not retry the installation token request when GitHub rejects it', function () {+ fakeGithubInstallationTokenResponses(Http::response(['message' => 'Bad credentials'], 401));++ expect(fn () => generateGithubInstallationToken($this->githubApp))+ ->toThrow(RuntimeException::class, 'Failed to get installation token for Private GitHub App with error: Bad credentials')+ ->and(githubInstallationTokenRequests())->toBe(1);+});++it('reports the HTTP status when GitHub keeps failing without an error message', function () {+ fakeGithubInstallationTokenResponses(Http::response('<html>Service Unavailable</html>', 503));++ expect(fn () => generateGithubInstallationToken($this->githubApp))+ ->toThrow(RuntimeException::class, 'Failed to get installation token for Private GitHub App with error: HTTP 503')+ ->and(githubInstallationTokenRequests())->toBe(3);+});