Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(deployments): retry transient GitHub token and clone failures

Andras Bacsai00678a12f0c2
diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php
index ec25b44b98..fe01e5b32f 100644
--- a/app/Jobs/ApplicationDeploymentJob.php
+++ b/app/Jobs/ApplicationDeploymentJob.php
@@ -3182,13 +3182,12 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
private function clone_repository()
{
- $importCommands = $this->generate_git_import_commands();
$this->application_deployment_queue->addLogEntry("\n----------------------------------------");
$this->application_deployment_queue->addLogEntry("Importing {$this->customRepository}:{$this->application->git_branch} (commit sha {$this->commit}) to {$this->basedir}.");
if ($this->pull_request_id !== 0) {
$this->application_deployment_queue->addLogEntry("Checking out tag pull/{$this->pull_request_id}/head.");
}
- $this->execute_remote_command(...$this->gitCommandDefinitions($importCommands));
+ $this->import_repository();
$this->create_workdir();
$this->execute_remote_command(
[
@@ -3206,6 +3205,37 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
}
}
+ /**
+ * SSH exits with 255 when the connection drops, which can happen during a long clone. Git exits with
+ * 128 for its own errors, so only 255 is retried. The interrupted clone can still run in the helper
+ * container, so stop it and remove the partial checkout before cloning again.
+ */
+ private function import_repository(): void
+ {
+ $maxAttempts = config('constants.ssh.max_retries');
+
+ for ($attempt = 1; ; $attempt++) {
+ try {
+ $this->execute_remote_command(...$this->gitCommandDefinitions($this->generate_git_import_commands()));
+
+ return;
+ } catch (DeploymentException $e) {
+ if ($e->getCode() !== 255 || $attempt >= $maxAttempts) {
+ throw $e;
+ }
+
+ $delay = $this->calculateRetryDelay($attempt - 1);
+ $this->application_deployment_queue->addLogEntry('Lost the SSH connection while importing the repository. Retrying in '.$delay.'s (attempt '.($attempt + 1)."/{$maxAttempts}).", 'stderr');
+ Sleep::for($delay)->seconds();
+ $this->execute_remote_command([
+ executeInDocker($this->deployment_uuid, 'pkill git; rm -rf '.escapeshellarg($this->basedir)),
+ 'hidden' => true,
+ 'ignore_errors' => true,
+ ]);
+ }
+ }
+ }
+
private function generate_git_import_commands()
{
['commands' => $commands, 'branch' => $this->branch, 'fullRepoUrl' => $this->fullRepoUrl] = $this->application->generateGitImportCommands(
diff --git a/bootstrap/helpers/github.php b/bootstrap/helpers/github.php
index 48498c0135..edcbca88f6 100644
--- a/bootstrap/helpers/github.php
+++ b/bootstrap/helpers/github.php
@@ -4,6 +4,8 @@ use App\Models\GithubApp;
use App\Models\PrivateKey;
use Carbon\Carbon;
use Carbon\CarbonImmutable;
+use Illuminate\Http\Client\ConnectionException;
+use Illuminate\Http\Client\RequestException;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Str;
@@ -195,13 +197,20 @@ function generateGithubToken(GithubApp $source, string $type)
return match ($type) {
'jwt' => $jwt,
'installation' => (function () use ($source, $jwt) {
+ // GitHub sometimes answers with a 5xx page or drops the connection; a new token request is harmless.
$response = Http::GitSource($source->api_url)->withHeaders([
'Authorization' => "Bearer $jwt",
'Accept' => 'application/vnd.github.machine-man-preview+json',
- ])->post("{$source->api_url}/app/installations/{$source->installation_id}/access_tokens");
+ ])->retry(
+ 3,
+ fn (int $attempt) => $attempt * 1000,
+ fn (Throwable $exception) => $exception instanceof ConnectionException
+ || ($exception instanceof RequestException && $exception->response->serverError()),
+ throw: false,
+ )->post("{$source->api_url}/app/installations/{$source->installation_id}/access_tokens");
if (! $response->successful()) {
- $error = data_get($response->json(), 'message', 'no error message found');
+ $error = data_get($response->json(), 'message', "HTTP {$response->status()}");
if ($error === 'Not Found') {
$error = 'Repository not found. Is it moved or deleted?';
}
diff --git a/tests/Feature/ApplicationDeploymentImportRetryTest.php b/tests/Feature/ApplicationDeploymentImportRetryTest.php
new file mode 100644
index 0000000000..e05bb97f1c
--- /dev/null
+++ b/tests/Feature/ApplicationDeploymentImportRetryTest.php
@@ -0,0 +1,89 @@
+<?php
+
+use App\Exceptions\DeploymentException;
+use App\Jobs\ApplicationDeploymentJob;
+use App\Models\Application;
+use App\Models\ApplicationDeploymentQueue;
+use Illuminate\Support\Sleep;
+
+/**
+ * Records remote commands and fails the repository import with the queued exit codes.
+ */
+class ImportRetryDeploymentJob extends ApplicationDeploymentJob
+{
+ public array $commands = [];
+
+ public array $importExitCodes = [];
+
+ public function __construct() {}
+
+ public function execute_remote_command(...$commands): void
+ {
+ foreach ($commands as $command) {
+ $this->commands[] = $command[0];
+ if (str_contains($command[0], 'git clone') && $exitCode = array_shift($this->importExitCodes)) {
+ throw new DeploymentException('Command execution failed', $exitCode);
+ }
+ }
+ }
+}
+
+beforeEach(function () {
+ Sleep::fake();
+ config()->set('constants.ssh.max_retries', 3);
+
+ $application = Mockery::mock(Application::class)->makePartial();
+ $application->shouldReceive('generateGitImportCommands')->andReturn([
+ 'commands' => "docker exec deploy-uuid bash -c 'git clone repo /artifacts/deploy-uuid'",
+ 'branch' => 'main',
+ 'fullRepoUrl' => 'repo',
+ ]);
+
+ $queue = Mockery::mock(ApplicationDeploymentQueue::class)->makePartial();
+ $queue->shouldReceive('addLogEntry');
+
+ $this->job = new ImportRetryDeploymentJob;
+ foreach ([
+ 'application' => $application,
+ 'application_deployment_queue' => $queue,
+ 'deployment_uuid' => 'deploy-uuid',
+ 'basedir' => '/artifacts/deploy-uuid',
+ 'pull_request_id' => 0,
+ 'commit' => 'HEAD',
+ ] as $property => $value) {
+ (new ReflectionProperty(ApplicationDeploymentJob::class, $property))->setValue($this->job, $value);
+ }
+});
+
+function importRepository(ImportRetryDeploymentJob $job): void
+{
+ (new ReflectionMethod(ApplicationDeploymentJob::class, 'import_repository'))->invoke($job);
+}
+
+function cloneCommandCount(ImportRetryDeploymentJob $job): int
+{
+ return collect($job->commands)->filter(fn (string $command) => str_contains($command, 'git clone'))->count();
+}
+
+it('cleans up and clones again when the SSH connection drops during the import', function () {
+ $this->job->importExitCodes = [255];
+
+ importRepository($this->job);
+
+ expect(cloneCommandCount($this->job))->toBe(2)
+ ->and($this->job->commands[1])->toBe(executeInDocker('deploy-uuid', "pkill git; rm -rf '/artifacts/deploy-uuid'"));
+});
+
+it('does not retry the import when git itself fails', function () {
+ $this->job->importExitCodes = [128];
+
+ expect(fn () => importRepository($this->job))->toThrow(DeploymentException::class);
+ expect(cloneCommandCount($this->job))->toBe(1);
+});
+
+it('stops retrying the import after the configured SSH attempts', function () {
+ $this->job->importExitCodes = [255, 255, 255];
+
+ expect(fn () => importRepository($this->job))->toThrow(DeploymentException::class);
+ expect(cloneCommandCount($this->job))->toBe(3);
+});
diff --git a/tests/Feature/GithubAppInstallationTokenTest.php b/tests/Feature/GithubAppInstallationTokenTest.php
index bb83df0636..9ea349c460 100644
--- a/tests/Feature/GithubAppInstallationTokenTest.php
+++ b/tests/Feature/GithubAppInstallationTokenTest.php
@@ -6,10 +6,13 @@ use App\Models\Team;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Http\Client\Request;
use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Sleep;
uses(RefreshDatabase::class);
beforeEach(function () {
+ Sleep::fake();
+
$this->team = Team::factory()->create();
openssl_pkey_export(openssl_pkey_new([
@@ -36,24 +39,36 @@ beforeEach(function () {
'https://api.github.com/zen' => fn () => Http::response('Keep it logically awesome.', 200, [
'Date' => now()->toRfc7231String(),
]),
- 'https://api.github.com/app/installations/*/access_tokens' => Http::sequence()
- ->push(['token' => 'first-token'], 201)
- ->push(['token' => 'second-token'], 201),
]);
});
+function fakeGithubInstallationTokenResponses(mixed $response): void
+{
+ Http::fake(['https://api.github.com/app/installations/*/access_tokens' => $response]);
+}
+
+function fakeTwoGithubInstallationTokens(): void
+{
+ fakeGithubInstallationTokenResponses(Http::sequence()
+ ->push(['token' => 'first-token'], 201)
+ ->push(['token' => 'second-token'], 201));
+}
+
function githubInstallationTokenRequests(): int
{
return Http::recorded(fn (Request $request): bool => str_ends_with($request->url(), '/access_tokens'))->count();
}
it('reuses the installation token of the same GitHub App instance', function () {
+ fakeTwoGithubInstallationTokens();
+
expect(generateGithubInstallationToken($this->githubApp))->toBe('first-token')
->and(generateGithubInstallationToken($this->githubApp))->toBe('first-token')
->and(githubInstallationTokenRequests())->toBe(1);
});
it('requests a new installation token after the in-memory token expires', function () {
+ fakeTwoGithubInstallationTokens();
generateGithubInstallationToken($this->githubApp);
$this->travel(51)->minutes();
@@ -63,6 +78,7 @@ it('requests a new installation token after the in-memory token expires', functi
});
it('requests a new installation token when the GitHub App installation changes', function () {
+ fakeTwoGithubInstallationTokens();
generateGithubInstallationToken($this->githubApp);
$this->githubApp->installation_id = 11111;
@@ -72,8 +88,34 @@ it('requests a new installation token when the GitHub App installation changes',
});
it('does not share the installation token between GitHub App instances', function () {
+ fakeTwoGithubInstallationTokens();
generateGithubInstallationToken($this->githubApp);
expect(generateGithubInstallationToken(GithubApp::find($this->githubApp->id)))->toBe('second-token')
->and(githubInstallationTokenRequests())->toBe(2);
});
+
+it('retries the installation token request when GitHub returns a server error', function () {
+ fakeGithubInstallationTokenResponses(Http::sequence()
+ ->push('<html>Bad Gateway</html>', 502)
+ ->push(['token' => 'retried-token'], 201));
+
+ expect(generateGithubInstallationToken($this->githubApp))->toBe('retried-token')
+ ->and(githubInstallationTokenRequests())->toBe(2);
+});
+
+it('does not retry the installation token request when GitHub rejects it', function () {
+ fakeGithubInstallationTokenResponses(Http::response(['message' => 'Bad credentials'], 401));
+
+ expect(fn () => generateGithubInstallationToken($this->githubApp))
+ ->toThrow(RuntimeException::class, 'Failed to get installation token for Private GitHub App with error: Bad credentials')
+ ->and(githubInstallationTokenRequests())->toBe(1);
+});
+
+it('reports the HTTP status when GitHub keeps failing without an error message', function () {
+ fakeGithubInstallationTokenResponses(Http::response('<html>Service Unavailable</html>', 503));
+
+ expect(fn () => generateGithubInstallationToken($this->githubApp))
+ ->toThrow(RuntimeException::class, 'Failed to get installation token for Private GitHub App with error: HTTP 503')
+ ->and(githubInstallationTokenRequests())->toBe(3);
+});