Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(backup): explain why a dump stopped without an error

Andras Bacsaia836756e9b92
diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php
index 2e6d71f18b..15b9ff4bba 100644
--- a/app/Jobs/DatabaseBackupJob.php
+++ b/app/Jobs/DatabaseBackupJob.php
@@ -460,6 +460,10 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
// Local backup failed: the partial file is not tracked, so retention would never remove it.
deleteBackupsLocally($this->backup_location, $this->server);
$failureMessage = $this->redact($this->error_output ?? $this->backup_output ?? $e->getMessage());
+ $exitCodeReason = $this->exitCodeReason($e->getCode());
+ if ($exitCodeReason !== null) {
+ $failureMessage = $exitCodeReason."\n\n".$failureMessage;
+ }
if ($this->backup_log) {
$this->backup_log->update([
'status' => 'failed',
@@ -816,6 +820,24 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
}
}
+ /**
+ * Explains exit codes of a dump that was stopped from outside. The dump tool cannot report these
+ * itself, so its output ends after the last progress line without an error.
+ */
+ private function exitCodeReason(int|string $exitCode): ?string
+ {
+ return match ($exitCode) {
+ 137 => 'The backup stopped with exit code 137: the dump process or the database container was killed (SIGKILL). '
+ .'This is usually the out-of-memory killer: the dump runs inside the database container and shares its memory limit. '
+ .'Check the memory limit of the database and the kernel log (dmesg) on the server.',
+ 143 => 'The backup stopped with exit code 143: the dump process or the database container was terminated (SIGTERM), '
+ .'for example because the database was stopped or restarted during the backup.',
+ 255 => 'The backup stopped with exit code 255: the SSH connection to the server closed. '
+ .'Check the network connection and the SSH server log on the server.',
+ default => null,
+ };
+ }
+
private function add_to_backup_output($output): void
{
if ($this->backup_output) {
diff --git a/tests/Feature/BackupFailureExitCodeTest.php b/tests/Feature/BackupFailureExitCodeTest.php
new file mode 100644
index 0000000000..cd5b8d492b
--- /dev/null
+++ b/tests/Feature/BackupFailureExitCodeTest.php
@@ -0,0 +1,100 @@
+<?php
+
+use App\Events\BackupCreated;
+use App\Jobs\DatabaseBackupJob;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\ScheduledDatabaseBackup;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\StandaloneMongodb;
+use App\Models\Team;
+use App\Notifications\Database\BackupFailed;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Event;
+use Illuminate\Support\Facades\Notification;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Storage;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ Server::flushIdentityMap();
+ Storage::fake('ssh-keys');
+ Storage::fake('ssh-mux');
+ config(['constants.ssh.mux_enabled' => false, 'constants.ssh.retry_base_delay' => 0]);
+ Event::fake([BackupCreated::class]);
+ Notification::fake();
+ InstanceSettings::unguarded(fn () => InstanceSettings::query()->firstOrCreate(['id' => 0]));
+
+ $this->team = Team::factory()->create();
+ $this->team->emailNotificationSettings->update(['smtp_enabled' => true, 'backup_failure_email_notifications' => true]);
+ $server = Server::factory()->create([
+ 'team_id' => $this->team->id,
+ 'private_key_id' => PrivateKey::factory()->create(['team_id' => $this->team->id])->id,
+ ]);
+ $server->settings()->update(['is_reachable' => true, 'is_usable' => true]);
+ $environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $this->team->id])->id]);
+ $database = create_standalone_mongodb($environment->id, StandaloneDocker::where('server_id', $server->id)->firstOrFail(), [
+ 'status' => 'running:healthy',
+ ]);
+ $this->backup = ScheduledDatabaseBackup::create([
+ 'frequency' => '0 0 * * *',
+ 'save_s3' => false,
+ 'database_type' => StandaloneMongodb::class,
+ 'database_id' => $database->id,
+ 'team_id' => $this->team->id,
+ 'databases_to_backup' => null,
+ ]);
+});
+
+/**
+ * mongodump writes its progress to stderr. When the dump process or the database container is killed,
+ * the output stops after the last progress line and there is no "Failed:" line (issue #11549).
+ */
+function interruptedMongodump(int $exitCode): void
+{
+ Process::fake(function ($process) use ($exitCode) {
+ if (str_contains($process->command, 'mongodump')) {
+ return Process::result(
+ errorOutput: "2026-08-29T15:44:00.555+0000\tdone dumping `evolution.leaverequests` (0 documents)\n2026-08-29T15:44:00.555+0000\twriting `evolution.printagents` to `archive on stdout`",
+ exitCode: $exitCode,
+ );
+ }
+
+ return Process::result(output: '');
+ });
+}
+
+it('tells why a dump stopped without an error message', function (int $exitCode, string $reason) {
+ interruptedMongodump($exitCode);
+
+ (new DatabaseBackupJob($this->backup))->handle();
+
+ $execution = $this->backup->executions()->latest('id')->firstOrFail();
+ expect($execution->status)->toBe('failed')
+ ->and($execution->message)->toStartWith($reason)
+ ->and($execution->message)->toContain('writing `evolution.printagents` to `archive on stdout`');
+ Notification::assertSentTo($this->team, BackupFailed::class, fn (BackupFailed $notification) => str_starts_with((string) $notification->output, $reason));
+})->with([
+ 'dump or database killed' => [137, 'The backup stopped with exit code 137: the dump process or the database container was killed (SIGKILL).'],
+ 'dump terminated' => [143, 'The backup stopped with exit code 143: the dump process or the database container was terminated (SIGTERM)'],
+ 'SSH connection lost' => [255, 'The backup stopped with exit code 255: the SSH connection to the server closed.'],
+]);
+
+it('keeps the error of a dump that reports its own failure', function () {
+ Process::fake(function ($process) {
+ if (str_contains($process->command, 'mongodump')) {
+ return Process::result(errorOutput: 'Failed: error connecting to db server: server returned error on SASL authentication step', exitCode: 1);
+ }
+
+ return Process::result(output: '');
+ });
+
+ (new DatabaseBackupJob($this->backup))->handle();
+
+ expect($this->backup->executions()->latest('id')->firstOrFail()->message)
+ ->toBe('Failed: error connecting to db server: server returned error on SASL authentication step');
+});