Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(database-import): stop the restore when the import task fails or loses SSH

Andras Bacsai1ee34a5ab2ec
diff --git a/app/Actions/CoolifyTask/RunRemoteProcess.php b/app/Actions/CoolifyTask/RunRemoteProcess.php
index 5b936686ce..90affa8db2 100644
--- a/app/Actions/CoolifyTask/RunRemoteProcess.php
+++ b/app/Actions/CoolifyTask/RunRemoteProcess.php
@@ -9,6 +9,7 @@ use App\Jobs\ApplicationDeploymentJob;
use App\Models\Server;
use App\Support\DatabaseImport\DatabaseImportCleanup;
use App\Support\RemoteProcessCommand;
+use App\Support\ResourceStartActivity;
use App\Traits\BroadcastsToTeam;
use Illuminate\Contracts\Process\ProcessResult;
use Illuminate\Support\Facades\DB;
@@ -92,6 +93,28 @@ class RunRemoteProcess
$this->activity->save();
$processResult = $process->wait();
+
+ // A database import whose SSH command ended without the restore result keeps blocking the
+ // database until its restore is stopped. The stop event also cleans up, so the normal
+ // finish event must not run first.
+ if (array_key_exists($processResult->exitCode(), DatabaseImportCleanup::TRANSPORT_FAILURE_REASONS)
+ && $this->activity->properties->get('operation') === ResourceStartActivity::DATABASE_IMPORT_OPERATION
+ && blank(Activity::query()->whereKey($this->activity->getKey())->first()?->properties?->get(DatabaseImportCleanup::STOP_REQUESTED_PROPERTY))
+ && DatabaseImportCleanup::stopAfterTransportFailure($this->activity, $processResult->exitCode())) {
+ $this->activity->properties = $this->activity->properties->merge([
+ 'exitCode' => $processResult->exitCode(),
+ 'stdout' => $processResult->output(),
+ 'stderr' => $processResult->errorOutput(),
+ ]);
+ $this->activity->save();
+
+ if (! $this->ignore_errors) {
+ throw new \RuntimeException($processResult->errorOutput(), $processResult->exitCode());
+ }
+
+ return $processResult;
+ }
+
if ($this->activity->properties->get('status') === ProcessStatus::ERROR->value) {
$status = ProcessStatus::ERROR;
} else {
diff --git a/app/Actions/Database/StartDatabaseImport.php b/app/Actions/Database/StartDatabaseImport.php
index b91d63cc92..5e113d163c 100644
--- a/app/Actions/Database/StartDatabaseImport.php
+++ b/app/Actions/Database/StartDatabaseImport.php
@@ -31,8 +31,27 @@ class StartDatabaseImport
public const LOCK_SECONDS = 1800;
+ /**
+ * Added to the SSH command timeout, so the remote restore ends (or times out) before the job.
+ * The job stays this far below the Horizon worker timeout too.
+ */
+ public const JOB_TIMEOUT_MARGIN_SECONDS = 300;
+
public function __construct(private readonly DatabaseImportCommandBuilder $commands) {}
+ /**
+ * The CoolifyTask timeout of an import. A restore can run until the SSH command timeout, so the
+ * job must not time out before it. It stays below the Horizon worker timeout, which is below the
+ * queue retry_after, so the worker does not kill or run the job again while it runs.
+ */
+ public static function jobTimeoutSeconds(): int
+ {
+ $timeout = (int) config('constants.ssh.command_timeout') + self::JOB_TIMEOUT_MARGIN_SECONDS;
+ $workerTimeout = (int) config('horizon.defaults.s6.timeout');
+
+ return $workerTimeout > 0 ? min($timeout, $workerTimeout - self::JOB_TIMEOUT_MARGIN_SECONDS) : $timeout;
+ }
+
public static function lockKey(string $resourceUuid): string
{
return "database-import:{$resourceUuid}";
@@ -157,13 +176,15 @@ class StartDatabaseImport
$commandList[] = 'chmod +x '.escapeshellarg($scriptPath);
$commandList[] = 'docker cp '.escapeshellarg($scriptPath).' '.escapeshellarg("{$container}:{$scriptPath}");
$commandList[] = 'rm -f '.escapeshellarg($scriptPath);
- $commandList[] = 'docker exec '.escapeshellarg($container).' sh -c '.escapeshellarg($scriptPath);
+ // The restore reports its own 124 and 255 as 1: Coolify reads those exit codes of the SSH
+ // command as a local timeout or a lost connection and then stops the restore.
+ $commandList[] = 'docker exec '.escapeshellarg($container).' sh -c '.escapeshellarg('"$0"; status=$?; [ "$status" -ne 124 ] && [ "$status" -ne 255 ] || status=1; exit "$status"').' '.escapeshellarg($scriptPath);
// The operation properties are set when the activity is created: the CoolifyTask job can
// load and save the activity before a later update, which would drop them again.
// The cleanup data (names and paths only, no credentials) lets Coolify stop and clean up
// an import that it fails after a restart or because it is stale.
- return remote_process($commandList, $server, type_uuid: $resource->uuid, model: $resource, callEventOnFinish: 'DatabaseImportFinished', callEventData: $cleanup, properties: [
+ return remote_process($commandList, $server, type_uuid: $resource->uuid, model: $resource, callEventOnFinish: 'DatabaseImportFinished', callEventData: $cleanup, timeout: self::jobTimeoutSeconds(), properties: [
'operation' => ResourceStartActivity::DATABASE_IMPORT_OPERATION,
'resource_kind' => $resource instanceof ServiceDatabase ? 'service_database' : 'standalone_database',
'operation_uuid' => $operation,
diff --git a/app/Jobs/CoolifyTask.php b/app/Jobs/CoolifyTask.php
index c19b61e19f..44739cd3f1 100755
--- a/app/Jobs/CoolifyTask.php
+++ b/app/Jobs/CoolifyTask.php
@@ -36,13 +36,19 @@ class CoolifyTask implements ShouldBeEncrypted, ShouldQueue
/**
* Create a new job instance.
+ *
+ * @param int|null $timeout Job timeout in seconds; null keeps the default above.
*/
public function __construct(
public Activity $activity,
public bool $ignore_errors,
public $call_event_on_finish,
public $call_event_data,
+ ?int $timeout = null,
) {
+ if ($timeout !== null) {
+ $this->timeout = $timeout;
+ }
$this->onQueue('high');
}
@@ -96,6 +102,14 @@ class CoolifyTask implements ShouldBeEncrypted, ShouldQueue
'trace' => $exception?->getTraceAsString(),
]);
+ // A database import whose restore can still run in the database container stays in progress
+ // until its stop has run, so it keeps blocking other operations on the database.
+ if (DatabaseImportCleanup::stopAfterTaskFailure($this->activity, $exception?->getMessage() ?: 'Job permanently failed')) {
+ RemoteProcessCommand::forget($this->activity);
+
+ return;
+ }
+
// Update activity status to reflect permanent failure
// A stopped database import already has the message that explains the stop; the process
// error ("Terminated") would hide it.
diff --git a/app/Listeners/CleanupDatabaseImport.php b/app/Listeners/CleanupDatabaseImport.php
index a136aef55b..62f5c72404 100644
--- a/app/Listeners/CleanupDatabaseImport.php
+++ b/app/Listeners/CleanupDatabaseImport.php
@@ -111,6 +111,11 @@ SH;
if ($server && $commands !== []) {
instant_remote_process($commands, $server);
}
+
+ // A failed import task keeps blocking the database until its restore is stopped.
+ if ($claimed && ($event->data['stopRestore'] ?? false) === true) {
+ DatabaseImportCleanup::finishStop($operation);
+ }
} catch (Throwable $exception) {
if ($claimed) {
DatabaseImportCleanup::release($operation);
diff --git a/app/Support/DatabaseImport/DatabaseImportCleanup.php b/app/Support/DatabaseImport/DatabaseImportCleanup.php
index 9cbd7900eb..9b7c7a72dc 100644
--- a/app/Support/DatabaseImport/DatabaseImportCleanup.php
+++ b/app/Support/DatabaseImport/DatabaseImportCleanup.php
@@ -2,7 +2,9 @@
namespace App\Support\DatabaseImport;
+use App\Enums\ProcessStatus;
use App\Events\DatabaseImportFinished;
+use App\Support\ResourceStartActivity;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Log;
@@ -28,6 +30,23 @@ class DatabaseImportCleanup
*/
public const STOP_REQUESTED_PROPERTY = 'stop_requested_at';
+ /**
+ * Why a failed import task was stopped. The activity gets this error when the stop has run.
+ */
+ public const STOP_REASON_PROPERTY = 'stop_reason';
+
+ /**
+ * Exit codes of the local SSH command that do not come from the restore: ssh exits with 255
+ * when the connection fails and the local `timeout` wrapper exits with 124. The restore
+ * command in the database container reports its own 124 and 255 as 1.
+ *
+ * @var array<int, string>
+ */
+ public const TRANSPORT_FAILURE_REASONS = [
+ 124 => 'The import timed out.',
+ 255 => 'The connection to the server was lost.',
+ ];
+
public const CLAIM_TTL_SECONDS = 7 * 24 * 3600;
/**
@@ -100,6 +119,81 @@ class DatabaseImportCleanup
}
}
+ /**
+ * Stop a database import whose CoolifyTask failed (for example it timed out) before the import
+ * reported its end, because the restore can still run in the database container.
+ *
+ * The activity stays in progress, so the import still blocks a start, restart, import or backup
+ * of the database. The cleanup listener marks it as failed after the stop has run.
+ *
+ * @return bool True when the import is stopped this way; the caller must not mark the activity as failed.
+ */
+ public static function stopAfterTaskFailure(Activity $activity, string $reason): bool
+ {
+ if (data_get($activity, 'properties.operation') !== ResourceStartActivity::DATABASE_IMPORT_OPERATION) {
+ return false;
+ }
+
+ $current = Activity::query()->find($activity->getKey());
+ $status = data_get($current, 'properties.status');
+ if (! $current || ! in_array($status, [ProcessStatus::QUEUED->value, ProcessStatus::IN_PROGRESS->value], true)) {
+ return false;
+ }
+
+ if (self::stopRequested($current)) {
+ return true;
+ }
+
+ $cleanup = self::stored($current);
+ if ($cleanup === null) {
+ return false;
+ }
+
+ // Save the stop flag before the stop is queued, so the listener and a retried CoolifyTask see it.
+ $current->properties = $current->properties->merge([
+ self::STOP_REQUESTED_PROPERTY => now()->toIso8601String(),
+ self::STOP_REASON_PROPERTY => $reason,
+ ]);
+ $current->save();
+ $activity->properties = $current->properties;
+
+ self::queueStop($cleanup);
+
+ return true;
+ }
+
+ /**
+ * Stop a database import whose SSH command ended without the result of the restore (the
+ * connection was lost or the local timeout ended it). The restore runs with `docker exec`
+ * and does not stop with the SSH session, so it is handled like a failed task.
+ *
+ * @return bool True when the import is stopped this way; the caller must keep it in progress.
+ */
+ public static function stopAfterTransportFailure(Activity $activity, ?int $exitCode): bool
+ {
+ $reason = self::TRANSPORT_FAILURE_REASONS[$exitCode] ?? null;
+
+ return $reason !== null && self::stopAfterTaskFailure($activity, $reason);
+ }
+
+ /**
+ * Mark the imports of an operation as failed that wait for the stop of their failed task.
+ * Called by the cleanup listener after the stop has run.
+ */
+ public static function finishStop(string $operationUuid): void
+ {
+ Activity::query()
+ ->where('properties->operation_uuid', $operationUuid)
+ ->where('properties->operation', ResourceStartActivity::DATABASE_IMPORT_OPERATION)
+ ->whereIn('properties->status', [ProcessStatus::QUEUED->value, ProcessStatus::IN_PROGRESS->value])
+ ->get()
+ ->filter(fn (Activity $activity): bool => filled(data_get($activity, 'properties.'.self::STOP_REASON_PROPERTY)))
+ ->each(fn (Activity $activity) => ResourceStartActivity::markFailed(
+ $activity,
+ data_get($activity, 'properties.'.self::STOP_REASON_PROPERTY).' '.ResourceStartActivity::IMPORT_STOPPED_MESSAGE,
+ ));
+ }
+
public static function claimKey(string $operationUuid): string
{
return "database-import-cleanup:{$operationUuid}";
diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php
index 62c9ae7d8d..b583521221 100644
--- a/bootstrap/helpers/remoteProcess.php
+++ b/bootstrap/helpers/remoteProcess.php
@@ -28,6 +28,7 @@ function remote_process(
$callEventOnFinish = null,
$callEventData = null,
array $properties = [],
+ ?int $timeout = null,
): Activity {
$type = $type ?? ActivityTypes::INLINE->value;
$command = $command instanceof Collection ? $command->toArray() : $command;
@@ -72,6 +73,7 @@ function remote_process(
ignore_errors: $ignore_errors,
call_event_on_finish: $callEventOnFinish,
call_event_data: $callEventData,
+ timeout: $timeout,
));
$activity->refresh();
diff --git a/tests/Feature/DatabaseImportInterruptedCleanupTest.php b/tests/Feature/DatabaseImportInterruptedCleanupTest.php
index fa2e41cfd0..0ef66fed33 100644
--- a/tests/Feature/DatabaseImportInterruptedCleanupTest.php
+++ b/tests/Feature/DatabaseImportInterruptedCleanupTest.php
@@ -1,6 +1,7 @@
<?php
use App\Actions\CoolifyTask\RunRemoteProcess;
+use App\Actions\Database\StartDatabase;
use App\Actions\Database\StartDatabaseImport;
use App\Enums\ActivityTypes;
use App\Enums\ProcessStatus;
@@ -18,9 +19,11 @@ use App\Models\Team;
use App\Support\DatabaseImport\DatabaseImportCleanup;
use App\Support\DatabaseImport\DatabaseImportException;
use App\Support\DatabaseImport\DatabaseImportSource;
+use App\Support\RemoteProcessCommand;
use App\Support\ResourceStartActivity;
use Illuminate\Events\CallQueuedListener;
use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Process\Factory as ProcessFactory;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Queue;
use Illuminate\Support\Str;
@@ -29,6 +32,7 @@ use Spatie\Activitylog\Models\Activity;
uses(RefreshDatabase::class);
beforeEach(function () {
+ Server::flushIdentityMap();
config()->set('cache.default', 'array');
config()->set('constants.ssh.mux_enabled', false);
InstanceSettings::forceCreate(['id' => 0]);
@@ -63,6 +67,10 @@ beforeEach(function () {
Queue::fake();
});
+afterEach(function () {
+ Server::flushIdentityMap();
+});
+
/**
* @return array<string, mixed>
*/
@@ -265,3 +273,214 @@ it('keeps the stop message when the stopped import job fails', function () {
->and(data_get($activity, 'properties.error'))->toContain('The import was stopped')
->and(data_get($activity, 'properties.failed_at'))->not->toBeNull();
});
+
+test('an import task may run longer than the SSH command timeout while other remote tasks keep the default timeout', function (int $commandTimeout) {
+ config()->set('constants.ssh.command_timeout', $commandTimeout);
+ Process::swap(new ProcessFactory);
+ Process::fake([
+ '*stat -c %s*' => Process::result('2048'),
+ '*od -An -tx1*' => Process::result(' 2d 2d 20 50 6f 73'),
+ '*' => Process::result(''),
+ ]);
+
+ app(StartDatabaseImport::class)->handle($this->database, new DatabaseImportSource('server', path: '/srv/backups/app.sql'), $this->team->id);
+ remote_process(['echo ok'], $this->server);
+
+ $timeouts = Queue::pushed(CoolifyTask::class)->mapWithKeys(fn (CoolifyTask $job) => [
+ (data_get($job->activity, 'properties.operation') ?? 'other') => $job->timeout,
+ ]);
+ $workerTimeout = (int) config('horizon.defaults.s6.timeout');
+
+ expect($timeouts['other'])->toBe(600)
+ ->and($timeouts[ResourceStartActivity::DATABASE_IMPORT_OPERATION])->toBeGreaterThan(min($commandTimeout, $workerTimeout - 600))
+ ->and($timeouts[ResourceStartActivity::DATABASE_IMPORT_OPERATION])->toBeLessThan($workerTimeout)
+ ->and($workerTimeout)->toBeLessThan((int) config('queue.connections.redis.retry_after'));
+})->with([
+ 'default SSH command timeout' => 3600,
+ 'SSH command timeout above the Horizon worker timeout' => 200000,
+]);
+
+test('a failed import task stops the remote restore and blocks the database until the restore is stopped', function () {
+ $operation = (string) Str::uuid();
+ $cleanup = storedImportCleanup($this, $operation);
+ $activity = runningImportActivity($this, ProcessStatus::IN_PROGRESS, $cleanup);
+ $job = new CoolifyTask($activity, ignore_errors: false, call_event_on_finish: 'DatabaseImportFinished', call_event_data: $cleanup);
+
+ $job->failed(new RuntimeException('App\\Jobs\\CoolifyTask has timed out.'));
+
+ assertStopQueuedOnce($cleanup);
+ expect(data_get($activity->refresh(), 'properties.status'))->toBe(ProcessStatus::IN_PROGRESS->value)
+ ->and(StartDatabase::operationInProgressError($this->database))->toBe(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE);
+
+ (new CleanupDatabaseImport)->handle(new DatabaseImportFinished([...$cleanup, 'stopRestore' => true]));
+
+ Process::assertRan(fn ($process) => str_contains($process->command, "/tmp/restore_{$operation}")
+ && str_contains($process->command, 'Stopping the database import processes'));
+ $activity->refresh();
+ expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value)
+ ->and(data_get($activity, 'properties.error'))->toBe('App\\Jobs\\CoolifyTask has timed out. '.ResourceStartActivity::IMPORT_STOPPED_MESSAGE)
+ ->and(StartDatabase::operationInProgressError($this->database))->toBeNull();
+});
+
+test('a failed import task whose restore already ended is only cleaned up', function () {
+ $operation = (string) Str::uuid();
+ $cleanup = storedImportCleanup($this, $operation);
+ $activity = runningImportActivity($this, ProcessStatus::ERROR, $cleanup);
+ $job = new CoolifyTask($activity, ignore_errors: false, call_event_on_finish: 'DatabaseImportFinished', call_event_data: $cleanup);
+
+ $job->failed(new RuntimeException('psql: error'));
+
+ Queue::assertPushed(CallQueuedListener::class, fn (CallQueuedListener $listener) => $listener->data[0]->data === $cleanup);
+ expect(data_get($activity->refresh(), 'properties.error'))->toBe('psql: error')
+ ->and(DatabaseImportCleanup::stopRequested($activity))->toBeFalse();
+});
+
+/**
+ * A CoolifyTask of a running import whose SSH command ends with the given result.
+ *
+ * @return array{0: Activity, 1: CoolifyTask, 2: array<string, mixed>}
+ */
+function importTaskEndingWith(object $test, int $exitCode, string $errorOutput): array
+{
+ $operation = (string) Str::uuid();
+ $cleanup = storedImportCleanup($test, $operation);
+ $activity = runningImportActivity($test, ProcessStatus::IN_PROGRESS, $cleanup);
+ $activity->properties = $activity->properties->merge([
+ 'server_uuid' => $test->server->uuid,
+ 'command' => "docker exec {$test->database->uuid} sh -c /tmp/restore_{$operation}.sh",
+ ]);
+ $activity->save();
+
+ Process::fake(fn () => Process::result(errorOutput: $errorOutput, exitCode: $exitCode));
+
+ return [$activity, new CoolifyTask($activity->fresh(), ignore_errors: false, call_event_on_finish: 'DatabaseImportFinished', call_event_data: $cleanup), $cleanup];
+}
+
+test('an import whose SSH command ends without the restore result stops the restore and blocks the database until it is stopped', function (int $exitCode, string $reason) {
+ [$activity, $job, $cleanup] = importTaskEndingWith($this, $exitCode, "Connection to 10.0.0.1 closed by remote host.\n");
+
+ // The queue calls failed() after the exception, because the task allows one exception.
+ try {
+ $job->handle();
+ $this->fail('The task must fail.');
+ } catch (RuntimeException $exception) {
+ $job->failed($exception);
+ }
+
+ // Only the stop is queued; the normal cleanup without the stop must not run first.
+ assertStopQueuedOnce($cleanup);
+ $activity->refresh();
+ expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::IN_PROGRESS->value)
+ ->and(data_get($activity, 'properties.exitCode'))->toBe($exitCode)
+ ->and(StartDatabase::operationInProgressError($this->database))->toBe(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE);
+
+ // The server is reachable again when the queued stop runs.
+ Process::swap(new ProcessFactory);
+ Process::fake();
+ (new CleanupDatabaseImport)->handle(new DatabaseImportFinished([...$cleanup, 'stopRestore' => true]));
+
+ Process::assertRan(fn ($process) => str_contains($process->command, 'Stopping the database import processes'));
+ $activity->refresh();
+ expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value)
+ ->and(data_get($activity, 'properties.error'))->toBe("{$reason} ".ResourceStartActivity::IMPORT_STOPPED_MESSAGE)
+ ->and(RunRemoteProcess::decodeOutput($activity))->toContain($reason)
+ ->and(StartDatabase::operationInProgressError($this->database))->toBeNull();
+})->with([
+ 'SSH connection lost (ssh exit 255)' => [255, 'The connection to the server was lost.'],
+ 'local SSH command timeout (timeout exit 124)' => [124, 'The import timed out.'],
+]);
+
+test('an import whose restore fails by itself is failed and cleaned up without a stop', function () {
+ [$activity, $job, $cleanup] = importTaskEndingWith($this, 3, "psql: ERROR: syntax error\n");
+
+ expect(fn () => $job->handle())->toThrow(RuntimeException::class);
+
+ Queue::assertPushed(CallQueuedListener::class, 1);
+ Queue::assertPushed(CallQueuedListener::class, fn (CallQueuedListener $listener) => $listener->data[0]->data === $cleanup);
+ $activity->refresh();
+ expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value)
+ ->and(DatabaseImportCleanup::stopRequested($activity))->toBeFalse()
+ ->and(StartDatabase::operationInProgressError($this->database))->toBeNull();
+});
+
+test('a lost import whose stop also fails stays blocking until the stale rule stops it again', function () {
+ config()->set('constants.ssh.max_retries', 1);
+ [$activity, $job, $cleanup] = importTaskEndingWith($this, 255, "Connection to 10.0.0.1 closed by remote host.\n");
+ expect(fn () => $job->handle())->toThrow(RuntimeException::class);
+
+ // The server is still unreachable, so the stop fails and the listener releases its claim for a retry.
+ expect(fn () => (new CleanupDatabaseImport)->handle(new DatabaseImportFinished([...$cleanup, 'stopRestore' => true])))
+ ->toThrow(RuntimeException::class);
+ expect(data_get($activity->refresh(), 'properties.status'))->toBe(ProcessStatus::IN_PROGRESS->value)
+ ->and(StartDatabase::operationInProgressError($this->database))->toBe(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE);
+
+ $this->travel(ResourceStartActivity::importStaleAfterSeconds() + 60)->seconds();
+ Queue::fake();
+
+ expect(StartDatabase::operationInProgressError($this->database))->toBeNull();
+ assertStopQueuedOnce($cleanup);
+ expect(data_get($activity->refresh(), 'properties.error'))->toBe(ResourceStartActivity::STALE_MESSAGE.' '.ResourceStartActivity::IMPORT_STOPPED_MESSAGE);
+});
+
+test('a restore that exits like a lost SSH connection or timeout reports a normal failure, also for a non-root SSH user', function (string $user, int $restoreExitCode, int $expectedExitCode) {
+ $this->server->update(['user' => $user]);
+ Server::flushIdentityMap();
+ Process::swap(new ProcessFactory);
+ Process::fake([
+ '*stat -c %s*' => Process::result('2048'),
+ '*od -An -tx1*' => Process::result(' 2d 2d 20 50 6f 73'),
+ '*' => Process::result(''),
+ ]);
+
+ app(StartDatabaseImport::class)->handle($this->database->fresh(), new DatabaseImportSource('server', path: '/srv/backups/app.sql'), $this->team->id);
+
+ $task = Queue::pushed(CoolifyTask::class)->first();
+ $restoreLine = collect(explode("\n", RemoteProcessCommand::read($task->activity)))->last();
+ $scriptPath = data_get($task->activity, 'properties.'.DatabaseImportCleanup::PROPERTY.'.scriptPath');
+ expect($restoreLine)->toContain($scriptPath);
+
+ // Fake sudo and docker: `docker exec <container> ...` runs the restore script on this machine.
+ $dir = sys_get_temp_dir().'/coolify-restore-exit-'.Str::random(8);
+ mkdir($dir);
+ file_put_contents("{$dir}/sudo", "#!/bin/sh\nexec \"\$@\"\n");
+ file_put_contents("{$dir}/docker", "#!/bin/sh\n[ \"\$1\" = exec ] || exit 1\nshift 2\nexec \"\$@\"\n");
+ file_put_contents($scriptPath, "echo restoring\nexit {$restoreExitCode}\n");
+ chmod("{$dir}/sudo", 0755);
+ chmod("{$dir}/docker", 0755);
+ chmod($scriptPath, 0755);
+
+ try {
+ $restore = new Symfony\Component\Process\Process(['bash', '-c', $restoreLine], env: ['PATH' => "{$dir}:".getenv('PATH')]);
+ $restore->run();
+
+ expect($restore->getExitCode())->toBe($expectedExitCode)
+ ->and($restore->getOutput())->toContain('restoring');
+ } finally {
+ @unlink($scriptPath);
+ array_map('unlink', glob("{$dir}/*"));
+ rmdir($dir);
+ }
+})->with([
+ 'root, restore exits 255' => ['root', 255, 1],
+ 'root, restore exits 124' => ['root', 124, 1],
+ 'root, restore fails' => ['root', 3, 3],
+ 'root, restore succeeds' => ['root', 0, 0],
+ 'non-root, restore exits 255' => ['ubuntu', 255, 1],
+ 'non-root, restore fails' => ['ubuntu', 3, 3],
+])->skip(fn () => PHP_OS_FAMILY !== 'Linux', 'Needs a POSIX shell.');
+
+test('a normal remote task that loses its SSH connection fails as before without an import stop', function (int $exitCode) {
+ Queue::fake();
+ Process::fake(fn () => Process::result(errorOutput: "Connection to 10.0.0.1 closed by remote host.\n", exitCode: $exitCode));
+ $activity = remote_process(['docker ps'], $this->server, ignore_errors: true, callEventOnFinish: 'DatabaseImportFinished', callEventData: ['marker' => 'normal-task']);
+
+ (new CoolifyTask($activity->fresh(), ignore_errors: true, call_event_on_finish: 'DatabaseImportFinished', call_event_data: ['marker' => 'normal-task']))->handle();
+
+ $activity->refresh();
+ expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value)
+ ->and(data_get($activity, 'properties.exitCode'))->toBe($exitCode)
+ ->and(data_get($activity, 'properties.'.DatabaseImportCleanup::STOP_REQUESTED_PROPERTY))->toBeNull();
+ // The normal finish event runs, without a stop.
+ Queue::assertPushed(CallQueuedListener::class, fn (CallQueuedListener $job) => ($job->data[0] ?? null) instanceof DatabaseImportFinished
+ && $job->data[0]->data === ['marker' => 'normal-task']);
+})->with(['ssh exit 255' => 255, 'timeout exit 124' => 124]);