An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(database-import): stop the restore when the import task fails or loses SSH
Andras Bacsai1ee34a5ab2ec
diff --git a/app/Actions/CoolifyTask/RunRemoteProcess.php b/app/Actions/CoolifyTask/RunRemoteProcess.phpindex 5b936686ce..90affa8db2 100644--- a/app/Actions/CoolifyTask/RunRemoteProcess.php+++ b/app/Actions/CoolifyTask/RunRemoteProcess.php@@ -9,6 +9,7 @@ use App\Jobs\ApplicationDeploymentJob;use App\Models\Server;use App\Support\DatabaseImport\DatabaseImportCleanup;use App\Support\RemoteProcessCommand;+use App\Support\ResourceStartActivity;use App\Traits\BroadcastsToTeam;use Illuminate\Contracts\Process\ProcessResult;use Illuminate\Support\Facades\DB;@@ -92,6 +93,28 @@ class RunRemoteProcess$this->activity->save();$processResult = $process->wait();++ // A database import whose SSH command ended without the restore result keeps blocking the+ // database until its restore is stopped. The stop event also cleans up, so the normal+ // finish event must not run first.+ if (array_key_exists($processResult->exitCode(), DatabaseImportCleanup::TRANSPORT_FAILURE_REASONS)+ && $this->activity->properties->get('operation') === ResourceStartActivity::DATABASE_IMPORT_OPERATION+ && blank(Activity::query()->whereKey($this->activity->getKey())->first()?->properties?->get(DatabaseImportCleanup::STOP_REQUESTED_PROPERTY))+ && DatabaseImportCleanup::stopAfterTransportFailure($this->activity, $processResult->exitCode())) {+ $this->activity->properties = $this->activity->properties->merge([+ 'exitCode' => $processResult->exitCode(),+ 'stdout' => $processResult->output(),+ 'stderr' => $processResult->errorOutput(),+ ]);+ $this->activity->save();++ if (! $this->ignore_errors) {+ throw new \RuntimeException($processResult->errorOutput(), $processResult->exitCode());+ }++ return $processResult;+ }+if ($this->activity->properties->get('status') === ProcessStatus::ERROR->value) {$status = ProcessStatus::ERROR;} else {diff --git a/app/Actions/Database/StartDatabaseImport.php b/app/Actions/Database/StartDatabaseImport.phpindex b91d63cc92..5e113d163c 100644--- a/app/Actions/Database/StartDatabaseImport.php+++ b/app/Actions/Database/StartDatabaseImport.php@@ -31,8 +31,27 @@ class StartDatabaseImportpublic const LOCK_SECONDS = 1800;+ /**+ * Added to the SSH command timeout, so the remote restore ends (or times out) before the job.+ * The job stays this far below the Horizon worker timeout too.+ */+ public const JOB_TIMEOUT_MARGIN_SECONDS = 300;+public function __construct(private readonly DatabaseImportCommandBuilder $commands) {}+ /**+ * The CoolifyTask timeout of an import. A restore can run until the SSH command timeout, so the+ * job must not time out before it. It stays below the Horizon worker timeout, which is below the+ * queue retry_after, so the worker does not kill or run the job again while it runs.+ */+ public static function jobTimeoutSeconds(): int+ {+ $timeout = (int) config('constants.ssh.command_timeout') + self::JOB_TIMEOUT_MARGIN_SECONDS;+ $workerTimeout = (int) config('horizon.defaults.s6.timeout');++ return $workerTimeout > 0 ? min($timeout, $workerTimeout - self::JOB_TIMEOUT_MARGIN_SECONDS) : $timeout;+ }+public static function lockKey(string $resourceUuid): string{return "database-import:{$resourceUuid}";@@ -157,13 +176,15 @@ class StartDatabaseImport$commandList[] = 'chmod +x '.escapeshellarg($scriptPath);$commandList[] = 'docker cp '.escapeshellarg($scriptPath).' '.escapeshellarg("{$container}:{$scriptPath}");$commandList[] = 'rm -f '.escapeshellarg($scriptPath);- $commandList[] = 'docker exec '.escapeshellarg($container).' sh -c '.escapeshellarg($scriptPath);+ // The restore reports its own 124 and 255 as 1: Coolify reads those exit codes of the SSH+ // command as a local timeout or a lost connection and then stops the restore.+ $commandList[] = 'docker exec '.escapeshellarg($container).' sh -c '.escapeshellarg('"$0"; status=$?; [ "$status" -ne 124 ] && [ "$status" -ne 255 ] || status=1; exit "$status"').' '.escapeshellarg($scriptPath);// The operation properties are set when the activity is created: the CoolifyTask job can// load and save the activity before a later update, which would drop them again.// The cleanup data (names and paths only, no credentials) lets Coolify stop and clean up// an import that it fails after a restart or because it is stale.- return remote_process($commandList, $server, type_uuid: $resource->uuid, model: $resource, callEventOnFinish: 'DatabaseImportFinished', callEventData: $cleanup, properties: [+ return remote_process($commandList, $server, type_uuid: $resource->uuid, model: $resource, callEventOnFinish: 'DatabaseImportFinished', callEventData: $cleanup, timeout: self::jobTimeoutSeconds(), properties: ['operation' => ResourceStartActivity::DATABASE_IMPORT_OPERATION,'resource_kind' => $resource instanceof ServiceDatabase ? 'service_database' : 'standalone_database','operation_uuid' => $operation,diff --git a/app/Jobs/CoolifyTask.php b/app/Jobs/CoolifyTask.phpindex c19b61e19f..44739cd3f1 100755--- a/app/Jobs/CoolifyTask.php+++ b/app/Jobs/CoolifyTask.php@@ -36,13 +36,19 @@ class CoolifyTask implements ShouldBeEncrypted, ShouldQueue/*** Create a new job instance.+ *+ * @param int|null $timeout Job timeout in seconds; null keeps the default above.*/public function __construct(public Activity $activity,public bool $ignore_errors,public $call_event_on_finish,public $call_event_data,+ ?int $timeout = null,) {+ if ($timeout !== null) {+ $this->timeout = $timeout;+ }$this->onQueue('high');}@@ -96,6 +102,14 @@ class CoolifyTask implements ShouldBeEncrypted, ShouldQueue'trace' => $exception?->getTraceAsString(),]);+ // A database import whose restore can still run in the database container stays in progress+ // until its stop has run, so it keeps blocking other operations on the database.+ if (DatabaseImportCleanup::stopAfterTaskFailure($this->activity, $exception?->getMessage() ?: 'Job permanently failed')) {+ RemoteProcessCommand::forget($this->activity);++ return;+ }+// Update activity status to reflect permanent failure// A stopped database import already has the message that explains the stop; the process// error ("Terminated") would hide it.diff --git a/app/Listeners/CleanupDatabaseImport.php b/app/Listeners/CleanupDatabaseImport.phpindex a136aef55b..62f5c72404 100644--- a/app/Listeners/CleanupDatabaseImport.php+++ b/app/Listeners/CleanupDatabaseImport.php@@ -111,6 +111,11 @@ SH;if ($server && $commands !== []) {instant_remote_process($commands, $server);}++ // A failed import task keeps blocking the database until its restore is stopped.+ if ($claimed && ($event->data['stopRestore'] ?? false) === true) {+ DatabaseImportCleanup::finishStop($operation);+ }} catch (Throwable $exception) {if ($claimed) {DatabaseImportCleanup::release($operation);diff --git a/app/Support/DatabaseImport/DatabaseImportCleanup.php b/app/Support/DatabaseImport/DatabaseImportCleanup.phpindex 9cbd7900eb..9b7c7a72dc 100644--- a/app/Support/DatabaseImport/DatabaseImportCleanup.php+++ b/app/Support/DatabaseImport/DatabaseImportCleanup.php@@ -2,7 +2,9 @@namespace App\Support\DatabaseImport;+use App\Enums\ProcessStatus;use App\Events\DatabaseImportFinished;+use App\Support\ResourceStartActivity;use Illuminate\Support\Arr;use Illuminate\Support\Facades\Cache;use Illuminate\Support\Facades\Log;@@ -28,6 +30,23 @@ class DatabaseImportCleanup*/public const STOP_REQUESTED_PROPERTY = 'stop_requested_at';+ /**+ * Why a failed import task was stopped. The activity gets this error when the stop has run.+ */+ public const STOP_REASON_PROPERTY = 'stop_reason';++ /**+ * Exit codes of the local SSH command that do not come from the restore: ssh exits with 255+ * when the connection fails and the local `timeout` wrapper exits with 124. The restore+ * command in the database container reports its own 124 and 255 as 1.+ *+ * @var array<int, string>+ */+ public const TRANSPORT_FAILURE_REASONS = [+ 124 => 'The import timed out.',+ 255 => 'The connection to the server was lost.',+ ];+public const CLAIM_TTL_SECONDS = 7 * 24 * 3600;/**@@ -100,6 +119,81 @@ class DatabaseImportCleanup}}+ /**+ * Stop a database import whose CoolifyTask failed (for example it timed out) before the import+ * reported its end, because the restore can still run in the database container.+ *+ * The activity stays in progress, so the import still blocks a start, restart, import or backup+ * of the database. The cleanup listener marks it as failed after the stop has run.+ *+ * @return bool True when the import is stopped this way; the caller must not mark the activity as failed.+ */+ public static function stopAfterTaskFailure(Activity $activity, string $reason): bool+ {+ if (data_get($activity, 'properties.operation') !== ResourceStartActivity::DATABASE_IMPORT_OPERATION) {+ return false;+ }++ $current = Activity::query()->find($activity->getKey());+ $status = data_get($current, 'properties.status');+ if (! $current || ! in_array($status, [ProcessStatus::QUEUED->value, ProcessStatus::IN_PROGRESS->value], true)) {+ return false;+ }++ if (self::stopRequested($current)) {+ return true;+ }++ $cleanup = self::stored($current);+ if ($cleanup === null) {+ return false;+ }++ // Save the stop flag before the stop is queued, so the listener and a retried CoolifyTask see it.+ $current->properties = $current->properties->merge([+ self::STOP_REQUESTED_PROPERTY => now()->toIso8601String(),+ self::STOP_REASON_PROPERTY => $reason,+ ]);+ $current->save();+ $activity->properties = $current->properties;++ self::queueStop($cleanup);++ return true;+ }++ /**+ * Stop a database import whose SSH command ended without the result of the restore (the+ * connection was lost or the local timeout ended it). The restore runs with `docker exec`+ * and does not stop with the SSH session, so it is handled like a failed task.+ *+ * @return bool True when the import is stopped this way; the caller must keep it in progress.+ */+ public static function stopAfterTransportFailure(Activity $activity, ?int $exitCode): bool+ {+ $reason = self::TRANSPORT_FAILURE_REASONS[$exitCode] ?? null;++ return $reason !== null && self::stopAfterTaskFailure($activity, $reason);+ }++ /**+ * Mark the imports of an operation as failed that wait for the stop of their failed task.+ * Called by the cleanup listener after the stop has run.+ */+ public static function finishStop(string $operationUuid): void+ {+ Activity::query()+ ->where('properties->operation_uuid', $operationUuid)+ ->where('properties->operation', ResourceStartActivity::DATABASE_IMPORT_OPERATION)+ ->whereIn('properties->status', [ProcessStatus::QUEUED->value, ProcessStatus::IN_PROGRESS->value])+ ->get()+ ->filter(fn (Activity $activity): bool => filled(data_get($activity, 'properties.'.self::STOP_REASON_PROPERTY)))+ ->each(fn (Activity $activity) => ResourceStartActivity::markFailed(+ $activity,+ data_get($activity, 'properties.'.self::STOP_REASON_PROPERTY).' '.ResourceStartActivity::IMPORT_STOPPED_MESSAGE,+ ));+ }+public static function claimKey(string $operationUuid): string{return "database-import-cleanup:{$operationUuid}";diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.phpindex 62c9ae7d8d..b583521221 100644--- a/bootstrap/helpers/remoteProcess.php+++ b/bootstrap/helpers/remoteProcess.php@@ -28,6 +28,7 @@ function remote_process($callEventOnFinish = null,$callEventData = null,array $properties = [],+ ?int $timeout = null,): Activity {$type = $type ?? ActivityTypes::INLINE->value;$command = $command instanceof Collection ? $command->toArray() : $command;@@ -72,6 +73,7 @@ function remote_process(ignore_errors: $ignore_errors,call_event_on_finish: $callEventOnFinish,call_event_data: $callEventData,+ timeout: $timeout,));$activity->refresh();diff --git a/tests/Feature/DatabaseImportInterruptedCleanupTest.php b/tests/Feature/DatabaseImportInterruptedCleanupTest.phpindex fa2e41cfd0..0ef66fed33 100644--- a/tests/Feature/DatabaseImportInterruptedCleanupTest.php+++ b/tests/Feature/DatabaseImportInterruptedCleanupTest.php@@ -1,6 +1,7 @@<?phpuse App\Actions\CoolifyTask\RunRemoteProcess;+use App\Actions\Database\StartDatabase;use App\Actions\Database\StartDatabaseImport;use App\Enums\ActivityTypes;use App\Enums\ProcessStatus;@@ -18,9 +19,11 @@ use App\Models\Team;use App\Support\DatabaseImport\DatabaseImportCleanup;use App\Support\DatabaseImport\DatabaseImportException;use App\Support\DatabaseImport\DatabaseImportSource;+use App\Support\RemoteProcessCommand;use App\Support\ResourceStartActivity;use Illuminate\Events\CallQueuedListener;use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Process\Factory as ProcessFactory;use Illuminate\Support\Facades\Process;use Illuminate\Support\Facades\Queue;use Illuminate\Support\Str;@@ -29,6 +32,7 @@ use Spatie\Activitylog\Models\Activity;uses(RefreshDatabase::class);beforeEach(function () {+ Server::flushIdentityMap();config()->set('cache.default', 'array');config()->set('constants.ssh.mux_enabled', false);InstanceSettings::forceCreate(['id' => 0]);@@ -63,6 +67,10 @@ beforeEach(function () {Queue::fake();});+afterEach(function () {+ Server::flushIdentityMap();+});+/*** @return array<string, mixed>*/@@ -265,3 +273,214 @@ it('keeps the stop message when the stopped import job fails', function () {->and(data_get($activity, 'properties.error'))->toContain('The import was stopped')->and(data_get($activity, 'properties.failed_at'))->not->toBeNull();});++test('an import task may run longer than the SSH command timeout while other remote tasks keep the default timeout', function (int $commandTimeout) {+ config()->set('constants.ssh.command_timeout', $commandTimeout);+ Process::swap(new ProcessFactory);+ Process::fake([+ '*stat -c %s*' => Process::result('2048'),+ '*od -An -tx1*' => Process::result(' 2d 2d 20 50 6f 73'),+ '*' => Process::result(''),+ ]);++ app(StartDatabaseImport::class)->handle($this->database, new DatabaseImportSource('server', path: '/srv/backups/app.sql'), $this->team->id);+ remote_process(['echo ok'], $this->server);++ $timeouts = Queue::pushed(CoolifyTask::class)->mapWithKeys(fn (CoolifyTask $job) => [+ (data_get($job->activity, 'properties.operation') ?? 'other') => $job->timeout,+ ]);+ $workerTimeout = (int) config('horizon.defaults.s6.timeout');++ expect($timeouts['other'])->toBe(600)+ ->and($timeouts[ResourceStartActivity::DATABASE_IMPORT_OPERATION])->toBeGreaterThan(min($commandTimeout, $workerTimeout - 600))+ ->and($timeouts[ResourceStartActivity::DATABASE_IMPORT_OPERATION])->toBeLessThan($workerTimeout)+ ->and($workerTimeout)->toBeLessThan((int) config('queue.connections.redis.retry_after'));+})->with([+ 'default SSH command timeout' => 3600,+ 'SSH command timeout above the Horizon worker timeout' => 200000,+]);++test('a failed import task stops the remote restore and blocks the database until the restore is stopped', function () {+ $operation = (string) Str::uuid();+ $cleanup = storedImportCleanup($this, $operation);+ $activity = runningImportActivity($this, ProcessStatus::IN_PROGRESS, $cleanup);+ $job = new CoolifyTask($activity, ignore_errors: false, call_event_on_finish: 'DatabaseImportFinished', call_event_data: $cleanup);++ $job->failed(new RuntimeException('App\\Jobs\\CoolifyTask has timed out.'));++ assertStopQueuedOnce($cleanup);+ expect(data_get($activity->refresh(), 'properties.status'))->toBe(ProcessStatus::IN_PROGRESS->value)+ ->and(StartDatabase::operationInProgressError($this->database))->toBe(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE);++ (new CleanupDatabaseImport)->handle(new DatabaseImportFinished([...$cleanup, 'stopRestore' => true]));++ Process::assertRan(fn ($process) => str_contains($process->command, "/tmp/restore_{$operation}")+ && str_contains($process->command, 'Stopping the database import processes'));+ $activity->refresh();+ expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value)+ ->and(data_get($activity, 'properties.error'))->toBe('App\\Jobs\\CoolifyTask has timed out. '.ResourceStartActivity::IMPORT_STOPPED_MESSAGE)+ ->and(StartDatabase::operationInProgressError($this->database))->toBeNull();+});++test('a failed import task whose restore already ended is only cleaned up', function () {+ $operation = (string) Str::uuid();+ $cleanup = storedImportCleanup($this, $operation);+ $activity = runningImportActivity($this, ProcessStatus::ERROR, $cleanup);+ $job = new CoolifyTask($activity, ignore_errors: false, call_event_on_finish: 'DatabaseImportFinished', call_event_data: $cleanup);++ $job->failed(new RuntimeException('psql: error'));++ Queue::assertPushed(CallQueuedListener::class, fn (CallQueuedListener $listener) => $listener->data[0]->data === $cleanup);+ expect(data_get($activity->refresh(), 'properties.error'))->toBe('psql: error')+ ->and(DatabaseImportCleanup::stopRequested($activity))->toBeFalse();+});++/**+ * A CoolifyTask of a running import whose SSH command ends with the given result.+ *+ * @return array{0: Activity, 1: CoolifyTask, 2: array<string, mixed>}+ */+function importTaskEndingWith(object $test, int $exitCode, string $errorOutput): array+{+ $operation = (string) Str::uuid();+ $cleanup = storedImportCleanup($test, $operation);+ $activity = runningImportActivity($test, ProcessStatus::IN_PROGRESS, $cleanup);+ $activity->properties = $activity->properties->merge([+ 'server_uuid' => $test->server->uuid,+ 'command' => "docker exec {$test->database->uuid} sh -c /tmp/restore_{$operation}.sh",+ ]);+ $activity->save();++ Process::fake(fn () => Process::result(errorOutput: $errorOutput, exitCode: $exitCode));++ return [$activity, new CoolifyTask($activity->fresh(), ignore_errors: false, call_event_on_finish: 'DatabaseImportFinished', call_event_data: $cleanup), $cleanup];+}++test('an import whose SSH command ends without the restore result stops the restore and blocks the database until it is stopped', function (int $exitCode, string $reason) {+ [$activity, $job, $cleanup] = importTaskEndingWith($this, $exitCode, "Connection to 10.0.0.1 closed by remote host.\n");++ // The queue calls failed() after the exception, because the task allows one exception.+ try {+ $job->handle();+ $this->fail('The task must fail.');+ } catch (RuntimeException $exception) {+ $job->failed($exception);+ }++ // Only the stop is queued; the normal cleanup without the stop must not run first.+ assertStopQueuedOnce($cleanup);+ $activity->refresh();+ expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::IN_PROGRESS->value)+ ->and(data_get($activity, 'properties.exitCode'))->toBe($exitCode)+ ->and(StartDatabase::operationInProgressError($this->database))->toBe(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE);++ // The server is reachable again when the queued stop runs.+ Process::swap(new ProcessFactory);+ Process::fake();+ (new CleanupDatabaseImport)->handle(new DatabaseImportFinished([...$cleanup, 'stopRestore' => true]));++ Process::assertRan(fn ($process) => str_contains($process->command, 'Stopping the database import processes'));+ $activity->refresh();+ expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value)+ ->and(data_get($activity, 'properties.error'))->toBe("{$reason} ".ResourceStartActivity::IMPORT_STOPPED_MESSAGE)+ ->and(RunRemoteProcess::decodeOutput($activity))->toContain($reason)+ ->and(StartDatabase::operationInProgressError($this->database))->toBeNull();+})->with([+ 'SSH connection lost (ssh exit 255)' => [255, 'The connection to the server was lost.'],+ 'local SSH command timeout (timeout exit 124)' => [124, 'The import timed out.'],+]);++test('an import whose restore fails by itself is failed and cleaned up without a stop', function () {+ [$activity, $job, $cleanup] = importTaskEndingWith($this, 3, "psql: ERROR: syntax error\n");++ expect(fn () => $job->handle())->toThrow(RuntimeException::class);++ Queue::assertPushed(CallQueuedListener::class, 1);+ Queue::assertPushed(CallQueuedListener::class, fn (CallQueuedListener $listener) => $listener->data[0]->data === $cleanup);+ $activity->refresh();+ expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value)+ ->and(DatabaseImportCleanup::stopRequested($activity))->toBeFalse()+ ->and(StartDatabase::operationInProgressError($this->database))->toBeNull();+});++test('a lost import whose stop also fails stays blocking until the stale rule stops it again', function () {+ config()->set('constants.ssh.max_retries', 1);+ [$activity, $job, $cleanup] = importTaskEndingWith($this, 255, "Connection to 10.0.0.1 closed by remote host.\n");+ expect(fn () => $job->handle())->toThrow(RuntimeException::class);++ // The server is still unreachable, so the stop fails and the listener releases its claim for a retry.+ expect(fn () => (new CleanupDatabaseImport)->handle(new DatabaseImportFinished([...$cleanup, 'stopRestore' => true])))+ ->toThrow(RuntimeException::class);+ expect(data_get($activity->refresh(), 'properties.status'))->toBe(ProcessStatus::IN_PROGRESS->value)+ ->and(StartDatabase::operationInProgressError($this->database))->toBe(ResourceStartActivity::DATABASE_OPERATION_IN_PROGRESS_MESSAGE);++ $this->travel(ResourceStartActivity::importStaleAfterSeconds() + 60)->seconds();+ Queue::fake();++ expect(StartDatabase::operationInProgressError($this->database))->toBeNull();+ assertStopQueuedOnce($cleanup);+ expect(data_get($activity->refresh(), 'properties.error'))->toBe(ResourceStartActivity::STALE_MESSAGE.' '.ResourceStartActivity::IMPORT_STOPPED_MESSAGE);+});++test('a restore that exits like a lost SSH connection or timeout reports a normal failure, also for a non-root SSH user', function (string $user, int $restoreExitCode, int $expectedExitCode) {+ $this->server->update(['user' => $user]);+ Server::flushIdentityMap();+ Process::swap(new ProcessFactory);+ Process::fake([+ '*stat -c %s*' => Process::result('2048'),+ '*od -An -tx1*' => Process::result(' 2d 2d 20 50 6f 73'),+ '*' => Process::result(''),+ ]);++ app(StartDatabaseImport::class)->handle($this->database->fresh(), new DatabaseImportSource('server', path: '/srv/backups/app.sql'), $this->team->id);++ $task = Queue::pushed(CoolifyTask::class)->first();+ $restoreLine = collect(explode("\n", RemoteProcessCommand::read($task->activity)))->last();+ $scriptPath = data_get($task->activity, 'properties.'.DatabaseImportCleanup::PROPERTY.'.scriptPath');+ expect($restoreLine)->toContain($scriptPath);++ // Fake sudo and docker: `docker exec <container> ...` runs the restore script on this machine.+ $dir = sys_get_temp_dir().'/coolify-restore-exit-'.Str::random(8);+ mkdir($dir);+ file_put_contents("{$dir}/sudo", "#!/bin/sh\nexec \"\$@\"\n");+ file_put_contents("{$dir}/docker", "#!/bin/sh\n[ \"\$1\" = exec ] || exit 1\nshift 2\nexec \"\$@\"\n");+ file_put_contents($scriptPath, "echo restoring\nexit {$restoreExitCode}\n");+ chmod("{$dir}/sudo", 0755);+ chmod("{$dir}/docker", 0755);+ chmod($scriptPath, 0755);++ try {+ $restore = new Symfony\Component\Process\Process(['bash', '-c', $restoreLine], env: ['PATH' => "{$dir}:".getenv('PATH')]);+ $restore->run();++ expect($restore->getExitCode())->toBe($expectedExitCode)+ ->and($restore->getOutput())->toContain('restoring');+ } finally {+ @unlink($scriptPath);+ array_map('unlink', glob("{$dir}/*"));+ rmdir($dir);+ }+})->with([+ 'root, restore exits 255' => ['root', 255, 1],+ 'root, restore exits 124' => ['root', 124, 1],+ 'root, restore fails' => ['root', 3, 3],+ 'root, restore succeeds' => ['root', 0, 0],+ 'non-root, restore exits 255' => ['ubuntu', 255, 1],+ 'non-root, restore fails' => ['ubuntu', 3, 3],+])->skip(fn () => PHP_OS_FAMILY !== 'Linux', 'Needs a POSIX shell.');++test('a normal remote task that loses its SSH connection fails as before without an import stop', function (int $exitCode) {+ Queue::fake();+ Process::fake(fn () => Process::result(errorOutput: "Connection to 10.0.0.1 closed by remote host.\n", exitCode: $exitCode));+ $activity = remote_process(['docker ps'], $this->server, ignore_errors: true, callEventOnFinish: 'DatabaseImportFinished', callEventData: ['marker' => 'normal-task']);++ (new CoolifyTask($activity->fresh(), ignore_errors: true, call_event_on_finish: 'DatabaseImportFinished', call_event_data: ['marker' => 'normal-task']))->handle();++ $activity->refresh();+ expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value)+ ->and(data_get($activity, 'properties.exitCode'))->toBe($exitCode)+ ->and(data_get($activity, 'properties.'.DatabaseImportCleanup::STOP_REQUESTED_PROPERTY))->toBeNull();+ // The normal finish event runs, without a stop.+ Queue::assertPushed(CallQueuedListener::class, fn (CallQueuedListener $job) => ($job->data[0] ?? null) instanceof DatabaseImportFinished+ && $job->data[0]->data === ['marker' => 'normal-task']);+})->with(['ssh exit 255' => 255, 'timeout exit 124' => 124]);