An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(databases): stop and clean up interrupted or stale imports
Andras Bacsai8246492bb97f
diff --git a/app/Actions/CoolifyTask/RunRemoteProcess.php b/app/Actions/CoolifyTask/RunRemoteProcess.phpindex e0dacfbec8..93a46d75a2 100644--- a/app/Actions/CoolifyTask/RunRemoteProcess.php+++ b/app/Actions/CoolifyTask/RunRemoteProcess.php@@ -7,7 +7,8 @@ use App\Enums\ProcessStatus;use App\Helpers\SshMultiplexingHelper;use App\Jobs\ApplicationDeploymentJob;use App\Models\Server;-use Illuminate\Process\ProcessResult;+use App\Support\DatabaseImport\DatabaseImportCleanup;+use Illuminate\Contracts\Process\ProcessResult;use Illuminate\Support\Facades\DB;use Illuminate\Support\Facades\Log;use Illuminate\Support\Facades\Process;@@ -80,9 +81,13 @@ class RunRemoteProcess$status = ProcessStatus::IN_PROGRESS;$timeout = config('constants.ssh.command_timeout');$process = Process::timeout($timeout)->start($this->getCommand(), $this->handleOutput(...));- $this->activity->properties = $this->activity->properties->merge([+ // Start from the stored properties and save now, so later output writes only change the log and+ // never write back an old copy over a status that Coolify set meanwhile (for example a stopped import).+ $stored = Activity::query()->whereKey($this->activity->getKey())->first()?->properties ?? $this->activity->properties;+ $this->activity->properties = $stored->merge(['process_id' => $process->id(),]);+ $this->activity->save();$processResult = $process->wait();if ($this->activity->properties->get('status') === ProcessStatus::ERROR->value) {@@ -95,12 +100,25 @@ class RunRemoteProcess}}- $this->activity->properties = $this->activity->properties->merge([+ $properties = ['exitCode' => $processResult->exitCode(),'stdout' => $processResult->output(),'stderr' => $processResult->errorOutput(),'status' => $status->value,- ]);+ ];++ // Coolify can stop a database import while this job runs (restart or stale import). Keep the+ // stop status and message, because this job only has its own older copy of the properties.+ $stored = Activity::query()->whereKey($this->activity->getKey())->first()?->properties;+ if ($stored !== null && filled($stored->get(DatabaseImportCleanup::STOP_REQUESTED_PROPERTY))) {+ $properties = [+ ...$properties,+ ...$stored->only([DatabaseImportCleanup::STOP_REQUESTED_PROPERTY, 'error'])->all(),+ 'status' => ProcessStatus::ERROR->value,+ ];+ }++ $this->activity->properties = $this->activity->properties->merge($properties);$this->activity->save();if ($this->call_event_on_finish) {try {diff --git a/app/Actions/Database/StartDatabaseImport.php b/app/Actions/Database/StartDatabaseImport.phpindex 7439e9f1b0..642ff35216 100644--- a/app/Actions/Database/StartDatabaseImport.php+++ b/app/Actions/Database/StartDatabaseImport.php@@ -7,6 +7,7 @@ use App\Models\Server;use App\Models\ServiceDatabase;use App\Models\SwarmDocker;use App\Support\DatabaseBackupFileValidator;+use App\Support\DatabaseImport\DatabaseImportCleanup;use App\Support\DatabaseImport\DatabaseImportCommandBuilder;use App\Support\DatabaseImport\DatabaseImportException;use App\Support\DatabaseImport\DatabaseImportSource;@@ -92,7 +93,7 @@ class StartDatabaseImport$containerPath = "/tmp/restore_{$operation}";$scriptPath = "/tmp/restore_{$operation}.sh";$commandList = [];- $cleanup = ['container' => $container, 'containerTmpPath' => $containerPath, 'scriptPath' => $scriptPath, 'serverId' => $server->id];+ $cleanup = ['container' => $container, 'containerTmpPath' => $containerPath, 'scriptPath' => $scriptPath, 'serverId' => $server->id, 'operationUuid' => $operation];if ($source->type === 'upload') {$staged = $source->uploadId@@ -158,10 +159,13 @@ class StartDatabaseImport// The operation properties are set when the activity is created: the CoolifyTask job can// load and save the activity before a later update, which would drop them again.+ // The cleanup data (names and paths only, no credentials) lets Coolify stop and clean up+ // an import that it fails after a restart or because it is stale.return remote_process($commandList, $server, type_uuid: $resource->uuid, model: $resource, callEventOnFinish: 'DatabaseImportFinished', callEventData: $cleanup, properties: ['operation' => ResourceStartActivity::DATABASE_IMPORT_OPERATION,'resource_kind' => $resource instanceof ServiceDatabase ? 'service_database' : 'standalone_database','operation_uuid' => $operation,+ DatabaseImportCleanup::PROPERTY => DatabaseImportCleanup::storable($cleanup),]);}diff --git a/app/Http/Controllers/Api/DatabasesController.php b/app/Http/Controllers/Api/DatabasesController.phpindex 7b04422384..e828066742 100644--- a/app/Http/Controllers/Api/DatabasesController.php+++ b/app/Http/Controllers/Api/DatabasesController.php@@ -73,7 +73,7 @@ class DatabasesController extends Controller],responses: [new OA\Response(response: 202, description: 'Import queued'),- new OA\Response(response: 409, description: 'Import already active'),+ new OA\Response(response: 409, description: 'Another start, restart or import of this database is already in progress.'),new OA\Response(response: 422, ref: '#/components/responses/422'),])]diff --git a/app/Http/Controllers/Api/ServiceDatabasesController.php b/app/Http/Controllers/Api/ServiceDatabasesController.phpindex 1d2a602461..62d1292192 100644--- a/app/Http/Controllers/Api/ServiceDatabasesController.php+++ b/app/Http/Controllers/Api/ServiceDatabasesController.php@@ -53,7 +53,7 @@ class ServiceDatabasesController extends Controller],responses: [new OA\Response(response: 202, description: 'Import queued'),- new OA\Response(response: 409, description: 'Import already active'),+ new OA\Response(response: 409, description: 'Another start, restart or import of this database is already in progress.'),new OA\Response(response: 422, ref: '#/components/responses/422'),])]diff --git a/app/Jobs/CoolifyTask.php b/app/Jobs/CoolifyTask.phpindex ce535e036d..f25258b075 100755--- a/app/Jobs/CoolifyTask.php+++ b/app/Jobs/CoolifyTask.php@@ -4,6 +4,7 @@ namespace App\Jobs;use App\Actions\CoolifyTask\RunRemoteProcess;use App\Enums\ProcessStatus;+use App\Support\DatabaseImport\DatabaseImportCleanup;use Illuminate\Bus\Queueable;use Illuminate\Contracts\Queue\ShouldBeEncrypted;use Illuminate\Contracts\Queue\ShouldQueue;@@ -50,6 +51,12 @@ class CoolifyTask implements ShouldBeEncrypted, ShouldQueue*/public function handle(): void{+ // A database import that Coolify stopped after a restart must not run again when+ // the queue retries this job. Its cleanup is already queued.+ if (DatabaseImportCleanup::stopRequested($this->activity)) {+ return;+ }+$remote_process = resolve(RunRemoteProcess::class, ['activity' => $this->activity,'ignore_errors' => $this->ignore_errors,@@ -84,11 +91,14 @@ class CoolifyTask implements ShouldBeEncrypted, ShouldQueue]);// Update activity status to reflect permanent failure- $this->activity->properties = $this->activity->properties->merge([+ // A stopped database import already has the message that explains the stop; the process+ // error ("Terminated") would hide it.+ $stopRequested = DatabaseImportCleanup::stopRequested($this->activity);+ $this->activity->properties = $this->activity->properties->merge(array_filter(['status' => ProcessStatus::ERROR->value,- 'error' => $exception?->getMessage() ?? 'Job permanently failed',+ 'error' => $stopRequested ? null : ($exception?->getMessage() ?? 'Job permanently failed'),'failed_at' => now()->toIso8601String(),- ]);+ ], fn ($value) => $value !== null));$this->activity->save();// Dispatch cleanup event on failure (same as on success)diff --git a/app/Listeners/CleanupDatabaseImport.php b/app/Listeners/CleanupDatabaseImport.phpindex a3a36b8342..a136aef55b 100644--- a/app/Listeners/CleanupDatabaseImport.php+++ b/app/Listeners/CleanupDatabaseImport.php@@ -4,8 +4,11 @@ namespace App\Listeners;use App\Events\DatabaseImportFinished;use App\Models\Server;+use App\Support\DatabaseImport\DatabaseImportCleanup;use Illuminate\Contracts\Queue\ShouldQueue;use Illuminate\Support\Facades\Log;+use Illuminate\Support\Str;+use InvalidArgumentException;use Throwable;class CleanupDatabaseImport implements ShouldQueue@@ -15,13 +18,105 @@ class CleanupDatabaseImport implements ShouldQueue/** @var array<int, int> */public array $backoff = [5, 15, 30];+ /**+ * Rounds (one second each) of the restore stop script, and how many of them send SIGTERM before SIGKILL.+ */+ private const STOP_ROUNDS = 30;++ private const STOP_TERM_ROUNDS = 10;++ /**+ * Placeholders: __OPERATION_PATH__ (quoted), __ROUNDS__, __TERM_ROUNDS__.+ * The script has no command substitution, so the non-root sudo parser does not change it.+ */+ private const STOP_RESTORE_SCRIPT = <<<'SH'+op=__OPERATION_PATH__+info() {+ pp=; st=+ while read -r k v rest; do+ case "$k" in PPid:) pp=$v ;; State:) st=$v ;; esac+ done 2>/dev/null < "/proc/$1/status"+}+skip=" $$ "+p=$$+while [ "$p" -gt 1 ] 2>/dev/null; do+ info "$p"+ [ -n "$pp" ] || break+ skip="$skip$pp "+ p=$pp+done+scan() {+ all=; zombies=; pids=+ for d in /proc/[0-9]*; do+ p=${d#/proc/}+ case "$skip" in *" $p "*) continue ;; esac+ info "$p"+ [ -n "$pp" ] || continue+ all="$all $p:$pp"+ if [ "$st" = Z ]; then zombies="$zombies $p"; continue; fi+ if tr '\000' ' ' < "$d/cmdline" 2>/dev/null | grep -qF -- "$op"; then pids="$pids $p"; fi+ done+ found=1+ while [ -n "$found" ]; do+ found=+ for e in $all; do+ p=${e%%:*}+ case "$pids " in *" $p "*) continue ;; esac+ case "$pids " in *" ${e#*:} "*) pids="$pids $p"; found=1 ;; esac+ done+ done+}+roots=+for round in __ROUNDS__; do+ scan+ [ -n "$pids" ] || exit 0+ sig=TERM+ [ "$round" -le __TERM_ROUNDS__ ] || sig=KILL+ previous=$roots; roots=; targets=+ for e in $all; do+ p=${e%%:*}; pp=${e#*:}+ case "$pids " in *" $p "*) ;; *) continue ;; esac+ case "$zombies " in *" $p "*) continue ;; esac+ case "$all " in *":$p "*) continue ;; esac+ [ "$pp" != 1 ] || continue+ case "$pids " in+ *" $pp "*) targets="$targets $p" ;;+ *) roots="$roots $p"; case "$previous " in *" $p "*) targets="$targets $p" ;; esac ;;+ esac+ done+ if [ -n "$targets" ]; then+ echo "Stopping the database import processes ($sig):$targets"+ kill -$sig $targets 2>/dev/null+ fi+ sleep 1+done+echo "Some database import processes still run:$pids" >&2+exit 0+SH;+public function handle(DatabaseImportFinished $event): void{- $commands = $this->commands($event->data);- $server = Server::query()->find($event->data['serverId'] ?? null);+ $operation = $event->data['operationUuid'] ?? null;+ $claimed = is_string($operation) && Str::isUuid($operation);++ // The normal finish and a stop after a restart or a stale import can both send the event.+ if ($claimed && ! DatabaseImportCleanup::claim($operation)) {+ return;+ }++ try {+ $commands = $this->commands($event->data);+ $server = Server::query()->find($event->data['serverId'] ?? null);++ if ($server && $commands !== []) {+ instant_remote_process($commands, $server);+ }+ } catch (Throwable $exception) {+ if ($claimed) {+ DatabaseImportCleanup::release($operation);+ }- if ($server && $commands !== []) {- instant_remote_process($commands, $server);+ throw $exception;}}@@ -33,6 +128,10 @@ class CleanupDatabaseImport implements ShouldQueue{$commands = [];+ if (($data['stopRestore'] ?? false) === true && filled($data['container'] ?? null) && Str::isUuid($data['operationUuid'] ?? null)) {+ $commands[] = $this->stopRestoreCommand($data['container'], $data['operationUuid']);+ }+if (filled($data['containerName'] ?? null)) {$commands[] = 'docker rm -f '.escapeshellarg($data['containerName']).' 2>/dev/null || true';}@@ -56,6 +155,39 @@ class CleanupDatabaseImport implements ShouldQueuereturn $commands;}+ /**+ * Stops a restore that still runs in the database container after its SSH session ended.+ *+ * `docker exec` processes are not stopped when the SSH session ends. The script selects the+ * processes with the unique operation path (/tmp/restore_{uuid}) in their command line and+ * all their child processes (for example psql or pg_restore). It never selects itself or+ * its parent processes.+ *+ * Processes are stopped from the bottom up. A process gets a signal only when it has no+ * child processes (zombies count until they are reaped), so its live parent reaps it.+ * A process whose parent is PID 1 never gets a signal: PostgreSQL runs as PID 1 and restarts+ * all connections when an unknown child process ends because of a signal. A top-level+ * process (its parent is outside the selection) gets a signal only after it had no child+ * processes for one full round, because a shell between two commands can fork at any time.+ * The first rounds send SIGTERM, later rounds SIGKILL.+ *+ * It uses only sh built-ins, tr, grep and sleep, because database images can lack pkill/pgrep.+ */+ public function stopRestoreCommand(string $container, string $operationUuid): string+ {+ if (! Str::isUuid($operationUuid)) {+ throw new InvalidArgumentException('The database import operation id is invalid.');+ }++ $script = strtr(self::STOP_RESTORE_SCRIPT, [+ '__OPERATION_PATH__' => escapeshellarg("/tmp/restore_{$operationUuid}"),+ '__ROUNDS__' => implode(' ', range(1, self::STOP_ROUNDS)),+ '__TERM_ROUNDS__' => (string) self::STOP_TERM_ROUNDS,+ ]);++ return 'docker exec '.escapeshellarg($container).' sh -c '.escapeshellarg($script).' 2>/dev/null || true';+ }+public function failed(DatabaseImportFinished $event, Throwable $exception): void{Log::error('Database import cleanup failed', [diff --git a/app/Support/DatabaseImport/DatabaseImportCleanup.php b/app/Support/DatabaseImport/DatabaseImportCleanup.phpnew file mode 100644index 0000000000..9cbd7900eb--- /dev/null+++ b/app/Support/DatabaseImport/DatabaseImportCleanup.php@@ -0,0 +1,123 @@+<?php++namespace App\Support\DatabaseImport;++use App\Events\DatabaseImportFinished;+use Illuminate\Support\Arr;+use Illuminate\Support\Facades\Cache;+use Illuminate\Support\Facades\Log;+use Illuminate\Support\Str;+use Spatie\Activitylog\Models\Activity;+use Throwable;++/**+ * Cleanup of a database import: helper containers and temporary files on the server+ * and in the database container.+ *+ * The normal path sends the cleanup data through the encrypted CoolifyTask payload.+ * The import also stores the same data on its activity, so Coolify can stop and clean up+ * an import that it marks as failed after a restart or because it is stale.+ * The data has only ids, container names and paths. It must never have credentials.+ */+class DatabaseImportCleanup+{+ public const PROPERTY = 'cleanup';++ /**+ * Set on the activity when Coolify stopped the import. A retried CoolifyTask does not run it again.+ */+ public const STOP_REQUESTED_PROPERTY = 'stop_requested_at';++ public const CLAIM_TTL_SECONDS = 7 * 24 * 3600;++ /**+ * The only keys that are stored on the activity and sent to the cleanup listener.+ */+ private const KEYS = [+ 'container',+ 'containerTmpPath',+ 'scriptPath',+ 'serverId',+ 'operationUuid',+ 'containerName',+ 'serverTmpPath',+ 'credentialTmpPath',+ ];++ /**+ * @param array<string, mixed> $data+ * @return array<string, int|string>+ */+ public static function storable(array $data): array+ {+ return array_filter(+ Arr::only($data, self::KEYS),+ fn ($value): bool => is_string($value) || is_int($value),+ );+ }++ /**+ * The cleanup data stored on an import activity, or null if the activity has no usable data.+ *+ * @return array<string, int|string>|null+ */+ public static function stored(Activity $activity): ?array+ {+ $data = data_get($activity, 'properties.'.self::PROPERTY);+ if (! is_array($data)) {+ return null;+ }++ $data = self::storable($data);+ if (! is_int($data['serverId'] ?? null) || ! Str::isUuid($data['operationUuid'] ?? null)) {+ return null;+ }++ return $data;+ }++ public static function stopRequested(Activity $activity): bool+ {+ return filled(data_get($activity, 'properties.'.self::STOP_REQUESTED_PROPERTY));+ }++ /**+ * Queue the stop of the remote restore and the cleanup. It does not run SSH commands+ * inline, so it is safe during boot. Errors are logged and do not stop the caller.+ *+ * @param array<string, int|string> $data+ */+ public static function queueStop(array $data): void+ {+ try {+ event(new DatabaseImportFinished([...$data, 'stopRestore' => true]));+ } catch (Throwable $exception) {+ Log::warning('Could not queue the stop of an interrupted database import', [+ 'operationUuid' => $data['operationUuid'] ?? null,+ 'serverId' => $data['serverId'] ?? null,+ 'error' => $exception->getMessage(),+ ]);+ }+ }++ public static function claimKey(string $operationUuid): string+ {+ return "database-import-cleanup:{$operationUuid}";+ }++ /**+ * Only the first caller for an operation gets true, so the cleanup runs at most once per import.+ */+ public static function claim(string $operationUuid): bool+ {+ return Cache::add(self::claimKey($operationUuid), true, self::CLAIM_TTL_SECONDS);+ }++ /**+ * Release the claim after a failed attempt, so a retry of the cleanup can run.+ */+ public static function release(string $operationUuid): void+ {+ Cache::forget(self::claimKey($operationUuid));+ }+}diff --git a/app/Support/ResourceStartActivity.php b/app/Support/ResourceStartActivity.phpindex 25a75e09e2..64b969587e 100644--- a/app/Support/ResourceStartActivity.php+++ b/app/Support/ResourceStartActivity.php@@ -4,6 +4,7 @@ namespace App\Support;use App\Enums\ProcessStatus;use App\Models\Service;+use App\Support\DatabaseImport\DatabaseImportCleanup;use Illuminate\Support\Collection;use Spatie\Activitylog\Models\Activity;@@ -54,6 +55,16 @@ class ResourceStartActivitypublic const STALE_MESSAGE = 'Marked as failed: no progress was recorded for too long.';+ /**+ * Added to the reason when Coolify queued the stop of the remote restore and the cleanup.+ */+ public const IMPORT_STOPPED_MESSAGE = 'The import was stopped. The database may be partly restored.';++ /**+ * Added to the reason for an import without stored cleanup data (started before Coolify stored it).+ */+ public const IMPORT_PARTLY_RESTORED_MESSAGE = 'The database may be partly restored.';+public const SUPERSEDED_MESSAGE = 'Skipped: a newer start of this database was requested.';public const ALREADY_STARTING_MESSAGE = 'Skipped: another start of this database is still running.';@@ -141,17 +152,20 @@ class ResourceStartActivity{[$stale, $live] = $activities->partition(fn (Activity $activity): bool => self::isStale($activity));- $stale->each(fn (Activity $activity) => self::markFailed($activity, self::STALE_MESSAGE));+ $stale->each(fn (Activity $activity) => self::fail($activity, self::STALE_MESSAGE));return $live->values();}/*** Mark an activity as failed, append the reason to its log and stop log polling.+ *+ * @param array<string, mixed> $extraProperties*/- public static function markFailed(Activity $activity, string $message): void+ public static function markFailed(Activity $activity, string $message, array $extraProperties = []): void{$properties = [+ ...$extraProperties,'status' => ProcessStatus::ERROR->value,'error' => $message,'failed_at' => now()->toIso8601String(),@@ -186,11 +200,38 @@ class ResourceStartActivity|| $serviceUuids->contains(data_get($activity, 'properties.type_uuid')));- $interrupted->each(fn (Activity $activity) => self::markFailed($activity, self::INTERRUPTED_MESSAGE));+ $interrupted->each(fn (Activity $activity) => self::fail($activity, self::INTERRUPTED_MESSAGE));return $interrupted->count();}+ /**+ * Fail an interrupted or stale activity. For a database import, also queue the stop of the+ * restore that can still run in the database container, and the removal of its helper+ * container and temporary files. The stop is queued: this also runs during boot.+ */+ private static function fail(Activity $activity, string $reason): void+ {+ if (data_get($activity, 'properties.operation') !== self::DATABASE_IMPORT_OPERATION) {+ self::markFailed($activity, $reason);++ return;+ }++ $cleanup = DatabaseImportCleanup::stored($activity);+ if ($cleanup === null) {+ self::markFailed($activity, $reason.' '.self::IMPORT_PARTLY_RESTORED_MESSAGE);++ return;+ }++ // Save the stop flag before the stop is queued, so a retried CoolifyTask sees it.+ self::markFailed($activity, $reason.' '.self::IMPORT_STOPPED_MESSAGE, [+ DatabaseImportCleanup::STOP_REQUESTED_PROPERTY => now()->toIso8601String(),+ ]);+ DatabaseImportCleanup::queueStop($cleanup);+ }+/*** @param Collection<int, Activity> $activities* @return Collection<int, string>diff --git a/openapi.json b/openapi.jsonindex 7cf0d40f39..bb265f086f 100644--- a/openapi.json+++ b/openapi.json@@ -556,6 +556,11 @@"nullable": true,"description": "Custom internal container name."},+ "custom_container_name_prefix": {+ "type": "string",+ "nullable": true,+ "description": "Prefix for generated container names (prefix-20260908T141530). Slugified and unique across the instance."+ },"preview_url_template": {"type": "string","description": "Preview URL template."@@ -1142,6 +1147,11 @@"nullable": true,"description": "Custom internal container name."},+ "custom_container_name_prefix": {+ "type": "string",+ "nullable": true,+ "description": "Prefix for generated container names (prefix-20260908T141530). Slugified and unique across the instance."+ },"preview_url_template": {"type": "string","description": "Preview URL template."@@ -1728,6 +1738,11 @@"nullable": true,"description": "Custom internal container name."},+ "custom_container_name_prefix": {+ "type": "string",+ "nullable": true,+ "description": "Prefix for generated container names (prefix-20260908T141530). Slugified and unique across the instance."+ },"preview_url_template": {"type": "string","description": "Preview URL template."@@ -2209,6 +2224,11 @@"nullable": true,"description": "Custom internal container name."},+ "custom_container_name_prefix": {+ "type": "string",+ "nullable": true,+ "description": "Prefix for generated container names (prefix-20260908T141530). Slugified and unique across the instance."+ },"preview_url_template": {"type": "string","description": "Preview URL template."@@ -2670,6 +2690,11 @@"nullable": true,"description": "Custom internal container name."},+ "custom_container_name_prefix": {+ "type": "string",+ "nullable": true,+ "description": "Prefix for generated container names (prefix-20260908T141530). Slugified and unique across the instance."+ },"preview_url_template": {"type": "string","description": "Preview URL template."@@ -3376,6 +3401,11 @@"nullable": true,"description": "Custom internal container name."},+ "custom_container_name_prefix": {+ "type": "string",+ "nullable": true,+ "description": "Prefix for generated container names (prefix-20260908T141530). Slugified and unique across the instance."+ },"preview_url_template": {"type": "string","description": "Preview URL template."@@ -3516,12 +3546,24 @@{"name": "lines","in": "query",- "description": "Number of lines to show from the end of the logs.",+ "description": "Number of lines to show from the end of the logs. Use `all` to return all logs. `-1` remains available as a compatibility alias.","required": false,"schema": {- "type": "integer",- "format": "int32",- "default": 100+ "oneOf": [+ {+ "type": "integer",+ "format": "int32",+ "default": 100,+ "maximum": 10000,+ "minimum": -1+ },+ {+ "type": "string",+ "enum": [+ "all"+ ]+ }+ ]}},{@@ -6097,7 +6139,7 @@"description": "Import queued"},"409": {- "description": "Import already active"+ "description": "Another start, restart or import of this database is already in progress."},"422": {"$ref": "#\/components\/responses\/422"@@ -8140,12 +8182,24 @@{"name": "lines","in": "query",- "description": "Number of lines to show from the end of the logs.",+ "description": "Number of lines to show from the end of the logs. Use `all` to return all logs. `-1` remains available as a compatibility alias.","required": false,"schema": {- "type": "integer",- "format": "int32",- "default": 100+ "oneOf": [+ {+ "type": "integer",+ "format": "int32",+ "default": 100,+ "maximum": 10000,+ "minimum": -1+ },+ {+ "type": "string",+ "enum": [+ "all"+ ]+ }+ ]}},{@@ -8554,6 +8608,9 @@},"404": {"$ref": "#\/components\/responses\/404"+ },+ "409": {+ "description": "Another start, restart or import of this database is already in progress."}},"security": [@@ -8669,6 +8726,9 @@},"404": {"$ref": "#\/components\/responses\/404"+ },+ "409": {+ "description": "Another start, restart or import of this database is already in progress."}},"security": [@@ -16033,7 +16093,9 @@"schema": {"properties": {"is_sentinel_enabled": {- "type": "boolean"+ "description": "Sentinel is mandatory on regular managed servers.",+ "type": "boolean",+ "readOnly": true},"is_metrics_enabled": {"type": "boolean"@@ -16132,9 +16194,6 @@"application\/json": {"schema": {"properties": {- "is_sentinel_enabled": {- "type": "boolean"- },"is_metrics_enabled": {"type": "boolean"},@@ -16676,10 +16735,15 @@"example": "og888os","description": "The UUID of the private key."},- "is_build_server": {- "type": "boolean",- "example": false,- "description": "Is build server."+ "server_role": {+ "type": "string",+ "enum": [+ "deployment",+ "build",+ "both"+ ],+ "example": "both",+ "description": "Server role."},"instant_validate": {"type": "boolean",@@ -16889,9 +16953,14 @@"type": "string","description": "The UUID of the private key."},- "is_build_server": {- "type": "boolean",- "description": "Is build server."+ "server_role": {+ "type": "string",+ "enum": [+ "deployment",+ "build",+ "both"+ ],+ "description": "Server role."},"instant_validate": {"type": "boolean",@@ -17368,6 +17437,12 @@"null"]},+ "is_force_https_enabled": {+ "type": [+ "boolean",+ "null"+ ]+ },"max_restart_count": {"description": "Maximum Docker restart count before Coolify stops the container. Set to 0 to disable the limit.","type": [@@ -17375,12 +17450,6 @@"null"],"minimum": 0- },- "is_force_https_enabled": {- "type": [- "boolean",- "null"- ]}},"type": "object"@@ -17449,12 +17518,24 @@{"name": "lines","in": "query",- "description": "Number of lines to show from the end of the logs.",+ "description": "Number of lines to show from the end of the logs. Use `all` to return all logs. `-1` remains available as a compatibility alias.","required": false,"schema": {- "type": "integer",- "format": "int32",- "default": 100+ "oneOf": [+ {+ "type": "integer",+ "format": "int32",+ "default": 100,+ "maximum": 10000,+ "minimum": -1+ },+ {+ "type": "string",+ "enum": [+ "all"+ ]+ }+ ]}}],@@ -17520,11 +17601,24 @@{"name": "lines","in": "query",+ "description": "Number of lines to show from the end of the logs. Use `all` to return all logs. `-1` remains available as a compatibility alias.","required": false,"schema": {- "type": "integer",- "format": "int32",- "default": 100+ "oneOf": [+ {+ "type": "integer",+ "format": "int32",+ "default": 100,+ "maximum": 10000,+ "minimum": -1+ },+ {+ "type": "string",+ "enum": [+ "all"+ ]+ }+ ]}}],@@ -17852,7 +17946,7 @@"description": "Import queued"},"409": {- "description": "Import already active"+ "description": "Another start, restart or import of this database is already in progress."},"422": {"$ref": "#\/components\/responses\/422"@@ -18159,11 +18253,24 @@{"name": "lines","in": "query",+ "description": "Number of lines to show from the end of the logs. Use `all` to return all logs. `-1` remains available as a compatibility alias.","required": false,"schema": {- "type": "integer",- "format": "int32",- "default": 100+ "oneOf": [+ {+ "type": "integer",+ "format": "int32",+ "default": 100,+ "maximum": 10000,+ "minimum": -1+ },+ {+ "type": "string",+ "enum": [+ "all"+ ]+ }+ ]}}],@@ -18741,6 +18848,16 @@"type": "boolean","default": true}+ },+ {+ "name": "delete_from_coolify_only",+ "in": "query",+ "description": "Remove only Coolify metadata without deleting Docker resources.",+ "required": false,+ "schema": {+ "type": "boolean",+ "default": false+ }}],"responses": {@@ -18985,12 +19102,24 @@{"name": "lines","in": "query",- "description": "Number of lines to show from the end of the logs.",+ "description": "Number of lines to show from the end of the logs. Use `all` to return all logs. `-1` remains available as a compatibility alias.","required": false,"schema": {- "type": "integer",- "format": "int32",- "default": 100+ "oneOf": [+ {+ "type": "integer",+ "format": "int32",+ "default": 100,+ "maximum": 10000,+ "minimum": -1+ },+ {+ "type": "string",+ "enum": [+ "all"+ ]+ }+ ]}},{@@ -20379,8 +20508,8 @@"type": "boolean"},"is_shown_once": {- "type": "boolean",- "description": "If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values."+ "description": "If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values.",+ "type": "boolean"},"comment": {"type": [@@ -21417,6 +21546,62 @@"bearerAuth": []}]+ },+ "patch": {+ "tags": [+ "Teams"+ ],+ "summary": "Update authenticated team",+ "description": "Update settings for the team bound to the API token.",+ "operationId": "update-token-team",+ "requestBody": {+ "required": true,+ "content": {+ "application\/json": {+ "schema": {+ "required": [+ "is_build_server_fallback_enabled"+ ],+ "properties": {+ "is_build_server_fallback_enabled": {+ "type": "boolean",+ "description": "Whether deployments can fall back to the deployment server when no usable dedicated build server is available."+ }+ },+ "type": "object"+ }+ }+ }+ },+ "responses": {+ "200": {+ "description": "Updated team.",+ "content": {+ "application\/json": {+ "schema": {+ "$ref": "#\/components\/schemas\/Team"+ }+ }+ }+ },+ "401": {+ "$ref": "#\/components\/responses\/401"+ },+ "400": {+ "$ref": "#\/components\/responses\/400"+ },+ "403": {+ "description": "Forbidden."+ },+ "422": {+ "$ref": "#\/components\/responses\/422"+ }+ },+ "security": [+ {+ "bearerAuth": []+ }+ ]}},"\/team\/members": {@@ -22992,6 +23177,10 @@"type": "string","nullable": true},+ "custom_container_name_prefix": {+ "type": "string",+ "nullable": true+ },"is_container_label_escape_enabled": {"type": "boolean"},@@ -23383,8 +23572,13 @@"force_server_cleanup": {"type": "boolean"},- "is_build_server": {- "type": "boolean"+ "server_role": {+ "type": "string",+ "enum": [+ "deployment",+ "build",+ "both"+ ]},"is_cloudflare_tunnel": {"type": "boolean"@@ -23693,6 +23887,10 @@"type": "string","description": "The custom server limit."},+ "is_build_server_fallback_enabled": {+ "type": "boolean",+ "description": "Whether deployments can fall back to the deployment server when no usable dedicated build server is available."+ },"members": {"description": "The members of the team.","type": "array",diff --git a/openapi.yaml b/openapi.yamlindex 682b99e2e3..54dea6555d 100644--- a/openapi.yaml+++ b/openapi.yaml@@ -388,6 +388,10 @@ paths:type: stringnullable: truedescription: 'Custom internal container name.'+ custom_container_name_prefix:+ type: string+ nullable: true+ description: 'Prefix for generated container names (prefix-20260908T141530). Slugified and unique across the instance.'preview_url_template:type: stringdescription: 'Preview URL template.'@@ -770,6 +774,10 @@ paths:type: stringnullable: truedescription: 'Custom internal container name.'+ custom_container_name_prefix:+ type: string+ nullable: true+ description: 'Prefix for generated container names (prefix-20260908T141530). Slugified and unique across the instance.'preview_url_template:type: stringdescription: 'Preview URL template.'@@ -1152,6 +1160,10 @@ paths:type: stringnullable: truedescription: 'Custom internal container name.'+ custom_container_name_prefix:+ type: string+ nullable: true+ description: 'Prefix for generated container names (prefix-20260908T141530). Slugified and unique across the instance.'preview_url_template:type: stringdescription: 'Preview URL template.'@@ -1475,6 +1487,10 @@ paths:type: stringnullable: truedescription: 'Custom internal container name.'+ custom_container_name_prefix:+ type: string+ nullable: true+ description: 'Prefix for generated container names (prefix-20260908T141530). Slugified and unique across the instance.'preview_url_template:type: stringdescription: 'Preview URL template.'@@ -1784,6 +1800,10 @@ paths:type: stringnullable: truedescription: 'Custom internal container name.'+ custom_container_name_prefix:+ type: string+ nullable: true+ description: 'Prefix for generated container names (prefix-20260908T141530). Slugified and unique across the instance.'preview_url_template:type: stringdescription: 'Preview URL template.'@@ -2247,6 +2267,10 @@ paths:type: stringnullable: truedescription: 'Custom internal container name.'+ custom_container_name_prefix:+ type: string+ nullable: true+ description: 'Prefix for generated container names (prefix-20260908T141530). Slugified and unique across the instance.'preview_url_template:type: stringdescription: 'Preview URL template.'@@ -2314,12 +2338,20 @@ paths:-name: linesin: query- description: 'Number of lines to show from the end of the logs.'+ description: 'Number of lines to show from the end of the logs. Use `all` to return all logs. `-1` remains available as a compatibility alias.'required: falseschema:- type: integer- format: int32- default: 100+ oneOf:+ -+ type: integer+ format: int32+ default: 100+ maximum: 10000+ minimum: -1+ -+ type: string+ enum:+ - all-name: show_timestampsin: query@@ -2578,7 +2610,7 @@ paths:description: 'The flag to indicate if the environment variable is multiline.'is_shown_once:type: boolean- description: "If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values."+ description: 'If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values.'type: objectresponses:'201':@@ -2639,7 +2671,7 @@ paths:description: 'The flag to indicate if the environment variable is multiline.'is_shown_once:type: boolean- description: "If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values."+ description: 'If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values.'type: objectresponses:'201':@@ -2683,7 +2715,7 @@ paths:properties:data:type: array- items: { properties: { key: { type: string, description: 'The key of the environment variable.' }, value: { type: string, description: 'The value of the environment variable.' }, is_preview: { type: boolean, description: 'The flag to indicate if the environment variable is used in preview deployments.' }, is_literal: { type: boolean, description: 'The flag to indicate if the environment variable is a literal, nothing espaced.' }, is_multiline: { type: boolean, description: 'The flag to indicate if the environment variable is multiline.' }, is_shown_once: { type: boolean, description: "If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values." } }, type: object }+ items: { properties: { key: { type: string, description: 'The key of the environment variable.' }, value: { type: string, description: 'The value of the environment variable.' }, is_preview: { type: boolean, description: 'The flag to indicate if the environment variable is used in preview deployments.' }, is_literal: { type: boolean, description: 'The flag to indicate if the environment variable is a literal, nothing espaced.' }, is_multiline: { type: boolean, description: 'The flag to indicate if the environment variable is multiline.' }, is_shown_once: { type: boolean, description: 'If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values.' } }, type: object }type: objectresponses:'201':@@ -3916,7 +3948,7 @@ paths:'202':description: 'Import queued''409':- description: 'Import already active'+ description: 'Another start, restart or import of this database is already in progress.''422':$ref: '#/components/responses/422'security:@@ -5361,12 +5393,20 @@ paths:-name: linesin: query- description: 'Number of lines to show from the end of the logs.'+ description: 'Number of lines to show from the end of the logs. Use `all` to return all logs. `-1` remains available as a compatibility alias.'required: falseschema:- type: integer- format: int32- default: 100+ oneOf:+ -+ type: integer+ format: int32+ default: 100+ maximum: 10000+ minimum: -1+ -+ type: string+ enum:+ - all-name: show_timestampsin: query@@ -5612,6 +5652,8 @@ paths:$ref: '#/components/responses/400''404':$ref: '#/components/responses/404'+ '409':+ description: 'Another start, restart or import of this database is already in progress.'security:-bearerAuth: []@@ -5685,6 +5727,8 @@ paths:$ref: '#/components/responses/400''404':$ref: '#/components/responses/404'+ '409':+ description: 'Another start, restart or import of this database is already in progress.'security:-bearerAuth: []@@ -5756,7 +5800,7 @@ paths:description: 'The flag to indicate if the environment variable is multiline.'is_shown_once:type: boolean- description: "If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values."+ description: 'If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values.'type: objectresponses:'201':@@ -5816,7 +5860,7 @@ paths:description: 'The flag to indicate if the environment variable is multiline.'is_shown_once:type: boolean- description: "If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values."+ description: 'If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values.'type: objectresponses:'201':@@ -5862,7 +5906,7 @@ paths:properties:data:type: array- items: { properties: { key: { type: string, description: 'The key of the environment variable.' }, value: { type: string, description: 'The value of the environment variable.' }, is_literal: { type: boolean, description: 'The flag to indicate if the environment variable is a literal, nothing espaced.' }, is_multiline: { type: boolean, description: 'The flag to indicate if the environment variable is multiline.' }, is_shown_once: { type: boolean, description: "If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values." } }, type: object }+ items: { properties: { key: { type: string, description: 'The key of the environment variable.' }, value: { type: string, description: 'The value of the environment variable.' }, is_literal: { type: boolean, description: 'The flag to indicate if the environment variable is a literal, nothing espaced.' }, is_multiline: { type: boolean, description: 'The flag to indicate if the environment variable is multiline.' }, is_shown_once: { type: boolean, description: 'If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values.' } }, type: object }type: objectresponses:'201':@@ -10175,7 +10219,7 @@ paths:application/json:schema:properties:- is_sentinel_enabled: { type: boolean }+ is_sentinel_enabled: { description: 'Sentinel is mandatory on regular managed servers.', type: boolean, readOnly: true }is_metrics_enabled: { type: boolean }is_sentinel_debug_enabled: { type: boolean }sentinel_token: { description: 'Only present with read:sensitive.', type: string }@@ -10221,8 +10265,6 @@ paths:application/json:schema:properties:- is_sentinel_enabled:- type: booleanis_metrics_enabled:type: booleanis_sentinel_debug_enabled:@@ -10579,10 +10621,11 @@ paths:type: stringexample: og888osdescription: 'The UUID of the private key.'- is_build_server:- type: boolean- example: false- description: 'Is build server.'+ server_role:+ type: string+ enum: [deployment, build, both]+ example: both+ description: 'Server role.'instant_validate:type: booleanexample: false@@ -10717,9 +10760,10 @@ paths:private_key_uuid:type: stringdescription: 'The UUID of the private key.'- is_build_server:- type: boolean- description: 'Is build server.'+ server_role:+ type: string+ enum: [deployment, build, both]+ description: 'Server role.'instant_validate:type: booleandescription: 'Instant validate.'@@ -10996,12 +11040,12 @@ paths:type: [boolean, 'null']is_stripprefix_enabled:type: [boolean, 'null']+ is_force_https_enabled:+ type: [boolean, 'null']max_restart_count:description: 'Maximum Docker restart count before Coolify stops the container. Set to 0 to disable the limit.'type: [integer, 'null']minimum: 0- is_force_https_enabled:- type: [boolean, 'null']type: objectresponses:'200':@@ -11046,12 +11090,20 @@ paths:-name: linesin: query- description: 'Number of lines to show from the end of the logs.'+ description: 'Number of lines to show from the end of the logs. Use `all` to return all logs. `-1` remains available as a compatibility alias.'required: falseschema:- type: integer- format: int32- default: 100+ oneOf:+ -+ type: integer+ format: int32+ default: 100+ maximum: 10000+ minimum: -1+ -+ type: string+ enum:+ - allresponses:'200':description: Logs.@@ -11094,11 +11146,20 @@ paths:-name: linesin: query+ description: 'Number of lines to show from the end of the logs. Use `all` to return all logs. `-1` remains available as a compatibility alias.'required: falseschema:- type: integer- format: int32- default: 100+ oneOf:+ -+ type: integer+ format: int32+ default: 100+ maximum: 10000+ minimum: -1+ -+ type: string+ enum:+ - allresponses:'200':description: Logs.@@ -11313,7 +11374,7 @@ paths:'202':description: 'Import queued''409':- description: 'Import already active'+ description: 'Another start, restart or import of this database is already in progress.''422':$ref: '#/components/responses/422'security:@@ -11514,11 +11575,20 @@ paths:-name: linesin: query+ description: 'Number of lines to show from the end of the logs. Use `all` to return all logs. `-1` remains available as a compatibility alias.'required: falseschema:- type: integer- format: int32- default: 100+ oneOf:+ -+ type: integer+ format: int32+ default: 100+ maximum: 10000+ minimum: -1+ -+ type: string+ enum:+ - allresponses:'200':description: Logs.@@ -11869,6 +11939,14 @@ paths:schema:type: booleandefault: true+ -+ name: delete_from_coolify_only+ in: query+ description: 'Remove only Coolify metadata without deleting Docker resources.'+ required: false+ schema:+ type: boolean+ default: falseresponses:'200':description: 'Delete a service by UUID'@@ -11995,12 +12073,20 @@ paths:-name: linesin: query- description: 'Number of lines to show from the end of the logs.'+ description: 'Number of lines to show from the end of the logs. Use `all` to return all logs. `-1` remains available as a compatibility alias.'required: falseschema:- type: integer- format: int32- default: 100+ oneOf:+ -+ type: integer+ format: int32+ default: 100+ maximum: 10000+ minimum: -1+ -+ type: string+ enum:+ - all-name: show_timestampsin: query@@ -12098,7 +12184,7 @@ paths:description: 'The flag to indicate if the environment variable is multiline.'is_shown_once:type: boolean- description: "If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values."+ description: 'If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values.'type: objectresponses:'201':@@ -12161,7 +12247,7 @@ paths:description: 'The flag to indicate if the environment variable is multiline.'is_shown_once:type: boolean- description: "If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values."+ description: 'If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values.'type: objectresponses:'201':@@ -12207,7 +12293,7 @@ paths:properties:data:type: array- items: { properties: { key: { type: string, description: 'The key of the environment variable.' }, value: { type: string, description: 'The value of the environment variable.' }, is_preview: { type: boolean, description: 'The flag to indicate if the environment variable is used in preview deployments.' }, is_literal: { type: boolean, description: 'The flag to indicate if the environment variable is a literal, nothing espaced.' }, is_multiline: { type: boolean, description: 'The flag to indicate if the environment variable is multiline.' }, is_shown_once: { type: boolean, description: "If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values." } }, type: object }+ items: { properties: { key: { type: string, description: 'The key of the environment variable.' }, value: { type: string, description: 'The value of the environment variable.' }, is_preview: { type: boolean, description: 'The flag to indicate if the environment variable is used in preview deployments.' }, is_literal: { type: boolean, description: 'The flag to indicate if the environment variable is a literal, nothing espaced.' }, is_multiline: { type: boolean, description: 'The flag to indicate if the environment variable is multiline.' }, is_shown_once: { type: boolean, description: 'If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values.' } }, type: object }type: objectresponses:'201':@@ -12885,8 +12971,8 @@ paths:is_multiline:type: booleanis_shown_once:- type: booleandescription: 'If true, the saved value is hidden in the UI and API responses. MCP never returns environment variable values.'+ type: booleancomment:type: [string, 'null']type: object@@ -13581,6 +13667,42 @@ paths:security:-bearerAuth: []+ patch:+ tags:+ - Teams+ summary: 'Update authenticated team'+ description: 'Update settings for the team bound to the API token.'+ operationId: update-token-team+ requestBody:+ required: true+ content:+ application/json:+ schema:+ required:+ - is_build_server_fallback_enabled+ properties:+ is_build_server_fallback_enabled:+ type: boolean+ description: 'Whether deployments can fall back to the deployment server when no usable dedicated build server is available.'+ type: object+ responses:+ '200':+ description: 'Updated team.'+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/Team'+ '401':+ $ref: '#/components/responses/401'+ '400':+ $ref: '#/components/responses/400'+ '403':+ description: Forbidden.+ '422':+ $ref: '#/components/responses/422'+ security:+ -+ bearerAuth: []/team/members:get:tags:@@ -14702,6 +14824,9 @@ components:custom_internal_name:type: stringnullable: true+ custom_container_name_prefix:+ type: string+ nullable: trueis_container_label_escape_enabled:type: booleanis_env_sorting_enabled:@@ -14981,8 +15106,12 @@ components:type: booleanforce_server_cleanup:type: boolean- is_build_server:- type: boolean+ server_role:+ type: string+ enum:+ - deployment+ - build+ - bothis_cloudflare_tunnel:type: booleanis_jump_server:@@ -15201,6 +15330,9 @@ components:custom_server_limit:type: stringdescription: 'The custom server limit.'+ is_build_server_fallback_enabled:+ type: boolean+ description: 'Whether deployments can fall back to the deployment server when no usable dedicated build server is available.'members:description: 'The members of the team.'type: arraydiff --git a/tests/Feature/DatabaseImportInterruptedCleanupTest.php b/tests/Feature/DatabaseImportInterruptedCleanupTest.phpnew file mode 100644index 0000000000..fa2e41cfd0--- /dev/null+++ b/tests/Feature/DatabaseImportInterruptedCleanupTest.php@@ -0,0 +1,267 @@+<?php++use App\Actions\CoolifyTask\RunRemoteProcess;+use App\Actions\Database\StartDatabaseImport;+use App\Enums\ActivityTypes;+use App\Enums\ProcessStatus;+use App\Events\DatabaseImportFinished;+use App\Jobs\CoolifyTask;+use App\Listeners\CleanupDatabaseImport;+use App\Models\Environment;+use App\Models\InstanceSettings;+use App\Models\PrivateKey;+use App\Models\Project;+use App\Models\Server;+use App\Models\StandaloneDocker;+use App\Models\StandalonePostgresql;+use App\Models\Team;+use App\Support\DatabaseImport\DatabaseImportCleanup;+use App\Support\DatabaseImport\DatabaseImportException;+use App\Support\DatabaseImport\DatabaseImportSource;+use App\Support\ResourceStartActivity;+use Illuminate\Events\CallQueuedListener;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Facades\Process;+use Illuminate\Support\Facades\Queue;+use Illuminate\Support\Str;+use Spatie\Activitylog\Models\Activity;++uses(RefreshDatabase::class);++beforeEach(function () {+ config()->set('cache.default', 'array');+ config()->set('constants.ssh.mux_enabled', false);+ InstanceSettings::forceCreate(['id' => 0]);++ $this->team = Team::factory()->create();+ $this->privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]);+ $this->server = Server::factory()->create([+ 'team_id' => $this->team->id,+ 'private_key_id' => $this->privateKey->id,+ 'user' => 'root',+ ]);+ $this->destination = StandaloneDocker::firstOrCreate(+ ['server_id' => $this->server->id, 'network' => 'coolify'],+ ['uuid' => (string) Str::uuid(), 'name' => 'docker']+ );+ $this->project = Project::factory()->create(['team_id' => $this->team->id]);+ $this->environment = Environment::factory()->create(['project_id' => $this->project->id]);+ $this->database = StandalonePostgresql::create([+ 'uuid' => (string) Str::uuid(),+ 'name' => 'db',+ 'postgres_user' => 'postgres',+ 'postgres_password' => 'password',+ 'postgres_db' => 'db',+ 'image' => 'postgres:17',+ 'status' => 'running',+ 'environment_id' => $this->environment->id,+ 'destination_id' => $this->destination->id,+ 'destination_type' => $this->destination->getMorphClass(),+ ]);++ Process::fake();+ Queue::fake();+});++/**+ * @return array<string, mixed>+ */+function storedImportCleanup(object $test, string $operation): array+{+ return [+ 'container' => $test->database->uuid,+ 'containerTmpPath' => "/tmp/restore_{$operation}",+ 'scriptPath' => "/tmp/restore_{$operation}.sh",+ 'serverId' => $test->server->id,+ 'operationUuid' => $operation,+ 'containerName' => "s3-restore-{$operation}",+ ];+}++function runningImportActivity(object $test, ProcessStatus $status, ?array $cleanup, int $minutesSinceLastUpdate = 0): Activity+{+ $activity = activity()+ ->withProperties(array_filter([+ 'type' => ActivityTypes::INLINE->value,+ 'type_uuid' => $test->database->uuid,+ 'team_id' => $test->team->id,+ 'status' => $status->value,+ 'operation' => ResourceStartActivity::DATABASE_IMPORT_OPERATION,+ 'operation_uuid' => $cleanup['operationUuid'] ?? null,+ DatabaseImportCleanup::PROPERTY => $cleanup,+ ], fn ($value) => $value !== null))+ ->event(ActivityTypes::INLINE->value)+ ->log('[]');++ Activity::query()->whereKey($activity->id)->update([+ 'created_at' => now()->subMinutes($minutesSinceLastUpdate),+ 'updated_at' => now()->subMinutes($minutesSinceLastUpdate),+ ]);++ return $activity->refresh();+}++function assertStopQueuedOnce(array $expected): void+{+ Queue::assertPushed(CallQueuedListener::class, 1);+ Queue::assertPushed(CallQueuedListener::class, function (CallQueuedListener $job) use ($expected) {+ $event = $job->data[0] ?? null;++ return $job->class === CleanupDatabaseImport::class+ && $event instanceof DatabaseImportFinished+ && $event->data === [...$expected, 'stopRestore' => true];+ });+}++test('an interrupted import queues the stop and cleanup with the stored data once, without SSH at boot', function () {+ $operation = (string) Str::uuid();+ $cleanup = storedImportCleanup($this, $operation);+ $activity = runningImportActivity($this, ProcessStatus::IN_PROGRESS, $cleanup);++ expect(ResourceStartActivity::failInterrupted())->toBe(1)+ ->and(ResourceStartActivity::failInterrupted())->toBe(0);++ assertStopQueuedOnce($cleanup);+ Process::assertNothingRan();++ $activity->refresh();+ expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value)+ ->and(data_get($activity, 'properties.error'))->toBe(ResourceStartActivity::INTERRUPTED_MESSAGE.' '.ResourceStartActivity::IMPORT_STOPPED_MESSAGE)+ ->and(data_get($activity, 'properties.'.DatabaseImportCleanup::STOP_REQUESTED_PROPERTY))->not->toBeNull()+ ->and($activity->description)->toContain('The import was stopped. The database may be partly restored.');+});++test('a stale import queues the stop and cleanup with the stored data once', function () {+ $operation = (string) Str::uuid();+ $cleanup = storedImportCleanup($this, $operation);+ $minutes = intdiv(ResourceStartActivity::importStaleAfterSeconds(), 60) + 5;+ $stale = runningImportActivity($this, ProcessStatus::IN_PROGRESS, $cleanup, $minutes);++ foreach ([1, 2] as $attempt) {+ expect(fn () => app(StartDatabaseImport::class)->handle(+ $this->database,+ new DatabaseImportSource('server', path: 'backup.sql'),+ $this->team->id,+ ))->toThrow(DatabaseImportException::class, 'The server path is invalid.');+ }++ assertStopQueuedOnce($cleanup);+ Process::assertNothingRan();++ $stale->refresh();+ expect(data_get($stale, 'properties.status'))->toBe(ProcessStatus::ERROR->value)+ ->and(data_get($stale, 'properties.error'))->toBe(ResourceStartActivity::STALE_MESSAGE.' '.ResourceStartActivity::IMPORT_STOPPED_MESSAGE);+});++test('an import without stored cleanup data is failed without a stop', function () {+ $activity = runningImportActivity($this, ProcessStatus::QUEUED, null);++ expect(ResourceStartActivity::failInterrupted())->toBe(1);++ Queue::assertNotPushed(CallQueuedListener::class);+ expect(data_get($activity->refresh(), 'properties.error'))+ ->toBe(ResourceStartActivity::INTERRUPTED_MESSAGE.' '.ResourceStartActivity::IMPORT_PARTLY_RESTORED_MESSAGE)+ ->and(data_get($activity, 'properties.'.DatabaseImportCleanup::STOP_REQUESTED_PROPERTY))->toBeNull();+});++test('stored cleanup data with unexpected keys or an invalid operation is not used', function () {+ $operation = (string) Str::uuid();+ $cleanup = [...storedImportCleanup($this, $operation), 'secret' => 'do-not-send'];+ runningImportActivity($this, ProcessStatus::IN_PROGRESS, $cleanup);+ runningImportActivity($this, ProcessStatus::IN_PROGRESS, [...storedImportCleanup($this, $operation), 'operationUuid' => "x'; reboot; '"]);++ expect(ResourceStartActivity::failInterrupted())->toBe(2);++ assertStopQueuedOnce(storedImportCleanup($this, $operation));+});++test('a stopped import is not run again when its queued task is retried after a restart', function () {+ $operation = (string) Str::uuid();+ $activity = runningImportActivity($this, ProcessStatus::IN_PROGRESS, storedImportCleanup($this, $operation));+ ResourceStartActivity::failInterrupted();++ $job = new CoolifyTask($activity->refresh(), ignore_errors: false, call_event_on_finish: 'DatabaseImportFinished', call_event_data: storedImportCleanup($this, $operation));+ $job->handle();++ Process::assertNothingRan();+ expect(data_get($activity->refresh(), 'properties.status'))->toBe(ProcessStatus::ERROR->value);+ Queue::assertPushed(CallQueuedListener::class, 1);+});++test('the cleanup runs at most once per import', function () {+ $operation = (string) Str::uuid();+ $cleanup = storedImportCleanup($this, $operation);+ $listener = new CleanupDatabaseImport;++ $listener->handle(new DatabaseImportFinished([...$cleanup, 'stopRestore' => true]));+ $listener->handle(new DatabaseImportFinished($cleanup));+ $listener->handle(new DatabaseImportFinished([...$cleanup, 'stopRestore' => true]));++ Process::assertRanTimes(fn ($process) => str_contains($process->command, "s3-restore-{$operation}"), 1);+ Process::assertRan(fn ($process) => str_contains($process->command, "/tmp/restore_{$operation}")+ && str_contains($process->command, 'Stopping the database import processes'));+});++test('the cleanup of an import without an operation id still runs every time', function () {+ $cleanup = storedImportCleanup($this, (string) Str::uuid());+ unset($cleanup['operationUuid']);+ $listener = new CleanupDatabaseImport;++ $listener->handle(new DatabaseImportFinished($cleanup));+ $listener->handle(new DatabaseImportFinished($cleanup));++ Process::assertRanTimes(fn ($process) => str_contains($process->command, $cleanup['containerName']), 2);+});++it('keeps the stop message when the stopped import process ends afterwards', function () {+ $operation = (string) Str::uuid();+ $activity = runningImportActivity($this, ProcessStatus::IN_PROGRESS, storedImportCleanup($this, $operation));+ $activity->properties = $activity->properties->merge([+ 'server_uuid' => $this->server->uuid,+ 'command' => "docker exec {$this->database->uuid} sh -c /tmp/restore_{$operation}.sh",+ ]);+ $activity->save();++ // The running job holds its own copy of the activity while the stale rule stops the import.+ $jobCopy = Activity::query()->findOrFail($activity->id);+ $remoteProcess = new RunRemoteProcess(activity: $jobCopy, ignore_errors: true);++ // The stale rule stops the import while the process runs; the killed restore then prints output.+ Process::fake(function () use ($activity) {+ $stopped = Activity::query()->findOrFail($activity->id);+ $stopped->properties = $stopped->properties->merge([+ 'status' => ProcessStatus::ERROR->value,+ 'error' => 'Stale import. The import was stopped. The database may be partly restored.',+ DatabaseImportCleanup::STOP_REQUESTED_PROPERTY => now()->toIso8601String(),+ ]);+ $stopped->save();++ return Process::result(errorOutput: "Terminated\n", exitCode: 143);+ });+ $remoteProcess();++ $activity->refresh();+ expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value)+ ->and(data_get($activity, 'properties.error'))->toContain('The import was stopped')+ ->and(DatabaseImportCleanup::stopRequested($activity))->toBeTrue()+ ->and(data_get($activity, 'properties.exitCode'))->toBe(143)+ ->and(RunRemoteProcess::decodeOutput($activity))->toContain('Terminated');+});++it('keeps the stop message when the stopped import job fails', function () {+ $operation = (string) Str::uuid();+ $activity = runningImportActivity($this, ProcessStatus::ERROR, storedImportCleanup($this, $operation));+ $activity->properties = $activity->properties->merge([+ 'error' => 'Stale import. The import was stopped. The database may be partly restored.',+ DatabaseImportCleanup::STOP_REQUESTED_PROPERTY => now()->toIso8601String(),+ ]);+ $activity->save();++ $job = new CoolifyTask(activity: $activity->fresh(), ignore_errors: false, call_event_on_finish: null, call_event_data: null);+ $job->failed(new RuntimeException("Terminated\n"));++ $activity->refresh();+ expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value)+ ->and(data_get($activity, 'properties.error'))->toContain('The import was stopped')+ ->and(data_get($activity, 'properties.failed_at'))->not->toBeNull();+});diff --git a/tests/Feature/StaleStartActivityTest.php b/tests/Feature/StaleStartActivityTest.phpindex 6560fe0366..da664f781d 100644--- a/tests/Feature/StaleStartActivityTest.php+++ b/tests/Feature/StaleStartActivityTest.php@@ -182,7 +182,7 @@ it('marks interrupted database imports as failed on startup and leaves other opeforeach ([$queuedImport, $runningImport] as $activity) {$activity->refresh();expect(data_get($activity, 'properties.status'))->toBe(ProcessStatus::ERROR->value)- ->and(data_get($activity, 'properties.error'))->toBe('Interrupted by a Coolify restart.');+ ->and(data_get($activity, 'properties.error'))->toBe('Interrupted by a Coolify restart. The database may be partly restored.');}expect(data_get($finishedImport->refresh(), 'properties.status'))->toBe(ProcessStatus::FINISHED->value)diff --git a/tests/Feature/StartDatabaseImportConcurrencyTest.php b/tests/Feature/StartDatabaseImportConcurrencyTest.phpindex e238c2ae95..da0a47c3b5 100644--- a/tests/Feature/StartDatabaseImportConcurrencyTest.php+++ b/tests/Feature/StartDatabaseImportConcurrencyTest.php@@ -167,7 +167,7 @@ test('a stale import does not block a new import and is marked as failed', funct$stale->refresh();expect(data_get($stale, 'properties.status'))->toBe(ProcessStatus::ERROR->value)- ->and(data_get($stale, 'properties.error'))->toBe(ResourceStartActivity::STALE_MESSAGE)+ ->and(data_get($stale, 'properties.error'))->toBe(ResourceStartActivity::STALE_MESSAGE.' '.ResourceStartActivity::IMPORT_PARTLY_RESTORED_MESSAGE)->and(data_get($stale, 'properties.exitCode'))->toBe(1);})->with(['queued' => ProcessStatus::QUEUED,@@ -187,7 +187,7 @@ test('an import interrupted by a Coolify restart is failed at boot and no longer$interrupted->refresh();expect(data_get($interrupted, 'properties.status'))->toBe(ProcessStatus::ERROR->value)- ->and(data_get($interrupted, 'properties.error'))->toBe(ResourceStartActivity::INTERRUPTED_MESSAGE);+ ->and(data_get($interrupted, 'properties.error'))->toBe(ResourceStartActivity::INTERRUPTED_MESSAGE.' '.ResourceStartActivity::IMPORT_PARTLY_RESTORED_MESSAGE);expect(fn () => startImport($this->database, $this->team->id))->toThrow(DatabaseImportException::class, 'The server path is invalid.');diff --git a/tests/Feature/StartDatabaseImportS3CredentialsTest.php b/tests/Feature/StartDatabaseImportS3CredentialsTest.phpindex 356a16cc07..066b1476a5 100644--- a/tests/Feature/StartDatabaseImportS3CredentialsTest.php+++ b/tests/Feature/StartDatabaseImportS3CredentialsTest.php@@ -11,6 +11,7 @@ use App\Models\Server;use App\Models\StandaloneDocker;use App\Models\StandalonePostgresql;use App\Models\Team;+use App\Support\DatabaseImport\DatabaseImportCleanup;use App\Support\DatabaseImport\DatabaseImportSource;use Illuminate\Filesystem\FilesystemAdapter;use Illuminate\Filesystem\FilesystemManager;@@ -102,3 +103,56 @@ test('s3 import activity command does not contain storage key or secret', functi&& filled($cleanup['containerName'] ?? null);});});++test('s3 import activity stores the cleanup names and paths but no credential values', function () {+ $accessKey = 'AKIA_STORED_CLEANUP_KEY';+ $secret = 'stored/cleanup/SECRET/value';+ $endpoint = 'https://8.8.4.4';+ $storage = S3Storage::create([+ 'name' => 'Import S3',+ 'region' => 'us-east-1',+ 'key' => $accessKey,+ 'secret' => $secret,+ 'bucket' => 'test-bucket',+ 'endpoint' => $endpoint,+ 'is_usable' => true,+ 'team_id' => $this->team->id,+ ]);++ $disk = Mockery::mock(FilesystemAdapter::class);+ $disk->shouldReceive('exists')->once()->with('backups/restore.sql')->andReturn(true);+ $disk->shouldReceive('size')->once()->with('backups/restore.sql')->andReturn(1024);+ $filesystem = Mockery::mock(FilesystemManager::class, [app()])->makePartial();+ $filesystem->shouldReceive('build')->once()->andReturn($disk);+ Storage::swap($filesystem);++ Process::fake();+ Queue::fake();++ $activity = app(StartDatabaseImport::class)->handle(+ $this->database,+ new DatabaseImportSource('s3', path: 'backups/restore.sql', s3StorageUuid: $storage->uuid),+ $this->team->id,+ );++ $stored = $activity->getExtraProperty(DatabaseImportCleanup::PROPERTY);+ $operation = $activity->getExtraProperty('operation_uuid');++ expect($stored)->toBe([+ 'container' => $this->database->uuid,+ 'containerTmpPath' => "/tmp/restore_{$operation}",+ 'scriptPath' => "/tmp/restore_{$operation}.sh",+ 'serverId' => $this->server->id,+ 'operationUuid' => $operation,+ 'containerName' => "s3-restore-{$operation}",+ ]);++ $properties = json_encode($activity->properties);+ expect($properties)+ ->not->toContain($accessKey)+ ->not->toContain($secret)+ ->not->toContain($endpoint)+ ->not->toContain('S3_ACCESS_KEY=');++ Queue::assertPushed(CoolifyTask::class, fn (CoolifyTask $job) => $job->call_event_data === $stored);+});diff --git a/tests/Unit/DatabaseImport/CleanupDatabaseImportTest.php b/tests/Unit/DatabaseImport/CleanupDatabaseImportTest.phpindex 895440e7a0..2141276364 100644--- a/tests/Unit/DatabaseImport/CleanupDatabaseImportTest.php+++ b/tests/Unit/DatabaseImport/CleanupDatabaseImportTest.php@@ -2,9 +2,12 @@use App\Events\DatabaseImportFinished;use App\Listeners\CleanupDatabaseImport;+use App\Models\Server;use Illuminate\Contracts\Queue\ShouldQueue;use Illuminate\Foundation\Testing\RefreshDatabase;use Illuminate\Support\Facades\Event;+use Illuminate\Support\Str;+use Symfony\Component\Process\Process;use Tests\TestCase;uses(TestCase::class, RefreshDatabase::class);@@ -101,3 +104,100 @@ test('handle skips remote process when the server is missing', function () {expect(fn () => (new CleanupDatabaseImport)->handle($event))->not->toThrow(Throwable::class);});++test('adds the restore stop command only when a stop is requested for a valid operation', function () {+ $listener = new CleanupDatabaseImport;+ $operation = '0b6f2f4e-3a55-4d8c-9d42-6c1c6c1f5a10';+ $payload = importCleanupPayload([+ 'operationUuid' => $operation,+ 'containerTmpPath' => "/tmp/restore_{$operation}",+ 'scriptPath' => "/tmp/restore_{$operation}.sh",+ 'containerName' => "s3-restore-{$operation}",+ ]);++ expect(collect($listener->commands($payload))->filter(fn ($command) => str_contains($command, 'kill')))->toBeEmpty();++ $commands = $listener->commands([...$payload, 'stopRestore' => true]);+ expect($commands[0])->toBe($listener->stopRestoreCommand('postgres-abc', $operation))+ ->and($commands[1])->toBe('docker rm -f '.escapeshellarg("s3-restore-{$operation}").' 2>/dev/null || true')+ ->and($commands)->toHaveCount(4);++ foreach (["x'; reboot; '", '../../etc', '', 'op'] as $invalid) {+ expect(collect($listener->commands([...$payload, 'operationUuid' => $invalid, 'stopRestore' => true]))+ ->filter(fn ($command) => str_contains($command, 'kill')))->toBeEmpty();+ }+});++test('the restore stop command quotes the container and targets only the operation path', function () {+ $listener = new CleanupDatabaseImport;+ $operation = '0b6f2f4e-3a55-4d8c-9d42-6c1c6c1f5a10';+ $command = $listener->stopRestoreCommand("db'; reboot; '", $operation);++ expect($command)+ ->toStartWith('docker exec '.escapeshellarg("db'; reboot; '").' sh -c ')+ ->toEndWith(' 2>/dev/null || true')+ ->toContain("/tmp/restore_{$operation}")+ ->not->toContain('pkill')+ ->not->toContain('pgrep')+ ->not->toContain('$(');++ expect(fn () => $listener->stopRestoreCommand('db', 'not-a-uuid'))->toThrow(InvalidArgumentException::class);+});++test('the restore stop command passes the non-root sudo parser and stops only the matching restore', function () {+ $operation = (string) Str::uuid();+ $other = (string) Str::uuid();+ $dir = sys_get_temp_dir().'/coolify-stop-restore-'.Str::random(8);+ mkdir($dir);+ // Fake sudo and docker: `docker exec <container> ...` runs the command on this machine.+ file_put_contents("{$dir}/sudo", "#!/bin/sh\nexec \"\$@\"\n");+ file_put_contents("{$dir}/docker", "#!/bin/sh\n[ \"\$1\" = exec ] || exit 1\nshift 2\nexec \"\$@\"\n");+ chmod("{$dir}/sudo", 0755);+ chmod("{$dir}/docker", 0755);++ $server = Mockery::mock(Server::class)->makePartial();+ $server->shouldReceive('getAttribute')->with('user')->andReturn('ubuntu');+ $server->user = 'ubuntu';++ $parsed = parseCommandsByLineForSudo(collect([(new CleanupDatabaseImport)->stopRestoreCommand('postgres-abc', $operation)]), $server);+ expect($parsed)->toHaveCount(1)+ ->and($parsed[0])->toStartWith("sudo bash -c 'docker exec ");+ file_put_contents("{$dir}/stop.sh", implode("\n", $parsed)."\n");++ // The restore shell has the operation path in its command line; its children do not.+ // The shell must reap each stopped child itself: it exits by itself after the last one.+ $restore = new Process(['sh', '-c', "sleep 60 | cat; sleep 60; : /tmp/restore_{$operation}.sh"]);+ // A top-level process without children, for example a direct `docker exec` of a restore tool.+ $leaf = new Process(['bash', '-c', "exec -a 'restore-tool /tmp/restore_{$operation}' sleep 60"]);+ $unrelated = new Process(['sh', '-c', "sleep 60; : /tmp/restore_{$other}.sh"]);+ $restore->start();+ $leaf->start();+ $unrelated->start();++ try {+ usleep(300_000);+ $children = array_filter(explode("\n", trim((string) shell_exec('pgrep -P '.(int) $restore->getPid()))));+ expect($children)->toHaveCount(2);++ $stop = new Process(['bash', "{$dir}/stop.sh"], env: ['PATH' => "{$dir}:".getenv('PATH')]);+ $stop->setTimeout(60)->run();++ expect($stop->getExitCode())->toBe(0)+ ->and($stop->getOutput())->toContain('Stopping the database import processes (TERM):');+ usleep(200_000);+ $isAlive = fn (string $pid): bool => file_exists("/proc/{$pid}")+ && ! str_contains((string) @file_get_contents("/proc/{$pid}/status"), "State:\tZ");++ expect($restore->isRunning())->toBeFalse()+ ->and($restore->hasBeenSignaled())->toBeFalse()+ ->and($leaf->isRunning())->toBeFalse()+ ->and(collect($children)->filter($isAlive))->toBeEmpty()+ ->and($unrelated->isRunning())->toBeTrue();+ } finally {+ $restore->stop(0);+ $leaf->stop(0);+ $unrelated->stop(0);+ array_map('unlink', glob("{$dir}/*"));+ rmdir($dir);+ }+})->skip(fn () => PHP_OS_FAMILY !== 'Linux' || ! is_dir('/proc/self'), 'Needs Linux /proc.');diff --git a/tests/Unit/DatabaseImportOpenApiTest.php b/tests/Unit/DatabaseImportOpenApiTest.phpindex 94aa6f81d7..d3caa127f4 100644--- a/tests/Unit/DatabaseImportOpenApiTest.php+++ b/tests/Unit/DatabaseImportOpenApiTest.php@@ -80,3 +80,17 @@ test('constrains additional properties on each database import source branch', f->toMatchArray(['type' => 'boolean', 'default' => false]);}});++test('documents the 409 response of database start, restart, and import endpoints', function () {+ $document = json_decode((string) file_get_contents(__DIR__.'/../../openapi.json'), true, flags: JSON_THROW_ON_ERROR);+ $inProgress = 'Another start, restart or import of this database is already in progress.';++ foreach ([+ ['/databases/{uuid}/start', 'post'],+ ['/databases/{uuid}/restart', 'post'],+ ['/databases/{uuid}/imports', 'post'],+ ['/services/{uuid}/databases/{database_uuid}/imports', 'post'],+ ] as [$path, $method]) {+ expect($document['paths'][$path][$method]['responses']['409']['description'] ?? null)->toBe($inProgress);+ }+});