An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(sentinel): reduce resource churn from health flaps
Andras Bacsai626cfb4a2281
diff --git a/app/Actions/Server/ResourcesCheck.php b/app/Actions/Server/ResourcesCheck.phpindex e6b90ba380..cc11a2e07c 100644--- a/app/Actions/Server/ResourcesCheck.php+++ b/app/Actions/Server/ResourcesCheck.php@@ -3,9 +3,11 @@namespace App\Actions\Server;use App\Models\Application;+use App\Models\Server;use App\Models\ServiceApplication;use App\Models\ServiceDatabase;use App\Models\StandaloneClickhouse;+use App\Models\StandaloneDocker;use App\Models\StandaloneDragonfly;use App\Models\StandaloneKeydb;use App\Models\StandaloneMariadb;@@ -13,6 +15,8 @@ use App\Models\StandaloneMongodb;use App\Models\StandaloneMysql;use App\Models\StandalonePostgresql;use App\Models\StandaloneRedis;+use App\Models\SwarmDocker;+use Illuminate\Support\Collection;use Lorisleiva\Actions\Concerns\AsAction;class ResourcesCheck@@ -21,21 +25,73 @@ class ResourcesCheckpublic function handle(){- $seconds = 60;+ $seconds = config('constants.sentinel.resource_stale_seconds', 300);+ $staleServerIds = $this->staleSentinelServerIds($seconds);++ if ($staleServerIds->isEmpty()) {+ return;+ }++ [$standaloneDockerIds, $swarmDockerIds] = $this->destinationIdsForServers($staleServerIds);+try {- Application::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);- ServiceApplication::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);- ServiceDatabase::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);- StandalonePostgresql::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);- StandaloneRedis::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);- StandaloneMongodb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);- StandaloneMysql::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);- StandaloneMariadb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);- StandaloneKeydb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);- StandaloneDragonfly::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);- StandaloneClickhouse::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);+ Application::where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))+ ->where('status', 'not like', 'exited%')+ ->update(['status' => 'exited']);++ ServiceApplication::whereHas('service', fn ($query) => $query->whereIn('server_id', $staleServerIds))+ ->where('status', 'not like', 'exited%')+ ->update(['status' => 'exited']);++ ServiceDatabase::whereHas('service', fn ($query) => $query->whereIn('server_id', $staleServerIds))+ ->where('status', 'not like', 'exited%')+ ->update(['status' => 'exited']);++ collect([+ StandalonePostgresql::class,+ StandaloneRedis::class,+ StandaloneMongodb::class,+ StandaloneMysql::class,+ StandaloneMariadb::class,+ StandaloneKeydb::class,+ StandaloneDragonfly::class,+ StandaloneClickhouse::class,+ ])->each(function (string $databaseClass) use ($standaloneDockerIds, $swarmDockerIds) {+ $databaseClass::query()+ ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))+ ->where('status', 'not like', 'exited%')+ ->update(['status' => 'exited']);+ });} catch (\Throwable $e) {return handleError($e);}}++ private function staleSentinelServerIds(int $seconds): Collection+ {+ return Server::query()+ ->whereNotNull('sentinel_updated_at')+ ->where('sentinel_updated_at', '<', now()->subSeconds($seconds))+ ->whereHas('settings', fn ($query) => $query->where('is_sentinel_enabled', true))+ ->pluck('id');+ }++ private function destinationIdsForServers(Collection $serverIds): array+ {+ return [+ StandaloneDocker::whereIn('server_id', $serverIds)->pluck('id'),+ SwarmDocker::whereIn('server_id', $serverIds)->pluck('id'),+ ];+ }++ private function scopeDestination($query, Collection $standaloneDockerIds, Collection $swarmDockerIds): void+ {+ $query->where(function ($query) use ($standaloneDockerIds) {+ $query->where('destination_type', StandaloneDocker::class)+ ->whereIn('destination_id', $standaloneDockerIds);+ })->orWhere(function ($query) use ($swarmDockerIds) {+ $query->where('destination_type', SwarmDocker::class)+ ->whereIn('destination_id', $swarmDockerIds);+ });+ }}diff --git a/app/Http/Controllers/Api/SentinelController.php b/app/Http/Controllers/Api/SentinelController.phpindex 53b611afa5..df5c60d406 100644--- a/app/Http/Controllers/Api/SentinelController.php+++ b/app/Http/Controllers/Api/SentinelController.php@@ -143,11 +143,13 @@ class SentinelController extends Controller/*** Build a stable hash of container state.*- * Covers [name, state, health_status] only — metrics and- * filesystem_usage_root are excluded on purpose (disk % churns constantly- * and would defeat the hash; the storage check is separately cache-gated- * inside PushServerUpdateJob). Sorted by name so container ordering from- * Sentinel does not affect the hash.+ * Covers [name, state] only — metrics, filesystem_usage_root, and+ * health_status are excluded on purpose. Disk % churns constantly, and+ * health checks can flap between starting/healthy/unhealthy while the+ * container lifecycle state remains unchanged. Both would otherwise defeat+ * the hash and dispatch DB-heavy PushServerUpdateJob instances too often.+ * The force window still refreshes full state periodically. Sorted by name+ * so container ordering from Sentinel does not affect the hash.*/private function containerStateHash(array $data): string{@@ -155,7 +157,6 @@ class SentinelController extends Controller->map(fn ($c) => ['name' => data_get($c, 'name'),'state' => data_get($c, 'state'),- 'health_status' => data_get($c, 'health_status'),])->sortBy('name')->values()diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.phpindex e75509f62d..37c73f168e 100644--- a/app/Jobs/PushServerUpdateJob.php+++ b/app/Jobs/PushServerUpdateJob.php@@ -13,6 +13,16 @@ use App\Models\ApplicationPreview;use App\Models\Server;use App\Models\ServiceApplication;use App\Models\ServiceDatabase;+use App\Models\StandaloneClickhouse;+use App\Models\StandaloneDocker;+use App\Models\StandaloneDragonfly;+use App\Models\StandaloneKeydb;+use App\Models\StandaloneMariadb;+use App\Models\StandaloneMongodb;+use App\Models\StandaloneMysql;+use App\Models\StandalonePostgresql;+use App\Models\StandaloneRedis;+use App\Models\SwarmDocker;use App\Notifications\Container\ContainerRestarted;use App\Services\ContainerStatusAggregator;use App\Traits\CalculatesExcludedStatus;@@ -25,6 +35,7 @@ use Illuminate\Queue\Middleware\WithoutOverlapping;use Illuminate\Queue\SerializesModels;use Illuminate\Support\Collection;use Illuminate\Support\Facades\Cache;+use Illuminate\Support\Facades\DB;use Laravel\Horizon\Contracts\Silenced;class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced@@ -46,6 +57,18 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silencedpublic Collection $services;+ public Collection $applicationsById;++ public Collection $previewsByKey;++ public Collection $databasesByUuid;++ public Collection $servicesById;++ public Collection $serviceApplicationsById;++ public Collection $serviceDatabasesById;+public Collection $allApplicationIds;public Collection $allDatabaseUuids;@@ -103,6 +126,12 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced$this->allTcpProxyUuids = collect();$this->allServiceApplicationIds = collect();$this->allServiceDatabaseIds = collect();+ $this->applicationsById = collect();+ $this->previewsByKey = collect();+ $this->databasesByUuid = collect();+ $this->servicesById = collect();+ $this->serviceApplicationsById = collect();+ $this->serviceDatabasesById = collect();}public function handle()@@ -120,6 +149,12 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced$this->allTcpProxyUuids ??= collect();$this->allServiceApplicationIds ??= collect();$this->allServiceDatabaseIds ??= collect();+ $this->applicationsById ??= collect();+ $this->previewsByKey ??= collect();+ $this->databasesByUuid ??= collect();+ $this->servicesById ??= collect();+ $this->serviceApplicationsById ??= collect();+ $this->serviceDatabasesById ??= collect();// TODO: Swarm is not supported yetif (! $this->data) {@@ -151,13 +186,16 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silencedreturn;}- $this->applications = $this->server->applications();- $this->databases = $this->server->databases();- $this->previews = $this->server->previews();- // Eager load service applications and databases to avoid N+1 queries- $this->services = $this->server->services()- ->with(['applications:id,service_id', 'databases:id,service_id'])- ->get();+ $this->applications = $this->loadApplications();+ $this->databases = $this->loadDatabases();+ $this->previews = $this->loadPreviews();+ $this->services = $this->loadServices();+ $this->applicationsById = $this->applications->keyBy(fn ($application) => (string) $application->id);+ $this->previewsByKey = $this->previews->keyBy(fn ($preview) => $preview->application_id.':'.$preview->pull_request_id);+ $this->databasesByUuid = $this->databases->keyBy('uuid');+ $this->servicesById = $this->services->keyBy(fn ($service) => (string) $service->id);+ $this->serviceApplicationsById = $this->services->flatMap(fn ($service) => $service->applications)->keyBy(fn ($application) => (string) $application->id);+ $this->serviceDatabasesById = $this->services->flatMap(fn ($service) => $service->databases)->keyBy(fn ($database) => (string) $database->id);$this->allApplicationIds = $this->applications->filter(function ($application) {return $application->additional_servers_count === 0;@@ -170,9 +208,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced});$this->allDatabaseUuids = $this->databases->pluck('uuid');$this->allTcpProxyUuids = $this->databases->where('is_public', true)->pluck('uuid');- // Use eager-loaded relationships instead of querying in loop- $this->allServiceApplicationIds = $this->services->flatMap(fn ($service) => $service->applications->pluck('id'));- $this->allServiceDatabaseIds = $this->services->flatMap(fn ($service) => $service->databases->pluck('id'));+ $this->allServiceApplicationIds = $this->serviceApplicationsById->keys();+ $this->allServiceDatabaseIds = $this->serviceDatabasesById->keys();foreach ($this->containers as $container) {$containerStatus = data_get($container, 'state', 'exited');@@ -286,6 +323,142 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced$this->checkLogDrainContainer();}+ private function loadApplications(): Collection+ {+ [$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds();++ $applications = Application::withoutGlobalScope('withRelations')+ ->select([+ 'id',+ 'uuid',+ 'name',+ 'status',+ 'build_pack',+ 'docker_compose_raw',+ 'destination_id',+ 'destination_type',+ 'last_online_at',+ ])+ ->withCount('additional_servers')+ ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))+ ->get();++ $additionalApplicationIds = DB::table('additional_destinations')+ ->where('server_id', $this->server->id)+ ->pluck('application_id');++ if ($additionalApplicationIds->isNotEmpty()) {+ $applications = $applications->concat(+ Application::withoutGlobalScope('withRelations')+ ->select([+ 'id',+ 'uuid',+ 'name',+ 'status',+ 'build_pack',+ 'docker_compose_raw',+ 'destination_id',+ 'destination_type',+ 'last_online_at',+ ])+ ->withCount('additional_servers')+ ->whereIn('id', $additionalApplicationIds)+ ->get()+ );+ }++ return $applications->unique('id')->values();+ }++ private function loadPreviews(): Collection+ {+ $applicationIds = $this->applications->pluck('id');++ if ($applicationIds->isEmpty()) {+ return collect();+ }++ return ApplicationPreview::query()+ ->select([+ 'id',+ 'application_id',+ 'pull_request_id',+ 'status',+ 'last_online_at',+ ])+ ->whereIn('application_id', $applicationIds)+ ->get();+ }++ private function loadServices(): Collection+ {+ return $this->server->services()+ ->select([+ 'id',+ 'server_id',+ 'uuid',+ 'docker_compose_raw',+ ])+ ->with([+ 'applications:id,service_id,status,last_online_at',+ 'databases:id,service_id,status,last_online_at,is_public,name',+ ])+ ->get();+ }++ private function loadDatabases(): Collection+ {+ [$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds();+ $databaseColumns = [+ 'id',+ 'uuid',+ 'name',+ 'status',+ 'is_public',+ 'destination_id',+ 'destination_type',+ 'last_online_at',+ 'restart_count',+ 'last_restart_at',+ 'last_restart_type',+ ];++ return collect([+ StandalonePostgresql::class,+ StandaloneRedis::class,+ StandaloneMongodb::class,+ StandaloneMysql::class,+ StandaloneMariadb::class,+ StandaloneKeydb::class,+ StandaloneDragonfly::class,+ StandaloneClickhouse::class,+ ])->flatMap(function (string $databaseClass) use ($databaseColumns, $standaloneDockerIds, $swarmDockerIds) {+ return $databaseClass::query()+ ->select($databaseColumns)+ ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))+ ->get();+ })->filter(fn ($database) => data_get($database, 'name') !== 'coolify-db')->values();+ }++ private function serverDestinationIds(): array+ {+ return [+ StandaloneDocker::where('server_id', $this->server->id)->pluck('id'),+ SwarmDocker::where('server_id', $this->server->id)->pluck('id'),+ ];+ }++ private function scopeDestination($query, Collection $standaloneDockerIds, Collection $swarmDockerIds): void+ {+ $query->where(function ($query) use ($standaloneDockerIds) {+ $query->where('destination_type', StandaloneDocker::class)+ ->whereIn('destination_id', $standaloneDockerIds);+ })->orWhere(function ($query) use ($swarmDockerIds) {+ $query->where('destination_type', SwarmDocker::class)+ ->whereIn('destination_id', $swarmDockerIds);+ });+ }+private function aggregateMultiContainerStatuses(){if ($this->applicationContainerStatuses->isEmpty()) {@@ -293,7 +466,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced}foreach ($this->applicationContainerStatuses as $applicationId => $containerStatuses) {- $application = $this->applications->where('id', $applicationId)->first();+ $application = $this->applicationsById->get((string) $applicationId);if (! $application) {continue;}@@ -314,8 +487,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silencedif ($aggregatedStatus && $application->status !== $aggregatedStatus) {$application->status = $aggregatedStatus;$application->save();- } elseif ($aggregatedStatus) {- $application->update(['last_online_at' => now()]);}continue;@@ -330,8 +501,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silencedif ($aggregatedStatus && $application->status !== $aggregatedStatus) {$application->status = $aggregatedStatus;$application->save();- } elseif ($aggregatedStatus) {- $application->update(['last_online_at' => now()]);}}}@@ -350,7 +519,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silencedcontinue;}- $service = $this->services->where('id', $serviceId)->first();+ $service = $this->servicesById->get((string) $serviceId);if (! $service) {continue;}@@ -358,9 +527,9 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced// Get the service sub-resource (ServiceApplication or ServiceDatabase)$subResource = null;if ($subType === 'application') {- $subResource = $service->applications->where('id', $subId)->first();+ $subResource = $this->serviceApplicationsById->get((string) $subId);} elseif ($subType === 'database') {- $subResource = $service->databases->where('id', $subId)->first();+ $subResource = $this->serviceDatabasesById->get((string) $subId);}if (! $subResource) {@@ -382,8 +551,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silencedif ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {$subResource->status = $aggregatedStatus;$subResource->save();- } elseif ($aggregatedStatus) {- $subResource->update(['last_online_at' => now()]);}continue;@@ -399,39 +566,31 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silencedif ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {$subResource->status = $aggregatedStatus;$subResource->save();- } elseif ($aggregatedStatus) {- $subResource->update(['last_online_at' => now()]);}}}private function updateApplicationStatus(string $applicationId, string $containerStatus){- $application = $this->applications->where('id', $applicationId)->first();+ $application = $this->applicationsById->get((string) $applicationId);if (! $application) {return;}if ($application->status !== $containerStatus) {$application->status = $containerStatus;$application->save();- } else {- $application->update(['last_online_at' => now()]);}}private function updateApplicationPreviewStatus(string $applicationId, string $pullRequestId, string $containerStatus){- $application = $this->previews->where('application_id', $applicationId)- ->where('pull_request_id', $pullRequestId)- ->first();+ $application = $this->previewsByKey->get($applicationId.':'.$pullRequestId);if (! $application) {return;}if ($application->status !== $containerStatus) {$application->status = $containerStatus;$application->save();- } else {- $application->update(['last_online_at' => now()]);}}@@ -479,9 +638,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced$applicationId = $parts[0];$pullRequestId = $parts[1];- $applicationPreview = $this->previews->where('application_id', $applicationId)- ->where('pull_request_id', $pullRequestId)- ->first();+ $applicationPreview = $this->previewsByKey->get($applicationId.':'.$pullRequestId);if ($applicationPreview && ! str($applicationPreview->status)->startsWith('exited')) {$previewIdsToUpdate->push($applicationPreview->id);@@ -520,15 +677,13 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silencedprivate function updateDatabaseStatus(string $databaseUuid, string $containerStatus, bool $tcpProxy = false){- $database = $this->databases->where('uuid', $databaseUuid)->first();+ $database = $this->databasesByUuid->get($databaseUuid);if (! $database) {return;}if ($database->status !== $containerStatus) {$database->status = $containerStatus;$database->save();- } else {- $database->update(['last_online_at' => now()]);}if ($this->isRunning($containerStatus) && $tcpProxy) {$tcpProxyContainerFound = $this->containers->filter(function ($value, $key) use ($databaseUuid) {@@ -563,7 +718,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced}$notFoundDatabaseUuids->each(function ($databaseUuid) {- $database = $this->databases->where('uuid', $databaseUuid)->first();+ $database = $this->databasesByUuid->get($databaseUuid);if ($database) {if (! str($database->status)->startsWith('exited')) {$database->update([diff --git a/config/constants.php b/config/constants.phpindex dfff17542d..e967ca3bc5 100644--- a/config/constants.php+++ b/config/constants.php@@ -93,9 +93,15 @@ return ['sentinel' => [// How often (seconds) PushServerUpdateJob is force-dispatched even when- // the container state hash is unchanged. Keeps last_online_at,- // exited-detection and storage checks from going stale.+ // the container state hash is unchanged. Keeps exited-detection and+ // storage checks from going stale without writing every resource row on+ // every push.'push_force_interval_seconds' => env('SENTINEL_PUSH_FORCE_INTERVAL_SECONDS', 300),++ // How long a Sentinel-enabled server may go without a heartbeat before+ // ResourcesCheck considers its resources stale. Per-resource+ // last_online_at is only updated on real status changes, not every push.+ 'resource_stale_seconds' => env('SENTINEL_RESOURCE_STALE_SECONDS', 300),],'proxy' => [diff --git a/tests/Feature/PushServerUpdateJobLastOnlineTest.php b/tests/Feature/PushServerUpdateJobLastOnlineTest.phpindex 5d2fd6c6ac..d986d421dc 100644--- a/tests/Feature/PushServerUpdateJobLastOnlineTest.php+++ b/tests/Feature/PushServerUpdateJobLastOnlineTest.php@@ -1,17 +1,19 @@<?phpuse App\Jobs\PushServerUpdateJob;+use App\Models\Environment;+use App\Models\Project;use App\Models\Server;+use App\Models\StandaloneDocker;use App\Models\StandalonePostgresql;use App\Models\Team;use Illuminate\Foundation\Testing\RefreshDatabase;uses(RefreshDatabase::class);-test('database last_online_at is updated when status unchanged', function () {+test('database last_online_at is not updated when status is unchanged', function () {$team = Team::factory()->create();- $database = StandalonePostgresql::factory()->create([- 'team_id' => $team->id,+ $database = createPushUpdatePostgresql($team, ['status' => 'running:healthy','last_online_at' => now()->subMinutes(5),]);@@ -40,15 +42,13 @@ test('database last_online_at is updated when status unchanged', function () {$database->refresh();- // last_online_at should be updated even though status didn't change- expect($database->last_online_at->greaterThan($oldLastOnline))->toBeTrue();+ expect((string) $database->last_online_at)->toBe((string) $oldLastOnline);expect($database->status)->toBe('running:healthy');});test('database status is updated when container status changes', function () {$team = Team::factory()->create();- $database = StandalonePostgresql::factory()->create([- 'team_id' => $team->id,+ $database = createPushUpdatePostgresql($team, ['status' => 'exited',]);@@ -79,8 +79,7 @@ test('database status is updated when container status changes', function () {test('database is not marked exited when containers list is empty', function () {$team = Team::factory()->create();- $database = StandalonePostgresql::factory()->create([- 'team_id' => $team->id,+ $database = createPushUpdatePostgresql($team, ['status' => 'running:healthy',]);@@ -99,3 +98,31 @@ test('database is not marked exited when containers list is empty', function ()// Status should remain running, NOT be set to exitedexpect($database->status)->toBe('running:healthy');});++function createPushUpdatePostgresql(Team $team, array $attributes = []): StandalonePostgresql+{+ $lastOnlineAt = $attributes['last_online_at'] ?? null;+ unset($attributes['last_online_at']);++ $server = Server::factory()->create(['team_id' => $team->id]);+ $destination = StandaloneDocker::where('server_id', $server->id)->first()+ ?? StandaloneDocker::factory()->create(['server_id' => $server->id]);+ $project = Project::factory()->create(['team_id' => $team->id]);+ $environment = Environment::factory()->create(['project_id' => $project->id]);++ $database = StandalonePostgresql::create(array_merge([+ 'uuid' => (string) str()->uuid(),+ 'name' => 'postgres-'.str()->random(8),+ 'postgres_password' => 'secret',+ 'status' => 'exited',+ 'destination_id' => $destination->id,+ 'destination_type' => $destination->getMorphClass(),+ 'environment_id' => $environment->id,+ ], $attributes));++ if ($lastOnlineAt !== null) {+ $database->forceFill(['last_online_at' => $lastOnlineAt])->saveQuietly();+ }++ return $database;+}diff --git a/tests/Feature/PushServerUpdateJobTest.php b/tests/Feature/PushServerUpdateJobTest.phpindex d508d58aba..af667d023e 100644--- a/tests/Feature/PushServerUpdateJobTest.php+++ b/tests/Feature/PushServerUpdateJobTest.php@@ -4,17 +4,21 @@ use App\Jobs\PushServerUpdateJob;use App\Models\Server;use App\Models\Service;use App\Models\ServiceApplication;+use App\Models\Team;use Illuminate\Foundation\Testing\RefreshDatabase;uses(RefreshDatabase::class);test('containers with empty service subId are skipped', function () {- $server = Server::factory()->create();+ $team = Team::factory()->create();+ $server = Server::factory()->create(['team_id' => $team->id]);$service = Service::factory()->create(['server_id' => $server->id,]);- $serviceApp = ServiceApplication::factory()->create([+ $serviceApp = ServiceApplication::create(['service_id' => $service->id,+ 'uuid' => (string) str()->uuid(),+ 'name' => 'app-'.str()->random(8),]);$data = [@@ -44,12 +48,15 @@ test('containers with empty service subId are skipped', function () {});test('containers with valid service subId are processed', function () {- $server = Server::factory()->create();+ $team = Team::factory()->create();+ $server = Server::factory()->create(['team_id' => $team->id]);$service = Service::factory()->create(['server_id' => $server->id,]);- $serviceApp = ServiceApplication::factory()->create([+ $serviceApp = ServiceApplication::create(['service_id' => $service->id,+ 'uuid' => (string) str()->uuid(),+ 'name' => 'app-'.str()->random(8),]);$data = [diff --git a/tests/Feature/ResourcesCheckTest.php b/tests/Feature/ResourcesCheckTest.phpnew file mode 100644index 0000000000..9fbf0ce2dd--- /dev/null+++ b/tests/Feature/ResourcesCheckTest.php@@ -0,0 +1,78 @@+<?php++use App\Actions\Server\ResourcesCheck;+use App\Models\Application;+use App\Models\Environment;+use App\Models\Project;+use App\Models\Server;+use App\Models\StandaloneDocker;+use App\Models\Team;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Carbon;++uses(RefreshDatabase::class);++it('does not mark resources exited when sentinel is still reporting for the server', function () {+ Carbon::setTestNow(Carbon::create(2026, 5, 27, 12, 0, 0, 'UTC'));+ config(['constants.sentinel.resource_stale_seconds' => 300]);++ $application = resourcesCheckApplication([+ 'last_online_at' => now()->subHour(),+ 'status' => 'running:healthy',+ ], [+ 'sentinel_updated_at' => now()->subMinute(),+ ]);++ ResourcesCheck::run();++ $application->refresh();++ expect($application->status)->toBe('running:healthy');+});++it('marks resources exited when their sentinel server is stale', function () {+ Carbon::setTestNow(Carbon::create(2026, 5, 27, 12, 0, 0, 'UTC'));+ config(['constants.sentinel.resource_stale_seconds' => 300]);++ $application = resourcesCheckApplication([+ 'last_online_at' => now()->subHour(),+ 'status' => 'running:healthy',+ ], [+ 'sentinel_updated_at' => now()->subMinutes(10),+ ]);++ ResourcesCheck::run();++ $application->refresh();++ expect($application->status)->toBe('exited:unhealthy');+});++function resourcesCheckApplication(array $applicationAttributes = [], array $serverAttributes = []): Application+{+ $lastOnlineAt = $applicationAttributes['last_online_at'] ?? null;+ unset($applicationAttributes['last_online_at']);++ $team = Team::factory()->create();+ $server = Server::factory()->create(array_merge([+ 'team_id' => $team->id,+ ], $serverAttributes));+ $server->settings()->update(['is_sentinel_enabled' => true]);++ $destination = StandaloneDocker::where('server_id', $server->id)->first()+ ?? StandaloneDocker::factory()->create(['server_id' => $server->id]);+ $project = Project::factory()->create(['team_id' => $team->id]);+ $environment = Environment::factory()->create(['project_id' => $project->id]);++ $application = Application::factory()->create(array_merge([+ 'environment_id' => $environment->id,+ 'destination_id' => $destination->id,+ 'destination_type' => $destination->getMorphClass(),+ ], $applicationAttributes));++ if ($lastOnlineAt !== null) {+ $application->forceFill(['last_online_at' => $lastOnlineAt])->saveQuietly();+ }++ return $application;+}diff --git a/tests/Feature/SentinelPushDeduplicationTest.php b/tests/Feature/SentinelPushDeduplicationTest.phpindex c14d5c67e0..a7ecd5d4c9 100644--- a/tests/Feature/SentinelPushDeduplicationTest.php+++ b/tests/Feature/SentinelPushDeduplicationTest.php@@ -138,6 +138,16 @@ it('dispatches the job when container state changes', function () use ($running)Queue::assertPushed(PushServerUpdateJob::class, 2);});+it('ignores health status changes while container lifecycle state is unchanged', function () {+ $healthy = [['name' => 'app-1', 'state' => 'running', 'health_status' => 'healthy']];+ $unhealthy = [['name' => 'app-1', 'state' => 'running', 'health_status' => 'unhealthy']];++ pushSentinel($this->token, sentinelPayload($healthy))->assertOk();+ pushSentinel($this->token, sentinelPayload($unhealthy))->assertOk();++ Queue::assertPushed(PushServerUpdateJob::class, 1);+});+it('ignores disk percentage changes (excluded from the hash)', function () use ($running) {pushSentinel($this->token, sentinelPayload($running(), diskPercentage: 42.0))->assertOk();pushSentinel($this->token, sentinelPayload($running(), diskPercentage: 88.0))->assertOk();