Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(sentinel): reduce resource churn from health flaps

Andras Bacsai626cfb4a2281
diff --git a/app/Actions/Server/ResourcesCheck.php b/app/Actions/Server/ResourcesCheck.php
index e6b90ba380..cc11a2e07c 100644
--- a/app/Actions/Server/ResourcesCheck.php
+++ b/app/Actions/Server/ResourcesCheck.php
@@ -3,9 +3,11 @@
namespace App\Actions\Server;
use App\Models\Application;
+use App\Models\Server;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
use App\Models\StandaloneClickhouse;
+use App\Models\StandaloneDocker;
use App\Models\StandaloneDragonfly;
use App\Models\StandaloneKeydb;
use App\Models\StandaloneMariadb;
@@ -13,6 +15,8 @@ use App\Models\StandaloneMongodb;
use App\Models\StandaloneMysql;
use App\Models\StandalonePostgresql;
use App\Models\StandaloneRedis;
+use App\Models\SwarmDocker;
+use Illuminate\Support\Collection;
use Lorisleiva\Actions\Concerns\AsAction;
class ResourcesCheck
@@ -21,21 +25,73 @@ class ResourcesCheck
public function handle()
{
- $seconds = 60;
+ $seconds = config('constants.sentinel.resource_stale_seconds', 300);
+ $staleServerIds = $this->staleSentinelServerIds($seconds);
+
+ if ($staleServerIds->isEmpty()) {
+ return;
+ }
+
+ [$standaloneDockerIds, $swarmDockerIds] = $this->destinationIdsForServers($staleServerIds);
+
try {
- Application::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
- ServiceApplication::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
- ServiceDatabase::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
- StandalonePostgresql::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
- StandaloneRedis::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
- StandaloneMongodb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
- StandaloneMysql::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
- StandaloneMariadb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
- StandaloneKeydb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
- StandaloneDragonfly::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
- StandaloneClickhouse::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
+ Application::where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
+ ->where('status', 'not like', 'exited%')
+ ->update(['status' => 'exited']);
+
+ ServiceApplication::whereHas('service', fn ($query) => $query->whereIn('server_id', $staleServerIds))
+ ->where('status', 'not like', 'exited%')
+ ->update(['status' => 'exited']);
+
+ ServiceDatabase::whereHas('service', fn ($query) => $query->whereIn('server_id', $staleServerIds))
+ ->where('status', 'not like', 'exited%')
+ ->update(['status' => 'exited']);
+
+ collect([
+ StandalonePostgresql::class,
+ StandaloneRedis::class,
+ StandaloneMongodb::class,
+ StandaloneMysql::class,
+ StandaloneMariadb::class,
+ StandaloneKeydb::class,
+ StandaloneDragonfly::class,
+ StandaloneClickhouse::class,
+ ])->each(function (string $databaseClass) use ($standaloneDockerIds, $swarmDockerIds) {
+ $databaseClass::query()
+ ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
+ ->where('status', 'not like', 'exited%')
+ ->update(['status' => 'exited']);
+ });
} catch (\Throwable $e) {
return handleError($e);
}
}
+
+ private function staleSentinelServerIds(int $seconds): Collection
+ {
+ return Server::query()
+ ->whereNotNull('sentinel_updated_at')
+ ->where('sentinel_updated_at', '<', now()->subSeconds($seconds))
+ ->whereHas('settings', fn ($query) => $query->where('is_sentinel_enabled', true))
+ ->pluck('id');
+ }
+
+ private function destinationIdsForServers(Collection $serverIds): array
+ {
+ return [
+ StandaloneDocker::whereIn('server_id', $serverIds)->pluck('id'),
+ SwarmDocker::whereIn('server_id', $serverIds)->pluck('id'),
+ ];
+ }
+
+ private function scopeDestination($query, Collection $standaloneDockerIds, Collection $swarmDockerIds): void
+ {
+ $query->where(function ($query) use ($standaloneDockerIds) {
+ $query->where('destination_type', StandaloneDocker::class)
+ ->whereIn('destination_id', $standaloneDockerIds);
+ })->orWhere(function ($query) use ($swarmDockerIds) {
+ $query->where('destination_type', SwarmDocker::class)
+ ->whereIn('destination_id', $swarmDockerIds);
+ });
+ }
}
diff --git a/app/Http/Controllers/Api/SentinelController.php b/app/Http/Controllers/Api/SentinelController.php
index 53b611afa5..df5c60d406 100644
--- a/app/Http/Controllers/Api/SentinelController.php
+++ b/app/Http/Controllers/Api/SentinelController.php
@@ -143,11 +143,13 @@ class SentinelController extends Controller
/**
* Build a stable hash of container state.
*
- * Covers [name, state, health_status] only — metrics and
- * filesystem_usage_root are excluded on purpose (disk % churns constantly
- * and would defeat the hash; the storage check is separately cache-gated
- * inside PushServerUpdateJob). Sorted by name so container ordering from
- * Sentinel does not affect the hash.
+ * Covers [name, state] only — metrics, filesystem_usage_root, and
+ * health_status are excluded on purpose. Disk % churns constantly, and
+ * health checks can flap between starting/healthy/unhealthy while the
+ * container lifecycle state remains unchanged. Both would otherwise defeat
+ * the hash and dispatch DB-heavy PushServerUpdateJob instances too often.
+ * The force window still refreshes full state periodically. Sorted by name
+ * so container ordering from Sentinel does not affect the hash.
*/
private function containerStateHash(array $data): string
{
@@ -155,7 +157,6 @@ class SentinelController extends Controller
->map(fn ($c) => [
'name' => data_get($c, 'name'),
'state' => data_get($c, 'state'),
- 'health_status' => data_get($c, 'health_status'),
])
->sortBy('name')
->values()
diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php
index e75509f62d..37c73f168e 100644
--- a/app/Jobs/PushServerUpdateJob.php
+++ b/app/Jobs/PushServerUpdateJob.php
@@ -13,6 +13,16 @@ use App\Models\ApplicationPreview;
use App\Models\Server;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
+use App\Models\StandaloneClickhouse;
+use App\Models\StandaloneDocker;
+use App\Models\StandaloneDragonfly;
+use App\Models\StandaloneKeydb;
+use App\Models\StandaloneMariadb;
+use App\Models\StandaloneMongodb;
+use App\Models\StandaloneMysql;
+use App\Models\StandalonePostgresql;
+use App\Models\StandaloneRedis;
+use App\Models\SwarmDocker;
use App\Notifications\Container\ContainerRestarted;
use App\Services\ContainerStatusAggregator;
use App\Traits\CalculatesExcludedStatus;
@@ -25,6 +35,7 @@ use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\DB;
use Laravel\Horizon\Contracts\Silenced;
class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
@@ -46,6 +57,18 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
public Collection $services;
+ public Collection $applicationsById;
+
+ public Collection $previewsByKey;
+
+ public Collection $databasesByUuid;
+
+ public Collection $servicesById;
+
+ public Collection $serviceApplicationsById;
+
+ public Collection $serviceDatabasesById;
+
public Collection $allApplicationIds;
public Collection $allDatabaseUuids;
@@ -103,6 +126,12 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
$this->allTcpProxyUuids = collect();
$this->allServiceApplicationIds = collect();
$this->allServiceDatabaseIds = collect();
+ $this->applicationsById = collect();
+ $this->previewsByKey = collect();
+ $this->databasesByUuid = collect();
+ $this->servicesById = collect();
+ $this->serviceApplicationsById = collect();
+ $this->serviceDatabasesById = collect();
}
public function handle()
@@ -120,6 +149,12 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
$this->allTcpProxyUuids ??= collect();
$this->allServiceApplicationIds ??= collect();
$this->allServiceDatabaseIds ??= collect();
+ $this->applicationsById ??= collect();
+ $this->previewsByKey ??= collect();
+ $this->databasesByUuid ??= collect();
+ $this->servicesById ??= collect();
+ $this->serviceApplicationsById ??= collect();
+ $this->serviceDatabasesById ??= collect();
// TODO: Swarm is not supported yet
if (! $this->data) {
@@ -151,13 +186,16 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
return;
}
- $this->applications = $this->server->applications();
- $this->databases = $this->server->databases();
- $this->previews = $this->server->previews();
- // Eager load service applications and databases to avoid N+1 queries
- $this->services = $this->server->services()
- ->with(['applications:id,service_id', 'databases:id,service_id'])
- ->get();
+ $this->applications = $this->loadApplications();
+ $this->databases = $this->loadDatabases();
+ $this->previews = $this->loadPreviews();
+ $this->services = $this->loadServices();
+ $this->applicationsById = $this->applications->keyBy(fn ($application) => (string) $application->id);
+ $this->previewsByKey = $this->previews->keyBy(fn ($preview) => $preview->application_id.':'.$preview->pull_request_id);
+ $this->databasesByUuid = $this->databases->keyBy('uuid');
+ $this->servicesById = $this->services->keyBy(fn ($service) => (string) $service->id);
+ $this->serviceApplicationsById = $this->services->flatMap(fn ($service) => $service->applications)->keyBy(fn ($application) => (string) $application->id);
+ $this->serviceDatabasesById = $this->services->flatMap(fn ($service) => $service->databases)->keyBy(fn ($database) => (string) $database->id);
$this->allApplicationIds = $this->applications->filter(function ($application) {
return $application->additional_servers_count === 0;
@@ -170,9 +208,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
});
$this->allDatabaseUuids = $this->databases->pluck('uuid');
$this->allTcpProxyUuids = $this->databases->where('is_public', true)->pluck('uuid');
- // Use eager-loaded relationships instead of querying in loop
- $this->allServiceApplicationIds = $this->services->flatMap(fn ($service) => $service->applications->pluck('id'));
- $this->allServiceDatabaseIds = $this->services->flatMap(fn ($service) => $service->databases->pluck('id'));
+ $this->allServiceApplicationIds = $this->serviceApplicationsById->keys();
+ $this->allServiceDatabaseIds = $this->serviceDatabasesById->keys();
foreach ($this->containers as $container) {
$containerStatus = data_get($container, 'state', 'exited');
@@ -286,6 +323,142 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
$this->checkLogDrainContainer();
}
+ private function loadApplications(): Collection
+ {
+ [$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds();
+
+ $applications = Application::withoutGlobalScope('withRelations')
+ ->select([
+ 'id',
+ 'uuid',
+ 'name',
+ 'status',
+ 'build_pack',
+ 'docker_compose_raw',
+ 'destination_id',
+ 'destination_type',
+ 'last_online_at',
+ ])
+ ->withCount('additional_servers')
+ ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
+ ->get();
+
+ $additionalApplicationIds = DB::table('additional_destinations')
+ ->where('server_id', $this->server->id)
+ ->pluck('application_id');
+
+ if ($additionalApplicationIds->isNotEmpty()) {
+ $applications = $applications->concat(
+ Application::withoutGlobalScope('withRelations')
+ ->select([
+ 'id',
+ 'uuid',
+ 'name',
+ 'status',
+ 'build_pack',
+ 'docker_compose_raw',
+ 'destination_id',
+ 'destination_type',
+ 'last_online_at',
+ ])
+ ->withCount('additional_servers')
+ ->whereIn('id', $additionalApplicationIds)
+ ->get()
+ );
+ }
+
+ return $applications->unique('id')->values();
+ }
+
+ private function loadPreviews(): Collection
+ {
+ $applicationIds = $this->applications->pluck('id');
+
+ if ($applicationIds->isEmpty()) {
+ return collect();
+ }
+
+ return ApplicationPreview::query()
+ ->select([
+ 'id',
+ 'application_id',
+ 'pull_request_id',
+ 'status',
+ 'last_online_at',
+ ])
+ ->whereIn('application_id', $applicationIds)
+ ->get();
+ }
+
+ private function loadServices(): Collection
+ {
+ return $this->server->services()
+ ->select([
+ 'id',
+ 'server_id',
+ 'uuid',
+ 'docker_compose_raw',
+ ])
+ ->with([
+ 'applications:id,service_id,status,last_online_at',
+ 'databases:id,service_id,status,last_online_at,is_public,name',
+ ])
+ ->get();
+ }
+
+ private function loadDatabases(): Collection
+ {
+ [$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds();
+ $databaseColumns = [
+ 'id',
+ 'uuid',
+ 'name',
+ 'status',
+ 'is_public',
+ 'destination_id',
+ 'destination_type',
+ 'last_online_at',
+ 'restart_count',
+ 'last_restart_at',
+ 'last_restart_type',
+ ];
+
+ return collect([
+ StandalonePostgresql::class,
+ StandaloneRedis::class,
+ StandaloneMongodb::class,
+ StandaloneMysql::class,
+ StandaloneMariadb::class,
+ StandaloneKeydb::class,
+ StandaloneDragonfly::class,
+ StandaloneClickhouse::class,
+ ])->flatMap(function (string $databaseClass) use ($databaseColumns, $standaloneDockerIds, $swarmDockerIds) {
+ return $databaseClass::query()
+ ->select($databaseColumns)
+ ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
+ ->get();
+ })->filter(fn ($database) => data_get($database, 'name') !== 'coolify-db')->values();
+ }
+
+ private function serverDestinationIds(): array
+ {
+ return [
+ StandaloneDocker::where('server_id', $this->server->id)->pluck('id'),
+ SwarmDocker::where('server_id', $this->server->id)->pluck('id'),
+ ];
+ }
+
+ private function scopeDestination($query, Collection $standaloneDockerIds, Collection $swarmDockerIds): void
+ {
+ $query->where(function ($query) use ($standaloneDockerIds) {
+ $query->where('destination_type', StandaloneDocker::class)
+ ->whereIn('destination_id', $standaloneDockerIds);
+ })->orWhere(function ($query) use ($swarmDockerIds) {
+ $query->where('destination_type', SwarmDocker::class)
+ ->whereIn('destination_id', $swarmDockerIds);
+ });
+ }
+
private function aggregateMultiContainerStatuses()
{
if ($this->applicationContainerStatuses->isEmpty()) {
@@ -293,7 +466,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
}
foreach ($this->applicationContainerStatuses as $applicationId => $containerStatuses) {
- $application = $this->applications->where('id', $applicationId)->first();
+ $application = $this->applicationsById->get((string) $applicationId);
if (! $application) {
continue;
}
@@ -314,8 +487,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
$application->status = $aggregatedStatus;
$application->save();
- } elseif ($aggregatedStatus) {
- $application->update(['last_online_at' => now()]);
}
continue;
@@ -330,8 +501,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
$application->status = $aggregatedStatus;
$application->save();
- } elseif ($aggregatedStatus) {
- $application->update(['last_online_at' => now()]);
}
}
}
@@ -350,7 +519,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
continue;
}
- $service = $this->services->where('id', $serviceId)->first();
+ $service = $this->servicesById->get((string) $serviceId);
if (! $service) {
continue;
}
@@ -358,9 +527,9 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
// Get the service sub-resource (ServiceApplication or ServiceDatabase)
$subResource = null;
if ($subType === 'application') {
- $subResource = $service->applications->where('id', $subId)->first();
+ $subResource = $this->serviceApplicationsById->get((string) $subId);
} elseif ($subType === 'database') {
- $subResource = $service->databases->where('id', $subId)->first();
+ $subResource = $this->serviceDatabasesById->get((string) $subId);
}
if (! $subResource) {
@@ -382,8 +551,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
$subResource->status = $aggregatedStatus;
$subResource->save();
- } elseif ($aggregatedStatus) {
- $subResource->update(['last_online_at' => now()]);
}
continue;
@@ -399,39 +566,31 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
$subResource->status = $aggregatedStatus;
$subResource->save();
- } elseif ($aggregatedStatus) {
- $subResource->update(['last_online_at' => now()]);
}
}
}
private function updateApplicationStatus(string $applicationId, string $containerStatus)
{
- $application = $this->applications->where('id', $applicationId)->first();
+ $application = $this->applicationsById->get((string) $applicationId);
if (! $application) {
return;
}
if ($application->status !== $containerStatus) {
$application->status = $containerStatus;
$application->save();
- } else {
- $application->update(['last_online_at' => now()]);
}
}
private function updateApplicationPreviewStatus(string $applicationId, string $pullRequestId, string $containerStatus)
{
- $application = $this->previews->where('application_id', $applicationId)
- ->where('pull_request_id', $pullRequestId)
- ->first();
+ $application = $this->previewsByKey->get($applicationId.':'.$pullRequestId);
if (! $application) {
return;
}
if ($application->status !== $containerStatus) {
$application->status = $containerStatus;
$application->save();
- } else {
- $application->update(['last_online_at' => now()]);
}
}
@@ -479,9 +638,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
$applicationId = $parts[0];
$pullRequestId = $parts[1];
- $applicationPreview = $this->previews->where('application_id', $applicationId)
- ->where('pull_request_id', $pullRequestId)
- ->first();
+ $applicationPreview = $this->previewsByKey->get($applicationId.':'.$pullRequestId);
if ($applicationPreview && ! str($applicationPreview->status)->startsWith('exited')) {
$previewIdsToUpdate->push($applicationPreview->id);
@@ -520,15 +677,13 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
private function updateDatabaseStatus(string $databaseUuid, string $containerStatus, bool $tcpProxy = false)
{
- $database = $this->databases->where('uuid', $databaseUuid)->first();
+ $database = $this->databasesByUuid->get($databaseUuid);
if (! $database) {
return;
}
if ($database->status !== $containerStatus) {
$database->status = $containerStatus;
$database->save();
- } else {
- $database->update(['last_online_at' => now()]);
}
if ($this->isRunning($containerStatus) && $tcpProxy) {
$tcpProxyContainerFound = $this->containers->filter(function ($value, $key) use ($databaseUuid) {
@@ -563,7 +718,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
}
$notFoundDatabaseUuids->each(function ($databaseUuid) {
- $database = $this->databases->where('uuid', $databaseUuid)->first();
+ $database = $this->databasesByUuid->get($databaseUuid);
if ($database) {
if (! str($database->status)->startsWith('exited')) {
$database->update([
diff --git a/config/constants.php b/config/constants.php
index dfff17542d..e967ca3bc5 100644
--- a/config/constants.php
+++ b/config/constants.php
@@ -93,9 +93,15 @@ return [
'sentinel' => [
// How often (seconds) PushServerUpdateJob is force-dispatched even when
- // the container state hash is unchanged. Keeps last_online_at,
- // exited-detection and storage checks from going stale.
+ // the container state hash is unchanged. Keeps exited-detection and
+ // storage checks from going stale without writing every resource row on
+ // every push.
'push_force_interval_seconds' => env('SENTINEL_PUSH_FORCE_INTERVAL_SECONDS', 300),
+
+ // How long a Sentinel-enabled server may go without a heartbeat before
+ // ResourcesCheck considers its resources stale. Per-resource
+ // last_online_at is only updated on real status changes, not every push.
+ 'resource_stale_seconds' => env('SENTINEL_RESOURCE_STALE_SECONDS', 300),
],
'proxy' => [
diff --git a/tests/Feature/PushServerUpdateJobLastOnlineTest.php b/tests/Feature/PushServerUpdateJobLastOnlineTest.php
index 5d2fd6c6ac..d986d421dc 100644
--- a/tests/Feature/PushServerUpdateJobLastOnlineTest.php
+++ b/tests/Feature/PushServerUpdateJobLastOnlineTest.php
@@ -1,17 +1,19 @@
<?php
use App\Jobs\PushServerUpdateJob;
+use App\Models\Environment;
+use App\Models\Project;
use App\Models\Server;
+use App\Models\StandaloneDocker;
use App\Models\StandalonePostgresql;
use App\Models\Team;
use Illuminate\Foundation\Testing\RefreshDatabase;
uses(RefreshDatabase::class);
-test('database last_online_at is updated when status unchanged', function () {
+test('database last_online_at is not updated when status is unchanged', function () {
$team = Team::factory()->create();
- $database = StandalonePostgresql::factory()->create([
- 'team_id' => $team->id,
+ $database = createPushUpdatePostgresql($team, [
'status' => 'running:healthy',
'last_online_at' => now()->subMinutes(5),
]);
@@ -40,15 +42,13 @@ test('database last_online_at is updated when status unchanged', function () {
$database->refresh();
- // last_online_at should be updated even though status didn't change
- expect($database->last_online_at->greaterThan($oldLastOnline))->toBeTrue();
+ expect((string) $database->last_online_at)->toBe((string) $oldLastOnline);
expect($database->status)->toBe('running:healthy');
});
test('database status is updated when container status changes', function () {
$team = Team::factory()->create();
- $database = StandalonePostgresql::factory()->create([
- 'team_id' => $team->id,
+ $database = createPushUpdatePostgresql($team, [
'status' => 'exited',
]);
@@ -79,8 +79,7 @@ test('database status is updated when container status changes', function () {
test('database is not marked exited when containers list is empty', function () {
$team = Team::factory()->create();
- $database = StandalonePostgresql::factory()->create([
- 'team_id' => $team->id,
+ $database = createPushUpdatePostgresql($team, [
'status' => 'running:healthy',
]);
@@ -99,3 +98,31 @@ test('database is not marked exited when containers list is empty', function ()
// Status should remain running, NOT be set to exited
expect($database->status)->toBe('running:healthy');
});
+
+function createPushUpdatePostgresql(Team $team, array $attributes = []): StandalonePostgresql
+{
+ $lastOnlineAt = $attributes['last_online_at'] ?? null;
+ unset($attributes['last_online_at']);
+
+ $server = Server::factory()->create(['team_id' => $team->id]);
+ $destination = StandaloneDocker::where('server_id', $server->id)->first()
+ ?? StandaloneDocker::factory()->create(['server_id' => $server->id]);
+ $project = Project::factory()->create(['team_id' => $team->id]);
+ $environment = Environment::factory()->create(['project_id' => $project->id]);
+
+ $database = StandalonePostgresql::create(array_merge([
+ 'uuid' => (string) str()->uuid(),
+ 'name' => 'postgres-'.str()->random(8),
+ 'postgres_password' => 'secret',
+ 'status' => 'exited',
+ 'destination_id' => $destination->id,
+ 'destination_type' => $destination->getMorphClass(),
+ 'environment_id' => $environment->id,
+ ], $attributes));
+
+ if ($lastOnlineAt !== null) {
+ $database->forceFill(['last_online_at' => $lastOnlineAt])->saveQuietly();
+ }
+
+ return $database;
+}
diff --git a/tests/Feature/PushServerUpdateJobTest.php b/tests/Feature/PushServerUpdateJobTest.php
index d508d58aba..af667d023e 100644
--- a/tests/Feature/PushServerUpdateJobTest.php
+++ b/tests/Feature/PushServerUpdateJobTest.php
@@ -4,17 +4,21 @@ use App\Jobs\PushServerUpdateJob;
use App\Models\Server;
use App\Models\Service;
use App\Models\ServiceApplication;
+use App\Models\Team;
use Illuminate\Foundation\Testing\RefreshDatabase;
uses(RefreshDatabase::class);
test('containers with empty service subId are skipped', function () {
- $server = Server::factory()->create();
+ $team = Team::factory()->create();
+ $server = Server::factory()->create(['team_id' => $team->id]);
$service = Service::factory()->create([
'server_id' => $server->id,
]);
- $serviceApp = ServiceApplication::factory()->create([
+ $serviceApp = ServiceApplication::create([
'service_id' => $service->id,
+ 'uuid' => (string) str()->uuid(),
+ 'name' => 'app-'.str()->random(8),
]);
$data = [
@@ -44,12 +48,15 @@ test('containers with empty service subId are skipped', function () {
});
test('containers with valid service subId are processed', function () {
- $server = Server::factory()->create();
+ $team = Team::factory()->create();
+ $server = Server::factory()->create(['team_id' => $team->id]);
$service = Service::factory()->create([
'server_id' => $server->id,
]);
- $serviceApp = ServiceApplication::factory()->create([
+ $serviceApp = ServiceApplication::create([
'service_id' => $service->id,
+ 'uuid' => (string) str()->uuid(),
+ 'name' => 'app-'.str()->random(8),
]);
$data = [
diff --git a/tests/Feature/ResourcesCheckTest.php b/tests/Feature/ResourcesCheckTest.php
new file mode 100644
index 0000000000..9fbf0ce2dd
--- /dev/null
+++ b/tests/Feature/ResourcesCheckTest.php
@@ -0,0 +1,78 @@
+<?php
+
+use App\Actions\Server\ResourcesCheck;
+use App\Models\Application;
+use App\Models\Environment;
+use App\Models\Project;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Carbon;
+
+uses(RefreshDatabase::class);
+
+it('does not mark resources exited when sentinel is still reporting for the server', function () {
+ Carbon::setTestNow(Carbon::create(2026, 5, 27, 12, 0, 0, 'UTC'));
+ config(['constants.sentinel.resource_stale_seconds' => 300]);
+
+ $application = resourcesCheckApplication([
+ 'last_online_at' => now()->subHour(),
+ 'status' => 'running:healthy',
+ ], [
+ 'sentinel_updated_at' => now()->subMinute(),
+ ]);
+
+ ResourcesCheck::run();
+
+ $application->refresh();
+
+ expect($application->status)->toBe('running:healthy');
+});
+
+it('marks resources exited when their sentinel server is stale', function () {
+ Carbon::setTestNow(Carbon::create(2026, 5, 27, 12, 0, 0, 'UTC'));
+ config(['constants.sentinel.resource_stale_seconds' => 300]);
+
+ $application = resourcesCheckApplication([
+ 'last_online_at' => now()->subHour(),
+ 'status' => 'running:healthy',
+ ], [
+ 'sentinel_updated_at' => now()->subMinutes(10),
+ ]);
+
+ ResourcesCheck::run();
+
+ $application->refresh();
+
+ expect($application->status)->toBe('exited:unhealthy');
+});
+
+function resourcesCheckApplication(array $applicationAttributes = [], array $serverAttributes = []): Application
+{
+ $lastOnlineAt = $applicationAttributes['last_online_at'] ?? null;
+ unset($applicationAttributes['last_online_at']);
+
+ $team = Team::factory()->create();
+ $server = Server::factory()->create(array_merge([
+ 'team_id' => $team->id,
+ ], $serverAttributes));
+ $server->settings()->update(['is_sentinel_enabled' => true]);
+
+ $destination = StandaloneDocker::where('server_id', $server->id)->first()
+ ?? StandaloneDocker::factory()->create(['server_id' => $server->id]);
+ $project = Project::factory()->create(['team_id' => $team->id]);
+ $environment = Environment::factory()->create(['project_id' => $project->id]);
+
+ $application = Application::factory()->create(array_merge([
+ 'environment_id' => $environment->id,
+ 'destination_id' => $destination->id,
+ 'destination_type' => $destination->getMorphClass(),
+ ], $applicationAttributes));
+
+ if ($lastOnlineAt !== null) {
+ $application->forceFill(['last_online_at' => $lastOnlineAt])->saveQuietly();
+ }
+
+ return $application;
+}
diff --git a/tests/Feature/SentinelPushDeduplicationTest.php b/tests/Feature/SentinelPushDeduplicationTest.php
index c14d5c67e0..a7ecd5d4c9 100644
--- a/tests/Feature/SentinelPushDeduplicationTest.php
+++ b/tests/Feature/SentinelPushDeduplicationTest.php
@@ -138,6 +138,16 @@ it('dispatches the job when container state changes', function () use ($running)
Queue::assertPushed(PushServerUpdateJob::class, 2);
});
+it('ignores health status changes while container lifecycle state is unchanged', function () {
+ $healthy = [['name' => 'app-1', 'state' => 'running', 'health_status' => 'healthy']];
+ $unhealthy = [['name' => 'app-1', 'state' => 'running', 'health_status' => 'unhealthy']];
+
+ pushSentinel($this->token, sentinelPayload($healthy))->assertOk();
+ pushSentinel($this->token, sentinelPayload($unhealthy))->assertOk();
+
+ Queue::assertPushed(PushServerUpdateJob::class, 1);
+});
+
it('ignores disk percentage changes (excluded from the hash)', function () use ($running) {
pushSentinel($this->token, sentinelPayload($running(), diskPercentage: 42.0))->assertOk();
pushSentinel($this->token, sentinelPayload($running(), diskPercentage: 88.0))->assertOk();