Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

refactor(server): dispatch event for reachability notifications, drop retry loop

Andras Bacsaib8226be774eb
diff --git a/app/Jobs/ServerConnectionCheckJob.php b/app/Jobs/ServerConnectionCheckJob.php
index 7ce316dcd2..98ad60fffe 100644
--- a/app/Jobs/ServerConnectionCheckJob.php
+++ b/app/Jobs/ServerConnectionCheckJob.php
@@ -2,6 +2,7 @@
namespace App\Jobs;
+use App\Events\ServerReachabilityChanged;
use App\Helpers\SshMultiplexingHelper;
use App\Models\Server;
use App\Services\ConfigurationRepository;
@@ -43,6 +44,9 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
public function handle()
{
+ $wasReachable = (bool) $this->server->settings->is_reachable;
+ $wasNotified = (bool) $this->server->unreachable_notification_sent;
+
try {
// Check if server is disabled
if ($this->server->settings->force_disabled) {
@@ -84,6 +88,8 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
'server_ip' => $this->server->ip,
]);
+ $this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, false);
+
return;
}
@@ -99,6 +105,8 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
$this->server->update(['unreachable_count' => 0]);
}
+ $this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, true);
+
} catch (\Throwable $e) {
Log::error('ServerConnectionCheckJob failed', [
@@ -111,6 +119,8 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
]);
$this->server->increment('unreachable_count');
+ $this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, false);
+
return;
}
}
@@ -118,17 +128,41 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
public function failed(?\Throwable $exception): void
{
if ($exception instanceof TimeoutExceededException) {
+ $wasReachable = (bool) $this->server->settings->is_reachable;
+ $wasNotified = (bool) $this->server->unreachable_notification_sent;
+
$this->server->settings->update([
'is_reachable' => false,
'is_usable' => false,
]);
$this->server->increment('unreachable_count');
+ $this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, false);
+
// Delete the queue job so it doesn't appear in Horizon's failed list.
$this->job?->delete();
}
}
+ /**
+ * Fire ServerReachabilityChanged when state crosses the unreachable threshold (count >= 2)
+ * or when a previously-notified server recovers. Skips noise from single transient flaps.
+ */
+ private function dispatchReachabilityChangedIfNeeded(bool $wasReachable, bool $wasNotified, bool $isReachable): void
+ {
+ if ($isReachable) {
+ if (! $wasReachable || $wasNotified) {
+ ServerReachabilityChanged::dispatch($this->server);
+ }
+
+ return;
+ }
+
+ if ($this->server->unreachable_count >= 2 && ! $wasNotified) {
+ ServerReachabilityChanged::dispatch($this->server);
+ }
+ }
+
private function checkHetznerStatus(): void
{
$status = null;
diff --git a/app/Models/Server.php b/app/Models/Server.php
index 06426f2119..74e8ba5b08 100644
--- a/app/Models/Server.php
+++ b/app/Models/Server.php
@@ -1236,10 +1236,8 @@ $schema://$host {
$this->refresh();
$unreachableNotificationSent = (bool) $this->unreachable_notification_sent;
$isReachable = (bool) $this->settings->is_reachable;
- if ($isReachable === true) {
- $this->unreachable_count = 0;
- $this->save();
+ if ($isReachable === true) {
if ($unreachableNotificationSent === true) {
$this->sendReachableNotification();
}
@@ -1247,28 +1245,8 @@ $schema://$host {
return;
}
- $this->increment('unreachable_count');
-
- if ($this->unreachable_count === 1) {
- $this->settings->is_reachable = true;
- $this->settings->save();
-
- return;
- }
-
if ($this->unreachable_count >= 2 && ! $unreachableNotificationSent) {
- $failedChecks = 0;
- for ($i = 0; $i < 3; $i++) {
- $status = $this->serverStatus();
- sleep(5);
- if (! $status) {
- $failedChecks++;
- }
- }
-
- if ($failedChecks === 3 && ! $unreachableNotificationSent) {
- $this->sendUnreachableNotification();
- }
+ $this->sendUnreachableNotification();
}
}
diff --git a/tests/Feature/ServerReachabilityNotificationTest.php b/tests/Feature/ServerReachabilityNotificationTest.php
new file mode 100644
index 0000000000..e996ba0286
--- /dev/null
+++ b/tests/Feature/ServerReachabilityNotificationTest.php
@@ -0,0 +1,105 @@
+<?php
+
+use App\Events\ServerReachabilityChanged;
+use App\Models\Server;
+use App\Models\Team;
+use App\Notifications\Channels\EmailChannel;
+use App\Notifications\Server\Reachable;
+use App\Notifications\Server\Unreachable;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Notification;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ $this->team = Team::factory()->create();
+ $this->team->emailNotificationSettings()->update([
+ 'use_instance_email_settings' => true,
+ 'server_unreachable_email_notifications' => true,
+ 'server_reachable_email_notifications' => true,
+ ]);
+
+ $this->server = Server::factory()->create(['team_id' => $this->team->id]);
+
+ Notification::fake();
+});
+
+it('sends Unreachable notification when threshold reached and not yet notified', function () {
+ $this->server->settings()->update(['is_reachable' => false]);
+ $this->server->forceFill([
+ 'unreachable_count' => 2,
+ 'unreachable_notification_sent' => false,
+ ])->save();
+
+ ServerReachabilityChanged::dispatch($this->server->fresh());
+
+ Notification::assertSentTo($this->team, Unreachable::class);
+ expect($this->server->fresh()->unreachable_notification_sent)->toBeTrue();
+});
+
+it('does not send Unreachable on first transient failure (count=1)', function () {
+ $this->server->settings()->update(['is_reachable' => false]);
+ $this->server->forceFill([
+ 'unreachable_count' => 1,
+ 'unreachable_notification_sent' => false,
+ ])->save();
+
+ ServerReachabilityChanged::dispatch($this->server->fresh());
+
+ Notification::assertNothingSent();
+});
+
+it('does not send Unreachable when already notified', function () {
+ $this->server->settings()->update(['is_reachable' => false]);
+ $this->server->forceFill([
+ 'unreachable_count' => 5,
+ 'unreachable_notification_sent' => true,
+ ])->save();
+
+ ServerReachabilityChanged::dispatch($this->server->fresh());
+
+ Notification::assertNothingSent();
+});
+
+it('sends Reachable notification on recovery when previously notified', function () {
+ $this->server->settings()->update(['is_reachable' => true]);
+ $this->server->forceFill([
+ 'unreachable_count' => 0,
+ 'unreachable_notification_sent' => true,
+ ])->save();
+
+ $fresh = $this->server->fresh();
+ expect($fresh->unreachable_notification_sent)->toBeTrue();
+ expect((bool) $fresh->settings->is_reachable)->toBeTrue();
+
+ ServerReachabilityChanged::dispatch($fresh);
+
+ Notification::assertSentTo($this->team, Reachable::class);
+ expect($this->server->fresh()->unreachable_notification_sent)->toBeFalse();
+});
+
+it('does not send Reachable when never notified', function () {
+ $this->server->settings()->update(['is_reachable' => true]);
+ $this->server->forceFill([
+ 'unreachable_count' => 0,
+ 'unreachable_notification_sent' => false,
+ ])->save();
+
+ ServerReachabilityChanged::dispatch($this->server->fresh());
+
+ Notification::assertNothingSent();
+});
+
+it('routes Unreachable notification through EmailChannel when email toggle is on', function () {
+ $this->server->settings()->update(['is_reachable' => false]);
+ $this->server->forceFill([
+ 'unreachable_count' => 2,
+ 'unreachable_notification_sent' => false,
+ ])->save();
+
+ ServerReachabilityChanged::dispatch($this->server->fresh());
+
+ Notification::assertSentTo($this->team, Unreachable::class, function ($notification, $channels) {
+ return in_array(EmailChannel::class, $channels);
+ });
+});
diff --git a/tests/Unit/IsReachableChangedTest.php b/tests/Unit/IsReachableChangedTest.php
new file mode 100644
index 0000000000..76f9863bf8
--- /dev/null
+++ b/tests/Unit/IsReachableChangedTest.php
@@ -0,0 +1,61 @@
+<?php
+
+use App\Models\Server;
+
+afterEach(function () {
+ Mockery::close();
+});
+
+function makeServerForReachabilityTest(bool $isReachable, bool $notificationSent, int $unreachableCount): Server
+{
+ $settings = Mockery::mock();
+ $settings->is_reachable = $isReachable;
+
+ $server = Mockery::mock(Server::class)->makePartial()->shouldAllowMockingProtectedMethods();
+ $server->shouldReceive('refresh')->andReturnSelf();
+ $server->shouldReceive('getAttribute')->with('settings')->andReturn($settings);
+ $server->shouldReceive('getAttribute')->with('unreachable_notification_sent')->andReturn($notificationSent);
+ $server->shouldReceive('getAttribute')->with('unreachable_count')->andReturn($unreachableCount);
+
+ return $server;
+}
+
+it('sends Reachable notification when reachable and notification was previously sent', function () {
+ $server = makeServerForReachabilityTest(isReachable: true, notificationSent: true, unreachableCount: 0);
+ $server->shouldReceive('sendReachableNotification')->once();
+ $server->shouldNotReceive('sendUnreachableNotification');
+
+ $server->isReachableChanged();
+});
+
+it('does not send any notification when reachable and notification was never sent', function () {
+ $server = makeServerForReachabilityTest(isReachable: true, notificationSent: false, unreachableCount: 0);
+ $server->shouldNotReceive('sendReachableNotification');
+ $server->shouldNotReceive('sendUnreachableNotification');
+
+ $server->isReachableChanged();
+});
+
+it('sends Unreachable notification when count >= 2 and not yet notified', function () {
+ $server = makeServerForReachabilityTest(isReachable: false, notificationSent: false, unreachableCount: 2);
+ $server->shouldReceive('sendUnreachableNotification')->once();
+ $server->shouldNotReceive('sendReachableNotification');
+
+ $server->isReachableChanged();
+});
+
+it('does not send Unreachable notification on first transient failure (count=1)', function () {
+ $server = makeServerForReachabilityTest(isReachable: false, notificationSent: false, unreachableCount: 1);
+ $server->shouldNotReceive('sendUnreachableNotification');
+ $server->shouldNotReceive('sendReachableNotification');
+
+ $server->isReachableChanged();
+});
+
+it('does not double-send Unreachable when already notified', function () {
+ $server = makeServerForReachabilityTest(isReachable: false, notificationSent: true, unreachableCount: 5);
+ $server->shouldNotReceive('sendUnreachableNotification');
+ $server->shouldNotReceive('sendReachableNotification');
+
+ $server->isReachableChanged();
+});
diff --git a/tests/Unit/ServerBackoffTest.php b/tests/Unit/ServerBackoffTest.php
index bdcefb74fb..9f1f747d47 100644
--- a/tests/Unit/ServerBackoffTest.php
+++ b/tests/Unit/ServerBackoffTest.php
@@ -1,11 +1,13 @@
<?php
+use App\Events\ServerReachabilityChanged;
use App\Jobs\ServerCheckJob;
use App\Jobs\ServerConnectionCheckJob;
use App\Jobs\ServerManagerJob;
use App\Models\Server;
use Illuminate\Queue\TimeoutExceededException;
use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Event;
use Tests\TestCase;
uses(TestCase::class);
@@ -126,16 +128,21 @@ describe('shouldSkipDueToBackoff', function () {
describe('ServerConnectionCheckJob unreachable_count', function () {
it('increments unreachable_count on timeout', function () {
+ Event::fake([ServerReachabilityChanged::class]);
+
$settings = Mockery::mock();
+ $settings->is_reachable = true;
$settings->shouldReceive('update')
->with(['is_reachable' => false, 'is_usable' => false])
->once();
$server = Mockery::mock(Server::class)->makePartial()->shouldAllowMockingProtectedMethods();
$server->shouldReceive('getAttribute')->with('settings')->andReturn($settings);
+ $server->shouldReceive('getAttribute')->with('unreachable_notification_sent')->andReturn(false);
$server->shouldReceive('increment')->with('unreachable_count')->once();
$server->id = 1;
$server->name = 'test-server';
+ $server->unreachable_count = 1; // Will become 2 after increment in real code; mock keeps value as-is
$job = new ServerConnectionCheckJob($server);
$job->failed(new TimeoutExceededException);
@@ -152,6 +159,50 @@ describe('ServerConnectionCheckJob unreachable_count', function () {
});
});
+describe('ServerConnectionCheckJob ServerReachabilityChanged dispatch', function () {
+ // ServerReachabilityChanged's constructor calls $server->isReachableChanged() — verifying that
+ // call is a clean proxy for "the event was dispatched", and avoids serializing a Mockery proxy
+ // through the event dispatcher (which trips Eloquent static method lookups on the proxy class).
+ $invoke = function (bool $wasReachable, bool $wasNotified, bool $isReachable, int $unreachableCount, bool $expectDispatch) {
+ $server = Mockery::mock(Server::class)->makePartial()->shouldAllowMockingProtectedMethods();
+ $server->shouldReceive('getAttribute')->with('unreachable_count')->andReturn($unreachableCount);
+ $server->shouldReceive('getAttribute')->with('id')->andReturn(1);
+ if ($expectDispatch) {
+ $server->shouldReceive('isReachableChanged')->once()->andReturnNull();
+ } else {
+ $server->shouldNotReceive('isReachableChanged');
+ }
+
+ $job = new ServerConnectionCheckJob($server);
+ $method = new ReflectionMethod($job, 'dispatchReachabilityChangedIfNeeded');
+ $method->invoke($job, $wasReachable, $wasNotified, $isReachable);
+ };
+
+ it('dispatches event when count crosses unreachable threshold', function () use ($invoke) {
+ $invoke(true, false, false, 2, true);
+ });
+
+ it('does not dispatch on first transient failure (count=1)', function () use ($invoke) {
+ $invoke(true, false, false, 1, false);
+ });
+
+ it('does not dispatch when already notified and still unreachable', function () use ($invoke) {
+ $invoke(false, true, false, 5, false);
+ });
+
+ it('dispatches recovery event when previously unreachable', function () use ($invoke) {
+ $invoke(false, false, true, 0, true);
+ });
+
+ it('dispatches recovery event when previously notified', function () use ($invoke) {
+ $invoke(true, true, true, 0, true);
+ });
+
+ it('does not dispatch when consistently reachable and never notified', function () use ($invoke) {
+ $invoke(true, false, true, 0, false);
+ });
+});
+
describe('ServerCheckJob unreachable_count', function () {
it('increments unreachable_count on timeout', function () {
$server = Mockery::mock(Server::class)->makePartial()->shouldAllowMockingProtectedMethods();