Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(deployments): include scheduled jobs in progress checks

Andras Bacsai56497d50cf5e
diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php
index 81f2dbe726..90063c7c28 100644
--- a/bootstrap/helpers/shared.php
+++ b/bootstrap/helpers/shared.php
@@ -2,6 +2,8 @@
use App\Enums\ApplicationDeploymentStatus;
use App\Enums\ProxyTypes;
+use App\Jobs\DatabaseBackupJob;
+use App\Jobs\ScheduledTaskJob;
use App\Jobs\ServerFilesFromServerJob;
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
@@ -14,6 +16,10 @@ use App\Models\LocalFileVolume;
use App\Models\LocalPersistentVolume;
use App\Models\Project;
use App\Models\S3Storage;
+use App\Models\ScheduledDatabaseBackupExecution;
+use App\Models\ScheduledTaskExecution;
+use App\Models\ScheduledVolumeBackup;
+use App\Models\ScheduledVolumeBackupExecution;
use App\Models\Server;
use App\Models\Service;
use App\Models\ServiceApplication;
@@ -35,6 +41,7 @@ use App\Models\User;
use App\Support\DnsRecordHints;
use Carbon\CarbonImmutable;
use DanHarrin\LivewireRateLimiting\Exceptions\TooManyRequestsException;
+use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\ModelNotFoundException;
use Illuminate\Database\UniqueConstraintViolationException;
use Illuminate\Process\Pool;
@@ -2614,11 +2621,6 @@ function isAnyDeploymentInprogress(bool $showAll = false)
{
$runningJobs = ApplicationDeploymentQueue::where('horizon_job_worker', gethostname())->where('status', ApplicationDeploymentStatus::IN_PROGRESS->value)->get();
- if ($runningJobs->isEmpty()) {
- echo "No deployments in progress.\n";
- exit(0);
- }
-
$horizonJobIds = [];
$deploymentDetails = [];
@@ -2654,20 +2656,43 @@ function isAnyDeploymentInprogress(bool $showAll = false)
}
}
- if (count($horizonJobIds) === 0) {
- echo "No active deployments in progress (all jobs completed or failed).\n";
+ $scheduledJobCount = runningScheduledJobCount();
+
+ if (count($horizonJobIds) === 0 && $scheduledJobCount === 0) {
+ echo "No active deployments or scheduled jobs in progress.\n";
exit(0);
}
- echo formatRunningDeploymentsOutput(count($horizonJobIds), $deploymentDetails, $showAll);
+ echo formatRunningDeploymentsOutput(count($horizonJobIds), $deploymentDetails, $showAll, $scheduledJobCount);
exit(1);
}
-function formatRunningDeploymentsOutput(int $activeDeploymentCount, array $deploymentDetails = [], bool $showAll = false): string
+/**
+ * Counts scheduled task, database backup, and volume backup runs that are still running.
+ * A run that started longer ago than its job timeout is stale: the worker has stopped it.
+ */
+function runningScheduledJobCount(): int
+{
+ $startedWithin = fn (Model $execution, int $timeoutSeconds): bool => $execution->created_at?->gt(now()->subSeconds($timeoutSeconds)) ?? false;
+
+ $tasks = ScheduledTaskExecution::with('scheduledTask')->where('status', 'running')->get()
+ ->filter(fn (ScheduledTaskExecution $execution) => $startedWithin($execution, ($execution->scheduledTask?->timeout ?? 300) + ScheduledTaskJob::WORKER_TIMEOUT_MARGIN_SECONDS));
+
+ $databaseBackups = ScheduledDatabaseBackupExecution::with('scheduledDatabaseBackup')->where('status', 'running')->get()
+ ->filter(fn (ScheduledDatabaseBackupExecution $execution) => $startedWithin($execution, ($execution->scheduledDatabaseBackup?->timeout ?? 3600) + DatabaseBackupJob::WORKER_TIMEOUT_MARGIN_SECONDS));
+
+ $volumeBackups = ScheduledVolumeBackupExecution::with('scheduledVolumeBackup')->where('status', 'running')->get()
+ ->filter(fn (ScheduledVolumeBackupExecution $execution) => $startedWithin($execution, $execution->scheduledVolumeBackup?->timeout ?? ScheduledVolumeBackup::DEFAULT_TIMEOUT));
+
+ return $tasks->count() + $databaseBackups->count() + $volumeBackups->count();
+}
+
+function formatRunningDeploymentsOutput(int $activeDeploymentCount, array $deploymentDetails = [], bool $showAll = false, int $scheduledJobCount = 0): string
{
$output = "\n=== Running Deployments ===\n";
$output .= 'Total active deployments: '.$activeDeploymentCount."\n";
+ $output .= 'Total running scheduled jobs: '.$scheduledJobCount."\n";
if (! $showAll) {
return $output;
diff --git a/tests/Feature/RunningScheduledJobCountTest.php b/tests/Feature/RunningScheduledJobCountTest.php
new file mode 100644
index 0000000000..37edc5d6bc
--- /dev/null
+++ b/tests/Feature/RunningScheduledJobCountTest.php
@@ -0,0 +1,66 @@
+<?php
+
+use App\Models\ScheduledDatabaseBackup;
+use App\Models\ScheduledDatabaseBackupExecution;
+use App\Models\ScheduledTask;
+use App\Models\ScheduledTaskExecution;
+use App\Models\ScheduledVolumeBackup;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ $team = Team::factory()->create();
+
+ $this->task = ScheduledTask::factory()->create(['team_id' => $team->id, 'timeout' => 300]);
+ $this->databaseBackup = ScheduledDatabaseBackup::create([
+ 'frequency' => '0 0 * * *',
+ 'save_s3' => false,
+ 'database_type' => 'App\Models\StandalonePostgresql',
+ 'database_id' => 1,
+ 'team_id' => $team->id,
+ 'timeout' => 3600,
+ ]);
+ $this->volumeBackup = ScheduledVolumeBackup::create([
+ 'uuid' => fake()->uuid(),
+ 'backupable_type' => 'App\Models\LocalPersistentVolume',
+ 'backupable_id' => 1,
+ 'team_id' => $team->id,
+ 'frequency' => 'daily',
+ 'enabled' => true,
+ 'save_s3' => false,
+ 'timeout' => 7200,
+ ]);
+});
+
+it('counts running scheduled tasks, database backups, and volume backups', function () {
+ ScheduledTaskExecution::create(['scheduled_task_id' => $this->task->id, 'status' => 'running']);
+ ScheduledDatabaseBackupExecution::create(['scheduled_database_backup_id' => $this->databaseBackup->id, 'database_name' => 'db', 'status' => 'running']);
+ $this->volumeBackup->executions()->create();
+
+ expect(runningScheduledJobCount())->toBe(3);
+});
+
+it('does not count finished runs', function () {
+ ScheduledTaskExecution::create(['scheduled_task_id' => $this->task->id, 'status' => 'success']);
+ ScheduledDatabaseBackupExecution::create(['scheduled_database_backup_id' => $this->databaseBackup->id, 'database_name' => 'db', 'status' => 'failed']);
+ $this->volumeBackup->executions()->create(['status' => 'success']);
+
+ expect(runningScheduledJobCount())->toBe(0);
+});
+
+it('does not count running rows older than their job timeout', function () {
+ ScheduledTaskExecution::create(['scheduled_task_id' => $this->task->id, 'status' => 'running']);
+ ScheduledDatabaseBackupExecution::create(['scheduled_database_backup_id' => $this->databaseBackup->id, 'database_name' => 'db', 'status' => 'running']);
+ $this->volumeBackup->executions()->create();
+
+ $this->travel(30)->minutes();
+ expect(runningScheduledJobCount())->toBe(2);
+
+ $this->travel(40)->minutes();
+ expect(runningScheduledJobCount())->toBe(1);
+
+ $this->travel(60)->minutes();
+ expect(runningScheduledJobCount())->toBe(0);
+});
diff --git a/tests/Unit/RunningDeploymentsOutputTest.php b/tests/Unit/RunningDeploymentsOutputTest.php
index 62d367514c..308c67e108 100644
--- a/tests/Unit/RunningDeploymentsOutputTest.php
+++ b/tests/Unit/RunningDeploymentsOutputTest.php
@@ -11,7 +11,7 @@ it('shows only the active deployment count by default', function () {
]]);
expect($output)
- ->toBe("\n=== Running Deployments ===\nTotal active deployments: 1\n")
+ ->toBe("\n=== Running Deployments ===\nTotal active deployments: 1\nTotal running scheduled jobs: 0\n")
->not->toContain('secret-application')
->not->toContain('https://example.com/secret-deployment')
->not->toContain('[email protected]');