An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(cleanup): prune audit events in batches and delete old finished runner executions
Andras Bacsai5e92a7267cba
diff --git a/app/Console/Commands/CleanupDatabase.php b/app/Console/Commands/CleanupDatabase.phpindex eac3a33ebf..eb65007194 100644--- a/app/Console/Commands/CleanupDatabase.php+++ b/app/Console/Commands/CleanupDatabase.php@@ -2,8 +2,11 @@namespace App\Console\Commands;+use App\Enums\GithubRunnerStatus;use App\Models\AuditEvent;+use Closure;use Illuminate\Console\Command;+use Illuminate\Database\Query\Builder;use Illuminate\Support\Facades\DB;class CleanupDatabase extends Command@@ -43,22 +46,29 @@ class CleanupDatabase extends Command$this->cleanupTable('application_deployment_queues', 'created_at', now()->subDays($keep_days));$this->cleanupTable('scheduled_task_executions', 'created_at', now()->subDays($keep_days));++ $this->cleanupTable('github_runner_executions', 'created_at', now()->subDays($keep_days), fn (Builder $query) => $query+ ->whereNotIn('status', array_map(fn (GithubRunnerStatus $status): string => $status->value, GithubRunnerStatus::active())));}/*** Delete the rows older than the given value in batches, so a large table is not locked* by one long DELETE.+ *+ * @param (Closure(Builder): mixed)|null $constraint Limits the rows that may be deleted.*/- private function cleanupTable(string $table, string $column, mixed $olderThan): void+ private function cleanupTable(string $table, string $column, mixed $olderThan, ?Closure $constraint = null): void{- $count = DB::table($table)->where($column, '<', $olderThan)->count();+ $query = fn (): Builder => DB::table($table)->where($column, '<', $olderThan)->when($constraint, $constraint);++ $count = $query()->count();$this->line("Delete $count entries from $table.");if (! $this->option('yes')) {return;}do {- $deleted = DB::table($table)->where($column, '<', $olderThan)->limit(self::DELETE_BATCH_SIZE)->delete();+ $deleted = $query()->limit(self::DELETE_BATCH_SIZE)->delete();} while ($deleted > 0);}}diff --git a/app/Models/AuditEvent.php b/app/Models/AuditEvent.phpindex b966e55706..2742a3f667 100644--- a/app/Models/AuditEvent.php+++ b/app/Models/AuditEvent.php@@ -41,6 +41,8 @@ class AuditEvent extends Model'configuration_snapshot', 'configuration_diff', 'content', 'file_storage_content',];+ private const PRUNE_BATCH_SIZE = 1000;+protected $fillable = ['team_id','event',@@ -228,11 +230,23 @@ class AuditEvent extends Model->first()?->team()?->id;}+ /**+ * Deletes in batches, so a large table is not locked by one long DELETE.+ */public static function pruneExpired(): int{- return self::query()- ->where('created_at', '<', now()->subDays(90))- ->delete();+ $olderThan = now()->subDays(90);+ $total = 0;++ do {+ $deleted = self::query()+ ->where('created_at', '<', $olderThan)+ ->limit(self::PRUNE_BATCH_SIZE)+ ->delete();+ $total += $deleted;+ } while ($deleted > 0);++ return $total;}/**diff --git a/tests/Feature/CleanupDatabaseRunnerExecutionsTest.php b/tests/Feature/CleanupDatabaseRunnerExecutionsTest.phpnew file mode 100644index 0000000000..2db3949e50--- /dev/null+++ b/tests/Feature/CleanupDatabaseRunnerExecutionsTest.php@@ -0,0 +1,96 @@+<?php++use App\Enums\GithubRunnerStatus;+use App\Models\AuditEvent;+use App\Models\GithubApp;+use App\Models\GithubRunnerExecution;+use App\Models\Team;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Facades\DB;++uses(RefreshDatabase::class);++function cleanupRunnerTestExecution(GithubApp $githubApp, GithubRunnerStatus $status, int $ageInDays): GithubRunnerExecution+{+ static $jobId = 9000;++ $execution = GithubRunnerExecution::create([+ 'github_app_id' => $githubApp->id,+ 'trigger_workflow_job_id' => ++$jobId,+ 'status' => $status,+ ]);+ GithubRunnerExecution::query()->whereKey($execution->id)->toBase()->update(['created_at' => now()->subDays($ageInDays)]);++ return $execution;+}++beforeEach(function () {+ $team = Team::factory()->create();+ $this->githubApp = GithubApp::create([+ 'name' => 'cleanup-runner-app',+ 'api_url' => 'https://api.github.com',+ 'html_url' => 'https://github.com',+ 'custom_user' => 'git',+ 'custom_port' => 22,+ 'team_id' => $team->id,+ 'is_system_wide' => false,+ ]);+});++it('deletes old finished runner executions and keeps recent and unfinished ones', function () {+ $oldFinished = collect([+ GithubRunnerStatus::Completed,+ GithubRunnerStatus::Failed,+ GithubRunnerStatus::Cancelled,+ GithubRunnerStatus::TimedOut,+ ])->map(fn (GithubRunnerStatus $status) => cleanupRunnerTestExecution($this->githubApp, $status, 61));+ $recentFinished = cleanupRunnerTestExecution($this->githubApp, GithubRunnerStatus::Completed, 5);+ $oldUnfinished = collect(GithubRunnerStatus::active())+ ->map(fn (GithubRunnerStatus $status) => cleanupRunnerTestExecution($this->githubApp, $status, 61));++ $this->artisan('cleanup:database --yes')+ ->expectsOutputToContain('Delete 4 entries from github_runner_executions.')+ ->assertSuccessful();++ expect(GithubRunnerExecution::query()->pluck('id')->sort()->values()->all())+ ->toBe($oldUnfinished->push($recentFinished)->pluck('id')->sort()->values()->all())+ ->and($oldFinished)->toHaveCount(4);+});++it('keeps old finished runner executions in a dry run', function () {+ cleanupRunnerTestExecution($this->githubApp, GithubRunnerStatus::Completed, 61);++ $this->artisan('cleanup:database')+ ->expectsOutputToContain('Delete 1 entries from github_runner_executions.')+ ->assertSuccessful();++ expect(GithubRunnerExecution::query()->count())->toBe(1);+});++it('prunes expired audit events in batches', function () {+ $expired = now()->subDays(91);+ $rows = collect(range(1, 2500))->map(fn (int $i) => [+ 'event' => 'ui.project.updated',+ 'source' => 'ui',+ 'action' => 'updated',+ 'level' => 'info',+ 'actor_type' => 'user',+ 'description' => 'Project updated',+ 'created_at' => $expired,+ ]);+ $rows->chunk(500)->each(fn ($chunk) => DB::table('audit_events')->insert($chunk->values()->all()));+ $recent = AuditEvent::factory()->create(['created_at' => now()->subDays(89)]);++ $deleteQueries = 0;+ DB::listen(function ($query) use (&$deleteQueries): void {+ if (str_starts_with(strtolower($query->sql), 'delete from "audit_events"')) {+ $deleteQueries++;+ }+ });++ $deleted = AuditEvent::pruneExpired();++ expect($deleted)->toBe(2500)+ ->and(AuditEvent::query()->pluck('id')->all())->toBe([$recent->id])+ ->and($deleteQueries)->toBeGreaterThan(1);+});