An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
perf(databases): load Redis-family passwords once for database lists
Andras Bacsai86d84e21c0b2
diff --git a/app/Http/Controllers/Api/DatabasesController.php b/app/Http/Controllers/Api/DatabasesController.phpindex 06870f160b..10a213e5f3 100644--- a/app/Http/Controllers/Api/DatabasesController.php+++ b/app/Http/Controllers/Api/DatabasesController.php@@ -20,11 +20,15 @@ use App\Models\S3Storage;use App\Models\ScheduledDatabaseBackup;use App\Models\Server;use App\Models\StandaloneDocker;+use App\Models\StandaloneDragonfly;+use App\Models\StandaloneKeydb;use App\Models\StandalonePostgresql;+use App\Models\StandaloneRedis;use App\Models\StandaloneSqlite;use App\Models\SwarmDocker;use App\Support\ResourceStartActivity;use App\Support\ValidationPatterns;+use Illuminate\Database\Eloquent\Collection as EloquentCollection;use Illuminate\Database\Eloquent\Model;use Illuminate\Http\JsonResponse;use Illuminate\Http\Request;@@ -275,6 +279,14 @@ class DatabasesController extends Controller$databases = $databases->merge($project->databases($databaseRelations));}+ if ($request->attributes->get('can_read_sensitive', false) === true) {+ // Their connection URLs read REDIS_PASSWORD; load the variables once per type instead of per database.+ $databases+ ->filter(fn ($database) => $database instanceof StandaloneRedis || $database instanceof StandaloneKeydb || $database instanceof StandaloneDragonfly)+ ->groupBy(fn ($database) => $database::class)+ ->each(fn ($group) => (new EloquentCollection($group->all()))->load('runtime_environment_variables'));+ }+$databaseIds = $databases->pluck('id')->toArray();$backupConfigs = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)->with('latest_log')@@ -284,6 +296,7 @@ class DatabasesController extends Controller$databases = $databases->map(function ($database) use ($backupConfigs) {$database->backup_configs = $backupConfigs->get($database->getMorphClass().':'.$database->id, collect())->values();+ $database->makeHidden('runtime_environment_variables');return $this->removeSensitiveData($database);});diff --git a/app/Http/Controllers/Api/ProjectController.php b/app/Http/Controllers/Api/ProjectController.phpindex 16eff1ba18..ad515f3fc7 100644--- a/app/Http/Controllers/Api/ProjectController.php+++ b/app/Http/Controllers/Api/ProjectController.php@@ -167,10 +167,12 @@ class ProjectController extends Controllerif (! $environment) {return response()->json(['message' => 'Environment not found.'], 404);}- $environment = $environment->load(['applications', 'postgresqls', 'redis', 'mongodbs', 'mysqls', 'mariadbs', 'services']);+ // Redis connection URLs read REDIS_PASSWORD and REDIS_USERNAME; load the variables once, not per database.+ $environment = $environment->load(['applications', 'postgresqls', 'redis.runtime_environment_variables', 'mongodbs', 'mysqls', 'mariadbs', 'services']);collect(['applications', 'postgresqls', 'redis', 'mongodbs', 'mysqls', 'mariadbs', 'services'])->flatMap(fn (string $relation) => $environment->{$relation})->each(fn ($resource) => exposeSensitiveFields($resource));+ $environment->redis->each(fn ($redis) => $redis->makeHidden('runtime_environment_variables'));return response()->json(serializeApiResponse($environment));}diff --git a/app/Models/StandaloneDragonfly.php b/app/Models/StandaloneDragonfly.phpindex 15dea750f8..98f9ff5afc 100644--- a/app/Models/StandaloneDragonfly.php+++ b/app/Models/StandaloneDragonfly.php@@ -369,6 +369,19 @@ class StandaloneDragonfly extends BaseModelreturn $this->morphMany(EnvironmentVariable::class, 'resourceable');}+ /**+ * Uses the loaded runtime_environment_variables relation when present, so lists that eager+ * load it do not run a query per database.+ */+ private function runtimeEnvironmentVariable(string $key): ?EnvironmentVariable+ {+ if ($this->relationLoaded('runtime_environment_variables')) {+ return $this->runtime_environment_variables->firstWhere('key', $key);+ }++ return $this->runtime_environment_variables()->where('key', $key)->first();+ }+/*** The REDIS_PASSWORD variable that sets the server password instead of the stored password.* Databases created before this release (legacy_password_quoting) keep their v4.3.23 server@@ -376,7 +389,7 @@ class StandaloneDragonfly extends BaseModel*/public function serverPasswordEnvironmentVariable(): ?EnvironmentVariable{- $environmentVariable = $this->runtime_environment_variables()->where('key', 'REDIS_PASSWORD')->first();+ $environmentVariable = $this->runtimeEnvironmentVariable('REDIS_PASSWORD');if (! $environmentVariable) {return null;diff --git a/app/Models/StandaloneKeydb.php b/app/Models/StandaloneKeydb.phpindex 35818a8742..bc1e045335 100644--- a/app/Models/StandaloneKeydb.php+++ b/app/Models/StandaloneKeydb.php@@ -370,6 +370,19 @@ class StandaloneKeydb extends BaseModelreturn $this->morphMany(EnvironmentVariable::class, 'resourceable');}+ /**+ * Uses the loaded runtime_environment_variables relation when present, so lists that eager+ * load it do not run a query per database.+ */+ private function runtimeEnvironmentVariable(string $key): ?EnvironmentVariable+ {+ if ($this->relationLoaded('runtime_environment_variables')) {+ return $this->runtime_environment_variables->firstWhere('key', $key);+ }++ return $this->runtime_environment_variables()->where('key', $key)->first();+ }+/*** The REDIS_PASSWORD variable that sets the server password instead of the stored password.* Databases created before this release (legacy_password_quoting) keep their v4.3.23 server@@ -377,7 +390,7 @@ class StandaloneKeydb extends BaseModel*/public function serverPasswordEnvironmentVariable(): ?EnvironmentVariable{- $environmentVariable = $this->runtime_environment_variables()->where('key', 'REDIS_PASSWORD')->first();+ $environmentVariable = $this->runtimeEnvironmentVariable('REDIS_PASSWORD');if (! $environmentVariable) {return null;diff --git a/app/Models/StandaloneRedis.php b/app/Models/StandaloneRedis.phpindex 2e6f158cf1..b8bdbef49b 100644--- a/app/Models/StandaloneRedis.php+++ b/app/Models/StandaloneRedis.php@@ -384,6 +384,19 @@ class StandaloneRedis extends BaseModelreturn $this->morphMany(EnvironmentVariable::class, 'resourceable');}+ /**+ * Uses the loaded runtime_environment_variables relation when present, so lists that eager+ * load it do not run a query per database.+ */+ private function runtimeEnvironmentVariable(string $key): ?EnvironmentVariable+ {+ if ($this->relationLoaded('runtime_environment_variables')) {+ return $this->runtime_environment_variables->firstWhere('key', $key);+ }++ return $this->runtime_environment_variables()->where('key', $key)->first();+ }+/*** The REDIS_PASSWORD variable whose resolved value sets the server password. Databases created* before this release (legacy_password_quoting) keep their v4.3.23 server password, the stored@@ -391,7 +404,7 @@ class StandaloneRedis extends BaseModel*/public function serverPasswordEnvironmentVariable(): ?EnvironmentVariable{- $environmentVariable = $this->runtime_environment_variables()->where('key', 'REDIS_PASSWORD')->first();+ $environmentVariable = $this->runtimeEnvironmentVariable('REDIS_PASSWORD');if (! $environmentVariable) {return null;@@ -444,7 +457,7 @@ class StandaloneRedis extends BaseModel{return new Attribute(get: function () {- $password = $this->runtime_environment_variables()->where('key', 'REDIS_PASSWORD')->first();+ $password = $this->runtimeEnvironmentVariable('REDIS_PASSWORD');if (! $password) {return null;}@@ -459,7 +472,9 @@ class StandaloneRedis extends BaseModel{return new Attribute(get: function () {- $username = $this->runtime_environment_variables()->where('key', 'REDIS_USERNAME')->first();+ // A loaded relation does not hold a variable created by an earlier read, so check the table before creating one.+ $username = $this->runtimeEnvironmentVariable('REDIS_USERNAME')+ ?? $this->runtime_environment_variables()->where('key', 'REDIS_USERNAME')->first();if (! $username) {$this->runtime_environment_variables()->create(['key' => 'REDIS_USERNAME',diff --git a/tests/Feature/Api/RedisFamilyDatabaseListQueryCountTest.php b/tests/Feature/Api/RedisFamilyDatabaseListQueryCountTest.phpnew file mode 100644index 0000000000..bea7b1de5b--- /dev/null+++ b/tests/Feature/Api/RedisFamilyDatabaseListQueryCountTest.php@@ -0,0 +1,144 @@+<?php++use App\Models\Environment;+use App\Models\EnvironmentVariable;+use App\Models\InstanceSettings;+use App\Models\Project;+use App\Models\Server;+use App\Models\StandaloneDocker;+use App\Models\StandaloneDragonfly;+use App\Models\StandaloneKeydb;+use App\Models\StandaloneRedis;+use App\Models\Team;+use App\Models\User;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Facades\DB;++uses(RefreshDatabase::class);++beforeEach(function () {+ InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0], ['is_api_enabled' => true]));+ Server::flushIdentityMap();++ $this->team = Team::factory()->create();+ $user = User::factory()->create();+ $this->team->members()->attach($user->id, ['role' => 'owner']);+ session(['currentTeam' => $this->team]);+ $this->bearerToken = $user->createToken('test-token', ['root'])->plainTextToken;++ $server = Server::factory()->create(['team_id' => $this->team->id]);+ $this->destination = StandaloneDocker::query()->where('server_id', $server->id)->firstOrFail();+ $project = Project::factory()->create(['team_id' => $this->team->id]);+ $this->environment = Environment::factory()->create(['project_id' => $project->id]);+});++function createRedisFamilyDatabasesForQueryCount(int $environmentId, StandaloneDocker $destination, int $count): void+{+ foreach (range(1, $count) as $index) {+ $public = ['is_public' => true, 'public_port' => 30000 + $index];++ create_standalone_redis($environmentId, $destination, $public + ['image' => 'redis:7.2']);++ foreach (['create_standalone_keydb', 'create_standalone_dragonfly'] as $create) {+ $database = $create($environmentId, $destination, $public);+ EnvironmentVariable::create([+ 'key' => 'REDIS_PASSWORD',+ 'value' => "variable-password-{$index}",+ 'resourceable_type' => $database->getMorphClass(),+ 'resourceable_id' => $database->id,+ ]);+ }+ }+}++/**+ * Counts the queries of one database list request. The REDIS_USERNAME lookup of the StandaloneRedis+ * "retrieved" hook runs per model before any relation can be eager loaded, so it is not counted.+ * Pass $onlyTable to count only the queries that read that table.+ *+ * @return array{0: int, 1: array<int, array<string, mixed>>}+ */+function countRedisFamilyDatabaseListQueries(string $bearerToken, string $uri = '/api/v1/databases', ?string $onlyTable = null): array+{+ DB::flushQueryLog();+ DB::enableQueryLog();++ $response = test()->withHeaders(['Authorization' => 'Bearer '.$bearerToken])+ ->getJson($uri)+ ->assertOk();++ DB::disableQueryLog();++ $queries = collect(DB::getQueryLog())+ ->reject(fn (array $query) => str_contains($query['query'], 'audit_events') || in_array('REDIS_USERNAME', $query['bindings'], true))+ ->filter(fn (array $query) => $onlyTable === null || str_contains($query['query'], "from \"{$onlyTable}\""))+ ->count();++ return [$queries, $response->json()];+}++it('does not query environment variables per Redis, KeyDB and Dragonfly database in the database list', function () {+ createRedisFamilyDatabasesForQueryCount($this->environment->id, $this->destination, 1);+ // The first request also authenticates the token; measure warm requests only.+ countRedisFamilyDatabaseListQueries($this->bearerToken);+ [$queriesForOneEach] = countRedisFamilyDatabaseListQueries($this->bearerToken);++ createRedisFamilyDatabasesForQueryCount($this->environment->id, $this->destination, 4);+ [$queriesForFiveEach, $databases] = countRedisFamilyDatabaseListQueries($this->bearerToken);++ expect($databases)->toHaveCount(15)+ ->and($queriesForFiveEach)->toBe($queriesForOneEach);++ $modelsByUuid = collect([StandaloneRedis::class, StandaloneKeydb::class, StandaloneDragonfly::class])+ ->flatMap(fn (string $model) => $model::query()->get())+ ->keyBy('uuid');++ foreach ($databases as $database) {+ $model = $modelsByUuid[$database['uuid']];++ expect($database)->not->toHaveKey('runtime_environment_variables')+ ->and($database['internal_db_url'])->toBe($model->internal_db_url)+ ->and($database['external_db_url'])->toBe($model->external_db_url)+ ->and($database['internal_db_url'])->toStartWith('redis://')+ ->and($database['external_db_url'])->toStartWith('redis://');++ if (str_starts_with($database['name'], 'redis-database-')) {+ expect($database['internal_db_url'])->toStartWith('redis://default:'.rawurlencode($model->redis_password).'@');+ } else {+ expect($database['internal_db_url'])->toMatch('/^redis:\/\/:variable-password-\d@/');+ }+ }+});++it('does not query environment variables per Redis database in the environment details', function () {+ $uri = "/api/v1/projects/{$this->environment->project->uuid}/{$this->environment->uuid}";++ createRedisFamilyDatabasesForQueryCount($this->environment->id, $this->destination, 1);+ countRedisFamilyDatabaseListQueries($this->bearerToken, $uri);+ // The environment details lazy load each database's destination for external URLs; only variable reads are checked here.+ [$queriesForOne] = countRedisFamilyDatabaseListQueries($this->bearerToken, $uri, 'environment_variables');++ createRedisFamilyDatabasesForQueryCount($this->environment->id, $this->destination, 4);+ [$queriesForFive, $environment] = countRedisFamilyDatabaseListQueries($this->bearerToken, $uri, 'environment_variables');++ expect($environment['redis'])->toHaveCount(5)+ ->and($queriesForFive)->toBe($queriesForOne);++ foreach ($environment['redis'] as $database) {+ $model = StandaloneRedis::query()->where('uuid', $database['uuid'])->firstOrFail();++ expect($database)->not->toHaveKey('runtime_environment_variables')+ ->and($database['internal_db_url'])->toBe($model->internal_db_url)+ ->and($database['external_db_url'])->toBe($model->external_db_url);+ }+});++it('creates a missing Redis username variable only once when the variables are loaded', function () {+ $redis = create_standalone_redis($this->environment->id, $this->destination);+ $redis->runtime_environment_variables()->where('key', 'REDIS_USERNAME')->delete();+ $redis->load('runtime_environment_variables');++ expect($redis->redis_username)->toBe('default')+ ->and($redis->redis_username)->toBe('default')+ ->and($redis->runtime_environment_variables()->where('key', 'REDIS_USERNAME')->count())->toBe(1);+});