Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(infra): keep no bundled container for a datastore not in use
Yudhi Armyndharis2456031faee5
diff --git a/docs/05-database-design.md b/docs/05-database-design.mdindex d2fa1d5aa..d3c69f712 100644--- a/docs/05-database-design.md+++ b/docs/05-database-design.md@@ -89,7 +89,7 @@ others.#### Built-in PostgreSQL Orchestration-When using PostgreSQL Built-in mode (`DATABASE_TYPE=postgres` with `POSTGRES_BUILTIN=true`), `main.ts` starts the managed `postgres` container (creating it if it does not exist) **before** `NestFactory.create`. It has to run that early: the `data` connection is opened while Nest instantiates providers, before any `onModuleInit` hook, so a stopped `openwa-postgres` would otherwise fail every boot. The step is bounded at 15 seconds and never fails boot on its own. `DockerService.onModuleInit()` later runs the same orchestration for `redis` / `minio` when their own `REDIS_BUILTIN` / `MINIO_BUILTIN` flags are set.+When using PostgreSQL Built-in mode (`DATABASE_TYPE=postgres` with `POSTGRES_BUILTIN=true`), `main.ts` starts the managed `postgres` container (creating it if it does not exist) **before** `NestFactory.create`. It has to run that early: the `data` connection is opened while Nest instantiates providers, before any `onModuleInit` hook, so a stopped `openwa-postgres` would otherwise fail every boot. The step is bounded at 15 seconds and never fails boot on its own. `DockerService.onModuleInit()` later runs the same orchestration for `redis` / `minio` when their own `REDIS_BUILTIN` / `MINIO_BUILTIN` flags are set. A flag set for a datastore the configuration does not use starts nothing: `postgres` needs `DATABASE_TYPE=postgres`, `minio` needs `STORAGE_TYPE=s3`, and `redis` needs one of `REDIS_ENABLED`, `QUEUE_ENABLED` or `CACHE_ENABLED`.The container can still be coming up when the `data` connection first dials it, so that connection is configured with `retryAttempts: 10` and `retryDelay: 3000` (`src/app.module.ts`), giving the database roughly 30 seconds to become reachable.diff --git a/src/modules/docker/docker.service.spec.ts b/src/modules/docker/docker.service.spec.tsindex 0cf411785..661763001 100644--- a/src/modules/docker/docker.service.spec.ts+++ b/src/modules/docker/docker.service.spec.ts@@ -185,7 +185,17 @@ describe('DockerService.onModuleInit', () => {// The probe runs against whatever buildDockerOptions() points at, but the dockerode// constructor is the auto-mock — the "daemon" is whatever the test's fake says it is,// so no real socket is ever touched.- const ENV_KEYS = ['DOCKER_HOST', 'REDIS_BUILTIN', 'POSTGRES_BUILTIN', 'MINIO_BUILTIN'];+ const ENV_KEYS = [+ 'DOCKER_HOST',+ 'REDIS_BUILTIN',+ 'POSTGRES_BUILTIN',+ 'MINIO_BUILTIN',+ 'DATABASE_TYPE',+ 'STORAGE_TYPE',+ 'REDIS_ENABLED',+ 'QUEUE_ENABLED',+ 'CACHE_ENABLED',+ ];let savedEnv: Record<string, string | undefined> = {};const happyDocker = () => ({@@ -226,6 +236,7 @@ describe('DockerService.onModuleInit', () => {it('bootstraps the env-configured built-in profiles when the daemon is reachable', async () => {process.env.REDIS_BUILTIN = 'true';+ process.env.REDIS_ENABLED = 'true';const docker = happyDocker();DockerMock.mockImplementation(() => docker);const service = new DockerService();@@ -238,6 +249,7 @@ describe('DockerService.onModuleInit', () => {it('logs a warning but still resolves when bootstrap orchestration fails', async () => {process.env.REDIS_BUILTIN = 'true';+ process.env.REDIS_ENABLED = 'true';const docker = {...happyDocker(),pull: (_image: string, cb: (err: Error | null, stream: null) => void) => cb(new Error('pull denied'), null),@@ -253,6 +265,36 @@ describe('DockerService.onModuleInit', () => {expect(warn).toHaveBeenCalledWith(expect.stringContaining('[Bootstrap Orchestration] Issues'));expect(docker.createContainer).not.toHaveBeenCalled();});++ // A built-in flag starts its container only for a datastore this boot uses: a flag left beside a SQLite database,+ // local storage or a Redis nothing reaches would otherwise run a container on every boot that nothing dials.+ // Redis is reached by the queue and the cache as well as by REDIS_ENABLED.+ describe.each([+ ['openwa-postgres', 'POSTGRES_BUILTIN', [{ DATABASE_TYPE: 'postgres' }], [{}, { DATABASE_TYPE: 'sqlite' }]],+ [+ 'openwa-redis',+ 'REDIS_BUILTIN',+ [{ REDIS_ENABLED: 'true' }, { QUEUE_ENABLED: 'true' }, { CACHE_ENABLED: 'true' }],+ [{}, { REDIS_ENABLED: 'false', QUEUE_ENABLED: 'false', CACHE_ENABLED: 'false' }],+ ],+ ['openwa-minio', 'MINIO_BUILTIN', [{ STORAGE_TYPE: 's3' }], [{}, { STORAGE_TYPE: 'local' }]],+ ] as const)('%s', (container, flag, used, unused) => {+ const started = async (env: Record<string, string>): Promise<boolean> => {+ Object.assign(process.env, { [flag]: 'true' }, env);+ const docker = happyDocker();+ DockerMock.mockImplementation(() => docker);+ await new DockerService().onModuleInit();+ return docker.createContainer.mock.calls.some(([spec]) => (spec as { name: string }).name === container);+ };++ it.each(used.map(env => [JSON.stringify(env), env]))('is started when %s', async (_label, env) => {+ expect(await started(env)).toBe(true);+ });++ it.each(unused.map(env => [JSON.stringify(env), env]))('is not started when %s', async (_label, env) => {+ expect(await started(env)).toBe(false);+ });+ });});describe('prestartBuiltinDatabase', () => {diff --git a/src/modules/docker/docker.service.ts b/src/modules/docker/docker.service.tsindex 0dc3167c2..1dc9d8d56 100644--- a/src/modules/docker/docker.service.ts+++ b/src/modules/docker/docker.service.ts@@ -10,6 +10,16 @@ import { readGeneratedEnv } from '../infra/generated-env';*/export const MANAGED_DOCKER_PROFILES: readonly string[] = ['postgres', 'redis', 'minio'];+/**+ * Whether the configuration `read` describes uses the datastore a managed profile runs. Redis is reached by the+ * queue and the cache as well as by REDIS_ENABLED, each of which connects on its own.+ */+export function datastoreInUse(profile: string, read: (key: string) => string | undefined): boolean {+ if (profile === 'postgres') return read('DATABASE_TYPE') === 'postgres';+ if (profile === 'minio') return read('STORAGE_TYPE') === 's3';+ return ['REDIS_ENABLED', 'QUEUE_ENABLED', 'CACHE_ENABLED'].some(key => read(key) === 'true');+}+interface ContainerInfo {id: string;name: string;@@ -84,8 +94,8 @@ export class DockerService implements OnModuleInit {* Bootstrap orchestration: start built-in containers based on saved config.* Runs from onModuleInit, which Nest calls only after the data connection is up. When that* connection is the built-in database (DATABASE_TYPE=postgres), the postgres profile is a no-op- * here: a stopped container was already started by prestartBuiltinDatabase in main.ts. With- * POSTGRES_BUILTIN=true on any other DATABASE_TYPE, this is what starts or creates it.+ * here: a stopped container was already started by prestartBuiltinDatabase in main.ts. A flag set+ * for a datastore this boot does not use (see datastoreInUse) starts nothing.*/private async bootstrapOrchestration(): Promise<void> {if (!this.isAvailable) {@@ -94,15 +104,16 @@ export class DockerService implements OnModuleInit {}const profiles: string[] = [];+ const env = (key: string): string | undefined => process.env[key];// Check for built-in services from environment variables- if (process.env.REDIS_BUILTIN === 'true') {+ if (process.env.REDIS_BUILTIN === 'true' && datastoreInUse('redis', env)) {profiles.push('redis');}- if (process.env.POSTGRES_BUILTIN === 'true') {+ if (process.env.POSTGRES_BUILTIN === 'true' && datastoreInUse('postgres', env)) {profiles.push('postgres');}- if (process.env.MINIO_BUILTIN === 'true') {+ if (process.env.MINIO_BUILTIN === 'true' && datastoreInUse('minio', env)) {profiles.push('minio');}diff --git a/src/modules/infra/config-sections.ts b/src/modules/infra/config-sections.tsindex f28e3327d..70a902fb9 100644--- a/src/modules/infra/config-sections.ts+++ b/src/modules/infra/config-sections.ts@@ -1,7 +1,9 @@import { BadRequestException } from '@nestjs/common';import { DEFAULT_PUPPETEER_ARGS } from '../../config/configuration';+import { isEnvPinned } from '../../config/env-precedence';import { postgresSchemaError } from '../../config/env.validation';import type { EngineFactory } from '../../engine/engine.factory';+import { datastoreInUse } from '../docker';import { DatabaseConfigDto, EngineConfigDto, RedisConfigDto, StorageConfigDto } from './dto/save-config.dto';// Mutable accumulator threaded through the pipeline stages and per-section appliers extracted@@ -149,10 +151,16 @@ export function applyRedisSection(}}setSecret(updates, 'REDIS_PASSWORD', redis.password);- const redisEnabled = redis.enabled ?? existing.REDIS_ENABLED === 'true';const redisBuiltIn = redis.builtIn ?? existing.REDIS_BUILTIN === 'true';- if (redisEnabled && redisBuiltIn) {- profiles.push('redis');+ // What the next boot reads: a value the environment pins, else this save's, else the stored one. The queue+ // section is applied first, so its switch is in `updates` already.+ if (datastoreInUse('redis', key => (isEnvPinned(key) ? process.env[key] : (updates[key] ?? existing[key])))) {+ if (redisBuiltIn) profiles.push('redis');+ } else {+ // Nothing the next boot runs reaches Redis, so no bundled container may be started for it at boot or kept by+ // the restart, which keeps a container while the saved host still names it.+ updates.REDIS_BUILTIN = 'false';+ if ((updates.REDIS_HOST ?? existing.REDIS_HOST) === 'redis') staleKeys.add('REDIS_HOST');}}diff --git a/src/modules/infra/infra-config.controller.spec.ts b/src/modules/infra/infra-config.controller.spec.tsindex 22ea41384..a7fd216c5 100644--- a/src/modules/infra/infra-config.controller.spec.ts+++ b/src/modules/infra/infra-config.controller.spec.ts@@ -1005,6 +1005,72 @@ describe('InfraConfigController.saveConfig built-in/external mode flips and theexpect(env).not.toContain('REDIS_PASSWORD=');});+ // The page sends the flag it shows, so switching Redis off still carries builtIn true. Nothing the next boot runs+ // dials a Redis that is off, so no bundled container may be started or kept for it.+ describe('a Redis nothing uses keeps no bundled container', () => {+ const BUILTIN_REDIS_ENV =+ 'REDIS_ENABLED=true\nREDIS_BUILTIN=true\nREDIS_HOST=redis\nREDIS_PORT=6379\nQUEUE_ENABLED=true\n';+ const switchedOff = {+ redis: { enabled: false, builtIn: true, host: 'redis', port: '6379' },+ queue: { enabled: false },+ };++ afterEach(() => {+ delete process.env.REDIS_ENABLED;+ recordPinnedEnvKeys({});+ });++ it('switching Redis off clears the built-in flag and the bundled host', () => {+ const env = written(switchedOff, BUILTIN_REDIS_ENV);++ expect(env).toContain('REDIS_ENABLED=false');+ expect(env).toContain('REDIS_BUILTIN=false');+ expect(env).not.toMatch(/^REDIS_HOST=/m);+ });++ it('keeps an external host when Redis is switched off', () => {+ const env = written(+ { redis: { enabled: false, builtIn: false, host: 'cache.example.com' }, queue: { enabled: false } },+ 'REDIS_ENABLED=true\nREDIS_BUILTIN=false\nREDIS_HOST=cache.example.com\n',+ );++ expect(env).toContain('REDIS_BUILTIN=false');+ expect(env).toContain('REDIS_HOST=cache.example.com');+ });++ it('keeps the bundled Redis while the queue still uses it', () => {+ const env = written({ redis: { enabled: false, builtIn: true }, queue: { enabled: true } }, BUILTIN_REDIS_ENV);++ expect(env).toContain('REDIS_BUILTIN=true');+ expect(env).toContain('REDIS_HOST=redis');+ });++ it('keeps the bundled Redis while the environment pins REDIS_ENABLED on', () => {+ process.env.REDIS_ENABLED = 'true';+ recordPinnedEnvKeys({ REDIS_ENABLED: 'true' });++ const env = written(switchedOff, BUILTIN_REDIS_ENV);++ expect(env).toContain('REDIS_BUILTIN=true');+ expect(env).toContain('REDIS_HOST=redis');+ });++ it('asks for the redis profile only while something uses Redis', () => {+ const profilesOf = (config: unknown) => {+ (fs.existsSync as jest.Mock).mockReturnValue(true);+ (fs.readFileSync as jest.Mock).mockReturnValue(BUILTIN_REDIS_ENV);+ const { profiles } = newController().saveConfig(config as never);+ (fs.existsSync as jest.Mock).mockReturnValue(false);+ (fs.readFileSync as jest.Mock).mockReturnValue('');+ return profiles;+ };++ expect(profilesOf(switchedOff)).toEqual([]);+ expect(profilesOf({ redis: { enabled: false, builtIn: true }, queue: { enabled: true } })).toEqual(['redis']);+ expect(profilesOf({ redis: { enabled: true, builtIn: true } })).toEqual(['redis']);+ });+ });+it('rejects a placeholder Redis password at save time', () => {expectRejected({ redis: { enabled: true, builtIn: false, host: 'redis.example.com', password: 'password' } },@@ -1681,6 +1747,41 @@ describe('InfraConfigController.requestRestart constrains teardown to managed prexpect(stopped).toBe(false);});++ // The page's own save for switching a bundled service off, then Restart Now with the container it runs: the+ // file the save wrote is what the restart reads, and nothing in it may keep the container for a service that is+ // no longer in use.+ describe.each([+ ['database', { database: { type: 'sqlite', builtIn: true, host: 'postgres' } }],+ ['redis', { redis: { enabled: false, builtIn: true, host: 'redis', port: '6379' }, queue: { enabled: false } }],+ ['storage', { storage: { type: 'local', builtIn: true, s3Endpoint: 'http://minio:9000' } }],+ ] as const)('%s switched off from the page', (service, payload) => {+ it('is stopped by the restart that follows the save', async () => {+ const saved = [+ 'DATABASE_TYPE=postgres\nPOSTGRES_BUILTIN=true\nDATABASE_HOST=postgres\nDATABASE_PASSWORD=openwa',+ 'REDIS_ENABLED=true\nREDIS_BUILTIN=true\nREDIS_HOST=redis\nREDIS_PORT=6379\nQUEUE_ENABLED=true',+ 'STORAGE_TYPE=s3\nMINIO_BUILTIN=true\nS3_ENDPOINT=http://minio:9000\nS3_ACCESS_KEY_ID=minioadmin',+ 'S3_SECRET_ACCESS_KEY=minioadmin\n',+ ].join('\n');+ (fs.existsSync as jest.Mock).mockReturnValue(true);+ (fs.readFileSync as jest.Mock).mockReturnValue(saved);+ (fs.writeFileSync as jest.Mock).mockClear();+ recordPinnedEnvKeys({});+ const stopManagedService = jest.fn().mockResolvedValue(true);+ const controller = buildController({+ isDockerAvailable: () => true,+ stopManagedService,+ orchestrateProfiles: jest.fn().mockResolvedValue({}),+ });++ const { profiles } = controller.saveConfig(payload);+ const written = ((fs.writeFileSync as jest.Mock).mock.calls as Array<[string, string]>)[0][1];+ (fs.readFileSync as jest.Mock).mockReturnValue(written);+ await controller.requestRestart({ profiles, profilesToRemove: [PROFILE_OF[service]] });++ expect(stopManagedService).toHaveBeenCalledWith(PROFILE_OF[service]);+ });+ });});it('starts only allowlisted profiles, never an unknown entry (symmetry with teardown)', async () => {diff --git a/src/modules/infra/infra-config.controller.ts b/src/modules/infra/infra-config.controller.tsindex ef3fe077f..98de01cb7 100644--- a/src/modules/infra/infra-config.controller.ts+++ b/src/modules/infra/infra-config.controller.ts@@ -247,13 +247,14 @@ export class InfraConfigController {}// Redis and queue are independent sections: a payload carrying only one of them must- // not rewrite (or disable) the other's saved keys.- if (config.redis) {- applyRedisSection(config.redis, existing, ctx);- }+ // not rewrite (or disable) the other's saved keys. The queue goes first because it reaches Redis+ // too, and the Redis section reads its new switch.if (config.queue) {if (config.queue.enabled !== undefined) updates.QUEUE_ENABLED = config.queue.enabled ? 'true' : 'false';}+ if (config.redis) {+ applyRedisSection(config.redis, existing, ctx);+ }if (config.storage) {applyStorageSection(config.storage, existing, ctx);