Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(docker): recover after the daemon was unreachable at boot or lost
Yudhi Armyndharis28131a79dc21
diff --git a/src/modules/docker/docker.service.spec.ts b/src/modules/docker/docker.service.spec.tsindex 51cc858b1..171c52a55 100644--- a/src/modules/docker/docker.service.spec.ts+++ b/src/modules/docker/docker.service.spec.ts@@ -1,6 +1,6 @@import Docker from 'dockerode';import { DockerService, prestartBuiltinDatabase } from './docker.service';-import { recordPinnedEnvKeys } from '../../config/env-precedence';+import { recordOsEnvKeys, recordPinnedEnvKeys } from '../../config/env-precedence';import { HOST_SPELLINGS } from '../../../test/fixtures/builtin-mode-cases';// Prevent actual Docker connections on module init during tests@@ -49,7 +49,7 @@ describe('DockerService.getRunningBuiltinServices', () => {expect(await service.getRunningBuiltinServices()).toEqual({ database: 'compose', cache: 'openwa', storage: false });});- it('reports all false when no bundled containers are present (e.g. Docker unavailable)', async () => {+ it('reports all false when a live inventory contains no bundled containers', async () => {const service = new DockerService();jest.spyOn(service, 'listContainers').mockResolvedValue([]);expect(await service.getRunningBuiltinServices()).toEqual({ database: false, cache: false, storage: false });@@ -447,8 +447,8 @@ describe('DockerService.listContainers', () => {return service;};- it('returns an empty list when Docker is unavailable', async () => {- expect(await new DockerService().listContainers()).toEqual([]);+ it('returns no live inventory when Docker is unavailable', async () => {+ expect(await new DockerService().listContainers()).toBeNull();});it('maps only OpenWA containers (label or /openwa- name) to ContainerInfo', async () => {@@ -487,9 +487,9 @@ describe('DockerService.listContainers', () => {]);});- it('returns an empty list when the daemon listing fails', async () => {+ it('returns no live inventory when the daemon listing fails', async () => {const service = makeService(jest.fn().mockRejectedValue(new Error('daemon gone')));- expect(await service.listContainers()).toEqual([]);+ expect(await service.listContainers()).toBeNull();});});@@ -880,3 +880,126 @@ describe('DockerService.orchestrateProfiles', () => {expect(result.message).toBe('Failed to start postgres: boom');});});++describe('DockerService availability recovery', () => {+ const keys = [+ 'DOCKER_HOST',+ 'DATABASE_TYPE',+ 'STORAGE_TYPE',+ 'REDIS_ENABLED',+ 'QUEUE_ENABLED',+ 'CACHE_ENABLED',+ 'POSTGRES_BUILTIN',+ 'REDIS_BUILTIN',+ 'MINIO_BUILTIN',+ 'REDIS_HOST',+ 'DATABASE_PASSWORD',+ 'S3_ACCESS_KEY_ID',+ 'S3_SECRET_ACCESS_KEY',+ 'S3_ACCESS_KEY',+ 'S3_SECRET_KEY',+ ];+ let saved: Record<string, string | undefined>;++ beforeEach(() => {+ saved = Object.fromEntries(keys.map(key => [key, process.env[key]]));+ for (const key of keys) delete process.env[key];+ Object.assign(process.env, {+ DOCKER_HOST: 'tcp://proxy.invalid:2375',+ DATABASE_TYPE: 'sqlite',+ STORAGE_TYPE: 'local',+ POSTGRES_BUILTIN: 'true',+ REDIS_BUILTIN: 'true',+ MINIO_BUILTIN: 'true',+ REDIS_HOST: 'redis',+ DATABASE_PASSWORD: '',+ S3_ACCESS_KEY_ID: '',+ S3_SECRET_ACCESS_KEY: '',+ S3_ACCESS_KEY: '',+ S3_SECRET_KEY: '',+ });+ recordPinnedEnvKeys({});+ recordOsEnvKeys(process.env);+ jest.useFakeTimers();+ });++ afterEach(() => {+ for (const key of keys) {+ if (saved[key] === undefined) delete process.env[key];+ else process.env[key] = saved[key];+ }+ recordPinnedEnvKeys({});+ recordOsEnvKeys(process.env);+ DockerMock.mockReset();+ jest.useRealTimers();+ });++ it.each([+ [{ REDIS_ENABLED: 'true' }, false, ['openwa-redis']],+ [{ QUEUE_ENABLED: 'true' }, false, ['openwa-redis']],+ [{ CACHE_ENABLED: 'true' }, false, ['openwa-redis']],+ [{ REDIS_ENABLED: 'true' }, true, []],+ [{}, false, []],+ ] as const)(+ 'reconciles selected services once for %j with an external pin %s',+ async (selection, external, expected) => {+ Object.assign(process.env, selection);+ if (external) {+ process.env.REDIS_HOST = 'cache.external.invalid';+ recordPinnedEnvKeys({ REDIS_HOST: 'cache.external.invalid' });+ }+ let reconnect: (() => void) | undefined;+ let releaseStart: (() => void) | undefined;+ const reconnecting = new Promise<void>(resolve => (reconnect = resolve));+ const starting = new Promise<void>(resolve => (releaseStart = resolve));+ const created: string[] = [];+ const daemon = {+ ping: jest+ .fn()+ .mockRejectedValueOnce(new Error('proxy not ready'))+ .mockImplementation(() => reconnecting),+ listContainers: jest.fn().mockResolvedValue([]),+ getImage: () => ({ inspect: () => Promise.resolve({}) }),+ modem: {},+ createVolume: jest.fn().mockResolvedValue({}),+ createContainer: jest.fn((config: { name: string }) => {+ created.push(config.name);+ return Promise.resolve({ start: () => starting });+ }),+ };+ DockerMock.mockImplementation(() => daemon);+ const service = new DockerService();+ await service.onModuleInit();+ for (let i = 0; i < 10; i++) service.isDockerAvailable();+ expect(DockerMock).toHaveBeenCalledTimes(2);+ reconnect!();+ await jest.advanceTimersByTimeAsync(1);+ expect(created).toEqual(expected);+ for (let i = 0; i < 10; i++) service.isDockerAvailable();+ expect(created).toEqual(expected);+ expect(DockerMock).toHaveBeenCalledTimes(2);+ releaseStart!();+ await jest.advanceTimersByTimeAsync(1);+ expect(created).toEqual(expected);+ },+ );++ it('recovers a previously reachable local socket after its inventory fails', async () => {+ delete process.env.DOCKER_HOST;+ process.env.POSTGRES_BUILTIN = 'false';+ process.env.REDIS_BUILTIN = 'false';+ process.env.MINIO_BUILTIN = 'false';+ const daemon = {+ ping: jest.fn().mockResolvedValue(undefined),+ listContainers: jest.fn().mockRejectedValueOnce(new Error('daemon restarted')).mockResolvedValue([]),+ };+ DockerMock.mockImplementation(() => daemon);+ const service = new DockerService();+ await service.onModuleInit();+ await expect(service.listContainers()).resolves.toBeNull();+ expect(service.isDockerAvailable()).toBe(false);+ await jest.advanceTimersByTimeAsync(1);+ expect(service.isDockerAvailable()).toBe(true);+ await expect(service.listContainers()).resolves.toEqual([]);+ });+});diff --git a/src/modules/docker/docker.service.ts b/src/modules/docker/docker.service.tsindex 8268d101e..c3d34b010 100644--- a/src/modules/docker/docker.service.ts+++ b/src/modules/docker/docker.service.ts@@ -56,6 +56,8 @@ export class DockerService implements OnModuleInit {private readonly logger = createLogger(DockerService.name);private docker: Docker | null = null;private isAvailable = false;+ /** Whether a daemon has ever answered, so a later loss is a reconnect and not a deployment without Docker. */+ private hasConnected = false;private reinitInFlight = false;/** Visible for testing. */callTimeoutMs = DOCKER_CALL_TIMEOUT_MS;@@ -152,6 +154,7 @@ export class DockerService implements OnModuleInit {this.docker = new Docker(this.buildDockerOptions());await this.answered(abortSignal => this.abortable<unknown>('ping', abortSignal), 'a ping');this.isAvailable = true;+ this.hasConnected = true;this.logger.log('Docker API connected successfully');} catch (error) {this.logger.warn('Docker not available. Container orchestration disabled.', {@@ -180,25 +183,30 @@ export class DockerService implements OnModuleInit {* (DOCKER_HOST=tcp://...), the proxy container may not be accepting connections at* the moment onModuleInit runs (compose `service_started` doesn't wait for readiness).* If the first connect failed, retry it once in the background here so orchestration- * recovers without a process restart. Only for the DOCKER_HOST (proxy/tcp) case — a- * socket-based or docker-less deployment has no such race.+ * recovers without a process restart. Only for the DOCKER_HOST (proxy/tcp) case or a daemon+ * that answered before: a docker-less deployment has no such race. Built-in services the boot+ * skipped while Docker was unreachable are started once it answers.*/isDockerAvailable(): boolean {- if (!this.isAvailable && !this.reinitInFlight && process.env.DOCKER_HOST) {+ if (!this.isAvailable && !this.reinitInFlight && (process.env.DOCKER_HOST || this.hasConnected)) {this.reinitInFlight = true;- void this.initializeDocker().finally(() => {- this.reinitInFlight = false;- });+ void this.initializeDocker()+ .then(() => this.bootstrapOrchestration())+ .catch(error => this.logger.warn('Could not start built-in services after Docker recovery', String(error)))+ .finally(() => {+ this.reinitInFlight = false;+ });}return this.isAvailable;}/**- * List all OpenWA-related containers+ * List all OpenWA-related containers, or null when the daemon could not be asked: an empty list+ * means it was asked and none exist, which callers act on.*/- async listContainers(): Promise<ContainerInfo[]> {+ async listContainers(): Promise<ContainerInfo[] | null> {if (!this.docker || !this.isAvailable) {- return [];+ return null;}try {@@ -221,7 +229,8 @@ export class DockerService implements OnModuleInit {}));} catch (error) {this.logger.error('Failed to list containers', error);- return [];+ this.isAvailable = false;+ return null;}}@@ -230,10 +239,14 @@ export class DockerService implements OnModuleInit {* `com.openwa.service` label (`database` | `cache` | `storage`), and who created each: `'compose'` for one* Docker Compose created (it carries Compose's project label and belongs to the deployment), `'openwa'` for* one createService made. Lets the dashboard show the real built-in state instead of the saved intent. All- * false when Docker is unavailable or none run.+ * false when none run; null when Docker could not be observed.*/- async getRunningBuiltinServices(): Promise<Record<'database' | 'cache' | 'storage', false | 'compose' | 'openwa'>> {+ async getRunningBuiltinServices(): Promise<Record<+ 'database' | 'cache' | 'storage',+ false | 'compose' | 'openwa'+ > | null> {const containers = await this.listContainers();+ if (containers === null) return null;const creator = (svc: string): false | 'compose' | 'openwa' => {const running = containers.find(c =>diff --git a/src/modules/infra/infra-status.controller.spec.ts b/src/modules/infra/infra-status.controller.spec.tsindex 69c796a00..e20cf8da4 100644--- a/src/modules/infra/infra-status.controller.spec.ts+++ b/src/modules/infra/infra-status.controller.spec.ts@@ -634,3 +634,58 @@ describe('InfraStatusController.getStatus running bundled containers', () => {]);});});++describe('InfraStatusController unavailable Docker inventory', () => {+ let previousHost: string | undefined;+ beforeEach(() => {+ previousHost = process.env.REDIS_HOST;+ process.env.REDIS_HOST = 'redis';+ recordPinnedEnvKeys({});+ (fs.existsSync as jest.Mock).mockReturnValue(true);+ (fs.readFileSync as jest.Mock).mockReturnValue('REDIS_BUILTIN=true\n');+ });+ afterEach(() => {+ if (previousHost === undefined) delete process.env.REDIS_HOST;+ else process.env.REDIS_HOST = previousHost;+ recordPinnedEnvKeys({});+ (fs.existsSync as jest.Mock).mockReturnValue(false);+ (fs.readFileSync as jest.Mock).mockReturnValue('');+ });++ it('distinguishes an unavailable inventory from an observed empty list and later recovery', async () => {+ const running = { database: false, cache: 'openwa', storage: false };+ const inventory = jest+ .fn()+ .mockResolvedValueOnce(null)+ .mockResolvedValueOnce({+ database: false,+ cache: false,+ storage: false,+ })+ .mockResolvedValueOnce(running);+ const config = {+ get: (key: string, fallback?: unknown) =>+ key === 'redis.host' ? 'redis' : key === 'engine.type' ? 'baileys' : fallback,+ };+ const database = { isInitialized: true, query: jest.fn().mockResolvedValue([]) };+ const controller = new InfraStatusController(+ config as never,+ database as never,+ database as never,+ {} as never,+ { isDockerAvailable: () => true, getRunningBuiltinServices: inventory } as never,+ { isAvailable: jest.fn().mockResolvedValue(false) } as never,+ {} as never,+ );+ const unavailable = await controller.getStatus();+ const empty = await controller.getStatus();+ const recovered = await controller.getStatus();++ expect(unavailable.redis.builtIn).toBe(true);+ expect(unavailable.redis).not.toHaveProperty('running');+ expect(empty.redis.builtIn).toBe(false);+ expect(empty.redis.running).toBe(false);+ expect(recovered.redis.builtIn).toBe(true);+ expect(recovered.redis.running).toBe(true);+ });+});