Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(docker): recover after the daemon was unreachable at boot or lost

Yudhi Armyndharis28131a79dc21
diff --git a/src/modules/docker/docker.service.spec.ts b/src/modules/docker/docker.service.spec.ts
index 51cc858b1..171c52a55 100644
--- a/src/modules/docker/docker.service.spec.ts
+++ b/src/modules/docker/docker.service.spec.ts
@@ -1,6 +1,6 @@
import Docker from 'dockerode';
import { DockerService, prestartBuiltinDatabase } from './docker.service';
-import { recordPinnedEnvKeys } from '../../config/env-precedence';
+import { recordOsEnvKeys, recordPinnedEnvKeys } from '../../config/env-precedence';
import { HOST_SPELLINGS } from '../../../test/fixtures/builtin-mode-cases';
// Prevent actual Docker connections on module init during tests
@@ -49,7 +49,7 @@ describe('DockerService.getRunningBuiltinServices', () => {
expect(await service.getRunningBuiltinServices()).toEqual({ database: 'compose', cache: 'openwa', storage: false });
});
- it('reports all false when no bundled containers are present (e.g. Docker unavailable)', async () => {
+ it('reports all false when a live inventory contains no bundled containers', async () => {
const service = new DockerService();
jest.spyOn(service, 'listContainers').mockResolvedValue([]);
expect(await service.getRunningBuiltinServices()).toEqual({ database: false, cache: false, storage: false });
@@ -447,8 +447,8 @@ describe('DockerService.listContainers', () => {
return service;
};
- it('returns an empty list when Docker is unavailable', async () => {
- expect(await new DockerService().listContainers()).toEqual([]);
+ it('returns no live inventory when Docker is unavailable', async () => {
+ expect(await new DockerService().listContainers()).toBeNull();
});
it('maps only OpenWA containers (label or /openwa- name) to ContainerInfo', async () => {
@@ -487,9 +487,9 @@ describe('DockerService.listContainers', () => {
]);
});
- it('returns an empty list when the daemon listing fails', async () => {
+ it('returns no live inventory when the daemon listing fails', async () => {
const service = makeService(jest.fn().mockRejectedValue(new Error('daemon gone')));
- expect(await service.listContainers()).toEqual([]);
+ expect(await service.listContainers()).toBeNull();
});
});
@@ -880,3 +880,126 @@ describe('DockerService.orchestrateProfiles', () => {
expect(result.message).toBe('Failed to start postgres: boom');
});
});
+
+describe('DockerService availability recovery', () => {
+ const keys = [
+ 'DOCKER_HOST',
+ 'DATABASE_TYPE',
+ 'STORAGE_TYPE',
+ 'REDIS_ENABLED',
+ 'QUEUE_ENABLED',
+ 'CACHE_ENABLED',
+ 'POSTGRES_BUILTIN',
+ 'REDIS_BUILTIN',
+ 'MINIO_BUILTIN',
+ 'REDIS_HOST',
+ 'DATABASE_PASSWORD',
+ 'S3_ACCESS_KEY_ID',
+ 'S3_SECRET_ACCESS_KEY',
+ 'S3_ACCESS_KEY',
+ 'S3_SECRET_KEY',
+ ];
+ let saved: Record<string, string | undefined>;
+
+ beforeEach(() => {
+ saved = Object.fromEntries(keys.map(key => [key, process.env[key]]));
+ for (const key of keys) delete process.env[key];
+ Object.assign(process.env, {
+ DOCKER_HOST: 'tcp://proxy.invalid:2375',
+ DATABASE_TYPE: 'sqlite',
+ STORAGE_TYPE: 'local',
+ POSTGRES_BUILTIN: 'true',
+ REDIS_BUILTIN: 'true',
+ MINIO_BUILTIN: 'true',
+ REDIS_HOST: 'redis',
+ DATABASE_PASSWORD: '',
+ S3_ACCESS_KEY_ID: '',
+ S3_SECRET_ACCESS_KEY: '',
+ S3_ACCESS_KEY: '',
+ S3_SECRET_KEY: '',
+ });
+ recordPinnedEnvKeys({});
+ recordOsEnvKeys(process.env);
+ jest.useFakeTimers();
+ });
+
+ afterEach(() => {
+ for (const key of keys) {
+ if (saved[key] === undefined) delete process.env[key];
+ else process.env[key] = saved[key];
+ }
+ recordPinnedEnvKeys({});
+ recordOsEnvKeys(process.env);
+ DockerMock.mockReset();
+ jest.useRealTimers();
+ });
+
+ it.each([
+ [{ REDIS_ENABLED: 'true' }, false, ['openwa-redis']],
+ [{ QUEUE_ENABLED: 'true' }, false, ['openwa-redis']],
+ [{ CACHE_ENABLED: 'true' }, false, ['openwa-redis']],
+ [{ REDIS_ENABLED: 'true' }, true, []],
+ [{}, false, []],
+ ] as const)(
+ 'reconciles selected services once for %j with an external pin %s',
+ async (selection, external, expected) => {
+ Object.assign(process.env, selection);
+ if (external) {
+ process.env.REDIS_HOST = 'cache.external.invalid';
+ recordPinnedEnvKeys({ REDIS_HOST: 'cache.external.invalid' });
+ }
+ let reconnect: (() => void) | undefined;
+ let releaseStart: (() => void) | undefined;
+ const reconnecting = new Promise<void>(resolve => (reconnect = resolve));
+ const starting = new Promise<void>(resolve => (releaseStart = resolve));
+ const created: string[] = [];
+ const daemon = {
+ ping: jest
+ .fn()
+ .mockRejectedValueOnce(new Error('proxy not ready'))
+ .mockImplementation(() => reconnecting),
+ listContainers: jest.fn().mockResolvedValue([]),
+ getImage: () => ({ inspect: () => Promise.resolve({}) }),
+ modem: {},
+ createVolume: jest.fn().mockResolvedValue({}),
+ createContainer: jest.fn((config: { name: string }) => {
+ created.push(config.name);
+ return Promise.resolve({ start: () => starting });
+ }),
+ };
+ DockerMock.mockImplementation(() => daemon);
+ const service = new DockerService();
+ await service.onModuleInit();
+ for (let i = 0; i < 10; i++) service.isDockerAvailable();
+ expect(DockerMock).toHaveBeenCalledTimes(2);
+ reconnect!();
+ await jest.advanceTimersByTimeAsync(1);
+ expect(created).toEqual(expected);
+ for (let i = 0; i < 10; i++) service.isDockerAvailable();
+ expect(created).toEqual(expected);
+ expect(DockerMock).toHaveBeenCalledTimes(2);
+ releaseStart!();
+ await jest.advanceTimersByTimeAsync(1);
+ expect(created).toEqual(expected);
+ },
+ );
+
+ it('recovers a previously reachable local socket after its inventory fails', async () => {
+ delete process.env.DOCKER_HOST;
+ process.env.POSTGRES_BUILTIN = 'false';
+ process.env.REDIS_BUILTIN = 'false';
+ process.env.MINIO_BUILTIN = 'false';
+ const daemon = {
+ ping: jest.fn().mockResolvedValue(undefined),
+ listContainers: jest.fn().mockRejectedValueOnce(new Error('daemon restarted')).mockResolvedValue([]),
+ };
+ DockerMock.mockImplementation(() => daemon);
+ const service = new DockerService();
+ await service.onModuleInit();
+ await expect(service.listContainers()).resolves.toBeNull();
+ expect(service.isDockerAvailable()).toBe(false);
+ await jest.advanceTimersByTimeAsync(1);
+ expect(service.isDockerAvailable()).toBe(true);
+ await expect(service.listContainers()).resolves.toEqual([]);
+ });
+});
diff --git a/src/modules/docker/docker.service.ts b/src/modules/docker/docker.service.ts
index 8268d101e..c3d34b010 100644
--- a/src/modules/docker/docker.service.ts
+++ b/src/modules/docker/docker.service.ts
@@ -56,6 +56,8 @@ export class DockerService implements OnModuleInit {
private readonly logger = createLogger(DockerService.name);
private docker: Docker | null = null;
private isAvailable = false;
+ /** Whether a daemon has ever answered, so a later loss is a reconnect and not a deployment without Docker. */
+ private hasConnected = false;
private reinitInFlight = false;
/** Visible for testing. */
callTimeoutMs = DOCKER_CALL_TIMEOUT_MS;
@@ -152,6 +154,7 @@ export class DockerService implements OnModuleInit {
this.docker = new Docker(this.buildDockerOptions());
await this.answered(abortSignal => this.abortable<unknown>('ping', abortSignal), 'a ping');
this.isAvailable = true;
+ this.hasConnected = true;
this.logger.log('Docker API connected successfully');
} catch (error) {
this.logger.warn('Docker not available. Container orchestration disabled.', {
@@ -180,25 +183,30 @@ export class DockerService implements OnModuleInit {
* (DOCKER_HOST=tcp://...), the proxy container may not be accepting connections at
* the moment onModuleInit runs (compose `service_started` doesn't wait for readiness).
* If the first connect failed, retry it once in the background here so orchestration
- * recovers without a process restart. Only for the DOCKER_HOST (proxy/tcp) case — a
- * socket-based or docker-less deployment has no such race.
+ * recovers without a process restart. Only for the DOCKER_HOST (proxy/tcp) case or a daemon
+ * that answered before: a docker-less deployment has no such race. Built-in services the boot
+ * skipped while Docker was unreachable are started once it answers.
*/
isDockerAvailable(): boolean {
- if (!this.isAvailable && !this.reinitInFlight && process.env.DOCKER_HOST) {
+ if (!this.isAvailable && !this.reinitInFlight && (process.env.DOCKER_HOST || this.hasConnected)) {
this.reinitInFlight = true;
- void this.initializeDocker().finally(() => {
- this.reinitInFlight = false;
- });
+ void this.initializeDocker()
+ .then(() => this.bootstrapOrchestration())
+ .catch(error => this.logger.warn('Could not start built-in services after Docker recovery', String(error)))
+ .finally(() => {
+ this.reinitInFlight = false;
+ });
}
return this.isAvailable;
}
/**
- * List all OpenWA-related containers
+ * List all OpenWA-related containers, or null when the daemon could not be asked: an empty list
+ * means it was asked and none exist, which callers act on.
*/
- async listContainers(): Promise<ContainerInfo[]> {
+ async listContainers(): Promise<ContainerInfo[] | null> {
if (!this.docker || !this.isAvailable) {
- return [];
+ return null;
}
try {
@@ -221,7 +229,8 @@ export class DockerService implements OnModuleInit {
}));
} catch (error) {
this.logger.error('Failed to list containers', error);
- return [];
+ this.isAvailable = false;
+ return null;
}
}
@@ -230,10 +239,14 @@ export class DockerService implements OnModuleInit {
* `com.openwa.service` label (`database` | `cache` | `storage`), and who created each: `'compose'` for one
* Docker Compose created (it carries Compose's project label and belongs to the deployment), `'openwa'` for
* one createService made. Lets the dashboard show the real built-in state instead of the saved intent. All
- * false when Docker is unavailable or none run.
+ * false when none run; null when Docker could not be observed.
*/
- async getRunningBuiltinServices(): Promise<Record<'database' | 'cache' | 'storage', false | 'compose' | 'openwa'>> {
+ async getRunningBuiltinServices(): Promise<Record<
+ 'database' | 'cache' | 'storage',
+ false | 'compose' | 'openwa'
+ > | null> {
const containers = await this.listContainers();
+ if (containers === null) return null;
const creator = (svc: string): false | 'compose' | 'openwa' => {
const running = containers.find(
c =>
diff --git a/src/modules/infra/infra-status.controller.spec.ts b/src/modules/infra/infra-status.controller.spec.ts
index 69c796a00..e20cf8da4 100644
--- a/src/modules/infra/infra-status.controller.spec.ts
+++ b/src/modules/infra/infra-status.controller.spec.ts
@@ -634,3 +634,58 @@ describe('InfraStatusController.getStatus running bundled containers', () => {
]);
});
});
+
+describe('InfraStatusController unavailable Docker inventory', () => {
+ let previousHost: string | undefined;
+ beforeEach(() => {
+ previousHost = process.env.REDIS_HOST;
+ process.env.REDIS_HOST = 'redis';
+ recordPinnedEnvKeys({});
+ (fs.existsSync as jest.Mock).mockReturnValue(true);
+ (fs.readFileSync as jest.Mock).mockReturnValue('REDIS_BUILTIN=true\n');
+ });
+ afterEach(() => {
+ if (previousHost === undefined) delete process.env.REDIS_HOST;
+ else process.env.REDIS_HOST = previousHost;
+ recordPinnedEnvKeys({});
+ (fs.existsSync as jest.Mock).mockReturnValue(false);
+ (fs.readFileSync as jest.Mock).mockReturnValue('');
+ });
+
+ it('distinguishes an unavailable inventory from an observed empty list and later recovery', async () => {
+ const running = { database: false, cache: 'openwa', storage: false };
+ const inventory = jest
+ .fn()
+ .mockResolvedValueOnce(null)
+ .mockResolvedValueOnce({
+ database: false,
+ cache: false,
+ storage: false,
+ })
+ .mockResolvedValueOnce(running);
+ const config = {
+ get: (key: string, fallback?: unknown) =>
+ key === 'redis.host' ? 'redis' : key === 'engine.type' ? 'baileys' : fallback,
+ };
+ const database = { isInitialized: true, query: jest.fn().mockResolvedValue([]) };
+ const controller = new InfraStatusController(
+ config as never,
+ database as never,
+ database as never,
+ {} as never,
+ { isDockerAvailable: () => true, getRunningBuiltinServices: inventory } as never,
+ { isAvailable: jest.fn().mockResolvedValue(false) } as never,
+ {} as never,
+ );
+ const unavailable = await controller.getStatus();
+ const empty = await controller.getStatus();
+ const recovered = await controller.getStatus();
+
+ expect(unavailable.redis.builtIn).toBe(true);
+ expect(unavailable.redis).not.toHaveProperty('running');
+ expect(empty.redis.builtIn).toBe(false);
+ expect(empty.redis.running).toBe(false);
+ expect(recovered.redis.builtIn).toBe(true);
+ expect(recovered.redis.running).toBe(true);
+ });
+});