Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(integration): index dead letters by delivery id

Yudhi Armyndharis90d683d369b5
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4e635b6c2..21c60eccd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -43,6 +43,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- Index ingress dead letters by delivery id, so the reconciler's per-delivery checks stop scanning every dead letter of an instance; the first boot builds the index.
- Replay stranded ingress events past one whose stored payload is not valid JSON, instead of failing every sweep on it.
- Hand a chat back to the bot when the plugin instance that took it over is deleted, instead of silencing every other plugin on it.
- Keep the sessions of a plugin's other enabled instances active when creating, disabling or deleting a session-scoped instance retires an operator-set `*`.
diff --git a/docs/05-database-design.md b/docs/05-database-design.md
index b4f6c1180..c19b6e986 100644
--- a/docs/05-database-design.md
+++ b/docs/05-database-design.md
@@ -925,7 +925,8 @@ src/database/migrations/ # data connection (pluggable)
├── 1787000000000-AddWebhookDeliveryFailurePayload.ts # webhook_delivery_failures.payload (optional replay copy)
├── 1787100000000-AddSendIdempotencyKeys.ts # send_idempotency_keys (opt-in send retry guard)
├── 1787200000000-DeduplicateTerminalWebhookFailures.ts # one terminal record per webhook and idempotency key
-└── 1790812800000-AddMessageWindowIndexes.ts # compound timestamp indexes for message-time pagination
+├── 1790812800000-AddMessageWindowIndexes.ts # compound timestamp indexes for message-time pagination
+└── 1791100000000-AddIntegrationDeliveryFailureDeliveryIndex.ts # (pluginId, instanceId, deliveryId) index for the dead-letter guards
```
> [!NOTE]
diff --git a/src/database/migrations/1791100000000-AddIntegrationDeliveryFailureDeliveryIndex.ts b/src/database/migrations/1791100000000-AddIntegrationDeliveryFailureDeliveryIndex.ts
new file mode 100644
index 000000000..5de03fafa
--- /dev/null
+++ b/src/database/migrations/1791100000000-AddIntegrationDeliveryFailureDeliveryIndex.ts
@@ -0,0 +1,33 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+/**
+ * Composite index on `integration_delivery_failures (pluginId, instanceId, deliveryId)`, backing the
+ * per-delivery guards of the ingress reconciler (the open dead-letter count, the lowest-id tie-break and
+ * the retire update) and of the ingress processor. The only index was (pluginId, instanceId), so each
+ * guard read every dead letter of the instance: the cost grew with the dead-letter queue, made the
+ * retention hand-off quadratic, and on SQLite ran on the event loop.
+ *
+ * Hand-authored because `synchronize` is off for the data connection on PostgreSQL (and optional on
+ * SQLite). The explicit name matches the entity's @Index, so the synchronize and migration schema paths
+ * converge on one index. Idempotent via IF NOT EXISTS (supported by both dialects). The earlier
+ * (pluginId, instanceId) index stays: it is a prefix of this one, and dropping it is not worth the risk.
+ */
+export class AddIntegrationDeliveryFailureDeliveryIndex1791100000000 implements MigrationInterface {
+ name = 'AddIntegrationDeliveryFailureDeliveryIndex1791100000000';
+
+ public async up(queryRunner: QueryRunner): Promise<void> {
+ // Boot and CLI migrations run on a pool without the runtime statement_timeout (pg-boot-migrations.ts),
+ // and MigrationExecutor wraps a lone pending migration in its own transaction, so lift any role- or
+ // database-level default for this transaction only, exactly like AddWebhookDeliveryFailureLookupIndex.
+ if (queryRunner.dataSource.options.type === 'postgres') {
+ await queryRunner.query('SET LOCAL statement_timeout = 0');
+ }
+ await queryRunner.query(
+ `CREATE INDEX IF NOT EXISTS "IDX_integration_delivery_failures_instance_delivery" ON "integration_delivery_failures" ("pluginId", "instanceId", "deliveryId")`,
+ );
+ }
+
+ public async down(queryRunner: QueryRunner): Promise<void> {
+ await queryRunner.query(`DROP INDEX IF EXISTS "IDX_integration_delivery_failures_instance_delivery"`);
+ }
+}
diff --git a/src/database/migrations/__tests__/1791100000000-AddIntegrationDeliveryFailureDeliveryIndex.spec.ts b/src/database/migrations/__tests__/1791100000000-AddIntegrationDeliveryFailureDeliveryIndex.spec.ts
new file mode 100644
index 000000000..24ec82bb4
--- /dev/null
+++ b/src/database/migrations/__tests__/1791100000000-AddIntegrationDeliveryFailureDeliveryIndex.spec.ts
@@ -0,0 +1,74 @@
+import { DataSource } from 'typeorm';
+import { AddIntegrationFabric1781900000000 } from '../1781900000000-AddIntegrationFabric';
+import { AddIntegrationDeliveryFailureDeliveryIndex1791100000000 } from '../1791100000000-AddIntegrationDeliveryFailureDeliveryIndex';
+
+const INDEX = 'IDX_integration_delivery_failures_instance_delivery';
+
+describe('AddIntegrationDeliveryFailureDeliveryIndex migration', () => {
+ let ds: DataSource;
+
+ beforeEach(async () => {
+ ds = new DataSource({ type: 'better-sqlite3', database: ':memory:' });
+ await ds.initialize();
+ const runner = ds.createQueryRunner();
+ await new AddIntegrationFabric1781900000000().up(runner);
+ await runner.release();
+ // One instance holding many dead letters, which is the shape that made the guards scan.
+ const rows: string[] = [];
+ for (let i = 0; i < 200; i++) {
+ rows.push(`('id${i}', 'inbound', 'plug', 'inst', 'd${i}', 1, 'sandbox 5xx', 0)`);
+ }
+ await ds.query(
+ `INSERT INTO "integration_delivery_failures" ("id","direction","pluginId","instanceId","deliveryId","attempts","lastError","redriven") VALUES ${rows.join(',')}`,
+ );
+ });
+
+ afterEach(async () => {
+ await ds.destroy();
+ });
+
+ const indexNames = async (): Promise<string[]> => {
+ const rows = await ds.query<{ name: string }[]>(`PRAGMA index_list("integration_delivery_failures")`);
+ return rows.map(r => r.name).sort();
+ };
+
+ it('creates the per-delivery index next to the instance index', async () => {
+ const runner = ds.createQueryRunner();
+ await new AddIntegrationDeliveryFailureDeliveryIndex1791100000000().up(runner);
+
+ expect(await indexNames()).toEqual(expect.arrayContaining([INDEX, 'IDX_integration_delivery_failures_instance']));
+ });
+
+ it('is idempotent (re-running up is a no-op) and down() drops only its own index', async () => {
+ const runner = ds.createQueryRunner();
+ const migration = new AddIntegrationDeliveryFailureDeliveryIndex1791100000000();
+
+ await migration.up(runner);
+ await expect(migration.up(runner)).resolves.toBeUndefined();
+ expect(await indexNames()).toContain(INDEX);
+
+ await migration.down(runner);
+ const after = await indexNames();
+ expect(after).not.toContain(INDEX);
+ expect(after).toContain('IDX_integration_delivery_failures_instance');
+ });
+
+ it('the open dead-letter guard is served by the per-delivery index', async () => {
+ const runner = ds.createQueryRunner();
+ const guard =
+ `EXPLAIN QUERY PLAN SELECT COUNT(*) FROM "integration_delivery_failures" ` +
+ `WHERE "direction" = 'inbound' AND "pluginId" = 'plug' AND "instanceId" = 'inst' AND "deliveryId" = 'd7' AND "redriven" = 0`;
+
+ // Baseline first: the instance index narrows to the whole instance and the rest is filtered row by
+ // row, so the assertion below measures the new index and not a plan that already used deliveryId.
+ const before = (await ds.query<{ detail: string }[]>(guard)).map(r => r.detail).join(' ');
+ expect(before).toContain('IDX_integration_delivery_failures_instance');
+ expect(before).not.toContain('deliveryId=?');
+
+ await new AddIntegrationDeliveryFailureDeliveryIndex1791100000000().up(runner);
+
+ const after = (await ds.query<{ detail: string }[]>(guard)).map(r => r.detail).join(' ');
+ expect(after).toContain(INDEX);
+ expect(after).toContain('deliveryId=?');
+ });
+});
diff --git a/src/modules/integration/entities/integration-delivery-failure.entity.ts b/src/modules/integration/entities/integration-delivery-failure.entity.ts
index e5ae7baf3..915868104 100644
--- a/src/modules/integration/entities/integration-delivery-failure.entity.ts
+++ b/src/modules/integration/entities/integration-delivery-failure.entity.ts
@@ -7,6 +7,8 @@ import { NulFreeTransformer } from '../../../common/transformers/nul-free.transf
// webhook_delivery_failures. sessionId is provenance (no FK).
@Entity('integration_delivery_failures')
@Index('IDX_integration_delivery_failures_instance', ['pluginId', 'instanceId'])
+// The per-delivery guards (open dead-letter count, tie-break, retire) filter on deliveryId as well.
+@Index('IDX_integration_delivery_failures_instance_delivery', ['pluginId', 'instanceId', 'deliveryId'])
export class IntegrationDeliveryFailure {
@PrimaryGeneratedColumn('uuid')
id!: string;