Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(email): persist pubsub worker capacity across deploys (#9177)
gbirman940274fa00fb
diff --git a/infra/stacks/email-service/AGENTS.md b/infra/stacks/email-service/AGENTS.mdnew file mode 100644index 0000000000..2841c40cb8--- /dev/null+++ b/infra/stacks/email-service/AGENTS.md@@ -0,0 +1,35 @@+# Email service infrastructure++Owns the email API, pub/sub consumers, focus worker, scheduled handlers, queues,+attachment storage, and Redis. Runtime code lives in `services/email_service`.++## Worker capacity++- Set `pubsub_workers_desired_count`, `pubsub_workers_min_capacity`, and+ `pubsub_workers_max_capacity` in each `Pulumi.<stack>.yaml`. They control ECS+ tasks, must be positive integers, and must satisfy min <= desired <= max.+- Production is pinned to 8 tasks for backfill recovery; dev retains a desired+ count of 1 and autoscaling bounds of 1–2. Lower the production values after+ recovery. There is no expiry or automatic scale-down when the queue empties.+- Keep desired count and autoscaling bounds in sync when changing capacity.+ Deployments apply configuration changes but do not refresh Pulumi state;+ manual ECS adjustments can persist when Pulumi sees no change. Refresh state+ before reconciling out-of-band drift.+- Per-task `BACKFILL_QUEUE_WORKERS` comes from the email service's Doppler config+ through `APP_SECRETS_JSON`. At 24 per task, 8 tasks provide 192 backfill+ workers at steady state; rolling deployments can temporarily overlap old and+ new tasks. The legacy `backfill_queue_workers` Pulumi key is not wired into+ the task's runtime configuration.+- These tasks also consume inbox sync and Gmail operation queues. Changing task+ count scales their consumers and database connection pools as well.++## Checks++- `bun run --cwd infra check` from the repository root.+- `just check` from the repository root.++## Related++- [Infrastructure guide](../../../docs/development/infrastructure.md)+- [Pulumi rules](../../../.agents/rules/infra/pulumi.md)+- [Configuration rules](../../../.agents/rules/config-and-service-urls.md)diff --git a/infra/stacks/email-service/CLAUDE.md b/infra/stacks/email-service/CLAUDE.mdnew file mode 120000index 0000000000..47dc3e3d86--- /dev/null+++ b/infra/stacks/email-service/CLAUDE.md@@ -0,0 +1 @@+AGENTS.md\ No newline at end of filediff --git a/infra/stacks/email-service/Pulumi.dev.yaml b/infra/stacks/email-service/Pulumi.dev.yamlindex 170b938010..99af3dce80 100644--- a/infra/stacks/email-service/Pulumi.dev.yaml+++ b/infra/stacks/email-service/Pulumi.dev.yaml@@ -1,6 +1,9 @@config:aws-native:region: us-east-1aws:region: us-east-1+ email-service:pubsub_workers_desired_count: 1+ email-service:pubsub_workers_min_capacity: 1+ email-service:pubsub_workers_max_capacity: 2email-service:authentication_service_internal_api_key: authentication-service-internal-api-key-devemail-service:jwt_secret_key: fusionauth-jwt-secret-devemail-service:fusionauth_client_id: fusionauth-client-id-key-devdiff --git a/infra/stacks/email-service/Pulumi.prod.yaml b/infra/stacks/email-service/Pulumi.prod.yamlindex b3c7bc75df..476cabac0d 100644--- a/infra/stacks/email-service/Pulumi.prod.yaml+++ b/infra/stacks/email-service/Pulumi.prod.yaml@@ -1,6 +1,9 @@config:aws-native:region: us-east-1aws:region: us-east-1+ email-service:pubsub_workers_desired_count: 8+ email-service:pubsub_workers_min_capacity: 8+ email-service:pubsub_workers_max_capacity: 8email-service:authentication_service_internal_api_key: authentication-service-internal-api-key-prodemail-service:jwt_secret_key: fusionauth-jwt-secret-prodemail-service:fusionauth_client_id: fusionauth-client-id-key-proddiff --git a/infra/stacks/email-service/pubsub_workers.ts b/infra/stacks/email-service/pubsub_workers.tsindex d4e75a9288..adbdc3eef1 100644--- a/infra/stacks/email-service/pubsub_workers.ts+++ b/infra/stacks/email-service/pubsub_workers.ts@@ -10,6 +10,7 @@ import {import { EcrImage } from '../../packages/service';import {CLOUD_TRAIL_SNS_TOPIC_ARN,+ config,type DopplerEcsEnvironment,stack,} from '../../packages/shared';@@ -60,6 +61,25 @@ export class EmailPubSubWorkers extends pulumi.ComponentResource {this.clusterName = clusterName;this.role = role;+ const capacity = {+ desiredCount: Number(config.require('pubsub_workers_desired_count')),+ minCapacity: Number(config.require('pubsub_workers_min_capacity')),+ maxCapacity: Number(config.require('pubsub_workers_max_capacity')),+ };+ for (const [name, value] of Object.entries(capacity)) {+ if (!Number.isInteger(value) || value < 1) {+ throw new Error(`Pub/sub worker ${name} must be a positive integer`);+ }+ }+ if (+ capacity.desiredCount < capacity.minCapacity ||+ capacity.desiredCount > capacity.maxCapacity+ ) {+ throw new Error(+ 'Pub/sub worker desired count must be between min and max capacity'+ );+ }+// ecr imageconst image = new EcrImage(`${BASE_NAME}-ecr-image-${stack}`,@@ -139,7 +159,7 @@ export class EmailPubSubWorkers extends pulumi.ComponentResource {}`,},},- desiredCount: stack === 'prod' ? 5 : 1,+ desiredCount: capacity.desiredCount,},{parent: this,@@ -148,7 +168,7 @@ export class EmailPubSubWorkers extends pulumi.ComponentResource {this.service = service;- this.setupAutoScaling();+ this.setupAutoScaling(capacity);this.setupServiceAlarms();}@@ -184,14 +204,20 @@ export class EmailPubSubWorkers extends pulumi.ComponentResource {return serviceSg;}- setupAutoScaling() {+ setupAutoScaling({+ minCapacity,+ maxCapacity,+ }: {+ minCapacity: number;+ maxCapacity: number;+ }) {if (!this.service) return;const serviceScalableTarget = new aws.appautoscaling.Target(`${BASE_NAME}-scalable-target-${stack}`,{- maxCapacity: stack === 'prod' ? 10 : 2,- minCapacity: stack === 'prod' ? 5 : 1,+ maxCapacity,+ minCapacity,resourceId: pulumi.interpolate`service/${this.clusterName}/${this.service.service.name}`,scalableDimension: 'ecs:service:DesiredCount',serviceNamespace: 'ecs',