Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

26.3 KB
Raw Blame
# Services
# Freeze the Docker Compose project name so every checkout/worktree uses the
# same local containers, networks, and anonymous Compose-managed resources.
# Local setup is single-instance by design.
name: macro

include:
  - docker/docker-compose-databases.yml
  - infra/stacks/fusionauth-instance/docker-compose.yml

x-common-env: &common-env
  # Bound console logs even when a worker repeatedly fails.
  logging:
    driver: json-file
    options:
      max-size: "20m"
      max-file: "3"
  # The xtask orchestrator points this at a per-instance generated env file via
  # MACRO_ENV_FILE (run-local/run-dev). Defaults to .env for any legacy/manual
  # `docker compose` invocation, and is optional so a missing file is not fatal.
  env_file:
    - path: ${MACRO_ENV_FILE:-.env}
      required: false

x-rust-services-image: &rust-services-image
  image: macro-local-rust-services:dev
  build:
    context: .
    dockerfile: docker/Dockerfile.dev
    target: services_bundle

services:
  rust_services_image:
    profiles:
      - build-helper
    build:
      context: .
      dockerfile: docker/Dockerfile.dev
      target: services_bundle
    image: macro-local-rust-services:dev

  # ============================================================================
  # authentication-service (Port 8080)
  # Primary auth service - depends on FusionAuth, Redis, MacroDB
  # ============================================================================
  authentication-service:
    <<: [*common-env, *rust-services-image]
    command: ["/app/out/authentication_service"]
    ports:
      - "8080:8080"
    depends_on:
      - postgres
      - fusionauth
      - redis
    networks:
      - auth
      - databases
      - services
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

  # ============================================================================
  # connection_gateway (Port 8082)
  # WebSocket gateway - Redis only
  # ============================================================================
  connection_gateway:
    <<: [*common-env, *rust-services-image]
    command: ["/app/out/connection_gateway_service"]
    ports:
      - "8082:8080"
    depends_on:
      - redis
    networks:
      databases:
      services:
        aliases:
          - connection-gateway
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

  # ============================================================================
  # contacts_service (Port 8083)
  # Contact management - ContactsDB
  # ============================================================================
  contacts_service:
    <<: [*common-env, *rust-services-image]
    command: ["/app/out/contacts_service"]
    ports:
      - "8083:8080"
    networks:
      databases:
      services:
        aliases:
          - contacts-service
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

  # ============================================================================
  document_storage_service:
    <<: [*common-env, *rust-services-image]
    command: ["/app/out/document_storage_service"]
    ports:
      - "8086:8080"
    environment:
      # - OVERRIDE_SYNC_SERVICE_URL=https://sync-service-playground.macroverse.workers.dev
      # uncomment above, and comment out below, for when you want to use the playground sync service. also use
      # VITE_SYNC_SERVICE_HOST=sync-service-playground.macroverse.workers.dev VITE_LOCAL_SERVERS=ALL bun run dev
      # if you do so
      - OVERRIDE_SYNC_SERVICE_URL=http://sync-service:8787
      - OVERRIDE_LEXICAL_SERVICE_URL=http://lexical-service:8096
    depends_on:
      - redis
      - connection_gateway
      - authentication-service
      - sync_service
      - lexical_service
    networks:
      databases:
      services:
        aliases:
          - document-storage-service
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

  # ============================================================================
  # slack_import_worker (portless, opt-in profile: slack-import)
  # Build separately; never part of the default or run-dev bundle.
  # ============================================================================
  slack_import_worker:
    <<: *common-env
    profiles:
      - slack-import
    build:
      context: .
      dockerfile: docker/Dockerfile
      args:
        SERVICE_NAME: slack_import_worker
    stop_grace_period: 120s
    depends_on:
      - postgres
      - search_processing_service
      - connection_gateway
    networks:
      - databases
      - services

  # ============================================================================
  # document_upload_finalizer (local worker)
  # LocalStack S3 ObjectCreated -> SQS consumer for document upload finalization.
  # Production uses the document_upload_finalizer_handler Lambda.
  # ============================================================================
  document_upload_finalizer:
    <<: [*common-env, *rust-services-image]
    command: ["/app/out/document_upload_finalizer_local_worker"]
    environment:
      LOCAL_AWS_URL: ${LOCAL_AWS_URL:-http://localstack:4566}
      DOCUMENT_UPLOAD_FINALIZER_QUEUE_URL: ${DOCUMENT_UPLOAD_FINALIZER_QUEUE_URL:-http://localstack:4566/000000000000/document-upload-finalizer-queue}
    depends_on:
      - postgres
      - sync_service
      - lexical_service
    networks:
      - databases
      - services

  # ============================================================================
  # email_service (Port 8087)
  # Email processing and management
  # ============================================================================
  email_service:
    <<: [*common-env, *rust-services-image]
    command: ["/app/out/email_service"]
    ports:
      - "8087:8080"
    depends_on:
      - authentication-service
      - document_storage_service
      - connection_gateway
      - static_file_service
      - redis
    networks:
      databases:
      services:
        aliases:
          - email-service
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

  # ============================================================================
  # email_pubsub_workers (local worker)
  # Consumes the email SQS queues (backfill, scheduled, gmail sync/ops, sfs,
  # link manager). Runs as its own deployment in prod; locally it's its own
  # container alongside email_service. No HTTP port — pure SQS consumer.
  # ============================================================================
  email_pubsub_workers:
    <<: [*common-env, *rust-services-image]
    command: ["/app/out/pubsub_workers"]
    depends_on:
      - postgres
      - redis
      - authentication-service
      - document_storage_service
      - connection_gateway
      - static_file_service
      - email_service
    networks:
      - databases
      - services

  # ============================================================================
  # calendar_service (Port 8088)
  # Google Calendar sync + calendar API, and the local owner of /calendar.
  # ============================================================================
  calendar_service:
    <<: [*common-env, *rust-services-image]
    command: ["/app/out/calendar_service"]
    ports:
      - "8088:8080"
    depends_on:
      - postgres
      - redis
      - authentication-service
      - connection_gateway
    networks:
      databases:
      services:
        aliases:
          - calendar-service
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

  # ============================================================================
  # notification_service (Port 8089)
  # User notifications - NotificationDB
  # ============================================================================
  notification_service:
    <<: [*common-env, *rust-services-image]
    command: ["/app/out/notification_service"]
    ports:
      - "8089:8080"
    environment:
      LAST_ONLINE_REDIS_URI: ${LAST_ONLINE_REDIS_URI:-${REDIS_URI:-redis://redis:6379}}
      OVERRIDE_CONNECTION_GATEWAY_URL: http://connection-gateway:8080
    depends_on:
      - redis
      - authentication-service
      - connection_gateway
      - document_storage_service
    networks:
      databases:
      services:
        aliases:
          - notification-service
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

  # ============================================================================
  search_processing_service:
    <<: [*common-env, *rust-services-image]
    command: ["/app/out/search_processing_service"]
    ports:
      - "8092:8080"
    environment:
      - OVERRIDE_LEXICAL_SERVICE_URL=http://lexical-service:8096
    depends_on:
      - email_service
      - lexical_service
    networks:
      databases:
      services:
        aliases:
          - search-processing-service
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

  # ============================================================================
  # agent_harness_service (Port 8101)
  # Channel messages -> agent session trigger events -> sandboxed coding agent
  # sessions, plus the session control routes, which have to be served from the
  # process that owns the live transports they act on.
  # Daytona-backed sandboxes require DAYTONA_API_KEY; `just run_local` sets
  # DEV_DANGEROUS_LOCAL_CONTAINERS and runs sandboxes on the host Docker daemon
  # instead. Trigger processing and external sessions run without either.
  # ============================================================================
  agent_harness_service:
    <<: [*common-env, *rust-services-image]
    command: ["/app/out/agent_harness_service"]
    restart: on-failure
    # No host port: everything reaches this through the proxy, and a second
    # local instance would collide on a fixed publish.
    environment:
      # In-network gateway address; the local default (http://localhost:8082)
      # would point at this container itself.
      - OVERRIDE_CONNECTION_GATEWAY_URL=http://connection-gateway:8080
      # In-network lexical address; the local default (http://localhost:8096)
      # would point at this container itself.
      - OVERRIDE_LEXICAL_SERVICE_URL=http://lexical-service:8096
      # In-network sync address; the local default (http://localhost:8787)
      # would point at this container itself, so agent-created documents
      # never get their initial snapshot.
      - OVERRIDE_SYNC_SERVICE_URL=http://sync-service:8787
    volumes:
      # The local container provider drives the host's Docker daemon, so
      # sandboxes are siblings of this container rather than children. Harmless
      # under the Daytona provider, which never opens it.
      - /var/run/docker.sock:/var/run/docker.sock
    depends_on:
      - postgres
      - redis
      - kafka
      - connection_gateway
      - lexical_service
      - fusionauth
    networks:
      # `auth` too: public MCP boot resolves the Google identity provider in
      # FusionAuth, which lives on that network only.
      auth:
      databases:
      services:
        aliases:
          # Hyphens, because a sandbox's git reaches the egress proxy by this
          # name and git percent-encodes `_` in a host before matching
          # `credential.<url>.helper`. Under the compose service name the
          # scoped helper silently never fires and the clone prompts for a
          # password it has no terminal to read.
          - agent-harness-service

  # ============================================================================
  # whatsapp_session_service
  # Personal WhatsApp linked devices: one outbound websocket per account,
  # leased through Postgres advisory locks on a DATABASE_URL connection
  # (the local Postgres; nothing pools it). Nothing calls it, so no host
  # port and no proxy route. OVERRIDE_WHATSAPP_* stay unset: those are for
  # tests against the fake server.
  # ============================================================================
  whatsapp_session_service:
    <<: [*common-env, *rust-services-image]
    command: ["/app/out/whatsapp_session_service"]
    restart: on-failure
    # Drain releases accounts in batches before exiting, as on ECS.
    stop_grace_period: 120s
    depends_on:
      - postgres
    networks:
      databases:
      services:
        aliases:
          - whatsapp-session-service
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

  # ============================================================================
  static_file_service:
    <<: [*common-env, *rust-services-image]
    command: ["/app/out/static_file_service"]
    ports:
      - "8094:8080"
    networks:
      databases:
      services:
        aliases:
          - static-file-service
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/api/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

  # ============================================================================
  # static_file_cdn (Port 8100)
  # Local CloudFront emulation - routes to S3 and static_file_service
  # ============================================================================
  static_file_cdn:
    image: nginx:alpine
    ports:
      - "8100:80"
    volumes:
      - ./infra/local/nginx/static-file-cdn.conf:/etc/nginx/conf.d/default.conf:ro
    depends_on:
      - static_file_service
    networks:
      databases:
      services:
        aliases:
          - static-file-cdn

  # ============================================================================
  # unfurl_service (Port 8095)
  # URL unfurling - no dependencies
  # ============================================================================
  unfurl_service:
    <<: [*common-env, *rust-services-image]
    command: ["/app/out/unfurl_service"]
    ports:
      - "8095:8080"
    networks:
      services:
        aliases:
          - unfurl-service
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

  # ============================================================================
  # image_proxy_service (Port 8097)
  # Proxies external images to avoid CORS issues and IP leakage
  # ============================================================================
  image_proxy_service:
    <<: [*common-env, *rust-services-image]
    command: ["/app/out/image_proxy_service"]
    ports:
      - "8097:8080"
    networks:
      services:
        aliases:
          - image-proxy-service
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

  # ============================================================================
  # websocket_service (Port 6969)
  # Simple WebSocket service - responds with "ping" to all messages
  # ============================================================================
  websocket_service:
    build:
      context: .
      dockerfile: docker/websocket-service.Dockerfile
    ports:
      - "6969:6969"
    networks:
      services:
        aliases:
          - websocket-service

  # ============================================================================
  # sync_service (Port 8787)
  # Document sync service using Cloudflare Workers/Durable Objects (via miniflare)
  # ============================================================================
  sync_service:
    <<: *common-env
    build:
      context: .
      dockerfile: docker/sync-service.Dockerfile
    ports:
      - "8787:8787"
    volumes:
      - ./services/sync-service/src:/app/src
      # Durable Objects, KV, R2 and D1 must survive container replacement.
      - sync_state:/app/.wrangler/state
    environment:
      - INTERNAL_API_SECRET_KEY=${INTERNAL_API_SECRET_KEY}
      - DOCUMENT_PERMISSIONS_SECRET=${DOCUMENT_PERMISSION_JWT}
      - DSS_INTERNAL_AUTH_KEY=${DOCUMENT_STORAGE_SERVICE_AUTH_KEY}
      # This worker exports OTLP/JSON over HTTP, so it needs the
      # HTTP intake (:4318) — NOT the gRPC :4317 endpoint the
      # shared env (env_layer) injects for the .with_tonic() Rust services.
      # Pinning it here overrides the shared env_file value for this service.
      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
    networks:
      services:
        aliases:
          - sync-service
      databases:
    healthcheck:
      test:
        [
          "CMD",
          "node",
          "-e",
          "fetch('http://localhost:8787/health').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))",
        ]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s

  # ============================================================================
  # ai_editing_worker (Port 8933)
  # AI-powered document editing Cloudflare Worker (local wrangler dev)
  # ============================================================================
  ai_editing_worker:
    <<: *common-env
    build:
      context: .
      dockerfile: docker/ai-editing-worker.Dockerfile
    ports:
      - "8933:8933"
    volumes:
      - .:/app
      # .wrangler (incl. the local D1 trace shard) persists to the host via the
      # repository mount above, so `bun run traces --env local --local` works from the
      # host without docker exec. An anonymous volume used to shadow this path.
    depends_on:
      - sync_service
    networks:
      services:
        aliases:
          - ai-editing-worker

  # ============================================================================
  # analytics_proxy (Port 8098)
  # First-party analytics/telemetry proxy (Cloudflare Worker, local wrangler
  # dev). Fronts PostHog and OTLP telemetry so ad blockers don't drop them and
  # the browser never holds the Datadog API key. Locally the
  # /i/otlp route forwards to the otel-collector alias (jaeger/datadog profile);
  # if neither profile is up, trace exports just fail harmlessly.
  # ============================================================================
  analytics_proxy:
    <<: *common-env
    build:
      context: .
      dockerfile: docker/analytics-proxy.Dockerfile
    ports:
      - "8098:8098"
    volumes:
      - .:/app
    networks:
      services:
        aliases:
          - analytics-proxy

  # ============================================================================
  # lexical_service (Port 8096)
  # Lexical document conversion service - converts Lexical JSON to various formats
  # ============================================================================
  lexical_service:
    <<: *common-env
    build:
      context: .
      dockerfile: docker/lexical-service.Dockerfile
      args:
        GITHUB_PACKAGES_TOKEN: ${GITHUB_PACKAGES_TOKEN}
    ports:
      - "8096:8096"
    depends_on:
      - sync_service
    environment:
      - PORT=8096
      - INTERNAL_AUTH_KEY=${INTERNAL_API_SECRET_KEY}
      - SYNC_SERVICE_AUTH_KEY=${INTERNAL_API_SECRET_KEY}
      - SYNC_SERVICE_URL=http://sync-service:8787
    networks:
      services:
        aliases:
          - lexical-service
    healthcheck:
      test: ["CMD", "bun", "-e", "fetch('http://localhost:8096/health').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

  # ============================================================================
  # lgtm (Grafana UI 3001, OTLP 4317/4318) — profile: lgtm
  # Grafana + Tempo (traces) + Loki (logs) + Prometheus (metrics) in one
  # container: the default `--traces` backend. Fully local, no account needed.
  # UI at http://localhost:3001 (anonymous admin). The backing stores are also
  # published for direct API queries (agents/scripts): Tempo 3200, Loki 3100,
  # Prometheus 9090. Binds the same OTLP ports as jaeger/datadog-agent — run
  # exactly one collector at a time.
  # Start: docker compose --project-directory . -f docker/docker-compose.yml up -d lgtm
  # ============================================================================
  lgtm:
    profiles:
      - lgtm
    # Pinned: docker/lgtm/tempo-config.yaml is a copy of this image's internal
    # Tempo config plus one key — an unpinned bump could silently diverge.
    image: grafana/otel-lgtm:0.32.0
    # The all-in-one occasionally loses a component on first boot (observed:
    # Pyroscope readiness flake) and exits; it's the default collector, so let
    # Docker bring it back rather than leaving telemetry dark.
    restart: unless-stopped
    environment:
      # No login wall: agents and humans hit the UI/API without credentials —
      # anonymous requests get Admin, the login form and sign-out menu are
      # gone entirely. Local-only telemetry, nothing to protect.
      GF_AUTH_ANONYMOUS_ENABLED: "true"
      GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin"
      GF_AUTH_DISABLE_LOGIN_FORM: "true"
      GF_AUTH_BASIC_ENABLED: "false"
      GF_AUTH_DISABLE_SIGNOUT_MENU: "true"
      GF_USERS_ALLOW_SIGN_UP: "false"
    ports:
      - "3001:3000"
      - "4317:4317"
      - "4318:4318"
      - "3100:3100"
      - "3200:3200"
      - "9090:9090"
    volumes:
      # Keep telemetry across container restarts — traces/logs are the local
      # debugging record; losing them on a collector bounce defeats the point.
      - lgtm_data:/data
      # The image's Tempo config plus `query_frontend.mcp_server.enabled` —
      # Tempo's MCP server (TraceQL query tools), which mcp-grafana proxies
      # to agents via its proxied-tools discovery.
      - ./docker/lgtm/tempo-config.yaml:/otel-lgtm/tempo-config.yaml:ro,Z
    networks:
      services:
        aliases:
          - otel-collector

  # ============================================================================
  # headless-chrome (CDP port 9222, noVNC 6080) — profile: chrome
  # Remote Chrome for agents: connect over the Chrome DevTools Protocol at
  # http://localhost:9222 (chrome-devtools-mcp, Playwright connectOverCDP,
  # raw CDP), and WATCH what the agent does at http://localhost:6080/vnc.html
  # (Chromium renders to an Xvfb display streamed over noVNC — see
  # docker/headless-chrome/). Runs on the host network so pages can load the
  # host-run frontend dev server and proxy with plain localhost URLs — bridge
  # networking can't reach host ports on firewalled hosts (NixOS default).
  # Global and shared (one per machine), started by
  # `just run_local --with-chrome` / `just stack up --with-chrome`.
  # Start: docker compose --project-directory . -f docker/docker-compose.yml up -d headless-chrome
  # ============================================================================
  headless-chrome:
    profiles:
      - chrome
    build:
      context: .
      dockerfile: docker/headless-chrome/Dockerfile
    image: macro-local-headless-chrome:dev
    restart: unless-stopped
    # Reap orphans from the display stack (x11vnc/websockify) — Chromium is
    # PID 1's exec target and doesn't reap.
    init: true
    network_mode: host
    # Chromium runs sandboxed as a non-root user (no --no-sandbox); the
    # sandbox creates PID/network namespaces, which Docker's default seccomp
    # profile denies to unprivileged processes. SYS_ADMIN lets the setuid
    # sandbox helper do that setup while the default seccomp filter stays on
    # — verified narrower than the seccomp=unconfined alternative.
    cap_add:
      - SYS_ADMIN

  # ============================================================================
  # jaeger (UI port 16686, OTLP 4317/4318) — profile: jaeger
  # Fully local trace viewer, no account needed. UI at http://localhost:16686.
  # Start: docker compose --project-directory . -f docker/docker-compose.yml up -d jaeger
  # ============================================================================
  jaeger:
    profiles:
      - jaeger
    image: jaegertracing/all-in-one:latest
    environment:
      COLLECTOR_OTLP_ENABLED: "true"
    # Browser -> collector requests need CORS on the OTLP HTTP receiver.
    # allowed-headers must include Content-Type: jaeger's default rejects the
    # preflight the browser OTLP exporter sends, even with origins=*.
    command:
      [
        "--collector.otlp.http.cors.allowed-origins=*",
        "--collector.otlp.http.cors.allowed-headers=*",
      ]
    ports:
      - "16686:16686"
      - "4317:4317"
      - "4318:4318"
    networks:
      services:
        aliases:
          - otel-collector

  # ============================================================================
  # datadog-agent (OTLP 4317/4318) — profile: datadog
  # Forwards OTLP traces to Datadog (us5); no local UI. View at
  # us5.datadoghq.com -> APM -> Traces, filtered by env:local (DD_ENV).
  # Binds the same OTLP ports as jaeger — run one or the other, not both.
  # Start: DD_API_KEY=<key> docker compose --project-directory . -f docker/docker-compose.yml up -d datadog-agent
  # ============================================================================
  datadog-agent:
    profiles:
      - datadog
    image: gcr.io/datadoghq/agent:7
    environment:
      # Default-empty (not :?) so the var is only required when this profile
      # actually runs; the agent itself fails fast without a key.
      DD_API_KEY: ${DD_API_KEY:-}
      DD_SITE: us5.datadoghq.com
      DD_ENV: ${DD_ENV:-local}
      DD_HOSTNAME: local-dev
      DD_APM_ENABLED: "true"
    volumes:
      # Mounted config exists to enable CORS on the OTLP HTTP receiver
      # (browser -> agent); that setting has no env-var form.
      - ./docker/datadog-agent/datadog.yaml:/etc/datadog-agent/datadog.yaml:ro
    ports:
      - "4317:4317"
      - "4318:4318"
    networks:
      services:
        aliases:
          - otel-collector

volumes:
  sync_state:
  # Backing store for the `lgtm` collector (profile-gated, global project).
  lgtm_data:

networks:
  services:
    driver: bridge
  databases:
    external: true
  auth:
    external: true