Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

ci: speed up PR checks across Rust and web (#7588)

teo6df5d6bb5bd6
diff --git a/.github/workflows/code_check_cloud_storage.yml b/.github/workflows/code_check_cloud_storage.yml
index 4834e6b750..31dd4c1741 100644
--- a/.github/workflows/code_check_cloud_storage.yml
+++ b/.github/workflows/code_check_cloud_storage.yml
@@ -19,12 +19,12 @@ env:
- main
jobs:
path-check:
- runs-on: namespace-profile-linux-small
+ runs-on: namespace-profile-linux-small;overrides.cache-tag=nextest-filter
outputs:
should_run: ${{ steps.filter.outputs.should_run }}
+ doppler_candidates: ${{ steps.filter.outputs.doppler_candidates }}
rust_packages: ${{ steps.nextest-filter.outputs.rust_packages }}
skip_tests: ${{ steps.nextest-filter.outputs.skip_tests }}
- doppler_config_bins: ${{ steps.doppler-bins.outputs.doppler_config_bins }}
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
@@ -65,6 +65,13 @@ jobs:
- '.github/actions/setup-sccache/**'
- '.github/services-config.json'
- .github/workflows/code_check_cloud_storage.yml
+ doppler_candidates:
+ - '.github/services-config.json'
+ - '**/src/config.rs'
+ - '**/src/doppler_config.rs'
+ - '**/Cargo.toml'
+ - 'tooling/xtask/crates/xtask_doppler_bins/**'
+ - .github/workflows/code_check_cloud_storage.yml
- name: compute changed files
if: steps.filter.outputs.should_run == 'true'
run: |
@@ -77,8 +84,26 @@ jobs:
if [ -z "${GITHUB_BASE_REF:-}" ]; then
compare_rev="$(git rev-parse HEAD~1)"
else
- git fetch origin "$GITHUB_BASE_REF:refs/remotes/origin/$GITHUB_BASE_REF"
- if ! compare_rev="$(git merge-base "origin/${GITHUB_BASE_REF}" HEAD)"; then
+ git fetch --no-tags origin "$GITHUB_BASE_REF:refs/remotes/origin/$GITHUB_BASE_REF"
+
+ # The checkout holds only the PR merge commit and its two parents. Normally
+ # the first parent is on the base branch, so the merge-base is immediate.
+ # Stacked PRs can arrive with a merge commit built on another branch's tip;
+ # deepen the PR side until its history reaches the base branch instead of
+ # falling back to the full suite.
+ compare_rev=""
+ for deepen in 0 200 2000; do
+ if [ "$deepen" -gt 0 ]; then
+ echo "No merge-base with origin/${GITHUB_BASE_REF} yet; deepening PR history by ${deepen}"
+ git fetch --no-tags --deepen="$deepen" origin "$GITHUB_SHA" || break
+ fi
+ if compare_rev="$(git merge-base "origin/${GITHUB_BASE_REF}" HEAD)"; then
+ break
+ fi
+ compare_rev=""
+ done
+
+ if [ -z "$compare_rev" ]; then
echo "Unable to find merge-base for origin/${GITHUB_BASE_REF}; falling back to full test suite" >&2
: > /tmp/changed-files
exit 0
@@ -88,52 +113,87 @@ jobs:
printf '%s\n' "$compare_rev" > /tmp/base-revision
git diff --name-only --no-renames "$compare_rev" "$GITHUB_SHA" > /tmp/changed-files
shell: bash
- - id: doppler-bins
- name: compute affected Doppler config bins
- if: steps.filter.outputs.should_run == 'true'
- run: |
- set -euo pipefail
-
- doppler_config_bins="$(cargo run --manifest-path tooling/xtask/Cargo.toml -- doppler-bins /tmp/changed-files)"
- {
- echo 'doppler_config_bins<<__DOPPLER_CONFIG_BINS__'
- if [ -n "$doppler_config_bins" ]; then
- printf '%s\n' "$doppler_config_bins"
- fi
- echo '__DOPPLER_CONFIG_BINS__'
- } >> "$GITHUB_OUTPUT"
- shell: bash
+ - name: Mount nextest-filter binary cache
+ if: steps.filter.outputs.should_run == 'true' && ((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository)
+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0
+ with:
+ path: /home/runner/.cache/macro-nextest-filter
+ continue-on-error: true
- id: nextest-filter
name: compute nextest package filter
if: steps.filter.outputs.should_run == 'true'
run: |
set -euo pipefail
- # Writes `rust_packages` for the check/test jobs:
- # all — compile and run the whole workspace
- # none — no Rust packages to lint/test (Nix-only, top-level JSON, docs, …)
- # <names> — space-separated workspace packages (changed + reverse deps)
- #
- # `skip_tests=true` keeps clippy on a package set while the live-Postgres
- # test job stays skipped (used for `.sqlx`-only diffs).
+ # Writes for the check/test jobs:
+ # rust_packages — all | none | space-separated packages (changed + reverse deps)
+ # skip_tests — true keeps clippy on rust_packages while the live-Postgres
+ # test job stays skipped (`.sqlx`-only or clippy.toml-only diffs)
#
- # Root Cargo/toolchain/config edits force `all`. Cargo.lock is handled by
- # determinator's old/new graph comparison. flake.nix, the Nix shell action,
- # and this workflow file used to live on the full-suite list and turned every
- # dev-shell tweak into a full-suite rebuild.
+ # Toolchain and cargo/nextest config edits force `all`. Root Cargo.toml and
+ # Cargo.lock go to the xtask: determinator's old/new graph comparison scopes
+ # lockfile, member, and workspace-dependency changes, and the xtask returns
+ # `all` for any other root-manifest edit (profiles, lints, patches, …).
+ # Cross.toml and deny.toml configure other tools and select nothing here.
+ # clippy.toml changes lint results but not test outcomes (handled below).
+ # flake.nix, the Nix shell action, and this workflow file used to live on the
+ # full-suite list and turned every dev-shell tweak into a full-suite rebuild.
+
+ # The filter runs as a release binary cached per source tree: a debug build
+ # spends ~20 s in guppy on this runner, and compiling it took ~40 s more. The
+ # key covers every input of the build, so a hit is exactly what `cargo build`
+ # would produce. FILTER_CACHE_DIR is a Namespace cache volume for same-repo
+ # PRs and a plain scratch directory for forks.
+ filter_bin=""
+ ensure_filter_bin() {
+ [ -n "$filter_bin" ] && return 0
+ local key bin_dir staging
+ key="$(git rev-parse \
+ HEAD:tooling/xtask/crates/xtask_nextest_filter \
+ HEAD:tooling/xtask/crates/xtask_graph \
+ HEAD:tooling/xtask/crates/xtask_paths \
+ HEAD:Cargo.lock \
+ HEAD:rust-toolchain.toml | sha256sum | cut -c1-16)"
+ bin_dir="$FILTER_CACHE_DIR/bin/$key"
+ filter_bin="$bin_dir/xtask_nextest_filter"
+ if [ -x "$filter_bin" ]; then
+ echo "Using cached nextest-filter build $key"
+ touch "$bin_dir"
+ return 0
+ fi
- echo "skip_tests=false" >> "$GITHUB_OUTPUT"
+ echo "Building nextest-filter $key"
+ CARGO_TARGET_DIR="$FILTER_CACHE_DIR/target" \
+ cargo build --release --locked --quiet -p xtask_nextest_filter
+ staging="$(mktemp -d "$FILTER_CACHE_DIR/bin/.staging.XXXXXX")"
+ cp "$FILTER_CACHE_DIR/target/release/xtask_nextest_filter" "$staging/"
+ # Another job may have published the same key meanwhile; either copy works.
+ mv -T "$staging" "$bin_dir" 2>/dev/null || rm -rf "$staging"
+ # Drop builds no PR has used for a week (every Cargo.lock change adds one).
+ find "$FILTER_CACHE_DIR/bin" -mindepth 1 -maxdepth 1 -type d -mtime +7 \
+ -exec rm -rf {} + || true
+ }
+ mkdir -p "$FILTER_CACHE_DIR/bin"
+
+ emit() {
+ echo "rust_packages=$1" >> "$GITHUB_OUTPUT"
+ echo "skip_tests=$2" >> "$GITHUB_OUTPUT"
+ exit 0
+ }
if [ ! -s /tmp/changed-files ]; then
echo "Unknown or empty change set; running all tests"
- echo "rust_packages=all" >> "$GITHUB_OUTPUT"
- exit 0
+ emit all false
fi
- if grep -qE '^(Cargo\.toml|rust-toolchain\.toml|Cross\.toml|clippy\.toml|deny\.toml|\.cargo/.*|\.config/.*)$' /tmp/changed-files; then
- echo "Workspace-level Cargo/toolchain change detected; running all tests"
- echo "rust_packages=all" >> "$GITHUB_OUTPUT"
- exit 0
+ if grep -qE '^(rust-toolchain\.toml|\.cargo/.*|\.config/.*)$' /tmp/changed-files; then
+ echo "Toolchain or cargo config change detected; running all tests"
+ emit all false
+ fi
+
+ clippy_config_changed=false
+ if grep -qx 'clippy\.toml' /tmp/changed-files; then
+ clippy_config_changed=true
fi
# The test job compiles queries against live Postgres (SQLX_OFFLINE is unset
@@ -142,22 +202,33 @@ jobs:
# still compile against the cache rather than skip the check job.
if ! grep -qvE '^\.sqlx/' /tmp/changed-files; then
echo "Only .sqlx changes detected; clippy all, skip live-Postgres tests"
- echo "rust_packages=all" >> "$GITHUB_OUTPUT"
- echo "skip_tests=true" >> "$GITHUB_OUTPUT"
- exit 0
+ emit all true
fi
- packages="$(cargo run --manifest-path tooling/xtask/Cargo.toml -- nextest-filter /tmp/changed-files "$(< /tmp/base-revision)")"
+ ensure_filter_bin
+ packages="$("$filter_bin" /tmp/changed-files "$(< /tmp/base-revision)")"
+
+ # One package list drives both clippy and tests, so a clippy.toml edit can
+ # only skip tests when nothing else selected a package.
+ if [ "$clippy_config_changed" = true ]; then
+ if [ -z "$packages" ] || [ "$packages" = "none" ]; then
+ echo "Only clippy.toml affects Rust; clippy all, skip live-Postgres tests"
+ emit all true
+ fi
+ echo "clippy.toml changed alongside package changes; running all tests"
+ emit all false
+ fi
if [ -z "$packages" ] || [ "$packages" = "none" ]; then
echo "No package-specific Rust changes detected; running no tests"
- echo "rust_packages=none" >> "$GITHUB_OUTPUT"
- exit 0
+ emit none false
fi
echo "rust packages: $packages"
- echo "rust_packages=$packages" >> "$GITHUB_OUTPUT"
+ emit "$packages" false
shell: bash
+ env:
+ FILTER_CACHE_DIR: /home/runner/.cache/macro-nextest-filter
check:
needs:
- path-check
@@ -211,47 +282,6 @@ jobs:
else
echo "::warning::Namespace remote sccache setup failed; using local cache fallback"
fi
- - name: validate Doppler configs
- if: needs.path-check.outputs.doppler_config_bins != ''
- run: |
- set -euo pipefail
-
- if [ -z "${DOPPLER_TOKEN:-}" ]; then
- echo "DOPPLER_TOKEN secret is required to validate Doppler configs" >&2
- exit 1
- fi
-
- if [ -z "${RUSTC_WRAPPER:-}" ]; then
- echo "RUSTC_WRAPPER is required so Doppler config binaries build through sccache" >&2
- exit 1
- fi
-
- bins=()
- cargo_args=(build --locked --all-features)
- while IFS= read -r bin; do
- if [ -z "$bin" ]; then
- continue
- fi
-
- bins+=("$bin")
- cargo_args+=(--bin "$bin")
- done <<< "$DOPPLER_CONFIG_BINS"
-
- if [ "${#bins[@]}" -eq 0 ]; then
- echo "No Doppler config binaries to validate"
- exit 0
- fi
-
- echo "Building affected Doppler config binaries with RUSTC_WRAPPER=$RUSTC_WRAPPER"
- printf ' %s\n' "${bins[@]}"
-
- cargo "${cargo_args[@]}"
- for bin in "${bins[@]}"; do
- "./target/debug/$bin"
- done
- env:
- DOPPLER_CONFIG_BINS: ${{ needs.path-check.outputs.doppler_config_bins }}
- DOPPLER_TOKEN: ${{ secrets.DOPPLER_TOKEN }}
- name: fmt
run: cargo fmt --check
- name: clippy
@@ -306,6 +336,161 @@ jobs:
- name: Teardown Nix
if: always()
uses: './.github/actions/teardown-nix'
+ doppler-config:
+ needs:
+ - path-check
+ if: needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false && needs.path-check.outputs.doppler_candidates == 'true'
+ runs-on: namespace-profile-linux-rust-ci;overrides.cache-tag=sccache-ci
+ env:
+ RUSTFLAGS: '-Dwarnings -Dclippy::disallowed_methods -C link-arg=-fuse-ld=mold'
+ RUSTDOCFLAGS: '-Dwarnings'
+ SQLX_OFFLINE: 'true'
+ steps:
+ - name: Checkout
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+ with:
+ clean: 'false'
+ persist-credentials: 'false'
+ fetch-depth: '2'
+ - name: compute changed files
+ run: |
+ set -euo pipefail
+
+ # Compare against the merge-base so a deleted file still appears. `--no-renames`
+ # turns a rename into a delete+add pair so the old path is attributed to its
+ # package (otherwise git reports only the new name).
+
+ if [ -z "${GITHUB_BASE_REF:-}" ]; then
+ compare_rev="$(git rev-parse HEAD~1)"
+ else
+ git fetch --no-tags origin "$GITHUB_BASE_REF:refs/remotes/origin/$GITHUB_BASE_REF"
+
+ # The checkout holds only the PR merge commit and its two parents. Normally
+ # the first parent is on the base branch, so the merge-base is immediate.
+ # Stacked PRs can arrive with a merge commit built on another branch's tip;
+ # deepen the PR side until its history reaches the base branch instead of
+ # falling back to the full suite.
+ compare_rev=""
+ for deepen in 0 200 2000; do
+ if [ "$deepen" -gt 0 ]; then
+ echo "No merge-base with origin/${GITHUB_BASE_REF} yet; deepening PR history by ${deepen}"
+ git fetch --no-tags --deepen="$deepen" origin "$GITHUB_SHA" || break
+ fi
+ if compare_rev="$(git merge-base "origin/${GITHUB_BASE_REF}" HEAD)"; then
+ break
+ fi
+ compare_rev=""
+ done
+
+ if [ -z "$compare_rev" ]; then
+ echo "Unable to find merge-base for origin/${GITHUB_BASE_REF}; falling back to full test suite" >&2
+ : > /tmp/changed-files
+ exit 0
+ fi
+ fi
+
+ printf '%s\n' "$compare_rev" > /tmp/base-revision
+ git diff --name-only --no-renames "$compare_rev" "$GITHUB_SHA" > /tmp/changed-files
+ shell: bash
+ - name: Mount Namespace cache volume
+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0
+ with:
+ cache: rust
+ path: /nix
+ continue-on-error: true
+ - name: Setup Nix
+ uses: './.github/actions/setup-nix'
+ - name: Setup Nix dev shell
+ uses: './.github/actions/setup-nix-dev-shell'
+ - name: Configure Namespace remote sccache
+ if: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository
+ run: |-
+ set -euo pipefail
+ env_file="$(mktemp "$RUNNER_TEMP/namespace-sccache.XXXXXX")"
+ trap 'rm -f "$env_file"' EXIT
+ if nsc cache sccache setup --cache_name sccache-ci > "$env_file"; then
+ # Register credential values as masked BEFORE exporting: GitHub only
+ # masks `secrets.*`, so without this SCCACHE_WEBDAV_TOKEN — a broad,
+ # ~24h Namespace workspace token (registry + cache write) — printed
+ # verbatim in every subsequent step's env dump. Masking is selective by
+ # key name: masking non-secrets like the endpoint URL or key prefix
+ # would redact those strings everywhere in the logs.
+ while IFS= read -r line; do
+ k="${line%%=*}"
+ v="${line#*=}"
+ [ -n "$v" ] && [ "$k" != "$line" ] || continue
+ case "$k" in
+ *TOKEN*|*SECRET*|*PASSWORD*) echo "::add-mask::$v" ;;
+ esac
+ done < "$env_file"
+ cat "$env_file" >> "$GITHUB_ENV"
+ # Force the next compiler invocation to start a server with the new remote
+ # backend even if a setup hook happened to launch one already.
+ sccache --stop-server >/dev/null 2>&1 || true
+ else
+ echo "::warning::Namespace remote sccache setup failed; using local cache fallback"
+ fi
+ - id: doppler-bins
+ name: compute affected Doppler config bins
+ run: |
+ set -euo pipefail
+
+ doppler_config_bins="$(cargo run --manifest-path tooling/xtask/Cargo.toml -- doppler-bins /tmp/changed-files)"
+ {
+ echo 'doppler_config_bins<<__DOPPLER_CONFIG_BINS__'
+ if [ -n "$doppler_config_bins" ]; then
+ printf '%s\n' "$doppler_config_bins"
+ fi
+ echo '__DOPPLER_CONFIG_BINS__'
+ } >> "$GITHUB_OUTPUT"
+ shell: bash
+ - name: validate Doppler configs
+ if: steps.doppler-bins.outputs.doppler_config_bins != ''
+ run: |
+ set -euo pipefail
+
+ if [ -z "${DOPPLER_TOKEN:-}" ]; then
+ echo "DOPPLER_TOKEN secret is required to validate Doppler configs" >&2
+ exit 1
+ fi
+
+ if [ -z "${RUSTC_WRAPPER:-}" ]; then
+ echo "RUSTC_WRAPPER is required so Doppler config binaries build through sccache" >&2
+ exit 1
+ fi
+
+ bins=()
+ cargo_args=(build --locked --all-features)
+ while IFS= read -r bin; do
+ if [ -z "$bin" ]; then
+ continue
+ fi
+
+ bins+=("$bin")
+ cargo_args+=(--bin "$bin")
+ done <<< "$DOPPLER_CONFIG_BINS"
+
+ if [ "${#bins[@]}" -eq 0 ]; then
+ echo "No Doppler config binaries to validate"
+ exit 0
+ fi
+
+ echo "Building affected Doppler config binaries with RUSTC_WRAPPER=$RUSTC_WRAPPER"
+ printf ' %s\n' "${bins[@]}"
+
+ cargo "${cargo_args[@]}"
+ for bin in "${bins[@]}"; do
+ "./target/debug/$bin"
+ done
+ env:
+ DOPPLER_CONFIG_BINS: ${{ steps.doppler-bins.outputs.doppler_config_bins }}
+ DOPPLER_TOKEN: ${{ secrets.DOPPLER_TOKEN }}
+ - name: show sccache stats
+ if: always()
+ run: sccache --show-stats || true
+ - name: Teardown Nix
+ if: always()
+ uses: './.github/actions/teardown-nix'
test:
needs:
- path-check
@@ -400,7 +585,7 @@ jobs:
done
docker exec "$postgres_container" psql -U user -d macrodb -c "SHOW max_locks_per_transaction;"
- name: prepare tests
- run: just setup_test_envs && just initialize_dbs
+ run: just setup_test_envs && just initialize_dbs && just setup_test_template
- name: run tests
run: |
set -euo pipefail
@@ -454,6 +639,7 @@ jobs:
needs:
- path-check
- check
+ - doppler-config
- test
if: always()
name: Cloud Storage Status Check
@@ -463,11 +649,13 @@ jobs:
run: |
echo "path-check: ${{ needs.path-check.result }}"
echo "check: ${{ needs.check.result }}"
+ echo "doppler-config: ${{ needs.doppler-config.result }}"
echo "test: ${{ needs.test.result }}"
# Fail if any job failed (skipped and success are both OK)
if [[ "${{ needs.path-check.result }}" == "failure" ]] || \
[[ "${{ needs.check.result }}" == "failure" ]] || \
+ [[ "${{ needs.doppler-config.result }}" == "failure" ]] || \
[[ "${{ needs.test.result }}" == "failure" ]]; then
echo "❌ One or more jobs failed"
exit 1
diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml
deleted file mode 100644
index 6d8a75c72a..0000000000
--- a/.github/workflows/docs-check.yml
+++ /dev/null
@@ -1,84 +0,0 @@
-# DO NOT EDIT — regenerate with `cargo x workflows` (from the repository root).
-# Source: tooling/xtask/crates/xtask_workflows/src/workflows/docs_check.rs
-name: Docs Check
-'on':
- pull_request:
- branches:
- - main
- paths:
- - apps/docs/**
- - crates/**/*.rs
- - Cargo.toml
- - Cargo.lock
- - '.github/workflows/docs-check.yml'
-jobs:
- check-docs:
- name: Docs Generated Code Check
- runs-on: namespace-profile-linux-mid;overrides.cache-tag=web-ci
- steps:
- - name: Checkout
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
- with:
- clean: 'false'
- persist-credentials: 'false'
- - name: Mount Namespace cache volume
- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0
- with:
- cache: nix
- path: |-
- /home/runner/.bun/install/cache
- /home/runner/.cargo/registry
- /home/runner/.cargo/git
- continue-on-error: true
- - name: Setup Rust
- uses: './.github/actions/setup-rust'
- with:
- sccache: 'true'
- rust-cache: 'false'
- - name: Setup Bun
- uses: oven-sh/setup-bun@v2
- - name: Configure Namespace remote sccache
- if: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository
- run: |-
- set -euo pipefail
- env_file="$(mktemp "$RUNNER_TEMP/namespace-sccache.XXXXXX")"
- trap 'rm -f "$env_file"' EXIT
- if nsc cache sccache setup --cache_name web-ci > "$env_file"; then
- # Register credential values as masked BEFORE exporting: GitHub only
- # masks `secrets.*`, so without this SCCACHE_WEBDAV_TOKEN — a broad,
- # ~24h Namespace workspace token (registry + cache write) — printed
- # verbatim in every subsequent step's env dump. Masking is selective by
- # key name: masking non-secrets like the endpoint URL or key prefix
- # would redact those strings everywhere in the logs.
- while IFS= read -r line; do
- k="${line%%=*}"
- v="${line#*=}"
- [ -n "$v" ] && [ "$k" != "$line" ] || continue
- case "$k" in
- *TOKEN*|*SECRET*|*PASSWORD*) echo "::add-mask::$v" ;;
- esac
- done < "$env_file"
- cat "$env_file" >> "$GITHUB_ENV"
- # Force the next compiler invocation to start a server with the new remote
- # backend even if a setup hook happened to launch one already.
- sccache --stop-server >/dev/null 2>&1 || true
- else
- echo "::warning::Namespace remote sccache setup failed; using local cache fallback"
- fi
- - name: Regenerate MCP tool pages
- run: bun run generate:tools
- working-directory: apps/docs
- - name: show sccache stats
- if: always()
- run: sccache --show-stats || true
- - name: Verify generated docs are fresh
- run: |
- if [ -n "$(git status --porcelain -- apps/docs)" ]; then
- echo "apps/docs tool reference is stale. Run 'bun run generate:tools' in apps/docs and commit the result."
- git status --porcelain -- apps/docs
- git diff -- apps/docs | head -200
- exit 1
- fi
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}-check
- cancel-in-progress: true
diff --git a/.github/workflows/sdk-check.yml b/.github/workflows/sdk-check.yml
index 24d40616dc..cb857e8c57 100644
--- a/.github/workflows/sdk-check.yml
+++ b/.github/workflows/sdk-check.yml
@@ -7,11 +7,6 @@ name: SDK Check
- main
paths:
- packages/sdk/**
- - crates/**/*.rs
- - Cargo.toml
- - Cargo.lock
- - apps/web/scripts/generate-api-schema.ts
- - apps/web/scripts/services.ts
- '.github/workflows/sdk-check.yml'
push:
branches:
@@ -45,74 +40,6 @@ jobs:
- name: Check endpoint coverage
run: bun run coverage
working-directory: packages/sdk
- check-sdk:
- if: ${{ github.event_name == 'pull_request' }}
- name: SDK Generated Code Check
- runs-on: namespace-profile-linux-mid;overrides.cache-tag=web-ci
- steps:
- - name: Checkout
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
- with:
- clean: 'false'
- persist-credentials: 'false'
- - name: Mount Namespace cache volume
- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0
- with:
- cache: nix
- path: |-
- /home/runner/.bun/install/cache
- /home/runner/.cargo/registry
- /home/runner/.cargo/git
- continue-on-error: true
- - name: Setup Nix
- uses: './.github/actions/setup-nix'
- - name: Setup Prereqs
- uses: './.github/actions/setup-reqs-web'
- - name: Configure Namespace remote sccache
- if: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository
- run: |-
- set -euo pipefail
- env_file="$(mktemp "$RUNNER_TEMP/namespace-sccache.XXXXXX")"
- trap 'rm -f "$env_file"' EXIT
- if nsc cache sccache setup --cache_name web-ci > "$env_file"; then
- # Register credential values as masked BEFORE exporting: GitHub only
- # masks `secrets.*`, so without this SCCACHE_WEBDAV_TOKEN — a broad,
- # ~24h Namespace workspace token (registry + cache write) — printed
- # verbatim in every subsequent step's env dump. Masking is selective by
- # key name: masking non-secrets like the endpoint URL or key prefix
- # would redact those strings everywhere in the logs.
- while IFS= read -r line; do
- k="${line%%=*}"
- v="${line#*=}"
- [ -n "$v" ] && [ "$k" != "$line" ] || continue
- case "$k" in
- *TOKEN*|*SECRET*|*PASSWORD*) echo "::add-mask::$v" ;;
- esac
- done < "$env_file"
- cat "$env_file" >> "$GITHUB_ENV"
- # Force the next compiler invocation to start a server with the new remote
- # backend even if a setup hook happened to launch one already.
- sccache --stop-server >/dev/null 2>&1 || true
- else
- echo "::warning::Namespace remote sccache setup failed; using local cache fallback"
- fi
- - name: Regenerate SDK code
- run: just update-generated
- working-directory: packages/sdk
- - name: show sccache stats
- if: always()
- run: sccache --show-stats || true
- - name: Verify generated code is fresh
- run: |
- if [ -n "$(git status --porcelain -- packages/sdk)" ]; then
- echo "packages/sdk generated code is stale. Run 'just update-generated' in packages/sdk and commit the result."
- git status --porcelain -- packages/sdk
- git diff -- packages/sdk | head -200
- exit 1
- fi
- - name: Teardown Nix
- if: always()
- uses: './.github/actions/teardown-nix'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-check
cancel-in-progress: true
diff --git a/.github/workflows/web-app-check-main.yml b/.github/workflows/web-app-check-main.yml
index 02af6c9f61..142cf8e024 100644
--- a/.github/workflows/web-app-check-main.yml
+++ b/.github/workflows/web-app-check-main.yml
@@ -15,7 +15,7 @@ jobs:
runs-on: namespace-profile-linux-tiny-no-cache
outputs:
should_run: ${{ steps.filter.outputs.should_run }}
- api_changed: ${{ steps.filter.outputs.api_changed }}
+ codegen_changed: ${{ steps.filter.outputs.codegen_changed }}
steps:
- name: Checkout Repo
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10
@@ -45,21 +45,63 @@ jobs:
- 'services/lexical-service/**'
- 'crates/ai_tools/src/display_results/schema.generated.json'
- '.github/actions/setup-reqs-web/**'
- api_changed:
+ codegen_changed:
- 'crates/**/*.rs'
- 'services/**/*.rs'
- 'Cargo.toml'
- 'Cargo.lock'
- 'apps/web/scripts/generate-api-schema.ts'
- 'apps/web/scripts/services.ts'
+ - 'packages/sdk/**'
+ - 'apps/docs/**'
- '.github/actions/setup-reqs-web/**'
typescript:
needs:
- path-check
- if: needs.path-check.outputs.should_run == 'true' || needs.path-check.outputs.api_changed == 'true'
+ if: needs.path-check.outputs.should_run == 'true'
name: Typecheck
runs-on: namespace-profile-linux-mid;overrides.cache-tag=web-ci
steps:
+ - name: Checkout Repo
+ uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10
+ - name: Mount Namespace cache volume
+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0
+ with:
+ cache: nix
+ path: /home/runner/.bun/install/cache
+ continue-on-error: true
+ - name: Setup Nix
+ uses: './.github/actions/setup-nix'
+ - name: Setup Prereqs
+ uses: './.github/actions/setup-reqs-web'
+ - name: Check Dynamic UI Schema
+ run: bun run check-dynamic-ui-schema
+ working-directory: apps/web
+ - name: Check Types
+ run: bun run type-check
+ working-directory: apps/web
+ - name: Check Collaboration Package Types
+ run: bun run type-check
+ working-directory: packages/collaboration
+ - name: Check Lexical Service Types
+ run: bun run check
+ working-directory: services/lexical-service
+ - name: Test Lexical Service Endpoints
+ run: bun test src
+ working-directory: services/lexical-service
+ - name: Teardown Nix
+ if: always()
+ uses: './.github/actions/teardown-nix'
+ generated-code:
+ needs:
+ - path-check
+ if: needs.path-check.outputs.codegen_changed == 'true'
+ name: Generated Code Check
+ runs-on: namespace-profile-linux-mid;overrides.cache-tag=web-ci
+ env:
+ CARGO_PROFILE_DEV_DEBUG: '0'
+ RUSTFLAGS: '-C link-arg=-fuse-ld=mold'
+ steps:
- name: Checkout Repo
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10
- name: Mount Namespace cache volume
@@ -76,7 +118,7 @@ jobs:
- name: Setup Prereqs
uses: './.github/actions/setup-reqs-web'
- name: Configure Namespace remote sccache
- if: ((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && (needs.path-check.outputs.api_changed == 'true')
+ if: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository
run: |-
set -euo pipefail
env_file="$(mktemp "$RUNNER_TEMP/namespace-sccache.XXXXXX")"
@@ -103,12 +145,40 @@ jobs:
else
echo "::warning::Namespace remote sccache setup failed; using local cache fallback"
fi
+ - name: Build schema binaries
+ run: |
+ set -euo pipefail
+ read -ra openapi_bins <<< "$(cd apps/web && bun scripts/generate-api-schema.ts --print-cargo-bins)"
+ SQLX_OFFLINE=true cargo build "${openapi_bins[@]}" --bin gen_tool_schemas
- name: Generate API Types
- if: needs.path-check.outputs.api_changed == 'true'
run: bun run gen-api -- --check
working-directory: apps/web
+ - name: Regenerate SDK code
+ run: |
+ bun install --frozen-lockfile
+ bun run sync-specs
+ bun run generate
+ working-directory: packages/sdk
+ - name: Verify SDK generated code is fresh
+ run: |
+ if [ -n "$(git status --porcelain -- packages/sdk)" ]; then
+ echo "packages/sdk generated code is stale. Run 'just update-generated' in packages/sdk and commit the result."
+ git status --porcelain -- packages/sdk
+ git diff -- packages/sdk | head -200
+ exit 1
+ fi
+ - name: Regenerate MCP tool pages
+ run: bun run generate:tools
+ working-directory: apps/docs
+ - name: Verify docs tool reference is fresh
+ run: |
+ if [ -n "$(git status --porcelain -- apps/docs)" ]; then
+ echo "apps/docs tool reference is stale. Run 'bun run generate:tools' in apps/docs and commit the result."
+ git status --porcelain -- apps/docs
+ git diff -- apps/docs | head -200
+ exit 1
+ fi
- name: Check Specta Types
- if: needs.path-check.outputs.api_changed == 'true'
run: |
just gen-agent-fold-types
just gen-database-sql-types
@@ -118,23 +188,8 @@ jobs:
fi
working-directory: apps/web
- name: show sccache stats
- if: always() && needs.path-check.outputs.api_changed == 'true'
+ if: always()
run: sccache --show-stats || true
- - name: Check Dynamic UI Schema
- run: bun run check-dynamic-ui-schema
- working-directory: apps/web
- - name: Check Types
- run: bun run --bun --silent tsc --project ./tsconfig.json
- working-directory: apps/web
- - name: Check Collaboration Package Types
- run: bun run type-check
- working-directory: packages/collaboration
- - name: Check Lexical Service Types
- run: bun run check
- working-directory: services/lexical-service
- - name: Test Lexical Service Endpoints
- run: bun test src
- working-directory: services/lexical-service
- name: Teardown Nix
if: always()
uses: './.github/actions/teardown-nix'
@@ -172,7 +227,7 @@ jobs:
- path-check
if: needs.path-check.outputs.should_run == 'true'
name: Test
- runs-on: namespace-profile-linux-small;overrides.cache-tag=web-ci
+ runs-on: namespace-profile-linux-mid;overrides.cache-tag=web-ci
steps:
- name: Checkout Repo
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10
@@ -180,7 +235,9 @@ jobs:
uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0
with:
cache: nix
- path: /home/runner/.bun/install/cache
+ path: |-
+ /home/runner/.bun/install/cache
+ /home/runner/.cache/vitest-modules
continue-on-error: true
- name: Setup Nix
uses: './.github/actions/setup-nix'
@@ -193,7 +250,9 @@ jobs:
timeout-minutes: 15
working-directory: apps/web
- name: Test
- run: bunx vitest
+ run: bun run test
+ env:
+ VITEST_MODULE_CACHE_DIR: /home/runner/.cache/vitest-modules
working-directory: apps/web
- name: Sign-up Browser Tests
run: just test-signup-browser
@@ -246,6 +305,7 @@ jobs:
/home/runner/.cargo/registry
/home/runner/.cargo/git
/home/runner/.cache/.wasm-pack
+ /home/runner/.cache/wasm-pkgs
continue-on-error: true
- name: Setup Nix
uses: './.github/actions/setup-nix'
@@ -283,6 +343,8 @@ jobs:
run: sccache --start-server
- name: Build
run: just build-dev
+ env:
+ WASM_PKG_CACHE_DIR: /home/runner/.cache/wasm-pkgs
working-directory: apps/web
- name: show sccache stats
if: always()
@@ -294,6 +356,7 @@ jobs:
needs:
- path-check
- typescript
+ - generated-code
- biome-check
- test
- cycles
@@ -306,6 +369,7 @@ jobs:
run: |
echo "path-check: ${{ needs.path-check.result }}"
echo "typescript: ${{ needs.typescript.result }}"
+ echo "generated-code: ${{ needs.generated-code.result }}"
echo "biome-check: ${{ needs.biome-check.result }}"
echo "test: ${{ needs.test.result }}"
echo "cycles: ${{ needs.cycles.result }}"
@@ -314,6 +378,7 @@ jobs:
# Fail if any job failed (skipped and success are both OK)
if [[ "${{ needs.path-check.result }}" == "failure" ]] || \
[[ "${{ needs.typescript.result }}" == "failure" ]] || \
+ [[ "${{ needs.generated-code.result }}" == "failure" ]] || \
[[ "${{ needs.biome-check.result }}" == "failure" ]] || \
[[ "${{ needs.test.result }}" == "failure" ]] || \
[[ "${{ needs.cycles.result }}" == "failure" ]] || \
diff --git a/Cargo.lock b/Cargo.lock
index c3c2ae9fc5..79839c422b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -21812,6 +21812,7 @@ dependencies = [
"determinator",
"guppy",
"tempfile",
+ "toml 0.8.23",
"xtask_graph",
"xtask_paths",
]
diff --git a/Cargo.toml b/Cargo.toml
index d5db97c841..725a5d921d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -437,6 +437,19 @@ ammonia = "4.1.4"
html-escape = "0.2"
subtle = "2.6.1"
+# cache-turso's storage tests load thousands of rows into the embedded SQL engine
+# to assert VM step budgets. Unoptimized, the engine made them the slowest tests
+# in the Rust suite (34 s for the crate); at opt-level 2 the crate's tests take
+# 2.6 s. The assertions count VM steps, which optimization does not change.
+[profile.dev.package.turso_core]
+opt-level = 2
+
+[profile.dev.package.turso_parser]
+opt-level = 2
+
+[profile.dev.package.turso_ext]
+opt-level = 2
+
# Retain Rust DWARF for local cache profiling while keeping release-level LLVM
# optimization. wasm-pack's custom profile metadata prevents the final
# wasm-opt pass from discarding or rewriting these locations.
diff --git a/apps/web/justfile b/apps/web/justfile
index 7005674673..af00c8ad3d 100644
--- a/apps/web/justfile
+++ b/apps/web/justfile
@@ -244,7 +244,7 @@ cycles-ci:
# (gitignored). Unconditional — CI app builds must never skip it (schema or
# engine changes alter the wasm without a cache-wasm version bump).
build-cache-wasm:
- CARGO_TARGET_DIR="{{cache_wasm_target}}" wasm-pack build {{justfile_directory()}}/../../crates/client/cache-wasm --target web --release --out-dir {{justfile_directory()}}/src/lib/graphql-cache/wasm
+ scripts/cached-wasm-build.sh cache-wasm {{justfile_directory()}}/src/lib/graphql-cache/wasm -- env CARGO_TARGET_DIR="{{cache_wasm_target}}" wasm-pack build {{justfile_directory()}}/../../crates/client/cache-wasm --target web --release --out-dir {{justfile_directory()}}/src/lib/graphql-cache/wasm
# Build the production browser harness served by Vite preview.
build-cache-wasm-browser-production: build-cache-wasm build-cache-wasm-browser-test-hooks
@@ -278,7 +278,7 @@ build-cache-wasm-profile:
# (gitignored). `--no-default-features` drops the crate's `cli` feature, whose
# tokio/clap do not build for wasm32; the fold itself needs neither.
build-agent-fold-wasm:
- CARGO_TARGET_DIR="{{agent_fold_wasm_target}}" wasm-pack build {{justfile_directory()}}/../../crates/folds/agent_fold --target web --release --out-dir {{justfile_directory()}}/src/lib/core/agent-fold/wasm -- --no-default-features
+ scripts/cached-wasm-build.sh agent_fold {{justfile_directory()}}/src/lib/core/agent-fold/wasm -- env CARGO_TARGET_DIR="{{agent_fold_wasm_target}}" wasm-pack build {{justfile_directory()}}/../../crates/folds/agent_fold --target web --release --out-dir {{justfile_directory()}}/src/lib/core/agent-fold/wasm -- --no-default-features
# Dev fast path: build the agent-fold wasm pkg only when it is missing or any
# fold source is newer than the built artifact. A stale artifact folds with the
@@ -295,7 +295,7 @@ ensure-agent-fold-wasm:
# Build the database-sql wasm module (the SQL engine's staged `Query`) into
# src/lib/core/database-sql/wasm (gitignored).
build-database-sql-wasm:
- CARGO_TARGET_DIR="{{database_sql_wasm_target}}" wasm-pack build {{justfile_directory()}}/../../crates/database_sql --target web --release --out-dir {{justfile_directory()}}/src/lib/core/database-sql/wasm
+ scripts/cached-wasm-build.sh database_sql {{justfile_directory()}}/src/lib/core/database-sql/wasm -- env CARGO_TARGET_DIR="{{database_sql_wasm_target}}" wasm-pack build {{justfile_directory()}}/../../crates/database_sql --target web --release --out-dir {{justfile_directory()}}/src/lib/core/database-sql/wasm
# Dev fast path: build the database-sql wasm pkg only when it is missing or
# any source it is compiled from is newer than the built artifact: the
@@ -324,7 +324,7 @@ ensure-database-sql-wasm:
# worker by the pptx block) into src/lib/core/pptx-engine/wasm (gitignored).
# SIMD speeds up rasterization; every browser the app supports has it.
build-pptx-engine-wasm:
- CARGO_TARGET_DIR="{{pptx_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/pptx_engine --target web --release --out-dir {{justfile_directory()}}/src/lib/core/pptx-engine/wasm
+ scripts/cached-wasm-build.sh pptx_engine {{justfile_directory()}}/src/lib/core/pptx-engine/wasm -- env CARGO_TARGET_DIR="{{pptx_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/pptx_engine --target web --release --out-dir {{justfile_directory()}}/src/lib/core/pptx-engine/wasm
# Dev fast path: build the pptx-engine wasm pkg only when it is missing or any
# engine source is newer than the built artifact.
@@ -340,7 +340,7 @@ ensure-pptx-engine-wasm:
# Build the docx-engine wasm module (DOCX layout, rendering and editing, hosted
# in a worker by the write block) into src/lib/core/docx-engine/wasm (gitignored).
build-docx-engine-wasm:
- CARGO_TARGET_DIR="{{docx_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/docx_engine --target web --release --out-dir {{justfile_directory()}}/src/lib/core/docx-engine/wasm
+ scripts/cached-wasm-build.sh docx_engine {{justfile_directory()}}/src/lib/core/docx-engine/wasm -- env CARGO_TARGET_DIR="{{docx_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/docx_engine --target web --release --out-dir {{justfile_directory()}}/src/lib/core/docx-engine/wasm
# Dev fast path: build the docx-engine wasm pkg only when it is missing or any
# engine source (including the pptx engine it builds on) is newer than it.
@@ -357,7 +357,7 @@ ensure-docx-engine-wasm:
# Build the fig-engine wasm module (.fig parsing and tile rasterization, hosted
# in workers by the fig block) into src/lib/core/fig-engine/wasm (gitignored).
build-fig-engine-wasm:
- CARGO_TARGET_DIR="{{fig_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/fig_engine --target web --profile fig-engine-wasm --out-dir {{justfile_directory()}}/src/lib/core/fig-engine/wasm
+ scripts/cached-wasm-build.sh fig_engine {{justfile_directory()}}/src/lib/core/fig-engine/wasm -- env CARGO_TARGET_DIR="{{fig_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/fig_engine --target web --profile fig-engine-wasm --out-dir {{justfile_directory()}}/src/lib/core/fig-engine/wasm
# Dev fast path: build the fig-engine wasm pkg only when it is missing or any
# engine source is newer than the built artifact.
@@ -374,7 +374,7 @@ ensure-fig-engine-wasm:
# compositing, hosted in workers by the psd block) into
# src/lib/core/psd-engine/wasm (gitignored).
build-psd-engine-wasm:
- CARGO_TARGET_DIR="{{psd_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/psd_engine --target web --profile psd-engine-wasm --out-dir {{justfile_directory()}}/src/lib/core/psd-engine/wasm
+ scripts/cached-wasm-build.sh psd_engine {{justfile_directory()}}/src/lib/core/psd-engine/wasm -- env CARGO_TARGET_DIR="{{psd_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/psd_engine --target web --profile psd-engine-wasm --out-dir {{justfile_directory()}}/src/lib/core/psd-engine/wasm
# Dev fast path: build the psd-engine wasm pkg only when it is missing or any
# engine source (or the fig engine it draws text with) is newer than it.
@@ -392,7 +392,7 @@ ensure-psd-engine-wasm:
# rendering, hosted in workers by the ai block) into
# src/lib/core/ai-engine/wasm (gitignored).
build-ai-engine-wasm:
- CARGO_TARGET_DIR="{{ai_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/ai_engine --target web --profile ai-engine-wasm --out-dir {{justfile_directory()}}/src/lib/core/ai-engine/wasm
+ scripts/cached-wasm-build.sh ai_engine {{justfile_directory()}}/src/lib/core/ai-engine/wasm -- env CARGO_TARGET_DIR="{{ai_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/ai_engine --target web --profile ai-engine-wasm --out-dir {{justfile_directory()}}/src/lib/core/ai-engine/wasm
# Dev fast path: build the ai-engine wasm pkg only when it is missing or any
# engine source (or the fig engine it draws text with) is newer than it.
diff --git a/apps/web/knip.ts b/apps/web/knip.ts
index 74880e5eb8..61ee34d549 100644
--- a/apps/web/knip.ts
+++ b/apps/web/knip.ts
@@ -60,6 +60,8 @@ const config: KnipConfig = {
'libheif-js',
'@tailwindcss/vite',
'bebop',
+ // TypeScript 7's compiler, run by path from `type-check`.
+ 'typescript-native',
],
ignoreWorkspaces: ['../../packages/loro-mirror', '../../packages/sdk'],
diff --git a/apps/web/package.json b/apps/web/package.json
index b14c6ee532..53641f592c 100644
--- a/apps/web/package.json
+++ b/apps/web/package.json
@@ -10,7 +10,7 @@
"build": "vite build -c vite.config.ts && bun run postbuild:semver && bun run postbuild:bundle-manifest",
"postbuild:semver": "echo $npm_package_version+$(git rev-parse --short HEAD) > ./dist/semver.txt",
"postbuild:bundle-manifest": "bun ./scripts/write-bundle-manifest.mjs",
- "test": "bunx vitest run",
+ "test": "bunx vitest run --fsModuleCache",
"native:e2e:build": "bash tests/native/build.sh",
"native:e2e": "bash tests/native/run.sh",
"native:e2e:smoke": "bash tests/native/run.sh --smoke",
@@ -31,7 +31,7 @@
"ui-audit": "bun scripts/ui-audit.ts",
"gen-agent-fold-types": "just gen-agent-fold-types",
"gen-database-sql-types": "just gen-database-sql-types",
- "type-check": "tsc --noEmit --skipLibCheck --project tsconfig.json",
+ "type-check": "bun ../../node_modules/typescript-native/bin/tsc --noEmit --skipLibCheck --project tsconfig.json",
"check": "bun check-graphql-cache-schema && bun check-dynamic-ui-schema && bun type-check && bun biome check",
"preview": "vite preview -c vite.config.ts --outDir dist --base /app",
"preview:dev": "vite preview -c vite.config.ts --outDir dist --base /app",
@@ -219,6 +219,7 @@
"orval": "^7.21.0",
"typedoc": "^0.22.18",
"typescript": "^5.9.3",
+ "typescript-native": "npm:[email protected]",
"vite": "^6.4.1",
"vite-plugin-solid": "^2.11.8",
"vite-plugin-solid-svg": "^0.8.1",
diff --git a/apps/web/scripts/cached-wasm-build.sh b/apps/web/scripts/cached-wasm-build.sh
new file mode 100755
index 0000000000..0228c519f9
--- /dev/null
+++ b/apps/web/scripts/cached-wasm-build.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Runs a wasm package build, or restores its output from WASM_PKG_CACHE_DIR when
+# nothing the package is built from has changed. Without WASM_PKG_CACHE_DIR it
+# only runs the build, so local `just build-*` behaves as before.
+#
+# The cache key hashes the contents of the crate's workspace dependency closure
+# (.github/workspace-dep-closures.json), the root manifests and lockfile, the
+# toolchain pins, the shared assets build scripts read, and the build command.
+#
+# usage: cached-wasm-build.sh <closure package> <out dir> -- <build command...>
+set -euo pipefail
+
+package="$1"
+out_dir="$2"
+shift 2
+if [ "${1:-}" = "--" ]; then
+ shift
+fi
+
+if [ -z "${WASM_PKG_CACHE_DIR:-}" ]; then
+ exec "$@"
+fi
+
+root="$(git rev-parse --show-toplevel)"
+mapfile -t dirs < <(jq -r --arg p "$package" '.closures[$p] // empty | .[]' \
+ "$root/.github/workspace-dep-closures.json")
+if [ "${#dirs[@]}" -eq 0 ]; then
+ echo "no dependency closure for $package; building without the cache" >&2
+ exec "$@"
+fi
+
+key="$(
+ {
+ (cd "$root" && git ls-files -z -- "${dirs[@]}" Cargo.toml Cargo.lock \
+ rust-toolchain.toml flake.lock .cargo static_assets | xargs -0 sha256sum)
+ printf '%s\n' "$@"
+ } | sha256sum | cut -c1-32
+)"
+entry="$WASM_PKG_CACHE_DIR/$package-$key"
+
+if [ -d "$entry" ]; then
+ echo "$package: unchanged inputs, reusing the cached wasm package ($key)"
+ rm -rf "$out_dir"
+ mkdir -p "$(dirname "$out_dir")"
+ cp -a "$entry" "$out_dir"
+ touch "$entry"
+ exit 0
+fi
+
+"$@"
+
+mkdir -p "$WASM_PKG_CACHE_DIR"
+# Entries are small and keyed by content; drop ones no build has used for a week.
+find "$WASM_PKG_CACHE_DIR" -mindepth 1 -maxdepth 1 -mtime +7 -exec rm -rf {} + || true
+tmp="$(mktemp -d "$WASM_PKG_CACHE_DIR/.tmp-XXXXXX")"
+cp -a "$out_dir/." "$tmp/"
+# A concurrent job may have stored the same key first; keep that one.
+mv -T "$tmp" "$entry" 2>/dev/null || rm -rf "$tmp"
diff --git a/apps/web/scripts/generate-api-schema.ts b/apps/web/scripts/generate-api-schema.ts
index 6bd9f881f3..3b7308ac30 100644
--- a/apps/web/scripts/generate-api-schema.ts
+++ b/apps/web/scripts/generate-api-schema.ts
@@ -2,6 +2,7 @@
// bun run scripts/generate-api-schema.ts <service-name> // generate for specific service
// bun run scripts/generate-api-schema.ts // generate for all services
// bun run scripts/generate-api-schema.ts --check // verify types are up to date (for CI)
+// bun run scripts/generate-api-schema.ts --print-cargo-bins // print the `--bin` args phase 1 builds (for CI)
//
// This script generates OpenAPI schemas by running the local Rust binaries
// instead of fetching from deployed services.
@@ -37,7 +38,12 @@ const getServiceClientsDir = () =>
path.resolve(import.meta.dirname, "../src/lib/service-clients");
// Parse arguments
const getTargetServices = () =>
- process.argv.slice(2).filter((arg) => arg !== "--check");
+ process.argv
+ .slice(2)
+ .filter((arg) => arg !== "--check" && arg !== "--print-cargo-bins");
+
+const openApiBinArgs = (crateNames: string[]) =>
+ crateNames.flatMap((crate) => ["--bin", `${crate}_openapi`]);
// Build all OpenAPI binaries in a single cargo invocation (cargo parallelizes internally).
async function buildOpenApiBinaries(
@@ -46,7 +52,7 @@ async function buildOpenApiBinaries(
): Promise<void> {
if (crateNames.length === 0) return;
- const binArgs = crateNames.flatMap((crate) => ["--bin", `${crate}_openapi`]);
+ const binArgs = openApiBinArgs(crateNames);
console.log(`Building ${crateNames.length} OpenAPI binaries...`);
console.log(`cargo build args: ${binArgs.join(" ")}`);
@@ -216,6 +222,13 @@ async function main() {
.map((s) => serviceToCrate[s.name])
.filter((crate): crate is string => !!crate);
+ // CI builds these together with other schema binaries in one cargo
+ // invocation, so the phase 1 build below finds them fresh.
+ if (process.argv.includes("--print-cargo-bins")) {
+ console.log(openApiBinArgs(crateNames).join(" "));
+ return;
+ }
+
console.log(`\nProcessing ${servicesToProcess.length} service(s)...\n`);
// Phase 1: Build all binaries (skipped when OPENAPI_BINS_DIR is set)
diff --git a/apps/web/scripts/generate-dcs-tools.ts b/apps/web/scripts/generate-dcs-tools.ts
index 23a0dd98bc..6d8d2150dd 100644
--- a/apps/web/scripts/generate-dcs-tools.ts
+++ b/apps/web/scripts/generate-dcs-tools.ts
@@ -59,7 +59,10 @@ type FrontendSchemas = z.infer<typeof FrontendSchemasValidator>;
async function buildAndRunSchemaGenerator(): Promise<void> {
console.log('Building gen_tool_schemas binary...');
- await $`cd ${rustWorkspaceDir} && SQLX_OFFLINE=true cargo build --package ai_tools --bin gen_tool_schemas`;
+ // No `--package`: like the OpenAPI bins and apps/docs' generator, resolve
+ // features across the workspace, so one build serves every caller instead
+ // of `--package ai_tools` compiling a second feature set of its deps.
+ await $`cd ${rustWorkspaceDir} && SQLX_OFFLINE=true cargo build --bin gen_tool_schemas`;
console.log('Build complete.');
await $`rm -rf ${path.join(aiToolsDir, 'schemas')}`.quiet();
diff --git a/apps/web/src/features/block-spreadsheet/README.md b/apps/web/src/features/block-spreadsheet/README.md
index f764c73799..74cb39a72c 100644
--- a/apps/web/src/features/block-spreadsheet/README.md
+++ b/apps/web/src/features/block-spreadsheet/README.md
@@ -463,13 +463,15 @@ group, SmartArt and an EMF, as Excel writes them) and corpus workbooks. Exports
which draws the same charts (surface charts aside, which it cannot draw) and
rebuilds pivot tables from edited data.
-`core/xlsx-corpus.test.ts` imports, recalculates, exports and reimports the
-real-world workbooks in `core/xlsx-fixtures/real-world/` (finance models,
-government statistics and open-source test suites; provenance and licenses are in
-its `manifest.json`). Its snapshot records counts, warnings, how many formulas
-recalculate to Excel's cached results, and the first round-trip difference, so a
-change in fidelity shows up in review. Set `XLSX_CORPUS_VERBOSE=1` to print
-mismatch examples.
+`core/xlsx-corpus-1.test.ts` through `core/xlsx-corpus-4.test.ts` import,
+recalculate, export and reimport the real-world workbooks in
+`core/xlsx-fixtures/real-world/` (finance models, government statistics and
+open-source test suites; provenance and licenses are in its `manifest.json`).
+Each file runs one slice of the corpus through `core/xlsx-corpus-suite.ts`, so
+the workbooks run on several Vitest workers. Their snapshots record counts,
+warnings, how many formulas recalculate to Excel's cached results, and the first
+round-trip difference, so a change in fidelity shows up in review. Set
+`XLSX_CORPUS_VERBOSE=1` to print mismatch examples.
## Verification
diff --git a/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-1.test.ts.snap b/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-1.test.ts.snap
new file mode 100644
index 0000000000..d4a8a7b73e
--- /dev/null
+++ b/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-1.test.ts.snap
@@ -0,0 +1,326 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`real-world XLSX corpus > bls-consumer-expenditure-2024-by-income… 1`] = `
+{
+ "calculation": {
+ "compared": 0,
+ "matched": 0,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 5323,
+ "formulas": 0,
+ "roundTrip": "identical",
+ "sheets": 1,
+ "styledCells": 5353,
+ "warnings": [
+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > fred-unrate-cpiaucsl-payems-export.xlsx 1`] = `
+{
+ "calculation": {
+ "compared": 0,
+ "matched": 0,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 4024,
+ "formulas": 0,
+ "roundTrip": "identical",
+ "sheets": 2,
+ "styledCells": 2965,
+ "warnings": [],
+}
+`;
+
+exports[`real-world XLSX corpus > github-alibhatti23-dcf-comps-lbo-merger… 1`] = `
+{
+ "calculation": {
+ "compared": 432,
+ "matched": 432,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 843,
+ "formulas": 432,
+ "roundTrip": "identical",
+ "sheets": 8,
+ "styledCells": 831,
+ "warnings": [
+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > ironcalc-financial-pmt.xlsx 1`] = `
+{
+ "calculation": {
+ "compared": 309,
+ "matched": 309,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 503,
+ "formulas": 309,
+ "roundTrip": "identical",
+ "sheets": 1,
+ "styledCells": 353,
+ "warnings": [],
+}
+`;
+
+exports[`real-world XLSX corpus > ironcalc-template-gantt-project-tracker… 1`] = `
+{
+ "calculation": {
+ "compared": 441,
+ "matched": 441,
+ "mismatchedFunctions": {},
+ "volatile": 3,
+ },
+ "cells": 629,
+ "formulas": 444,
+ "roundTrip": "identical",
+ "sheets": 4,
+ "styledCells": 5541,
+ "warnings": [
+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > libreoffice-forum-mso-en4-134670-sales-… 1`] = `
+{
+ "calculation": {
+ "compared": 516,
+ "matched": 398,
+ "mismatchedFunctions": {
+ "(no function)": 50,
+ "FREQUENCY": 19,
+ "IF": 19,
+ "IFERROR": 19,
+ "MATCH": 19,
+ "ROUND": 49,
+ "ROW": 19,
+ "SIGN": 19,
+ "SUM": 19,
+ },
+ "volatile": 0,
+ },
+ "cells": 10300,
+ "formulas": 516,
+ "roundTrip": "identical",
+ "sheets": 5,
+ "styledCells": 10319,
+ "warnings": [
+ "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",
+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Text rotation, shrink-to-fit and right-to-left layout are not imported.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > obr-efo-ready-reckoner-spring-2025.xlsx 1`] = `
+{
+ "calculation": {
+ "compared": 4601,
+ "matched": 4601,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 8608,
+ "formulas": 4601,
+ "roundTrip": "identical",
+ "sheets": 27,
+ "styledCells": 12262,
+ "warnings": [
+ "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",
+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
+ "Hyperlinks are imported as display text; link targets are not retained.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Rich text is imported as plain text with the cell’s formatting.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > openxmlsdk-business-startup-costs-pl-te… 1`] = `
+{
+ "calculation": {
+ "compared": 214,
+ "matched": 214,
+ "mismatchedFunctions": {},
+ "volatile": 4,
+ },
+ "cells": 566,
+ "formulas": 218,
+ "roundTrip": "identical",
+ "sheets": 6,
+ "styledCells": 3731,
+ "warnings": [
+ "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",
+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
+ "Font sizes are rounded and limited to 8–36 pt.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Rich text is imported as plain text with the cell’s formatting.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > packt-wazobia-3-statement-dcf-valuation… 1`] = `
+{
+ "calculation": {
+ "compared": 1118,
+ "matched": 1118,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 1843,
+ "formulas": 1118,
+ "roundTrip": "identical",
+ "sheets": 6,
+ "styledCells": 3821,
+ "warnings": [
+ "Excel sheet protection is not imported; Macro sharing controls determine edit access.",
+ "External workbook links are not supported; formulas that use them keep their last calculated values.",
+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Names that refer to other workbooks are not imported.",
+ "What-if data tables are imported as their last calculated values.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > poi-formula-eval-test-data.xlsx 1`] = `
+{
+ "calculation": {
+ "compared": 1281,
+ "matched": 1226,
+ "mismatchedFunctions": {
+ "ADDRESS": 1,
+ "AND": 1,
+ "AVEDEV": 3,
+ "DATE": 5,
+ "DAY": 2,
+ "DEVSQ": 2,
+ "INDEX": 1,
+ "LOOKUP": 1,
+ "MAXA": 1,
+ "MINA": 1,
+ "MODE": 4,
+ "MONTH": 2,
+ "STDEV": 2,
+ "SUMIF": 1,
+ "SUMPRODUCT": 1,
+ "SUMSQ": 2,
+ "SUMX2MY2": 6,
+ "SUMX2PY2": 6,
+ "SUMXMY2": 5,
+ "T": 2,
+ "VAR": 2,
+ "VARP": 2,
+ "YEAR": 2,
+ },
+ "volatile": 14,
+ },
+ "cells": 4160,
+ "formulas": 1295,
+ "roundTrip": "identical",
+ "sheets": 4,
+ "styledCells": 591,
+ "warnings": [
+ "Hyperlinks are imported as display text; link targets are not retained.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",
+ "Text rotation, shrink-to-fit and right-to-left layout are not imported.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > sec-jpmorgan-10k-fy2024-financial-repor… 1`] = `
+{
+ "calculation": {
+ "compared": 0,
+ "matched": 0,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 30568,
+ "formulas": 0,
+ "roundTrip": "identical",
+ "sheets": 234,
+ "styledCells": 30568,
+ "warnings": [
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Text longer than 10,000 characters was shortened to fit Macro cells.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > sheetjs-formula-stress-test.xlsx 1`] = `
+{
+ "calculation": {
+ "compared": 439,
+ "matched": 415,
+ "mismatchedFunctions": {
+ "AREAS": 1,
+ "BESSELY": 1,
+ "CHAR": 2,
+ "CODE": 1,
+ "CONVERT": 1,
+ "DATEVALUE": 1,
+ "ECMA.CEILING": 1,
+ "FREQUENCY": 1,
+ "GROWTH": 1,
+ "IMDIV": 1,
+ "IMSUM": 1,
+ "LINEST": 2,
+ "LOGEST": 2,
+ "MINVERSE": 1,
+ "MMULT": 1,
+ "PHONETIC": 1,
+ "TEXT": 1,
+ "TREND": 2,
+ "XIRR": 1,
+ "XNPV": 1,
+ },
+ "volatile": 8,
+ },
+ "cells": 1845,
+ "formulas": 447,
+ "roundTrip": "identical",
+ "sheets": 10,
+ "styledCells": 164,
+ "warnings": [
+ "Macro cannot calculate these Excel functions yet, so their formulas show errors: ECMA.CEILING, JIS, PHONETIC, YEN.",
+ "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > zenodo-surewave-floating-pv-cost-and-fi… 1`] = `
+{
+ "calculation": {
+ "compared": 1888,
+ "matched": 1888,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 3671,
+ "formulas": 1888,
+ "roundTrip": "identical",
+ "sheets": 12,
+ "styledCells": 42476,
+ "warnings": [
+ "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",
+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
+ "Hyperlinks are imported as display text; link targets are not retained.",
+ "Macro cannot calculate these Excel functions yet, so their formulas show errors: __XLUDF.UNSUPPORTED.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ ],
+}
+`;
diff --git a/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-2.test.ts.snap b/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-2.test.ts.snap
new file mode 100644
index 0000000000..2ed90a6c17
--- /dev/null
+++ b/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-2.test.ts.snap
@@ -0,0 +1,291 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`real-world XLSX corpus > bea-gdp-key-source-data-2q2026-third.xl… 1`] = `
+{
+ "calculation": {
+ "compared": 0,
+ "matched": 0,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 7816,
+ "formulas": 0,
+ "roundTrip": "identical",
+ "sheets": 4,
+ "styledCells": 9255,
+ "warnings": [
+ "Hyperlinks are imported as display text; link targets are not retained.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Rich text is imported as plain text with the cell’s formatting.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > closedxml-template-table-source-pivot-t… 1`] = `
+{
+ "calculation": {
+ "compared": 1450,
+ "matched": 1449,
+ "mismatchedFunctions": {
+ "XLL.DSSDATA": 1,
+ },
+ "volatile": 0,
+ },
+ "cells": 1789,
+ "formulas": 1450,
+ "roundTrip": "identical",
+ "sheets": 28,
+ "styledCells": 2126,
+ "warnings": [
+ "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",
+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
+ "Macro cannot calculate these Excel functions yet, so their formulas show errors: _XLL.DSSDATA.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > ironcalc-template-travel-expenses-track… 1`] = `
+{
+ "calculation": {
+ "compared": 40,
+ "matched": 40,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 382,
+ "formulas": 40,
+ "roundTrip": "identical",
+ "sheets": 2,
+ "styledCells": 1475,
+ "warnings": [
+ "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",
+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > libreoffice-missing-path-external-shift… 1`] = `
+{
+ "calculation": {
+ "compared": 4027,
+ "matched": 4027,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 8334,
+ "formulas": 4027,
+ "roundTrip": "identical",
+ "sheets": 3,
+ "styledCells": 8397,
+ "warnings": [
+ "External workbook links are not supported; formulas that use them keep their last calculated values.",
+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Names that refer to other workbooks are not imported.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > obr-public-finances-databank-2026-09.xl… 1`] = `
+{
+ "calculation": {
+ "compared": 5938,
+ "matched": 5938,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 14387,
+ "formulas": 5938,
+ "roundTrip": "identical",
+ "sheets": 7,
+ "styledCells": 20760,
+ "warnings": [
+ "Chart sheets and other non-grid sheets are not imported.",
+ "External workbook links are not supported; formulas that use them keep their last calculated values.",
+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Names that refer to other workbooks are not imported.",
+ "Rich text is imported as plain text with the cell’s formatting.",
+ "Special font effects are simplified to the supported text formatting.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > ons-business-demography-2024-reference-… 1`] = `
+{
+ "calculation": {
+ "compared": 1808,
+ "matched": 1808,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 94578,
+ "formulas": 1808,
+ "roundTrip": "identical",
+ "sheets": 45,
+ "styledCells": 110323,
+ "warnings": [
+ "Hyperlinks are imported as display text; link targets are not retained.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Rich text is imported as plain text with the cell’s formatting.",
+ "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > packt-project-finance-model.xlsx 1`] = `
+{
+ "calculation": {
+ "compared": 2973,
+ "matched": 2973,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 4088,
+ "formulas": 2973,
+ "roundTrip": "identical",
+ "sheets": 10,
+ "styledCells": 5437,
+ "warnings": [
+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > poi-number-format-tests-date1904.xlsx 1`] = `
+{
+ "calculation": {
+ "compared": 325,
+ "matched": 88,
+ "mismatchedFunctions": {
+ "TEXT": 237,
+ },
+ "volatile": 0,
+ },
+ "cells": 1314,
+ "formulas": 325,
+ "roundTrip": "identical",
+ "sheets": 2,
+ "styledCells": 1322,
+ "warnings": [
+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
+ "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > poi-structured-refs-lots-with-lookups-b… 1`] = `
+{
+ "calculation": {
+ "compared": 3476,
+ "matched": 3476,
+ "mismatchedFunctions": {},
+ "volatile": 3798,
+ },
+ "cells": 37696,
+ "formulas": 7274,
+ "roundTrip": "identical",
+ "sheets": 5,
+ "styledCells": 4022,
+ "warnings": [
+ "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",
+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
+ "External workbook links are not supported; formulas that use them keep their last calculated values.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Names that refer to other workbooks are not imported.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > sba-table-of-size-standards-2023.xlsx 1`] = `
+{
+ "calculation": {
+ "compared": 0,
+ "matched": 0,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 5873,
+ "formulas": 0,
+ "roundTrip": "identical",
+ "sheets": 6,
+ "styledCells": 5739,
+ "warnings": [
+ "Excel filters are retained for export; Macro displays all rows.",
+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
+ "Hyperlinks are imported as display text; link targets are not retained.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Rich text is imported as plain text with the cell’s formatting.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > sec-educational-development-corp-10k-fy… 1`] = `
+{
+ "calculation": {
+ "compared": 0,
+ "matched": 0,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 3835,
+ "formulas": 0,
+ "roundTrip": "identical",
+ "sheets": 81,
+ "styledCells": 3835,
+ "warnings": [
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Text longer than 10,000 characters was shortened to fit Macro cells.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > sheetjs-metadata-2019-dynamic-arrays-ri… 1`] = `
+{
+ "calculation": {
+ "compared": 8,
+ "matched": 1,
+ "mismatchedFunctions": {
+ "FV": 6,
+ "UNIQUE": 1,
+ },
+ "volatile": 0,
+ },
+ "cells": 30,
+ "formulas": 8,
+ "roundTrip": "identical",
+ "sheets": 1,
+ "styledCells": 4,
+ "warnings": [
+ "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",
+ "Linked data types and pictures in cells are imported as their displayed values.",
+ "Macro cannot calculate these Excel functions yet, so their formulas show errors: _FV.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > zenodo-debt-capacity-calculator.xlsx 1`] = `
+{
+ "calculation": {
+ "compared": 110,
+ "matched": 110,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 225,
+ "formulas": 110,
+ "roundTrip": "identical",
+ "sheets": 2,
+ "styledCells": 328,
+ "warnings": [
+ "External workbook links are not supported; formulas that use them keep their last calculated values.",
+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Names that refer to other workbooks are not imported.",
+ "Rich text is imported as plain text with the cell’s formatting.",
+ ],
+}
+`;
diff --git a/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-3.test.ts.snap b/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-3.test.ts.snap
new file mode 100644
index 0000000000..4f0206ef47
--- /dev/null
+++ b/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-3.test.ts.snap
@@ -0,0 +1,286 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`real-world XLSX corpus > census-monthly-retail-trade-sales-1992-… 1`] = `
+{
+ "calculation": {
+ "compared": 0,
+ "matched": 0,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 52106,
+ "formulas": 0,
+ "roundTrip": "identical",
+ "sheets": 35,
+ "styledCells": 60252,
+ "warnings": [
+ "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > closedxml-data-table-formula-what-if.xl… 1`] = `
+{
+ "calculation": {
+ "compared": 9,
+ "matched": 9,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 111,
+ "formulas": 9,
+ "roundTrip": "identical",
+ "sheets": 9,
+ "styledCells": 4,
+ "warnings": [
+ "What-if data tables are imported as their last calculated values.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > ironcalc-example-pivots-charts-comments… 1`] = `
+{
+ "calculation": {
+ "compared": 36,
+ "matched": 36,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 154,
+ "formulas": 36,
+ "roundTrip": "identical",
+ "sheets": 10,
+ "styledCells": 128,
+ "warnings": [
+ "Chart sheets and other non-grid sheets are not imported.",
+ "Diagonal borders are not imported.",
+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Patterned fills are shown as a blended solid color.",
+ "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",
+ "Rich text is imported as plain text with the cell’s formatting.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > irs-soi-2023-partnerships-table3-balanc… 1`] = `
+{
+ "calculation": {
+ "compared": 0,
+ "matched": 0,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 722,
+ "formulas": 0,
+ "roundTrip": "identical",
+ "sheets": 1,
+ "styledCells": 8677,
+ "warnings": [
+ "Font sizes are rounded and limited to 8–36 pt.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > libreoffice-functions-excel-2010.xlsx 1`] = `
+{
+ "calculation": {
+ "compared": 517,
+ "matched": 513,
+ "mismatchedFunctions": {
+ "AND": 1,
+ "RANK.EQ": 1,
+ "ROUND": 2,
+ },
+ "volatile": 0,
+ },
+ "cells": 973,
+ "formulas": 517,
+ "roundTrip": "identical",
+ "sheets": 2,
+ "styledCells": 281,
+ "warnings": [
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > nrel-sam-ppa-single-owner-project-finan… 1`] = `
+{
+ "calculation": {
+ "compared": 16248,
+ "matched": 16158,
+ "mismatchedFunctions": {
+ "(no function)": 64,
+ "SUM": 26,
+ },
+ "volatile": 0,
+ },
+ "cells": 18067,
+ "formulas": 16248,
+ "roundTrip": "identical",
+ "sheets": 3,
+ "styledCells": 28010,
+ "warnings": [
+ "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",
+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Rich text is imported as plain text with the cell’s formatting.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > packt-loan-amortization-schedule.xlsx 1`] = `
+{
+ "calculation": {
+ "compared": 96,
+ "matched": 96,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 138,
+ "formulas": 96,
+ "roundTrip": "identical",
+ "sheets": 1,
+ "styledCells": 138,
+ "warnings": [],
+}
+`;
+
+exports[`real-world XLSX corpus > poi-cvlkra-kyc-file-structure-tika3163.… 1`] = `
+{
+ "calculation": {
+ "compared": 103,
+ "matched": 103,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 2987,
+ "formulas": 103,
+ "roundTrip": "identical",
+ "sheets": 30,
+ "styledCells": 52242,
+ "warnings": [
+ "Excel sheet protection is not imported; Macro sharing controls determine edit access.",
+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
+ "Hyperlinks are imported as display text; link targets are not retained.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Rich text is imported as plain text with the cell’s formatting.",
+ "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > poi-data-validation-table-range-county-… 1`] = `
+{
+ "calculation": {
+ "compared": 87,
+ "matched": 87,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 6615,
+ "formulas": 87,
+ "roundTrip": "identical",
+ "sheets": 2,
+ "styledCells": 176,
+ "warnings": [
+ "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",
+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
+ "Font sizes are rounded and limited to 8–36 pt.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > sheetjs-large-strings.xlsx 1`] = `
+{
+ "calculation": {
+ "compared": 21492,
+ "matched": 21492,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 56913,
+ "formulas": 21492,
+ "roundTrip": "identical",
+ "sheets": 3,
+ "styledCells": 35238,
+ "warnings": [],
+}
+`;
+
+exports[`real-world XLSX corpus > worldbank-commodity-prices-pink-sheet-m… 1`] = `
+{
+ "calculation": {
+ "compared": 0,
+ "matched": 0,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 72988,
+ "formulas": 0,
+ "roundTrip": "identical",
+ "sheets": 5,
+ "styledCells": 85343,
+ "warnings": [
+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Special font effects are simplified to the supported text formatting.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > zenodo-ccs-cement-post-tax-dcf-model.xl… 1`] = `
+{
+ "calculation": {
+ "compared": 1752,
+ "matched": 1752,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 16138,
+ "formulas": 1752,
+ "roundTrip": "identical",
+ "sheets": 12,
+ "styledCells": 20467,
+ "warnings": [
+ "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",
+ "Excel filters are retained for export; Macro displays all rows.",
+ "Hyperlinks are imported as display text; link targets are not retained.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Patterned fills are shown as a blended solid color.",
+ "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",
+ "Rich text is imported as plain text with the cell’s formatting.",
+ "Text rotation, shrink-to-fit and right-to-left layout are not imported.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > zenodo-circular-economy-business-case-c… 1`] = `
+{
+ "calculation": {
+ "compared": 2825,
+ "matched": 2825,
+ "mismatchedFunctions": {},
+ "volatile": 3,
+ },
+ "cells": 4665,
+ "formulas": 2828,
+ "roundTrip": "identical",
+ "sheets": 14,
+ "styledCells": 20074,
+ "warnings": [
+ "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",
+ "Excel sheet protection is not imported; Macro sharing controls determine edit access.",
+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Rich text is imported as plain text with the cell’s formatting.",
+ ],
+}
+`;
diff --git a/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-4.test.ts.snap b/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-4.test.ts.snap
new file mode 100644
index 0000000000..667b7efbd7
--- /dev/null
+++ b/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-4.test.ts.snap
@@ -0,0 +1,288 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`real-world XLSX corpus > census-susb-2022-us-sector-large-employ… 1`] = `
+{
+ "calculation": {
+ "compared": 0,
+ "matched": 0,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 2464,
+ "formulas": 0,
+ "roundTrip": "identical",
+ "sheets": 2,
+ "styledCells": 2483,
+ "warnings": [
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Rich text is imported as plain text with the cell’s formatting.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > exceljs-test-issue-877-spare-parts-inve… 1`] = `
+{
+ "calculation": {
+ "compared": 89,
+ "matched": 89,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 977,
+ "formulas": 89,
+ "roundTrip": "identical",
+ "sheets": 5,
+ "styledCells": 3786,
+ "warnings": [
+ "Excel sheet protection is not imported; Macro sharing controls determine edit access.",
+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
+ "Hyperlinks are imported as display text; link targets are not retained.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Rich text is imported as plain text with the cell’s formatting.",
+ "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",
+ "Text rotation, shrink-to-fit and right-to-left layout are not imported.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > figshare-ucl-social-enterprise-financia… 1`] = `
+{
+ "calculation": {
+ "compared": 1861,
+ "matched": 1861,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 65799,
+ "formulas": 1861,
+ "roundTrip": "identical",
+ "sheets": 17,
+ "styledCells": 101113,
+ "warnings": [
+ "Excel filters are retained for export; Macro displays all rows.",
+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Pivot tables over other workbooks or data connections show their last values in Macro; the download keeps them with the data Excel saved, to refresh in Excel.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > gsa-per-diem-rates-fy2026-master.xlsx 1`] = `
+{
+ "calculation": {
+ "compared": 0,
+ "matched": 0,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 4927,
+ "formulas": 0,
+ "roundTrip": "identical",
+ "sheets": 1,
+ "styledCells": 5209,
+ "warnings": [],
+}
+`;
+
+exports[`real-world XLSX corpus > ironcalc-tables-structured-references.x… 1`] = `
+{
+ "calculation": {
+ "compared": 201,
+ "matched": 192,
+ "mismatchedFunctions": {
+ "SUBTOTAL": 9,
+ },
+ "volatile": 4,
+ },
+ "cells": 831,
+ "formulas": 205,
+ "roundTrip": "identical",
+ "sheets": 9,
+ "styledCells": 327,
+ "warnings": [
+ "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",
+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
+ "Some table references could not be converted; those formulas show their last calculated values.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > ironcalc-template-invoice.xlsx 1`] = `
+{
+ "calculation": {
+ "compared": 23,
+ "matched": 23,
+ "mismatchedFunctions": {},
+ "volatile": 2,
+ },
+ "cells": 96,
+ "formulas": 25,
+ "roundTrip": "identical",
+ "sheets": 3,
+ "styledCells": 230,
+ "warnings": [
+ "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",
+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > libreoffice-column-style-autofilter-lin… 1`] = `
+{
+ "calculation": {
+ "compared": 27,
+ "matched": 27,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 62015,
+ "formulas": 27,
+ "roundTrip": "identical",
+ "sheets": 24,
+ "styledCells": 3694,
+ "warnings": [
+ "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",
+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
+ "Pivot tables over other workbooks or data connections show their last values in Macro; the download keeps them with the data Excel saved, to refresh in Excel.",
+ "Some pivot tables over other workbooks or data connections are downloaded without the data Excel saved with them; Excel fills them again when they refresh.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > libreoffice-forum-mso-de-104083-pivot-w… 1`] = `
+{
+ "calculation": {
+ "compared": 2303,
+ "matched": 2303,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 9524,
+ "formulas": 2303,
+ "roundTrip": "identical",
+ "sheets": 2,
+ "styledCells": 12208,
+ "warnings": [
+ "Excel filters are retained for export; Macro displays all rows.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > nrel-sam-residential-solar-cash-flow.xl… 1`] = `
+{
+ "calculation": {
+ "compared": 22431,
+ "matched": 22431,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 23003,
+ "formulas": 22431,
+ "roundTrip": "identical",
+ "sheets": 2,
+ "styledCells": 32545,
+ "warnings": [
+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Patterned fills are shown as a blended solid color.",
+ "Rich text is imported as plain text with the cell’s formatting.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > packt365-dynamic-array-functions-workbo… 1`] = `
+{
+ "calculation": {
+ "compared": 302,
+ "matched": 301,
+ "mismatchedFunctions": {
+ "FILTER": 1,
+ },
+ "volatile": 0,
+ },
+ "cells": 1239,
+ "formulas": 302,
+ "roundTrip": "identical",
+ "sheets": 3,
+ "styledCells": 1411,
+ "warnings": [
+ "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",
+ "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",
+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
+ "External workbook links are not supported; formulas that use them keep their last calculated values.",
+ "Linked data types and pictures in cells are imported as their displayed values.",
+ "Names that refer to other workbooks are not imported.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > poi-average-tax-rates-ireland.xlsx 1`] = `
+{
+ "calculation": {
+ "compared": 0,
+ "matched": 0,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 1110,
+ "formulas": 0,
+ "roundTrip": "identical",
+ "sheets": 3,
+ "styledCells": 1360,
+ "warnings": [
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > poi-conditional-formatting-samples.xlsx 1`] = `
+{
+ "calculation": {
+ "compared": 352,
+ "matched": 351,
+ "mismatchedFunctions": {
+ "SUBTOTAL": 1,
+ },
+ "volatile": 0,
+ },
+ "cells": 1723,
+ "formulas": 352,
+ "roundTrip": "identical",
+ "sheets": 18,
+ "styledCells": 1838,
+ "warnings": [
+ "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",
+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Rich text is imported as plain text with the cell’s formatting.",
+ "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",
+ ],
+}
+`;
+
+exports[`real-world XLSX corpus > zenodo-offshore-wind-lcoe-dcf-model.xlsx 1`] = `
+{
+ "calculation": {
+ "compared": 4000,
+ "matched": 4000,
+ "mismatchedFunctions": {},
+ "volatile": 0,
+ },
+ "cells": 5012,
+ "formulas": 4000,
+ "roundTrip": "identical",
+ "sheets": 9,
+ "styledCells": 6665,
+ "warnings": [
+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
+ "Hyperlinks are imported as display text; link targets are not retained.",
+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
+ "Rich text is imported as plain text with the cell’s formatting.",
+ "Some table references could not be converted; those formulas show their last calculated values.",
+ ],
+}
+`;
diff --git a/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus.test.ts.snap b/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus.test.ts.snap
deleted file mode 100644
index 56f1140dc3..0000000000
--- a/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus.test.ts.snap
+++ /dev/null
@@ -1,1188 +0,0 @@
-// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
-
-exports[`real-world XLSX corpus > bea-gdp-key-source-data-2q2026-third.xl… 1`] = `
-{
- "calculation": {
- "compared": 0,
- "matched": 0,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 7816,
- "formulas": 0,
- "roundTrip": "identical",
- "sheets": 4,
- "styledCells": 9255,
- "warnings": [
- "Hyperlinks are imported as display text; link targets are not retained.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Rich text is imported as plain text with the cell’s formatting.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > bls-consumer-expenditure-2024-by-income… 1`] = `
-{
- "calculation": {
- "compared": 0,
- "matched": 0,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 5323,
- "formulas": 0,
- "roundTrip": "identical",
- "sheets": 1,
- "styledCells": 5353,
- "warnings": [
- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > census-monthly-retail-trade-sales-1992-… 1`] = `
-{
- "calculation": {
- "compared": 0,
- "matched": 0,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 52106,
- "formulas": 0,
- "roundTrip": "identical",
- "sheets": 35,
- "styledCells": 60252,
- "warnings": [
- "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > census-susb-2022-us-sector-large-employ… 1`] = `
-{
- "calculation": {
- "compared": 0,
- "matched": 0,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 2464,
- "formulas": 0,
- "roundTrip": "identical",
- "sheets": 2,
- "styledCells": 2483,
- "warnings": [
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Rich text is imported as plain text with the cell’s formatting.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > closedxml-data-table-formula-what-if.xl… 1`] = `
-{
- "calculation": {
- "compared": 9,
- "matched": 9,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 111,
- "formulas": 9,
- "roundTrip": "identical",
- "sheets": 9,
- "styledCells": 4,
- "warnings": [
- "What-if data tables are imported as their last calculated values.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > closedxml-template-table-source-pivot-t… 1`] = `
-{
- "calculation": {
- "compared": 1450,
- "matched": 1449,
- "mismatchedFunctions": {
- "XLL.DSSDATA": 1,
- },
- "volatile": 0,
- },
- "cells": 1789,
- "formulas": 1450,
- "roundTrip": "identical",
- "sheets": 28,
- "styledCells": 2126,
- "warnings": [
- "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",
- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
- "Macro cannot calculate these Excel functions yet, so their formulas show errors: _XLL.DSSDATA.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > exceljs-test-issue-877-spare-parts-inve… 1`] = `
-{
- "calculation": {
- "compared": 89,
- "matched": 89,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 977,
- "formulas": 89,
- "roundTrip": "identical",
- "sheets": 5,
- "styledCells": 3786,
- "warnings": [
- "Excel sheet protection is not imported; Macro sharing controls determine edit access.",
- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
- "Hyperlinks are imported as display text; link targets are not retained.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Rich text is imported as plain text with the cell’s formatting.",
- "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",
- "Text rotation, shrink-to-fit and right-to-left layout are not imported.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > figshare-ucl-social-enterprise-financia… 1`] = `
-{
- "calculation": {
- "compared": 1861,
- "matched": 1861,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 65799,
- "formulas": 1861,
- "roundTrip": "identical",
- "sheets": 17,
- "styledCells": 101113,
- "warnings": [
- "Excel filters are retained for export; Macro displays all rows.",
- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Pivot tables over other workbooks or data connections show their last values in Macro; the download keeps them with the data Excel saved, to refresh in Excel.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > fred-unrate-cpiaucsl-payems-export.xlsx 1`] = `
-{
- "calculation": {
- "compared": 0,
- "matched": 0,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 4024,
- "formulas": 0,
- "roundTrip": "identical",
- "sheets": 2,
- "styledCells": 2965,
- "warnings": [],
-}
-`;
-
-exports[`real-world XLSX corpus > github-alibhatti23-dcf-comps-lbo-merger… 1`] = `
-{
- "calculation": {
- "compared": 432,
- "matched": 432,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 843,
- "formulas": 432,
- "roundTrip": "identical",
- "sheets": 8,
- "styledCells": 831,
- "warnings": [
- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > gsa-per-diem-rates-fy2026-master.xlsx 1`] = `
-{
- "calculation": {
- "compared": 0,
- "matched": 0,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 4927,
- "formulas": 0,
- "roundTrip": "identical",
- "sheets": 1,
- "styledCells": 5209,
- "warnings": [],
-}
-`;
-
-exports[`real-world XLSX corpus > ironcalc-example-pivots-charts-comments… 1`] = `
-{
- "calculation": {
- "compared": 36,
- "matched": 36,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 154,
- "formulas": 36,
- "roundTrip": "identical",
- "sheets": 10,
- "styledCells": 128,
- "warnings": [
- "Chart sheets and other non-grid sheets are not imported.",
- "Diagonal borders are not imported.",
- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Patterned fills are shown as a blended solid color.",
- "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",
- "Rich text is imported as plain text with the cell’s formatting.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > ironcalc-financial-pmt.xlsx 1`] = `
-{
- "calculation": {
- "compared": 309,
- "matched": 309,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 503,
- "formulas": 309,
- "roundTrip": "identical",
- "sheets": 1,
- "styledCells": 353,
- "warnings": [],
-}
-`;
-
-exports[`real-world XLSX corpus > ironcalc-tables-structured-references.x… 1`] = `
-{
- "calculation": {
- "compared": 201,
- "matched": 192,
- "mismatchedFunctions": {
- "SUBTOTAL": 9,
- },
- "volatile": 4,
- },
- "cells": 831,
- "formulas": 205,
- "roundTrip": "identical",
- "sheets": 9,
- "styledCells": 327,
- "warnings": [
- "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",
- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
- "Some table references could not be converted; those formulas show their last calculated values.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > ironcalc-template-gantt-project-tracker… 1`] = `
-{
- "calculation": {
- "compared": 441,
- "matched": 441,
- "mismatchedFunctions": {},
- "volatile": 3,
- },
- "cells": 629,
- "formulas": 444,
- "roundTrip": "identical",
- "sheets": 4,
- "styledCells": 5541,
- "warnings": [
- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > ironcalc-template-invoice.xlsx 1`] = `
-{
- "calculation": {
- "compared": 23,
- "matched": 23,
- "mismatchedFunctions": {},
- "volatile": 2,
- },
- "cells": 96,
- "formulas": 25,
- "roundTrip": "identical",
- "sheets": 3,
- "styledCells": 230,
- "warnings": [
- "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",
- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > ironcalc-template-travel-expenses-track… 1`] = `
-{
- "calculation": {
- "compared": 40,
- "matched": 40,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 382,
- "formulas": 40,
- "roundTrip": "identical",
- "sheets": 2,
- "styledCells": 1475,
- "warnings": [
- "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",
- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > irs-soi-2023-partnerships-table3-balanc… 1`] = `
-{
- "calculation": {
- "compared": 0,
- "matched": 0,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 722,
- "formulas": 0,
- "roundTrip": "identical",
- "sheets": 1,
- "styledCells": 8677,
- "warnings": [
- "Font sizes are rounded and limited to 8–36 pt.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > libreoffice-column-style-autofilter-lin… 1`] = `
-{
- "calculation": {
- "compared": 27,
- "matched": 27,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 62015,
- "formulas": 27,
- "roundTrip": "identical",
- "sheets": 24,
- "styledCells": 3694,
- "warnings": [
- "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",
- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
- "Pivot tables over other workbooks or data connections show their last values in Macro; the download keeps them with the data Excel saved, to refresh in Excel.",
- "Some pivot tables over other workbooks or data connections are downloaded without the data Excel saved with them; Excel fills them again when they refresh.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > libreoffice-forum-mso-de-104083-pivot-w… 1`] = `
-{
- "calculation": {
- "compared": 2303,
- "matched": 2303,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 9524,
- "formulas": 2303,
- "roundTrip": "identical",
- "sheets": 2,
- "styledCells": 12208,
- "warnings": [
- "Excel filters are retained for export; Macro displays all rows.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > libreoffice-forum-mso-en4-134670-sales-… 1`] = `
-{
- "calculation": {
- "compared": 516,
- "matched": 398,
- "mismatchedFunctions": {
- "(no function)": 50,
- "FREQUENCY": 19,
- "IF": 19,
- "IFERROR": 19,
- "MATCH": 19,
- "ROUND": 49,
- "ROW": 19,
- "SIGN": 19,
- "SUM": 19,
- },
- "volatile": 0,
- },
- "cells": 10300,
- "formulas": 516,
- "roundTrip": "identical",
- "sheets": 5,
- "styledCells": 10319,
- "warnings": [
- "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",
- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Text rotation, shrink-to-fit and right-to-left layout are not imported.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > libreoffice-functions-excel-2010.xlsx 1`] = `
-{
- "calculation": {
- "compared": 517,
- "matched": 513,
- "mismatchedFunctions": {
- "AND": 1,
- "RANK.EQ": 1,
- "ROUND": 2,
- },
- "volatile": 0,
- },
- "cells": 973,
- "formulas": 517,
- "roundTrip": "identical",
- "sheets": 2,
- "styledCells": 281,
- "warnings": [
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > libreoffice-missing-path-external-shift… 1`] = `
-{
- "calculation": {
- "compared": 4027,
- "matched": 4027,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 8334,
- "formulas": 4027,
- "roundTrip": "identical",
- "sheets": 3,
- "styledCells": 8397,
- "warnings": [
- "External workbook links are not supported; formulas that use them keep their last calculated values.",
- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Names that refer to other workbooks are not imported.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > nrel-sam-ppa-single-owner-project-finan… 1`] = `
-{
- "calculation": {
- "compared": 16248,
- "matched": 16158,
- "mismatchedFunctions": {
- "(no function)": 64,
- "SUM": 26,
- },
- "volatile": 0,
- },
- "cells": 18067,
- "formulas": 16248,
- "roundTrip": "identical",
- "sheets": 3,
- "styledCells": 28010,
- "warnings": [
- "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",
- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Rich text is imported as plain text with the cell’s formatting.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > nrel-sam-residential-solar-cash-flow.xl… 1`] = `
-{
- "calculation": {
- "compared": 22431,
- "matched": 22431,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 23003,
- "formulas": 22431,
- "roundTrip": "identical",
- "sheets": 2,
- "styledCells": 32545,
- "warnings": [
- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Patterned fills are shown as a blended solid color.",
- "Rich text is imported as plain text with the cell’s formatting.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > obr-efo-ready-reckoner-spring-2025.xlsx 1`] = `
-{
- "calculation": {
- "compared": 4601,
- "matched": 4601,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 8608,
- "formulas": 4601,
- "roundTrip": "identical",
- "sheets": 27,
- "styledCells": 12262,
- "warnings": [
- "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",
- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
- "Hyperlinks are imported as display text; link targets are not retained.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Rich text is imported as plain text with the cell’s formatting.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > obr-public-finances-databank-2026-09.xl… 1`] = `
-{
- "calculation": {
- "compared": 5938,
- "matched": 5938,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 14387,
- "formulas": 5938,
- "roundTrip": "identical",
- "sheets": 7,
- "styledCells": 20760,
- "warnings": [
- "Chart sheets and other non-grid sheets are not imported.",
- "External workbook links are not supported; formulas that use them keep their last calculated values.",
- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Names that refer to other workbooks are not imported.",
- "Rich text is imported as plain text with the cell’s formatting.",
- "Special font effects are simplified to the supported text formatting.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > ons-business-demography-2024-reference-… 1`] = `
-{
- "calculation": {
- "compared": 1808,
- "matched": 1808,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 94578,
- "formulas": 1808,
- "roundTrip": "identical",
- "sheets": 45,
- "styledCells": 110323,
- "warnings": [
- "Hyperlinks are imported as display text; link targets are not retained.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Rich text is imported as plain text with the cell’s formatting.",
- "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > openxmlsdk-business-startup-costs-pl-te… 1`] = `
-{
- "calculation": {
- "compared": 214,
- "matched": 214,
- "mismatchedFunctions": {},
- "volatile": 4,
- },
- "cells": 566,
- "formulas": 218,
- "roundTrip": "identical",
- "sheets": 6,
- "styledCells": 3731,
- "warnings": [
- "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",
- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
- "Font sizes are rounded and limited to 8–36 pt.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Rich text is imported as plain text with the cell’s formatting.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > packt-loan-amortization-schedule.xlsx 1`] = `
-{
- "calculation": {
- "compared": 96,
- "matched": 96,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 138,
- "formulas": 96,
- "roundTrip": "identical",
- "sheets": 1,
- "styledCells": 138,
- "warnings": [],
-}
-`;
-
-exports[`real-world XLSX corpus > packt-project-finance-model.xlsx 1`] = `
-{
- "calculation": {
- "compared": 2973,
- "matched": 2973,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 4088,
- "formulas": 2973,
- "roundTrip": "identical",
- "sheets": 10,
- "styledCells": 5437,
- "warnings": [
- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > packt-wazobia-3-statement-dcf-valuation… 1`] = `
-{
- "calculation": {
- "compared": 1118,
- "matched": 1118,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 1843,
- "formulas": 1118,
- "roundTrip": "identical",
- "sheets": 6,
- "styledCells": 3821,
- "warnings": [
- "Excel sheet protection is not imported; Macro sharing controls determine edit access.",
- "External workbook links are not supported; formulas that use them keep their last calculated values.",
- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Names that refer to other workbooks are not imported.",
- "What-if data tables are imported as their last calculated values.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > packt365-dynamic-array-functions-workbo… 1`] = `
-{
- "calculation": {
- "compared": 302,
- "matched": 301,
- "mismatchedFunctions": {
- "FILTER": 1,
- },
- "volatile": 0,
- },
- "cells": 1239,
- "formulas": 302,
- "roundTrip": "identical",
- "sheets": 3,
- "styledCells": 1411,
- "warnings": [
- "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",
- "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",
- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
- "External workbook links are not supported; formulas that use them keep their last calculated values.",
- "Linked data types and pictures in cells are imported as their displayed values.",
- "Names that refer to other workbooks are not imported.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > poi-average-tax-rates-ireland.xlsx 1`] = `
-{
- "calculation": {
- "compared": 0,
- "matched": 0,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 1110,
- "formulas": 0,
- "roundTrip": "identical",
- "sheets": 3,
- "styledCells": 1360,
- "warnings": [
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > poi-conditional-formatting-samples.xlsx 1`] = `
-{
- "calculation": {
- "compared": 352,
- "matched": 351,
- "mismatchedFunctions": {
- "SUBTOTAL": 1,
- },
- "volatile": 0,
- },
- "cells": 1723,
- "formulas": 352,
- "roundTrip": "identical",
- "sheets": 18,
- "styledCells": 1838,
- "warnings": [
- "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",
- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Rich text is imported as plain text with the cell’s formatting.",
- "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > poi-cvlkra-kyc-file-structure-tika3163.… 1`] = `
-{
- "calculation": {
- "compared": 103,
- "matched": 103,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 2987,
- "formulas": 103,
- "roundTrip": "identical",
- "sheets": 30,
- "styledCells": 52242,
- "warnings": [
- "Excel sheet protection is not imported; Macro sharing controls determine edit access.",
- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
- "Hyperlinks are imported as display text; link targets are not retained.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Rich text is imported as plain text with the cell’s formatting.",
- "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > poi-data-validation-table-range-county-… 1`] = `
-{
- "calculation": {
- "compared": 87,
- "matched": 87,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 6615,
- "formulas": 87,
- "roundTrip": "identical",
- "sheets": 2,
- "styledCells": 176,
- "warnings": [
- "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",
- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
- "Font sizes are rounded and limited to 8–36 pt.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > poi-formula-eval-test-data.xlsx 1`] = `
-{
- "calculation": {
- "compared": 1281,
- "matched": 1226,
- "mismatchedFunctions": {
- "ADDRESS": 1,
- "AND": 1,
- "AVEDEV": 3,
- "DATE": 5,
- "DAY": 2,
- "DEVSQ": 2,
- "INDEX": 1,
- "LOOKUP": 1,
- "MAXA": 1,
- "MINA": 1,
- "MODE": 4,
- "MONTH": 2,
- "STDEV": 2,
- "SUMIF": 1,
- "SUMPRODUCT": 1,
- "SUMSQ": 2,
- "SUMX2MY2": 6,
- "SUMX2PY2": 6,
- "SUMXMY2": 5,
- "T": 2,
- "VAR": 2,
- "VARP": 2,
- "YEAR": 2,
- },
- "volatile": 14,
- },
- "cells": 4160,
- "formulas": 1295,
- "roundTrip": "identical",
- "sheets": 4,
- "styledCells": 591,
- "warnings": [
- "Hyperlinks are imported as display text; link targets are not retained.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",
- "Text rotation, shrink-to-fit and right-to-left layout are not imported.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > poi-number-format-tests-date1904.xlsx 1`] = `
-{
- "calculation": {
- "compared": 325,
- "matched": 88,
- "mismatchedFunctions": {
- "TEXT": 237,
- },
- "volatile": 0,
- },
- "cells": 1314,
- "formulas": 325,
- "roundTrip": "identical",
- "sheets": 2,
- "styledCells": 1322,
- "warnings": [
- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
- "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > poi-structured-refs-lots-with-lookups-b… 1`] = `
-{
- "calculation": {
- "compared": 3476,
- "matched": 3476,
- "mismatchedFunctions": {},
- "volatile": 3798,
- },
- "cells": 37696,
- "formulas": 7274,
- "roundTrip": "identical",
- "sheets": 5,
- "styledCells": 4022,
- "warnings": [
- "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",
- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
- "External workbook links are not supported; formulas that use them keep their last calculated values.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Names that refer to other workbooks are not imported.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > sba-table-of-size-standards-2023.xlsx 1`] = `
-{
- "calculation": {
- "compared": 0,
- "matched": 0,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 5873,
- "formulas": 0,
- "roundTrip": "identical",
- "sheets": 6,
- "styledCells": 5739,
- "warnings": [
- "Excel filters are retained for export; Macro displays all rows.",
- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
- "Hyperlinks are imported as display text; link targets are not retained.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Rich text is imported as plain text with the cell’s formatting.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > sec-educational-development-corp-10k-fy… 1`] = `
-{
- "calculation": {
- "compared": 0,
- "matched": 0,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 3835,
- "formulas": 0,
- "roundTrip": "identical",
- "sheets": 81,
- "styledCells": 3835,
- "warnings": [
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Text longer than 10,000 characters was shortened to fit Macro cells.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > sec-jpmorgan-10k-fy2024-financial-repor… 1`] = `
-{
- "calculation": {
- "compared": 0,
- "matched": 0,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 30568,
- "formulas": 0,
- "roundTrip": "identical",
- "sheets": 234,
- "styledCells": 30568,
- "warnings": [
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Text longer than 10,000 characters was shortened to fit Macro cells.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > sheetjs-formula-stress-test.xlsx 1`] = `
-{
- "calculation": {
- "compared": 439,
- "matched": 415,
- "mismatchedFunctions": {
- "AREAS": 1,
- "BESSELY": 1,
- "CHAR": 2,
- "CODE": 1,
- "CONVERT": 1,
- "DATEVALUE": 1,
- "ECMA.CEILING": 1,
- "FREQUENCY": 1,
- "GROWTH": 1,
- "IMDIV": 1,
- "IMSUM": 1,
- "LINEST": 2,
- "LOGEST": 2,
- "MINVERSE": 1,
- "MMULT": 1,
- "PHONETIC": 1,
- "TEXT": 1,
- "TREND": 2,
- "XIRR": 1,
- "XNPV": 1,
- },
- "volatile": 8,
- },
- "cells": 1845,
- "formulas": 447,
- "roundTrip": "identical",
- "sheets": 10,
- "styledCells": 164,
- "warnings": [
- "Macro cannot calculate these Excel functions yet, so their formulas show errors: ECMA.CEILING, JIS, PHONETIC, YEN.",
- "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > sheetjs-large-strings.xlsx 1`] = `
-{
- "calculation": {
- "compared": 21492,
- "matched": 21492,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 56913,
- "formulas": 21492,
- "roundTrip": "identical",
- "sheets": 3,
- "styledCells": 35238,
- "warnings": [],
-}
-`;
-
-exports[`real-world XLSX corpus > sheetjs-metadata-2019-dynamic-arrays-ri… 1`] = `
-{
- "calculation": {
- "compared": 8,
- "matched": 1,
- "mismatchedFunctions": {
- "FV": 6,
- "UNIQUE": 1,
- },
- "volatile": 0,
- },
- "cells": 30,
- "formulas": 8,
- "roundTrip": "identical",
- "sheets": 1,
- "styledCells": 4,
- "warnings": [
- "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",
- "Linked data types and pictures in cells are imported as their displayed values.",
- "Macro cannot calculate these Excel functions yet, so their formulas show errors: _FV.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > worldbank-commodity-prices-pink-sheet-m… 1`] = `
-{
- "calculation": {
- "compared": 0,
- "matched": 0,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 72988,
- "formulas": 0,
- "roundTrip": "identical",
- "sheets": 5,
- "styledCells": 85343,
- "warnings": [
- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Special font effects are simplified to the supported text formatting.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > zenodo-ccs-cement-post-tax-dcf-model.xl… 1`] = `
-{
- "calculation": {
- "compared": 1752,
- "matched": 1752,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 16138,
- "formulas": 1752,
- "roundTrip": "identical",
- "sheets": 12,
- "styledCells": 20467,
- "warnings": [
- "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",
- "Excel filters are retained for export; Macro displays all rows.",
- "Hyperlinks are imported as display text; link targets are not retained.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Patterned fills are shown as a blended solid color.",
- "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",
- "Rich text is imported as plain text with the cell’s formatting.",
- "Text rotation, shrink-to-fit and right-to-left layout are not imported.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > zenodo-circular-economy-business-case-c… 1`] = `
-{
- "calculation": {
- "compared": 2825,
- "matched": 2825,
- "mismatchedFunctions": {},
- "volatile": 3,
- },
- "cells": 4665,
- "formulas": 2828,
- "roundTrip": "identical",
- "sheets": 14,
- "styledCells": 20074,
- "warnings": [
- "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",
- "Excel sheet protection is not imported; Macro sharing controls determine edit access.",
- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Rich text is imported as plain text with the cell’s formatting.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > zenodo-debt-capacity-calculator.xlsx 1`] = `
-{
- "calculation": {
- "compared": 110,
- "matched": 110,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 225,
- "formulas": 110,
- "roundTrip": "identical",
- "sheets": 2,
- "styledCells": 328,
- "warnings": [
- "External workbook links are not supported; formulas that use them keep their last calculated values.",
- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Names that refer to other workbooks are not imported.",
- "Rich text is imported as plain text with the cell’s formatting.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > zenodo-offshore-wind-lcoe-dcf-model.xlsx 1`] = `
-{
- "calculation": {
- "compared": 4000,
- "matched": 4000,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 5012,
- "formulas": 4000,
- "roundTrip": "identical",
- "sheets": 9,
- "styledCells": 6665,
- "warnings": [
- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",
- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
- "Hyperlinks are imported as display text; link targets are not retained.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- "Rich text is imported as plain text with the cell’s formatting.",
- "Some table references could not be converted; those formulas show their last calculated values.",
- ],
-}
-`;
-
-exports[`real-world XLSX corpus > zenodo-surewave-floating-pv-cost-and-fi… 1`] = `
-{
- "calculation": {
- "compared": 1888,
- "matched": 1888,
- "mismatchedFunctions": {},
- "volatile": 0,
- },
- "cells": 3671,
- "formulas": 1888,
- "roundTrip": "identical",
- "sheets": 12,
- "styledCells": 42476,
- "warnings": [
- "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",
- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",
- "Hyperlinks are imported as display text; link targets are not retained.",
- "Macro cannot calculate these Excel functions yet, so their formulas show errors: __XLUDF.UNSUPPORTED.",
- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",
- ],
-}
-`;
diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-1.test.ts b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-1.test.ts
new file mode 100644
index 0000000000..d6b5c42360
--- /dev/null
+++ b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-1.test.ts
@@ -0,0 +1,3 @@
+import { describeCorpusSlice } from './xlsx-corpus-suite';
+
+describeCorpusSlice(1);
diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-2.test.ts b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-2.test.ts
new file mode 100644
index 0000000000..53f595762c
--- /dev/null
+++ b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-2.test.ts
@@ -0,0 +1,3 @@
+import { describeCorpusSlice } from './xlsx-corpus-suite';
+
+describeCorpusSlice(2);
diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-3.test.ts b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-3.test.ts
new file mode 100644
index 0000000000..86fc4f9585
--- /dev/null
+++ b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-3.test.ts
@@ -0,0 +1,3 @@
+import { describeCorpusSlice } from './xlsx-corpus-suite';
+
+describeCorpusSlice(3);
diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-4.test.ts b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-4.test.ts
new file mode 100644
index 0000000000..cbf2362b07
--- /dev/null
+++ b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-4.test.ts
@@ -0,0 +1,3 @@
+import { describeCorpusSlice } from './xlsx-corpus-suite';
+
+describeCorpusSlice(4);
diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-suite.ts b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-suite.ts
new file mode 100644
index 0000000000..dba820ac31
--- /dev/null
+++ b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-suite.ts
@@ -0,0 +1,152 @@
+import { readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { initSync } from '@ironcalc/wasm';
+import { createInitializedSpreadsheetCalculator } from '@macro-inc/spreadsheet/calculation';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import type { SpreadsheetCalculator, WorkbookCalculation } from './calculation';
+import type { WorkbookFileData } from './workbook-file-types';
+import { decodeXlsx, encodeXlsx } from './xlsx-codec';
+import {
+ type CalculationFidelity,
+ calculateImported,
+ calculationFidelity,
+ corpusBytes,
+ corpusEntries,
+} from './xlsx-fixtures/corpus';
+
+/**
+ * The real-world corpus runs from `xlsx-corpus-1.test.ts` through
+ * `xlsx-corpus-4.test.ts`, one slice each, so Vitest spreads the workbooks
+ * across workers instead of running them serially in one file for a minute.
+ * A slice takes every `CORPUS_SLICES`th manifest entry, so new workbooks
+ * spread across the files.
+ */
+export const CORPUS_SLICES = 4;
+
+type CorpusSummary = {
+ error?: string;
+ sheets?: number;
+ cells?: number;
+ formulas?: number;
+ styledCells?: number;
+ calculation?: Omit<CalculationFidelity, 'examples'> | string;
+ roundTrip?: 'identical' | string;
+ warnings?: string[];
+};
+
+function counts(workbook: WorkbookFileData) {
+ const cells = workbook.sheets.flatMap((sheet) => Object.values(sheet.cells));
+ return {
+ sheets: workbook.sheets.length,
+ cells: cells.filter((cell) => cell.value !== '').length,
+ formulas: cells.filter(
+ (cell) => cell.value.startsWith('=') && cell.format !== 'text'
+ ).length,
+ styledCells: cells.filter((cell) => Object.keys(cell).length > 1).length,
+ };
+}
+
+/** The first difference between two imports, or `identical`. */
+function roundTripDifference(
+ first: WorkbookFileData,
+ second: WorkbookFileData
+): string {
+ if (first.sheets.length !== second.sheets.length)
+ return `sheet count ${first.sheets.length} → ${second.sheets.length}`;
+ for (const [index, sheet] of first.sheets.entries()) {
+ const other = second.sheets[index];
+ if (sheet.name !== other.name) return `sheet name ${sheet.name}`;
+ const addresses = new Set([
+ ...Object.keys(sheet.cells),
+ ...Object.keys(other.cells),
+ ]);
+ for (const address of addresses) {
+ const before = JSON.stringify(sheet.cells[address] ?? null);
+ const after = JSON.stringify(other.cells[address] ?? null);
+ if (before !== after)
+ return `${sheet.name}!${address} ${before} → ${after}`;
+ }
+ if (JSON.stringify(sheet.metadata) !== JSON.stringify(other.metadata))
+ return `${sheet.name} metadata`;
+ if (
+ JSON.stringify(sheet.columnWidths) !== JSON.stringify(other.columnWidths)
+ )
+ return `${sheet.name} column widths`;
+ }
+ return 'identical';
+}
+
+/**
+ * Imports, recalculates and round-trips slice `slice` (1-based) of the corpus.
+ * `corpusEntries` fails when a workbook lacks provenance in `manifest.json`.
+ */
+export function describeCorpusSlice(slice: number) {
+ if (!Number.isInteger(slice) || slice < 1 || slice > CORPUS_SLICES)
+ throw new Error(`Corpus slice must be 1–${CORPUS_SLICES}, got ${slice}`);
+ const entries = corpusEntries().filter(
+ (_, index) => index % CORPUS_SLICES === slice - 1
+ );
+
+ let calculator: SpreadsheetCalculator;
+ beforeAll(() => {
+ initSync({
+ module: readFileSync(
+ createRequire(import.meta.url).resolve('@ironcalc/wasm/wasm_bg.wasm')
+ ),
+ });
+ calculator = createInitializedSpreadsheetCalculator();
+ });
+ afterAll(() => calculator.dispose());
+
+ // Snapshot names stay `real-world XLSX corpus > <file>` in every slice.
+ describe('real-world XLSX corpus', () => {
+ it.each(entries)(
+ '$file',
+ async (entry) => {
+ const bytes = corpusBytes(entry);
+ const summary: CorpusSummary = {};
+ let imported: WorkbookFileData;
+ try {
+ imported = await decodeXlsx(bytes);
+ } catch (error) {
+ summary.error =
+ error instanceof Error ? error.message : String(error);
+ expect(summary).toMatchSnapshot();
+ return;
+ }
+ Object.assign(summary, counts(imported));
+ summary.warnings = [...imported.warnings].sort();
+ let values: WorkbookCalculation = {};
+ if (entry.skipCalculation) summary.calculation = entry.skipCalculation;
+ else {
+ values = calculateImported(calculator, imported, bytes);
+ const { examples, ...calculation } = calculationFidelity(
+ bytes,
+ imported,
+ values
+ );
+ summary.calculation = calculation;
+ if (process.env.XLSX_CORPUS_VERBOSE)
+ console.info(entry.file, JSON.stringify(calculation), examples);
+ }
+ try {
+ const exported = await encodeXlsx({
+ sheets: imported.sheets.map((sheet, index) => ({
+ ...sheet,
+ values: values[String(index)],
+ })),
+ images: imported.images,
+ });
+ summary.roundTrip = roundTripDifference(
+ imported,
+ await decodeXlsx(exported.bytes)
+ );
+ } catch (error) {
+ summary.roundTrip = `export failed: ${error instanceof Error ? error.message : String(error)}`;
+ }
+ expect(summary).toMatchSnapshot();
+ },
+ 120_000
+ );
+ });
+}
diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-corpus.test.ts b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus.test.ts
deleted file mode 100644
index 37951e0fa3..0000000000
--- a/apps/web/src/features/block-spreadsheet/core/xlsx-corpus.test.ts
+++ /dev/null
@@ -1,129 +0,0 @@
-import { readFileSync } from 'node:fs';
-import { createRequire } from 'node:module';
-import { initSync } from '@ironcalc/wasm';
-import { createInitializedSpreadsheetCalculator } from '@macro-inc/spreadsheet/calculation';
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import type { SpreadsheetCalculator, WorkbookCalculation } from './calculation';
-import type { WorkbookFileData } from './workbook-file-types';
-import { decodeXlsx, encodeXlsx } from './xlsx-codec';
-import {
- type CalculationFidelity,
- calculateImported,
- calculationFidelity,
- corpusBytes,
- corpusEntries,
-} from './xlsx-fixtures/corpus';
-
-let calculator: SpreadsheetCalculator;
-beforeAll(() => {
- initSync({
- module: readFileSync(
- createRequire(import.meta.url).resolve('@ironcalc/wasm/wasm_bg.wasm')
- ),
- });
- calculator = createInitializedSpreadsheetCalculator();
-});
-afterAll(() => calculator.dispose());
-
-type CorpusSummary = {
- error?: string;
- sheets?: number;
- cells?: number;
- formulas?: number;
- styledCells?: number;
- calculation?: Omit<CalculationFidelity, 'examples'> | string;
- roundTrip?: 'identical' | string;
- warnings?: string[];
-};
-
-function counts(workbook: WorkbookFileData) {
- const cells = workbook.sheets.flatMap((sheet) => Object.values(sheet.cells));
- return {
- sheets: workbook.sheets.length,
- cells: cells.filter((cell) => cell.value !== '').length,
- formulas: cells.filter(
- (cell) => cell.value.startsWith('=') && cell.format !== 'text'
- ).length,
- styledCells: cells.filter((cell) => Object.keys(cell).length > 1).length,
- };
-}
-
-/** The first difference between two imports, or `identical`. */
-function roundTripDifference(
- first: WorkbookFileData,
- second: WorkbookFileData
-): string {
- if (first.sheets.length !== second.sheets.length)
- return `sheet count ${first.sheets.length} → ${second.sheets.length}`;
- for (const [index, sheet] of first.sheets.entries()) {
- const other = second.sheets[index];
- if (sheet.name !== other.name) return `sheet name ${sheet.name}`;
- const addresses = new Set([
- ...Object.keys(sheet.cells),
- ...Object.keys(other.cells),
- ]);
- for (const address of addresses) {
- const before = JSON.stringify(sheet.cells[address] ?? null);
- const after = JSON.stringify(other.cells[address] ?? null);
- if (before !== after)
- return `${sheet.name}!${address} ${before} → ${after}`;
- }
- if (JSON.stringify(sheet.metadata) !== JSON.stringify(other.metadata))
- return `${sheet.name} metadata`;
- if (
- JSON.stringify(sheet.columnWidths) !== JSON.stringify(other.columnWidths)
- )
- return `${sheet.name} column widths`;
- }
- return 'identical';
-}
-
-describe('real-world XLSX corpus', () => {
- it.each(corpusEntries())(
- '$file',
- async (entry) => {
- const bytes = corpusBytes(entry);
- const summary: CorpusSummary = {};
- let imported: WorkbookFileData;
- try {
- imported = await decodeXlsx(bytes);
- } catch (error) {
- summary.error = error instanceof Error ? error.message : String(error);
- expect(summary).toMatchSnapshot();
- return;
- }
- Object.assign(summary, counts(imported));
- summary.warnings = [...imported.warnings].sort();
- let values: WorkbookCalculation = {};
- if (entry.skipCalculation) summary.calculation = entry.skipCalculation;
- else {
- values = calculateImported(calculator, imported, bytes);
- const { examples, ...calculation } = calculationFidelity(
- bytes,
- imported,
- values
- );
- summary.calculation = calculation;
- if (process.env.XLSX_CORPUS_VERBOSE)
- console.info(entry.file, JSON.stringify(calculation), examples);
- }
- try {
- const exported = await encodeXlsx({
- sheets: imported.sheets.map((sheet, index) => ({
- ...sheet,
- values: values[String(index)],
- })),
- images: imported.images,
- });
- summary.roundTrip = roundTripDifference(
- imported,
- await decodeXlsx(exported.bytes)
- );
- } catch (error) {
- summary.roundTrip = `export failed: ${error instanceof Error ? error.message : String(error)}`;
- }
- expect(summary).toMatchSnapshot();
- },
- 120_000
- );
-});
diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.md b/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.md
index e763c1f2a9..9a09ad93b4 100644
--- a/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.md
+++ b/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.md
@@ -16,7 +16,7 @@ Additional generated-in-test workbooks exercise the 1904 date system, stored Exc
## Real-world corpus
-`real-world/` holds published workbooks that Excel, LibreOffice and other producers wrote: financial models (project finance, DCF, LBO, loan schedules, three-statement models), small-business and government statistics workbooks, and files from open-source spreadsheet test suites. `corpus.ts` lists them from `real-world/manifest.json` and fails if a workbook has no recorded provenance. `../xlsx-corpus.test.ts` imports, recalculates and round-trips each one and snapshots the results.
+`real-world/` holds published workbooks that Excel, LibreOffice and other producers wrote: financial models (project finance, DCF, LBO, loan schedules, three-statement models), small-business and government statistics workbooks, and files from open-source spreadsheet test suites. `corpus.ts` lists them from `real-world/manifest.json` and fails if a workbook has no recorded provenance. `../xlsx-corpus-suite.ts` imports, recalculates and round-trips each one and snapshots the results, split across `../xlsx-corpus-1.test.ts` through `../xlsx-corpus-4.test.ts` so the workbooks run in parallel.
Every file is redistributable. `manifest.json` records each file's source page and license:
diff --git a/apps/web/src/lib/queries/favorites/graphql.ts b/apps/web/src/lib/queries/favorites/graphql.ts
index 5d6bf96ba3..f791dd84e2 100644
--- a/apps/web/src/lib/queries/favorites/graphql.ts
+++ b/apps/web/src/lib/queries/favorites/graphql.ts
@@ -1,6 +1,7 @@
import {
createUrqlMutation,
createUrqlQuery,
+ type UrqlMutationExecutorArgs,
type UrqlQueryResult,
} from '@app/lib/urql-solid';
import { optimisticMutationDispositionOf } from '@graphql-cache/exchange/optimistic';
@@ -171,7 +172,10 @@ function createFavoriteMutation<
>(() => ({
mutation: options.mutation,
client: getGraphqlSoupClient(),
- execute: async ({ client, input }) => {
+ execute: async ({
+ client,
+ input,
+ }: UrqlMutationExecutorArgs<Data, Variables, Input>) => {
const result = await options.execute(client, input);
// Validate before onSuccess, so malformed responses take the error
// lifecycle too. Server-side failures are already transport errors,
diff --git a/apps/web/src/lib/service-clients/service-storage/util/uploadDocx.ts b/apps/web/src/lib/service-clients/service-storage/util/uploadDocx.ts
index 849af4d9f7..2b722851cc 100644
--- a/apps/web/src/lib/service-clients/service-storage/util/uploadDocx.ts
+++ b/apps/web/src/lib/service-clients/service-storage/util/uploadDocx.ts
@@ -1,4 +1,5 @@
import { ENABLE_DOCX_TO_PDF } from '@core/constant/featureFlags';
+import type { MinimalWebSocket } from '@macro-inc/collaboration/websocket/platform/minimal-websocket';
import {
createWebsocketPromiseChain,
JobTypeEnum,
@@ -7,7 +8,7 @@ import {
const PROMISE_TIMEOUT_MS = 15000; // 15 seconds
-export function uploadDocx(ws: WebSocket) {
+export function uploadDocx(ws: MinimalWebSocket) {
const websocketChain = createWebsocketPromiseChain(
{
jobType: JobTypeEnum.DocxUpload,
diff --git a/apps/web/src/lib/service-clients/service-storage/util/websocketPromiseChain.ts b/apps/web/src/lib/service-clients/service-storage/util/websocketPromiseChain.ts
index 3517ab7949..e8670ddc97 100644
--- a/apps/web/src/lib/service-clients/service-storage/util/websocketPromiseChain.ts
+++ b/apps/web/src/lib/service-clients/service-storage/util/websocketPromiseChain.ts
@@ -1,5 +1,5 @@
import type { ResultError } from '@core/util/result';
-
+import type { MinimalWebSocket } from '@macro-inc/collaboration/websocket/platform/minimal-websocket';
import { err, ok, type Result } from 'neverthrow';
import { v7 as uuid7 } from 'uuid';
@@ -30,7 +30,7 @@ export function createWebsocketPromiseChain<
jobType: JobTypeEnum;
getInputData: (...args: FunctionInputArgType) => any | Promise<any>;
timeoutMs: number;
- ws: WebSocket;
+ ws: MinimalWebSocket;
},
listenersChain: L
) {
diff --git a/apps/web/vite.base.ts b/apps/web/vite.base.ts
index 5ea4476bdd..601b538077 100644
--- a/apps/web/vite.base.ts
+++ b/apps/web/vite.base.ts
@@ -187,6 +187,9 @@ export const createAppViteConfig = (): UserConfigFn => {
outDir: 'dist',
emptyOutDir: true,
minify: !NO_MINIFY,
+ // The gzip-size console report re-compresses every chunk, adding ~20 s
+ // to each build; nothing reads it.
+ reportCompressedSize: false,
rollupOptions: {
input: {
app: resolve(__dirname, 'index.html'),
diff --git a/apps/web/vitest.config.ts b/apps/web/vitest.config.ts
index c9a93464c7..1743d7416a 100644
--- a/apps/web/vitest.config.ts
+++ b/apps/web/vitest.config.ts
@@ -1,9 +1,55 @@
+import { createHash } from 'node:crypto';
+import {
+ mkdirSync,
+ readdirSync,
+ readFileSync,
+ rmSync,
+ statSync,
+ writeFileSync,
+} from 'node:fs';
+import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import solidPlugin from 'vite-plugin-solid';
import solidSvg from 'vite-plugin-solid-svg';
import tsconfigPaths from 'vite-tsconfig-paths';
import { configDefaults, defineConfig } from 'vitest/config';
+const MODULE_CACHE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
+
+/**
+ * Directory for Vitest's filesystem module cache, which `--fsModuleCache`
+ * (the `test` script) turns on for every project. Cached transforms embed
+ * resolved import paths, so entries live under a key of the lockfile and
+ * tsconfig; key directories unused for a week are deleted. CI points
+ * `VITEST_MODULE_CACHE_DIR` at a cache volume that persists across jobs.
+ */
+function moduleCachePath(): string {
+ const root =
+ process.env.VITEST_MODULE_CACHE_DIR ??
+ fileURLToPath(new URL('./.cache/vitest-modules', import.meta.url));
+ const key = createHash('sha256')
+ .update(readFileSync(new URL('../../bun.lock', import.meta.url)))
+ .update(readFileSync(new URL('./tsconfig.json', import.meta.url)))
+ .digest('hex')
+ .slice(0, 16);
+ const dir = join(root, key);
+ mkdirSync(dir, { recursive: true });
+ writeFileSync(join(dir, '.last-used'), '');
+ for (const entry of readdirSync(root)) {
+ if (entry === key) continue;
+ const stale = join(root, entry);
+ try {
+ const lastUsed = statSync(join(stale, '.last-used')).mtimeMs;
+ if (Date.now() - lastUsed > MODULE_CACHE_MAX_AGE_MS) {
+ rmSync(stale, { recursive: true, force: true });
+ }
+ } catch {
+ // Not a key directory, or another run removed it first.
+ }
+ }
+ return dir;
+}
+
export default defineConfig({
plugins: [
tsconfigPaths(),
@@ -29,6 +75,8 @@ export default defineConfig({
},
test: {
exclude: [...configDefaults.exclude],
+ // Projects without their own path fall back to the root's.
+ fsModuleCachePath: moduleCachePath(),
projects: [
{
extends: './src/features/scheduling/vitest.config.ts',
diff --git a/bun.lock b/bun.lock
index dd790338a1..8b85f3cc6f 100644
--- a/bun.lock
+++ b/bun.lock
@@ -217,6 +217,7 @@
"orval": "^7.21.0",
"typedoc": "^0.22.18",
"typescript": "^5.9.3",
+ "typescript-native": "npm:[email protected]",
"vite": "^6.4.1",
"vite-plugin-solid": "^2.11.8",
"vite-plugin-solid-svg": "^0.8.1",
@@ -249,6 +250,7 @@
"bebop-tools": "3.1.3",
"jsdom": "^26.1.0",
"typescript": "^5.9.3",
+ "typescript-native": "npm:[email protected]",
"vite-plugin-solid": "^2.11.8",
"vite-plugin-wasm": "^3.5.0",
"vitest": "5.0.1",
@@ -1842,6 +1844,46 @@
"@types/yauzl": ["@types/[email protected]", "", { "dependencies": { "@types/node": "*" } }, "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q=="],
+ "@typescript/typescript-aix-ppc64": ["@typescript/[email protected]", "", { "os": "aix", "cpu": "ppc64" }, "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ=="],
+
+ "@typescript/typescript-darwin-arm64": ["@typescript/[email protected]", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA=="],
+
+ "@typescript/typescript-darwin-x64": ["@typescript/[email protected]", "", { "os": "darwin", "cpu": "x64" }, "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA=="],
+
+ "@typescript/typescript-freebsd-arm64": ["@typescript/[email protected]", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ=="],
+
+ "@typescript/typescript-freebsd-x64": ["@typescript/[email protected]", "", { "os": "freebsd", "cpu": "x64" }, "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw=="],
+
+ "@typescript/typescript-linux-arm": ["@typescript/[email protected]", "", { "os": "linux", "cpu": "arm" }, "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ=="],
+
+ "@typescript/typescript-linux-arm64": ["@typescript/[email protected]", "", { "os": "linux", "cpu": "arm64" }, "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ=="],
+
+ "@typescript/typescript-linux-loong64": ["@typescript/[email protected]", "", { "os": "linux", "cpu": "none" }, "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ=="],
+
+ "@typescript/typescript-linux-mips64el": ["@typescript/[email protected]", "", { "os": "linux", "cpu": "none" }, "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA=="],
+
+ "@typescript/typescript-linux-ppc64": ["@typescript/[email protected]", "", { "os": "linux", "cpu": "ppc64" }, "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA=="],
+
+ "@typescript/typescript-linux-riscv64": ["@typescript/[email protected]", "", { "os": "linux", "cpu": "none" }, "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ=="],
+
+ "@typescript/typescript-linux-s390x": ["@typescript/[email protected]", "", { "os": "linux", "cpu": "s390x" }, "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw=="],
+
+ "@typescript/typescript-linux-x64": ["@typescript/[email protected]", "", { "os": "linux", "cpu": "x64" }, "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A=="],
+
+ "@typescript/typescript-netbsd-arm64": ["@typescript/[email protected]", "", { "os": "none", "cpu": "arm64" }, "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA=="],
+
+ "@typescript/typescript-netbsd-x64": ["@typescript/[email protected]", "", { "os": "none", "cpu": "x64" }, "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA=="],
+
+ "@typescript/typescript-openbsd-arm64": ["@typescript/[email protected]", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ=="],
+
+ "@typescript/typescript-openbsd-x64": ["@typescript/[email protected]", "", { "os": "openbsd", "cpu": "x64" }, "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg=="],
+
+ "@typescript/typescript-sunos-x64": ["@typescript/[email protected]", "", { "os": "sunos", "cpu": "x64" }, "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g=="],
+
+ "@typescript/typescript-win32-arm64": ["@typescript/[email protected]", "", { "os": "win32", "cpu": "arm64" }, "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ=="],
+
+ "@typescript/typescript-win32-x64": ["@typescript/[email protected]", "", { "os": "win32", "cpu": "x64" }, "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g=="],
+
"@typespec/ts-http-runtime": ["@typespec/[email protected]", "", { "dependencies": { "http-proxy-agent": "^7.0.0", "https-proxy-agent": "^7.0.0", "tslib": "^2.6.2" } }, "sha512-jIXhD0eWQ1JA6ln/5Dltyx22UxWNrw0hZmhy2rlv6m6KgF7kplHx3g0fzi09lNmTJQRR91OlemYp3xFnvDK9og=="],
"@ungap/structured-clone": ["@ungap/[email protected]", "", {}, "sha512-60YRaenCQcVjYEKOcG824+DRGGIQ3VKErcBoAEDJZz5bKIs2ZG+X/H9Nk+Q6EVkwJk5QNApxbrc5QtBSwtrXAg=="],
@@ -3638,6 +3680,8 @@
"typescript": ["[email protected]", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
+ "typescript-native": ["[email protected]", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="],
+
"uc.micro": ["[email protected]", "", {}, "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A=="],
"uhyphen": ["[email protected]", "", {}, "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="],
diff --git a/crates/ai_tools/src/test/booking_links.rs b/crates/ai_tools/src/test/booking_links.rs
index 72fecff9f2..ddd43d1e3c 100644
--- a/crates/ai_tools/src/test/booking_links.rs
+++ b/crates/ai_tools/src/test/booking_links.rs
@@ -25,6 +25,11 @@ use model_user::UserContext;
use rootcause::Report;
use uuid::Uuid;
+/// CI clones every test database from a `template1` that already holds the macrodb
+/// schema (`just setup_test_template`). This test builds its own schema, so it
+/// starts from an empty `public` schema.
+const EMPTY_PUBLIC_SCHEMA: &str = "DROP SCHEMA public CASCADE; CREATE SCHEMA public;";
+
#[derive(Clone)]
struct NoAuth;
impl MacroAuthorizationService for NoAuth {
@@ -90,6 +95,10 @@ async fn execute(context: Context, tool: &str, args: Value) -> Result<Value, Str
#[sqlx::test(migrations = false)]
async fn confirmed_booking_is_saved_and_bookable_without_touching_other_links(pool: sqlx::PgPool) {
+ sqlx::raw_sql(EMPTY_PUBLIC_SCHEMA)
+ .execute(&pool)
+ .await
+ .unwrap();
sqlx::raw_sql("CREATE TABLE \"User\" (id text PRIMARY KEY); CREATE TABLE team (id uuid PRIMARY KEY); INSERT INTO \"User\" VALUES ('macro|[email protected]');").execute(&pool).await.unwrap();
sqlx::raw_sql(include_str!(
"../../../macro_db_client/migrations/20260918164303_calendar_scheduling.sql"
diff --git a/crates/calendar_scheduling/src/inbound/router/test.rs b/crates/calendar_scheduling/src/inbound/router/test.rs
index c161ce3e6b..cb74a298e5 100644
--- a/crates/calendar_scheduling/src/inbound/router/test.rs
+++ b/crates/calendar_scheduling/src/inbound/router/test.rs
@@ -7,6 +7,11 @@ use model_user::UserContext;
use rootcause::Report;
use std::sync::atomic::{AtomicBool, Ordering};
+/// CI clones every test database from a `template1` that already holds the macrodb
+/// schema (`just setup_test_template`). These tests build their own schema, so they
+/// start from an empty `public` schema.
+const EMPTY_PUBLIC_SCHEMA: &str = "DROP SCHEMA public CASCADE; CREATE SCHEMA public;";
+
#[derive(Clone)]
struct Identity;
impl MacroAuthorizationService for Identity {
@@ -76,6 +81,10 @@ impl Calendars for Calendar {
#[sqlx::test(migrations = false)]
async fn http_booking_receipt_recovery_reschedule_cancel_and_team_auth(pool: sqlx::PgPool) {
+ sqlx::raw_sql(EMPTY_PUBLIC_SCHEMA)
+ .execute(&pool)
+ .await
+ .unwrap();
sqlx::raw_sql("CREATE TABLE \"User\" (id text PRIMARY KEY); CREATE TABLE team (id uuid PRIMARY KEY); INSERT INTO \"User\" VALUES ('macro|[email protected]');").execute(&pool).await.unwrap();
sqlx::raw_sql(include_str!(
"../../../../macro_db_client/migrations/20260918164303_calendar_scheduling.sql"
diff --git a/crates/calendar_scheduling/src/outbound/postgres/test.rs b/crates/calendar_scheduling/src/outbound/postgres/test.rs
index 11993863c3..6ab0419775 100644
--- a/crates/calendar_scheduling/src/outbound/postgres/test.rs
+++ b/crates/calendar_scheduling/src/outbound/postgres/test.rs
@@ -1,7 +1,16 @@
use super::*;
use chrono::Duration;
+/// CI clones every test database from a `template1` that already holds the macrodb
+/// schema (`just setup_test_template`). These tests build their own schema, so they
+/// start from an empty `public` schema.
+const EMPTY_PUBLIC_SCHEMA: &str = "DROP SCHEMA public CASCADE; CREATE SCHEMA public;";
+
async fn setup(pool: &PgPool) {
+ sqlx::raw_sql(EMPTY_PUBLIC_SCHEMA)
+ .execute(pool)
+ .await
+ .unwrap();
sqlx::raw_sql(
"CREATE TABLE \"User\" (id text PRIMARY KEY); CREATE TABLE team (id uuid PRIMARY KEY);",
)
diff --git a/crates/codex_cloud_agents/src/outbound/postgres_journal/test.rs b/crates/codex_cloud_agents/src/outbound/postgres_journal/test.rs
index f21891c7ec..4bbb40d619 100644
--- a/crates/codex_cloud_agents/src/outbound/postgres_journal/test.rs
+++ b/crates/codex_cloud_agents/src/outbound/postgres_journal/test.rs
@@ -1,6 +1,11 @@
use super::*;
use uuid::Uuid;
+/// CI clones every test database from a `template1` that already holds the macrodb
+/// schema (`just setup_test_template`). These tests build their own schema, so they
+/// start from an empty `public` schema.
+const EMPTY_PUBLIC_SCHEMA: &str = "DROP SCHEMA public CASCADE; CREATE SCHEMA public;";
+
#[sqlx::test(migrations = false)]
async fn journal_survives_takeover_and_rejects_stale_or_foreign_attachments(pool: PgPool) {
tables(&pool).await;
@@ -114,6 +119,10 @@ async fn journal_survives_takeover_and_rejects_stale_or_foreign_attachments(pool
}
async fn tables(pool: &PgPool) {
+ sqlx::raw_sql(EMPTY_PUBLIC_SCHEMA)
+ .execute(pool)
+ .await
+ .unwrap();
sqlx::raw_sql("CREATE TABLE agent_session(id uuid PRIMARY KEY, manager_replica_id uuid, manager_fence bigint NOT NULL)")
.execute(pool).await.unwrap();
sqlx::raw_sql(include_str!(
diff --git a/crates/cursor_cloud_agents/src/outbound/postgres_journal/test.rs b/crates/cursor_cloud_agents/src/outbound/postgres_journal/test.rs
index 04a0dada3b..7f242b5fae 100644
--- a/crates/cursor_cloud_agents/src/outbound/postgres_journal/test.rs
+++ b/crates/cursor_cloud_agents/src/outbound/postgres_journal/test.rs
@@ -1,8 +1,17 @@
use super::*;
use uuid::Uuid;
+/// CI clones every test database from a `template1` that already holds the macrodb
+/// schema (`just setup_test_template`). These tests build their own schema, so they
+/// start from an empty `public` schema.
+const EMPTY_PUBLIC_SCHEMA: &str = "DROP SCHEMA public CASCADE; CREATE SCHEMA public;";
+
#[sqlx::test(migrations = false)]
async fn append_is_ordered_fenced_and_session_scoped(pool: PgPool) {
+ sqlx::raw_sql(EMPTY_PUBLIC_SCHEMA)
+ .execute(&pool)
+ .await
+ .unwrap();
sqlx::raw_sql(
"CREATE TABLE agent_session(id uuid PRIMARY KEY, manager_replica_id uuid, manager_fence bigint NOT NULL);
CREATE TABLE agent_session_log(id uuid PRIMARY KEY, agent_session_id uuid NOT NULL REFERENCES agent_session(id));
diff --git a/crates/macro_db_client/tests/normalize_macro_bot_mention_ids.rs b/crates/macro_db_client/tests/normalize_macro_bot_mention_ids.rs
index 64827d34bf..c8ea53ad0d 100644
--- a/crates/macro_db_client/tests/normalize_macro_bot_mention_ids.rs
+++ b/crates/macro_db_client/tests/normalize_macro_bot_mention_ids.rs
@@ -1,5 +1,10 @@
use sqlx::{Pool, Postgres};
+/// CI clones every test database from a `template1` that already holds the macrodb
+/// schema (`just setup_test_template`). These tests build their own schema, so they
+/// start from an empty `public` schema.
+const EMPTY_PUBLIC_SCHEMA: &str = "DROP SCHEMA public CASCADE; CREATE SCHEMA public;";
+
const NORMALIZED_MACRO_BOT_ID: &str = "bot|00000000-0000-0000-0000-00000000a1a1";
const OTHER_UUID: &str = "11111111-1111-1111-1111-111111111111";
const MIGRATION: &str =
@@ -180,6 +185,7 @@ async fn migration_normalizes_bare_uuid_message_senders(
}
async fn create_minimal_tables(pool: &Pool<Postgres>) -> anyhow::Result<()> {
+ sqlx::raw_sql(EMPTY_PUBLIC_SCHEMA).execute(pool).await?;
sqlx::raw_sql(
r#"
CREATE TABLE comms_messages (
diff --git a/crates/macro_db_migrator/src/test.rs b/crates/macro_db_migrator/src/test.rs
index e2031a95c4..aab977d63b 100644
--- a/crates/macro_db_migrator/src/test.rs
+++ b/crates/macro_db_migrator/src/test.rs
@@ -2,9 +2,18 @@ use super::MACRO_DB_MIGRATIONS;
use sqlx::{PgPool, migrate::Migrator};
use std::borrow::Cow;
+/// CI clones every test database from a `template1` that already holds the macrodb
+/// schema (`just setup_test_template`). These tests build their own schema, so they
+/// start from an empty `public` schema.
+const EMPTY_PUBLIC_SCHEMA: &str = "DROP SCHEMA public CASCADE; CREATE SCHEMA public;";
+
const SCHEMA_DROP_VERSION: i64 = 20261002150422;
async fn before_drop(pool: &PgPool) {
+ sqlx::raw_sql(EMPTY_PUBLIC_SCHEMA)
+ .execute(pool)
+ .await
+ .unwrap();
let migrator = Migrator {
migrations: Cow::Owned(
MACRO_DB_MIGRATIONS
diff --git a/crates/notification_state/src/postgres/test/migration.rs b/crates/notification_state/src/postgres/test/migration.rs
index 06efafe0bd..fd049f048b 100644
--- a/crates/notification_state/src/postgres/test/migration.rs
+++ b/crates/notification_state/src/postgres/test/migration.rs
@@ -1,6 +1,11 @@
use crate::NotificationState;
use sqlx::{Connection, PgPool};
+/// CI clones every test database from a `template1` that already holds the macrodb
+/// schema (`just setup_test_template`). These tests build their own schema, so they
+/// start from an empty `public` schema.
+const EMPTY_PUBLIC_SCHEMA: &str = "DROP SCHEMA public CASCADE; CREATE SCHEMA public;";
+
const LEGACY_SCHEMA: &str = include_str!(
"../../../../macro_db_client/migrations/20260126170641_create_notification_tables.sql"
);
@@ -20,6 +25,7 @@ async fn migration_normalizes_legacy_states_without_changing_timestamps(
) -> Result<(), sqlx::Error> {
// DDL and the pre-cutover fixture cannot be checked against the current
// development schema. Execute the real migration scripts in an isolated DB.
+ sqlx::raw_sql(EMPTY_PUBLIC_SCHEMA).execute(&pool).await?;
sqlx::raw_sql(LEGACY_SCHEMA).execute(&pool).await?;
sqlx::raw_sql(LEGACY_INDEXES).execute(&pool).await?;
sqlx::raw_sql(
@@ -77,6 +83,7 @@ async fn migration_normalizes_legacy_states_without_changing_timestamps(
#[sqlx::test(migrations = false)]
async fn state_defaults_and_constraints_are_enforced(pool: PgPool) -> Result<(), sqlx::Error> {
+ sqlx::raw_sql(EMPTY_PUBLIC_SCHEMA).execute(&pool).await?;
sqlx::raw_sql(LEGACY_SCHEMA).execute(&pool).await?;
sqlx::raw_sql(LEGACY_INDEXES).execute(&pool).await?;
sqlx::raw_sql(UP).execute(&pool).await?;
diff --git a/docs/DATABASE_DEVELOPMENT.md b/docs/DATABASE_DEVELOPMENT.md
index 1482120b5c..b683871be6 100644
--- a/docs/DATABASE_DEVELOPMENT.md
+++ b/docs/DATABASE_DEVELOPMENT.md
@@ -35,6 +35,20 @@ On **Cursor Cloud**, installation already prepares the database and test envs.
Start the infrastructure with `bash .cursor/infra.sh`; see [Cursor Cloud](CURSOR_CLOUD.md).
Apply any new migrations before testing.
+### Pre-migrated test template
+
+CI runs `just setup_test_template` after `initialize_dbs`. It migrates `template1`,
+which Postgres clones for every new database, so each `#[sqlx::test]` database
+starts with the MacroDB schema instead of replaying every migration. That
+replay used to be nearly all of the Rust test suite's runtime. Run the recipe
+locally only against a Postgres dedicated to one checkout; every database created
+on that server afterwards gets the schema too.
+
+Tests that build their own schema (`#[sqlx::test(migrations = false)]`, or a
+crate's test-only schema fixture) must first run
+`DROP SCHEMA public CASCADE; CREATE SCHEMA public;`, since their database may
+arrive pre-migrated.
+
## Safe database schema changes
### Entity tables and reusable storage
diff --git a/nix-support/node_modules-hashes.json b/nix-support/node_modules-hashes.json
index 6839c2ba30..7cf66fb599 100644
--- a/nix-support/node_modules-hashes.json
+++ b/nix-support/node_modules-hashes.json
@@ -1,6 +1,6 @@
{
"nodeModules": {
- "aarch64-darwin": "sha256-tjUS6Z66ykQJ+FmBB+TYmrxbur3wu7f5iJbnaZNVieA=",
- "x86_64-linux": "sha256-egI40KcIhudsvS0EkjSMOi0gnAy6cKwG/vfxi7W8Wow="
+ "aarch64-darwin": "sha256-9bFwe5o3iBLBrlBlRQ3hmeq0A7VjmDbsxGTjOpFnSqY=",
+ "x86_64-linux": "sha256-9WeocJXG6/po7vLNPjue7JUErg1K7nJ5Q7X5yZ6XrpE="
}
}
diff --git a/nix/cloud-storage.nix b/nix/cloud-storage.nix
index 1f732c893c..f128d44d27 100644
--- a/nix/cloud-storage.nix
+++ b/nix/cloud-storage.nix
@@ -55,8 +55,10 @@
# `.sh` is required so `include_str!` of
# `crates/agent_harness/container/ensure_ready.sh` survives the prune;
# `.ttf` and `.xml` for the fonts and preset shape definitions that
- # `crates/pptx_engine` embeds with `include_bytes!`.
- assetFilter = path: _type: builtins.match ".*\\.(md|html|txt|json|canvas|sql|sh|ttf|xml)$" path != null;
+ # `crates/pptx_engine` embeds with `include_bytes!`; `.graphql` for the
+ # `static_assets/schema.graphql` that `cache-core`'s build script reads;
+ # `.jsonl` for the recorded sessions `agent_fold` tests embed.
+ assetFilter = path: _type: builtins.match ".*\\.(md|html|txt|json|jsonl|canvas|sql|sh|ttf|xml|graphql)$" path != null;
# Rust local HTTP clients embed only the public proxy CA, never its keys.
localCaFilter = path: _type: pkgs.lib.hasSuffix "/infra/local/certs/ca.pem" (toString path);
binFilter = path: _type: builtins.match ".*\\.bin$" path != null;
@@ -226,7 +228,9 @@
doCheck = false;
doInstallCargoArtifacts = true;
cargoExtraArgs = "--locked --all-features --workspace --lib --exclude sync_service";
- RUSTFLAGS = "-Dwarnings" + pkgs.lib.optionalString isLinux " -C link-arg=-fuse-ld=mold";
+ # Keep RUSTFLAGS identical to cargoArtifacts: any difference changes
+ # cargo's fingerprint and recompiles every dependency in this layer.
+ # The clippy checks enforce `-D warnings`.
RUSTDOCFLAGS = "-Dwarnings";
}
);
diff --git a/packages/collaboration/package.json b/packages/collaboration/package.json
index fe534d90c3..15c79c4834 100644
--- a/packages/collaboration/package.json
+++ b/packages/collaboration/package.json
@@ -15,7 +15,7 @@
"generate-sync-schema": "bun scripts/generate-schema.ts",
"lint": "biome check .",
"test": "vitest run --config vitest.config.ts",
- "type-check": "tsc --noEmit --project tsconfig.json"
+ "type-check": "bun ../../node_modules/typescript-native/bin/tsc --noEmit --project tsconfig.json"
},
"dependencies": {
"@loro-mirror/core": "workspace:*",
@@ -38,6 +38,7 @@
"bebop-tools": "3.1.3",
"jsdom": "^26.1.0",
"typescript": "^5.9.3",
+ "typescript-native": "npm:[email protected]",
"vite-plugin-solid": "^2.11.8",
"vite-plugin-wasm": "^3.5.0",
"vitest": "5.0.1",
diff --git a/packages/sdk/RELEASING.md b/packages/sdk/RELEASING.md
index 48badbcae4..29bee1e62b 100644
--- a/packages/sdk/RELEASING.md
+++ b/packages/sdk/RELEASING.md
@@ -34,8 +34,9 @@ npm pack --dry-run --ignore-scripts
The compiled JavaScript and declaration files referenced by `exports` must
appear in the package listing. `coverage` checks endpoint wrappers, not test
coverage. `SDK Check` runs the same package suite on the PR and again on the
-merge to `main`, and additionally checks generated-code freshness on the PR; if
-it reports staleness, run `just update-generated` and review the diff. Run
+merge to `main`. Generated-code freshness is checked on the PR by the web app
+workflow's `Generated Code Check`; if it reports staleness, run
+`just update-generated` and review the diff. Run
`just check` from the repository root before committing.
Forgetting to bump publishes nothing, which is the intended failure mode: open a
diff --git a/services/authentication_service/src/api/context.rs b/services/authentication_service/src/api/context.rs
index eb092f47bd..02e8852462 100644
--- a/services/authentication_service/src/api/context.rs
+++ b/services/authentication_service/src/api/context.rs
@@ -58,7 +58,7 @@ use sqlx::PgPool;
use tokio_util::task::TaskTracker;
use crate::microsoft_token_cipher::MicrosoftTokenCipher;
-use authentication_service::service::signup_policy::SignupPolicy;
+use crate::service::signup_policy::SignupPolicy;
use cursor_api_key::cipher::CursorApiKeyCipher;
pub(crate) type NotificationIngressType = SqsNotificationIngress<SqsQueue>;
@@ -148,16 +148,14 @@ pub(crate) struct ApiContext {
pub macro_cache_client: Arc<MacroCache>,
pub stripe_client: Arc<stripe::Client>,
pub subscription_checkout: Arc<
- authentication_service::service::subscription_checkout::CheckoutService<
- authentication_service::outbound::subscription_checkout::StripeCheckoutGateway<
- GtmInviteServiceType,
- >,
+ crate::service::subscription_checkout::CheckoutService<
+ crate::outbound::subscription_checkout::StripeCheckoutGateway<GtmInviteServiceType>,
>,
>,
pub document_storage_service_client:
Arc<document_storage_service_client::DocumentStorageServiceClient>,
pub user_deletion: Arc<
- authentication_service::outbound::user_deletion::UserDeletionAdapter<
+ crate::outbound::user_deletion::UserDeletionAdapter<
TeamsServiceType,
onboarding::outbound::pg_onboarding_repo::PgOnboardingRepo,
>,
diff --git a/services/authentication_service/src/api/email/verify_email_link.rs b/services/authentication_service/src/api/email/verify_email_link.rs
index 4c839a9818..8de7813495 100644
--- a/services/authentication_service/src/api/email/verify_email_link.rs
+++ b/services/authentication_service/src/api/email/verify_email_link.rs
@@ -7,7 +7,7 @@ use axum::{
use macro_middleware::tracking::ClientIp;
use crate::api::{context::ApiContext, utils::default_redirect_url};
-use authentication_service::service::user::create_user::create_user_profile;
+use crate::service::user::create_user::create_user_profile;
use model::response::{EmptyResponse, ErrorResponse};
diff --git a/services/authentication_service/src/api/user/stripe/create_checkout_session_v2.rs b/services/authentication_service/src/api/user/stripe/create_checkout_session_v2.rs
index 825dc13666..3dd6cfdf7e 100644
--- a/services/authentication_service/src/api/user/stripe/create_checkout_session_v2.rs
+++ b/services/authentication_service/src/api/user/stripe/create_checkout_session_v2.rs
@@ -1,4 +1,4 @@
-use authentication_service::service::subscription_checkout::CheckoutRequest;
+use crate::service::subscription_checkout::CheckoutRequest;
use axum::{Json, extract::State};
use entity_access::domain::models::OwnerTeamRole;
use entity_access::domain::ports::EntityAccessService;
diff --git a/services/authentication_service/src/api/user/stripe/shared.rs b/services/authentication_service/src/api/user/stripe/shared.rs
index 0022044c4f..3b79750356 100644
--- a/services/authentication_service/src/api/user/stripe/shared.rs
+++ b/services/authentication_service/src/api/user/stripe/shared.rs
@@ -1,4 +1,4 @@
-use authentication_service::{
+use crate::{
outbound::subscription_checkout::StripeCheckoutError,
service::subscription_checkout::CheckoutError,
};
diff --git a/services/authentication_service/src/api/webhooks/user/create_user_webhook.rs b/services/authentication_service/src/api/webhooks/user/create_user_webhook.rs
index b104dae883..2e7cee8919 100644
--- a/services/authentication_service/src/api/webhooks/user/create_user_webhook.rs
+++ b/services/authentication_service/src/api/webhooks/user/create_user_webhook.rs
@@ -4,8 +4,8 @@ mod test;
use analytics_client::{MetaActionSource, MetaUserData};
use std::{collections::HashSet, future::Future};
+use crate::service::signup_policy::SignupPolicy;
use anyhow::Context;
-use authentication_service::service::signup_policy::SignupPolicy;
use axum::{
extract::{self, State},
http::StatusCode,
@@ -14,6 +14,10 @@ use axum::{
use macro_authorization::{InternalOnly, MacroAuthorizationExtractor};
use rand::Rng;
+use crate::service::user::create_user::create_user;
+use crate::service::user::support_channel_welcome::{
+ AuthorizedSupportChannelMessages, post_support_channel_welcome,
+};
use crate::{
api::{
context::{ApiContext, AuthorizationService},
@@ -21,10 +25,6 @@ use crate::{
},
rate_limit_config::RATE_LIMIT_CONFIG,
};
-use authentication_service::service::user::create_user::create_user;
-use authentication_service::service::user::support_channel_welcome::{
- AuthorizedSupportChannelMessages, post_support_channel_welcome,
-};
use channels::domain::{
models::{ChannelType, CreateChannelRequest},
ports::ChannelService,
diff --git a/services/authentication_service/src/api/webhooks/user/create_user_webhook/test.rs b/services/authentication_service/src/api/webhooks/user/create_user_webhook/test.rs
index f5bde3eafc..8d0b83859d 100644
--- a/services/authentication_service/src/api/webhooks/user/create_user_webhook/test.rs
+++ b/services/authentication_service/src/api/webhooks/user/create_user_webhook/test.rs
@@ -3,7 +3,7 @@ use std::sync::{
atomic::{AtomicUsize, Ordering},
};
-use authentication_service::service::signup_policy::SignupPolicy;
+use crate::service::signup_policy::SignupPolicy;
use axum::http::StatusCode;
use macro_user_id::email::Email;
use model::authentication::webhooks::{
diff --git a/services/authentication_service/src/api/webhooks/user/delete_user_webhook.rs b/services/authentication_service/src/api/webhooks/user/delete_user_webhook.rs
index 6d1c1bb7e6..a5b786a903 100644
--- a/services/authentication_service/src/api/webhooks/user/delete_user_webhook.rs
+++ b/services/authentication_service/src/api/webhooks/user/delete_user_webhook.rs
@@ -232,7 +232,7 @@ async fn delete_user(
.iter()
.map(|id| MacroUserIdStr::try_from(id.clone()))
.collect::<Result<Vec<_>, _>>()?;
- authentication_service::service::user::delete_user::delete_user_data(
+ crate::service::user::delete_user::delete_user_data(
ctx.user_deletion.as_ref(),
&macro_user.id,
&users,
diff --git a/services/authentication_service/src/config.rs b/services/authentication_service/src/config.rs
index 5afdaa6beb..75a644a3a8 100644
--- a/services/authentication_service/src/config.rs
+++ b/services/authentication_service/src/config.rs
@@ -1,7 +1,7 @@
use std::sync::LazyLock;
+use crate::service::signup_policy::SignupPolicy;
use anyhow::Context;
-use authentication_service::service::signup_policy::SignupPolicy;
use database_env_vars::{DatabaseUrl, RedisUri};
use gtm_invite::domain::models::{GtmInviteConfig, PromoCode};
use macro_auth::InternalApiKey;
@@ -287,8 +287,9 @@ impl Config {
self.signup_policy_for_environment(self.environment)
}
- /// Resolves the signup policy for an explicit environment.
- pub(crate) fn signup_policy_for_environment(
+ /// Resolves the signup policy for an explicit environment. Public for the
+ /// Doppler config check binary, which validates both environments.
+ pub fn signup_policy_for_environment(
&self,
environment: Environment,
) -> anyhow::Result<SignupPolicy> {
diff --git a/services/authentication_service/src/config/test.rs b/services/authentication_service/src/config/test.rs
index 7430e009a6..28490f8fb3 100644
--- a/services/authentication_service/src/config/test.rs
+++ b/services/authentication_service/src/config/test.rs
@@ -1,4 +1,4 @@
-use authentication_service::service::signup_policy::SignupPolicyDenial;
+use crate::service::signup_policy::SignupPolicyDenial;
use super::*;
diff --git a/services/authentication_service/src/doppler_config.rs b/services/authentication_service/src/doppler_config.rs
index 1593ecc815..e053ffe5d3 100644
--- a/services/authentication_service/src/doppler_config.rs
+++ b/services/authentication_service/src/doppler_config.rs
@@ -1,10 +1,6 @@
-#![allow(unused)]
-#![recursion_limit = "256"]
-
+use authentication_service::Config;
use macro_env::Environment;
-mod config;
-
const DOPPLER_PROJECT: &str = "authentication-service";
#[tokio::main]
@@ -16,7 +12,7 @@ async fn main() -> anyhow::Result<()> {
.build()
.expect("able to grab doppler project");
- let dev_config = dev.load::<config::Config>().await?;
+ let dev_config = dev.load::<Config>().await?;
dev_config.signup_policy_for_environment(Environment::Develop)?;
let prd = doppler_config::DopplerConfig::builder()
@@ -26,7 +22,7 @@ async fn main() -> anyhow::Result<()> {
.build()
.expect("able to grab doppler project");
- let prd_config = prd.load::<config::Config>().await?;
+ let prd_config = prd.load::<Config>().await?;
prd_config.signup_policy_for_environment(Environment::Production)?;
Ok(())
diff --git a/services/authentication_service/src/lib.rs b/services/authentication_service/src/lib.rs
index d83bf12274..601b1458ea 100644
--- a/services/authentication_service/src/lib.rs
+++ b/services/authentication_service/src/lib.rs
@@ -1,2 +1,19 @@
+#![recursion_limit = "256"]
+//! Authentication service. The service binary, the OpenAPI generator, and the
+//! Doppler config check all link this library, so the service compiles once
+//! and caches as one rlib.
+
+mod api;
+mod config;
+mod generate_password;
+mod microsoft_token_cipher;
pub mod outbound;
+mod rate_limit_config;
+mod server;
pub mod service;
+
+/// The service's OpenAPI document, rendered by the `*_openapi` binary.
+pub use api::swagger::ApiDoc;
+/// The service configuration, loaded from Doppler by the config check binary.
+pub use config::Config;
+pub use server::run;
diff --git a/services/authentication_service/src/main.rs b/services/authentication_service/src/main.rs
index 6e140fdcf6..de7eff7e0b 100644
--- a/services/authentication_service/src/main.rs
+++ b/services/authentication_service/src/main.rs
@@ -1,655 +1,11 @@
+// `run()`'s future type is deep enough to need the library's recursion limit
+// wherever it is awaited.
#![recursion_limit = "256"]
-use analytics_client::{
- AnalyticsClient, AnalyticsClientConfig, GoogleAnalyticsConfig, MetaConfig, PostHogConfig,
-};
-use anyhow::{Context, anyhow};
-use channels::{
- domain::{
- service::ChannelServiceImpl,
- side_effects::{ChannelSideEffectService, SpawnedChannelEventDispatcher},
- },
- outbound::{
- connection_gateway_realtime::ConnectionGatewayChannelRealtimePublisher,
- contacts_dispatcher::ContactsChannelDispatcher,
- notification_sender::NotificationChannelSender,
- pg_channel_reference_share_permissions::PgChannelReferenceSharePermissions,
- pg_channels_repo::PgChannelsRepo, pg_side_effect_context::PgChannelSideEffectContext,
- },
-};
-use config::{Config, Environment};
-use connection_gateway_client::ConnectionGatewayClient;
-use contacts::{domain::service::SqsContactsIngress, outbound::ingress::SqsContactsQueue};
-use document_storage_service_client::DocumentStorageServiceClient;
-use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};
-use foreign_entity::{
- domain::service::ForeignEntityServiceImpl,
- outbound::pg_foreign_entity_repo::PgForeignEntityRepo,
-};
-use github::{
- domain::service::{GithubLinkConfig, GithubLinkServiceImpl},
- outbound::{
- github_auth_client::GithubAuthImpl, github_oauth_client::GithubOauthImpl,
- pg_github_repo::PgGithubRepo,
- },
-};
-use github_pull_requests::{
- domain::service::GithubPullRequestServiceImpl,
- outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo,
-};
-use loops_client::LoopsClient;
-use macro_auth::middleware::decode_jwt::JwtValidationArgs;
-use macro_authorization::{
- InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationState,
- PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,
-};
-use macro_entrypoint::MacroEntrypoint;
-use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
-use macro_service_urls::{
- AppServiceUrl, ConnectionGatewayUrl, DocumentStorageServiceUrl, EmailServiceUrl,
-};
-use native_app_service::{
- domain::{models::PlatformData, service::NativeAppServiceImpl},
- outbound::DefaultBundleFetcher,
-};
-use notification::outbound::queue::SqsQueue;
-use notification::{
- domain::service::SqsNotificationIngress, outbound::rate_limit::RedisRateLimitAdapter,
-};
-use rate_limit::domain::service::RateLimitServiceImpl;
-use roles_and_permissions::{
- domain::service::UserRolesAndPermissionsServiceImpl, outbound::pgpool::MacroDB,
-};
-use secretsmanager_client::SecretManager;
-use sqlx::postgres::PgPoolOptions;
-use teams::{
- domain::team_service::TeamServiceImpl,
- outbound::{
- contacts_enqueuer::ContactsIngressEnqueuer, customer_repo::CustomerRepositoryImpl,
- team_analytics::AnalyticsClientTeamAnalytics, team_repo::TeamRepositoryImpl,
- },
-};
-
-use gtm_invite::{
- domain::service::GtmInviteServiceImpl, outbound::pg_gtm_invite_repo::PgGtmInviteRepo,
-};
-use referral::{
- domain::service::ReferralServiceImpl,
- outbound::{pg_referral_repo::PgReferralRepo, stripe_discount_client::StripeDiscountClient},
-};
-
-use crate::{
- api::context::{
- ApiContext, AuthorizationService, MacroApiTokenContext, MacroApiTokenExpirySeconds,
- MacroApiTokenIssuer, MacroApiTokenPrivateSecretKey, StripeWebhookSecretKey,
- },
- microsoft_token_cipher::{
- EnvelopeMicrosoftTokenCipher, KmsDataKeyProvider, MicrosoftTokenCipher,
- },
-};
-use std::{sync::Arc, time::Duration};
-use tokio_util::task::TaskTracker;
-mod api;
-mod config;
-mod generate_password;
-mod microsoft_token_cipher;
-mod rate_limit_config;
+use macro_entrypoint::MacroEntrypoint;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
MacroEntrypoint::default().init();
- let env = Environment::new_or_prod();
-
- // One SDK config is sufficient for every AWS client in this process.
- let aws_config = macro_aws_config::get_macro_aws_config().await;
- let secretsmanager_client = secretsmanager_client::SecretsManager::new(
- aws_sdk_secretsmanager::Client::new(&aws_config),
- );
-
- // Parse our configuration from the environment.
- let config = Config::from_env().context("expected to be able to generate config")?;
- let signup_policy = Arc::new(
- config
- .signup_policy()
- .context("invalid signup policy configuration")?,
- );
- let gtm_invite_config = config
- .gtm_invite_config()
- .context("invalid GTM invite link configuration")?;
- let microsoft_credentials = config
- .microsoft_credentials()
- .context("invalid Microsoft OAuth configuration")?;
- let microsoft_token_cipher = microsoft_credentials.as_ref().map(|credentials| {
- Arc::new(EnvelopeMicrosoftTokenCipher::new(KmsDataKeyProvider::new(
- aws_sdk_kms::Client::new(&aws_config),
- credentials.token_kms_key_id.clone(),
- ))) as Arc<dyn MicrosoftTokenCipher>
- });
-
- // A separate KMS key from the Microsoft one by design: sharing it would
- // grant whatever decrypts Cursor keys access to the key protecting
- // everyone's mailbox credentials.
- let cursor_api_key_cipher = Arc::new(cursor_api_key::cipher::KmsCursorApiKeyCipher::new(
- cursor_api_key::cipher::AwsKmsCiphertexts::new(
- aws_sdk_kms::Client::new(&aws_config),
- config.cursor_api_key_kms_key_id()?,
- ),
- )) as Arc<dyn cursor_api_key::cipher::CursorApiKeyCipher>;
-
- let internal_api_key = config.internal_api_key.clone();
-
- let stripe_webhook_secret = secretsmanager_client
- .get_maybe_secret_value(env, StripeWebhookSecretKey::new()?)
- .await?;
-
- tracing::trace!("initialized config");
-
- let (min_connections, max_connections): (u32, u32) = match config.environment {
- Environment::Production => (5, 25),
- Environment::Develop => (1, 25),
- Environment::Local => (1, 10),
- };
-
- let db = PgPoolOptions::new()
- .min_connections(min_connections)
- .max_connections(max_connections)
- .connect(&config.database_url)
- .await
- .context("could not connect to db")?;
-
- tracing::trace!(
- min_connections,
- max_connections,
- "initialized db connection"
- );
-
- // Macro API token
- let macro_api_token_private_key = secretsmanager_client
- .get_maybe_secret_value(config.environment, MacroApiTokenPrivateSecretKey::new()?)
- .await?;
-
- let fusionauth_api_key = match config.environment {
- Environment::Local => config.fusionauth_api_key_secret_key.to_string().clone(),
- _ => secretsmanager_client
- .get_secret_value(&config.fusionauth_api_key_secret_key)
- .await
- .context("unable to get secret")?
- .to_string(),
- };
-
- let fusionauth_client_secret = match config.environment {
- Environment::Local => config.fusionauth_client_secret_key.to_string().clone(),
- _ => secretsmanager_client
- .get_secret_value(&config.fusionauth_client_secret_key)
- .await
- .context("unable to get secret")?
- .to_string(),
- };
-
- let stripe_client_secret = match config.environment {
- Environment::Local => config.stripe_secret_key.to_string().clone(),
- _ => secretsmanager_client
- .get_secret_value(&config.stripe_secret_key)
- .await
- .context("unable to get secret")?
- .to_string(),
- };
-
- let google_client_secret = match config.environment {
- Environment::Local => config.google_client_secret_key.to_string().clone(),
- _ => secretsmanager_client
- .get_secret_value(&config.google_client_secret_key)
- .await
- .context("unable to get google client secret")?
- .to_string(),
- };
-
- let fusionauth_public_url = config
- .fusionauth_public_url
- .value()
- .unwrap_or(config.fusionauth_base_url.as_ref())
- .to_owned();
- let auth_client = fusionauth::FusionAuthClient::new(
- fusionauth_api_key,
- config.fusionauth_client_id.to_string().clone(),
- fusionauth_client_secret,
- config.fusionauth_base_url.to_string().clone(),
- config.fusionauth_oauth_redirect_uri.to_string().clone(),
- config.google_client_id.to_string().clone(),
- google_client_secret,
- )
- .with_public_url(fusionauth_public_url);
- let auth_client = match microsoft_credentials {
- Some(credentials) => auth_client.with_microsoft_credentials(
- credentials.client_id,
- credentials.client_secret,
- credentials.tenant_id,
- ),
- None => auth_client,
- };
- tracing::trace!("initialized auth client");
-
- let document_storage_service_client = DocumentStorageServiceClient::new(
- config.service_internal_auth_key.to_string().clone(),
- DocumentStorageServiceUrl::new()?.to_string(),
- );
- tracing::trace!("initialized document storage service client");
-
- let email_service_client =
- email::outbound::EmailServiceHttpClient::new(EmailServiceUrl::new()?.to_string());
- tracing::trace!("initialized email service client");
-
- let macro_cache_client =
- macro_cache_client::MacroCache::new(config.redis_uri.to_string().as_str());
-
- tracing::trace!("initialized redis client");
-
- let stripe_client = stripe::Client::new(stripe_client_secret);
- tracing::trace!("initialized stripe client");
-
- // `from_env` routes to local SMTP (Mailpit) when SMTP_HOST is set, else SES.
- let ses_client = ses_client::Ses::from_env(
- aws_sdk_sesv2::Client::new(&aws_config),
- &config.environment.to_string(),
- );
-
- let jwt_args =
- JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)
- .await?;
- let authorization_state = MacroAuthorizationState::new(Arc::new(AuthorizationService::new(
- MacroAuthJwtValidator::new(jwt_args.clone()),
- InternalAuthConfig {
- api_key: internal_api_key.to_string(),
- default_user_id: None,
- },
- macro_authorization::NoBotAuthorizer,
- PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),
- )));
-
- let redis_client = redis::Client::open(config.redis_uri.to_string().as_str())
- .context("failed to create redis client")?;
- let redis_multiplexed_conn = redis_client
- .get_multiplexed_async_connection()
- .await
- .context("failed to get multiplexed redis connection")?;
-
- let notification_queue = macro_queues::NotificationIngressQueue::new();
- let search_event_queue = macro_queues::SearchEventQueue::new();
- let link_manager_queue = macro_queues::LinkManagerQueue::new();
- let email_backfill_queue = macro_queues::EmailBackfillQueue::new();
- let ingress_queue = SqsQueue::new(
- aws_sdk_sqs::Client::new(&macro_aws_config::get_macro_aws_config().await),
- notification_queue.to_string(),
- );
- let notification_ingress_service = SqsNotificationIngress {
- queue: ingress_queue,
- };
- tracing::trace!("initialized notification ingress service");
-
- let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(&aws_config))
- .search_event_queue(&search_event_queue)
- .email_link_manager_queue(&link_manager_queue)
- .email_backfill_queue(&email_backfill_queue);
- tracing::trace!("initialized sqs client");
-
- // Initialize analytics client with configured providers
- let analytics_client = Arc::new(AnalyticsClient::new(AnalyticsClientConfig {
- google_analytics: config
- .ga_measurement_id
- .value()
- .zip(config.ga_api_secret.value())
- .map(|(measurement_id, api_secret)| {
- tracing::info!("configuring Google Analytics");
- GoogleAnalyticsConfig {
- measurement_id: measurement_id.to_string(),
- api_secret: api_secret.to_string(),
- }
- }),
- meta: config
- .meta_pixel_id
- .value()
- .zip(config.meta_access_token.value())
- .map(|(pixel_id, access_token)| {
- tracing::info!("configuring Meta Conversions API");
- MetaConfig {
- pixel_id: pixel_id.to_string(),
- access_token: access_token.to_string(),
- test_event_code: config.meta_test_event_code.value().map(str::to_string),
- }
- }),
- posthog: config.posthog_api_key.value().map(|api_key| {
- tracing::info!("configuring PostHog");
- PostHogConfig {
- api_key: api_key.to_string(),
- host: config
- .posthog_host
- .value()
- .map(str::to_string)
- .unwrap_or_else(|| "https://us.i.posthog.com".to_string()),
- }
- }),
- }));
- tracing::trace!("initialized analytics client");
-
- // Initialize Loops client. Production and develop sign-ups are added to
- // Loops; on localhost, or wherever no API key is configured, this is a
- // no-op. Note there is one Loops audience — a develop sign-up creates a
- // real contact and can trigger live workflows, so leave the key unset in
- // any environment that should not send.
- let loops_client = match (config.environment, config.loops_api_key.value()) {
- (Environment::Production | Environment::Develop, Some(api_key)) => {
- tracing::info!("configuring Loops");
- LoopsClient::new(api_key.to_string())
- }
- _ => LoopsClient::noop(),
- };
- tracing::trace!("initialized loops client");
-
- let user_roles_and_permissions_macro_db = MacroDB::new(db.clone());
-
- let user_roles_and_permissions_service = UserRolesAndPermissionsServiceImpl::new(
- user_roles_and_permissions_macro_db.clone(),
- user_roles_and_permissions_macro_db,
- );
-
- let stripe_prices = api::StripePrices {
- premium: config.stripe_price_id.to_string(),
- max: config
- .stripe_max_price_id
- .value()
- .filter(|id| !id.trim().is_empty())
- .map(str::to_owned),
- };
- if stripe_prices.max.is_none() {
- tracing::warn!("STRIPE_MAX_PRICE_ID is not set; the Max plan cannot be sold");
- }
-
- let teams_repo_impl = TeamRepositoryImpl::new(db.clone());
- let customer_repo_impl =
- CustomerRepositoryImpl::new(stripe_client.clone(), stripe_prices.seat_prices());
- let favorites_service = favorites::domain::service::FavoritesServiceImpl::new(
- favorites::outbound::pg_favorites_repo::PgFavoritesRepo::new(db.clone()),
- );
- let team_crm_settings_repo_impl =
- teams::outbound::team_crm_settings_repo::TeamCrmSettingsRepositoryImpl::new(db.clone());
-
- let notification_ingress_service = Arc::new(notification_ingress_service);
-
- let crm_enqueuer = teams::outbound::crm_enqueuer::SqsCrmEnqueuer::new(sqs_client.clone());
- let sqs_client = Arc::new(sqs_client);
- let contacts_ingress = Arc::new(SqsContactsIngress {
- queue: SqsContactsQueue::new(
- aws_sdk_sqs::Client::new(&aws_config),
- macro_queues::ContactsQueue::new().to_string(),
- ),
- });
- let contacts_enqueuer = ContactsIngressEnqueuer::new(contacts_ingress.clone());
- let team_analytics = AnalyticsClientTeamAnalytics::new(analytics_client.clone());
- let event_broker_tracker = TaskTracker::new();
- let macro_event_broker = MacroEventBrokerService::new(
- KafkaEventPublisher::new(config.kafka_brokers.as_ref())
- .context("failed to create kafka event publisher")?,
- event_broker_tracker.clone(),
- );
- let entity_access_service_impl = Arc::new(EntityAccessServiceImpl::new(
- PgAccessRepository::new(db.clone()),
- ));
- let connection_gateway_client = Arc::new(ConnectionGatewayClient::new(
- config.service_internal_auth_key.to_string(),
- ConnectionGatewayUrl::new()?.to_string(),
- ));
- // Authentication creates channels and posts support welcome messages in-process, so its
- // channel service must dispatch the same realtime, notification, contact, and broker side
- // effects as the document-storage channel API. Channel broker events drive live search
- // indexing.
- let channel_side_effects = ChannelSideEffectService::new(
- PgChannelSideEffectContext::new(db.clone()),
- ConnectionGatewayChannelRealtimePublisher::new(connection_gateway_client.clone()),
- NotificationChannelSender::new(notification_ingress_service.clone()),
- ContactsChannelDispatcher::new(contacts_ingress),
- )
- .with_macro_event_broker(macro_event_broker.clone());
- let channel_event_dispatcher = SpawnedChannelEventDispatcher::new(channel_side_effects);
- let channel_service = ChannelServiceImpl::with_dependencies(
- PgChannelsRepo::new(db.clone()),
- channel_event_dispatcher.clone(),
- PgChannelReferenceSharePermissions::new(db.clone(), entity_access_service_impl.clone()),
- );
- // The welcome message goes through the shared message service so it gets the
- // same persistence and delivery as every other channel message.
- let channel_messages: Arc<dyn messages::domain::api::MessageCommands> = Arc::new(
- messages::domain::service::MessageService::new(
- messages::outbound::pg_message_repo::PgMessageRepository::new(db.clone())
- .with_initiatives(initiative::domain::lookup::InitiativeLookup::new(
- initiative::outbound::PgInitiativeRepo::new(db.clone()),
- ))
- .with_crm(crm::outbound::lookup::PgCrmParentReader::new(db.clone())),
- messages::domain::effects::MessageEffects::new(
- messages::outbound::broker::BrokerMessagePublisher::new(macro_event_broker.clone()),
- messages::domain::ports::NoMessageEventPublisher,
- channels::domain::message_delivery::ChannelMessageDelivery::new(
- PgChannelsRepo::new(db.clone()),
- channel_event_dispatcher,
- PgChannelReferenceSharePermissions::new(
- db.clone(),
- entity_access_service_impl.clone(),
- ),
- messages::outbound::connection_gateway::ConnectionGatewayMessages(
- connection_gateway_client,
- ),
- ),
- ),
- )
- .with_group_recipients(channels::domain::group_mentions::ChannelGroupRecipients(
- PgChannelsRepo::new(db.clone()),
- ))
- .with_references(
- messages::outbound::entity_access_audience::EntityAccessMessageReferences(
- (*entity_access_service_impl).clone(),
- ),
- ),
- );
-
- let foreign_entity_service =
- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone()));
-
- let github_link_service_impl = GithubLinkServiceImpl::new(
- PgGithubRepo::new(db.clone()),
- GithubOauthImpl::default(),
- GithubAuthImpl::new(auth_client.clone(), redis_multiplexed_conn),
- GithubPullRequestServiceImpl::new(
- foreign_entity_service,
- PgGithubPullRequestRepo::new(db.clone()),
- ),
- GithubLinkConfig {
- client_id: config.github_client_id.to_string(),
- client_secret: config.github_client_secret.to_string(),
- idp_id: config.github_idp_id.to_string(),
- },
- );
-
- let rate_limit = RateLimitServiceImpl {
- repo: RedisRateLimitAdapter {
- redis: redis_client,
- },
- };
- let referral_service = ReferralServiceImpl {
- repo: PgReferralRepo::new(db.clone()),
- discount_client: StripeDiscountClient::new(
- stripe_client.clone(),
- 10000, /*100$ credit, in cents*/
- ),
- notification_ingress: notification_ingress_service.clone(),
- };
- let gtm_invite_service = GtmInviteServiceImpl {
- repo: PgGtmInviteRepo::new(db.clone()),
- config: gtm_invite_config,
- };
-
- let codex_connection = if let Some(key_id) = config.codex_oauth_kms_key_id() {
- let cipher = Arc::new(codex_connection::outbound::cipher::EnvelopeCipher::new(
- aws_sdk_kms::Client::new(&aws_config),
- key_id,
- )?);
- let repository = Arc::new(
- codex_connection::outbound::postgres::PostgresRepository::new(db.clone(), cipher),
- );
- let provider = codex_cloud_agents::outbound::openai::OpenAi::new()
- .map_err(|_| anyhow::anyhow!("failed to initialize Codex OAuth client"))?;
- Some(
- Arc::new(codex_connection::domain::ConnectionServiceImpl::new(
- repository, provider,
- )) as Arc<dyn codex_connection::domain::ConnectionService>,
- )
- } else {
- None
- };
-
- let stripe_client = Arc::new(stripe_client);
- let gtm_invite_service = Arc::new(gtm_invite_service);
- let subscription_checkout = Arc::new(
- authentication_service::service::subscription_checkout::CheckoutService::new(
- authentication_service::outbound::subscription_checkout::StripeCheckoutGateway::new(
- db.clone(),
- stripe_client.clone(),
- gtm_invite_service.clone(),
- stripe_prices.seat_prices(),
- ),
- ),
- );
- let ai_payment_gateway = ai_billing::outbound::StripePaymentGateway::new(stripe_client.clone());
- let ai_pricing = config.ai_pricing();
- let ai_billing_service = Arc::new(
- ai_billing::domain::BillingServiceImpl::new(
- ai_billing::outbound::RolesTeamsEntitlementSource::new(
- user_roles_and_permissions_service.clone(),
- teams_repo_impl.clone(),
- ),
- ai_billing::outbound::PgUsageReader::new(db.clone()),
- ai_billing::outbound::PgBillingRepo::new(db.clone(), ai_pricing),
- ai_payment_gateway.clone(),
- ai_pricing,
- )
- .with_enforcement(config.enable_ai_usage_enforcement)
- .with_billing(config.enable_ai_usage_billing),
- );
- let teams_service_impl = TeamServiceImpl::new_with_analytics(
- teams_repo_impl.clone(),
- customer_repo_impl,
- channel_service.clone(),
- user_roles_and_permissions_service.clone(),
- notification_ingress_service.clone(),
- crm_enqueuer,
- team_crm_settings_repo_impl,
- team_analytics,
- )
- .with_contacts_enqueuer(contacts_enqueuer)
- .with_event_broker(macro_event_broker)
- .with_open_seat_release((*ai_billing_service).clone());
- let teams_service = Arc::new(teams_service_impl);
- let document_storage_service_client = Arc::new(document_storage_service_client);
- // The harness and scheduled-action services validate the fleet-wide
- // internal key, not this service's own inbound key.
- let user_deletion = Arc::new(
- authentication_service::outbound::user_deletion::UserDeletionAdapter::new(
- db.clone(),
- document_storage_service_client.clone(),
- teams_service.clone(),
- onboarding::outbound::pg_onboarding_repo::PgOnboardingRepo::new(db.clone()),
- stripe_client.clone(),
- config.service_internal_auth_key.to_string(),
- macro_service_urls::AgentHarnessServiceUrl::new()?.to_string(),
- macro_service_urls::ScheduledActionServiceUrl::new()?.to_string(),
- )
- .map_err(|error| anyhow::anyhow!("{error:?}"))?,
- );
-
- let server_result = api::setup_and_serve(
- ApiContext {
- db,
- github_link_service: Arc::new(github_link_service_impl),
- auth_client: Arc::new(auth_client),
- microsoft_token_cipher,
- cursor_api_key_cipher,
- codex_connection,
- macro_cache_client: Arc::new(macro_cache_client),
- stripe_client,
- subscription_checkout,
- document_storage_service_client,
- user_deletion,
- email_service_client: Arc::new(email_service_client),
- ses_client: Arc::new(ses_client),
- notification_ingress_service,
- sqs_client,
- environment: config.environment,
- signup_policy,
- rate_limit_service: rate_limit,
- calendar_scope_enabled: config.calendar_scope_enabled,
- jwt_args,
- authorization_state,
- token_context: MacroApiTokenContext {
- issuer: MacroApiTokenIssuer::new()?,
- macro_api_token_private_key,
- expiry_seconds: MacroApiTokenExpirySeconds::new()?
- .as_ref()
- .parse()
- .context("failed to parse MACRO_API_TOKEN_EXPIRY_SECONDS as usize")?,
- },
- internal_api_key,
- stripe_webhook_secret,
- user_roles_and_permissions_service: Arc::new(user_roles_and_permissions_service),
- teams_service,
- channel_service: Arc::new(channel_service),
- channel_messages,
- favorites_service: Arc::new(favorites_service),
- entity_access_service: entity_access_service_impl,
- referral_service: Arc::new(referral_service),
- gtm_invite_service,
- native_app_service: Arc::new(NativeAppServiceImpl {
- bundle_fetcher: DefaultBundleFetcher::new(
- AppServiceUrl::new_for_environment(config.environment)
- .context("failed to resolve app service URL")?
- .parse_url()
- .context("failed to parse app service URL")?,
- ),
- bundle_policy: native_app_service::domain::models::BundleUpdatePolicy::from_env()
- .map_err(|err| {
- anyhow!("failed to load bundle update policy: {err}")
- })?,
- platform_data: PlatformData {
- ios_development_team_id: IOS_DEVELOPMENT_TEAM_ID.to_string(),
- ios_app_bundle_id: IOS_APP_BUNDLE_ID.to_string(),
- },
- }),
- loops_client: Arc::new(loops_client),
- analytics_client,
- stripe_prices,
- ai_billing_service,
- ai_payment_gateway: Arc::new(ai_payment_gateway),
- },
- config.port,
- )
- .await;
-
- tracing::info!("waiting for event broker publishes to drain");
- event_broker_tracker.close();
- match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {
- Ok(()) => tracing::info!("event broker publishes drained"),
- Err(error) => {
- tracing::warn!(
- error=?error,
- timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),
- "timed out waiting for event broker publishes to drain"
- );
- }
- }
-
- server_result
+ authentication_service::run().await
}
-
-const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
-
-// SAFETY: this is not a secret value
-const IOS_DEVELOPMENT_TEAM_ID: &str = "TY74Q77JBD";
-// SAFETY: this is not a secret value
-const IOS_APP_BUNDLE_ID: &str = "com.macro.app.prod";
diff --git a/services/authentication_service/src/openapi.rs b/services/authentication_service/src/openapi.rs
index 196a3f0025..60a543a982 100644
--- a/services/authentication_service/src/openapi.rs
+++ b/services/authentication_service/src/openapi.rs
@@ -1,17 +1,6 @@
-#![allow(unused)]
-
-mod api;
-mod config;
-mod generate_password;
-mod microsoft_token_cipher;
-mod rate_limit_config;
-mod service;
-
+use authentication_service::ApiDoc;
use utoipa::OpenApi;
fn main() {
- println!(
- "{}",
- api::swagger::ApiDoc::openapi().to_pretty_json().unwrap()
- );
+ println!("{}", ApiDoc::openapi().to_pretty_json().unwrap());
}
diff --git a/services/authentication_service/src/server.rs b/services/authentication_service/src/server.rs
new file mode 100644
index 0000000000..9c068785d8
--- /dev/null
+++ b/services/authentication_service/src/server.rs
@@ -0,0 +1,651 @@
+//! The service's composition root: reads config and wires every adapter and
+//! domain service into the HTTP server.
+
+use analytics_client::{
+ AnalyticsClient, AnalyticsClientConfig, GoogleAnalyticsConfig, MetaConfig, PostHogConfig,
+};
+use anyhow::{Context, anyhow};
+use channels::{
+ domain::{
+ service::ChannelServiceImpl,
+ side_effects::{ChannelSideEffectService, SpawnedChannelEventDispatcher},
+ },
+ outbound::{
+ connection_gateway_realtime::ConnectionGatewayChannelRealtimePublisher,
+ contacts_dispatcher::ContactsChannelDispatcher,
+ notification_sender::NotificationChannelSender,
+ pg_channel_reference_share_permissions::PgChannelReferenceSharePermissions,
+ pg_channels_repo::PgChannelsRepo, pg_side_effect_context::PgChannelSideEffectContext,
+ },
+};
+use connection_gateway_client::ConnectionGatewayClient;
+use contacts::{domain::service::SqsContactsIngress, outbound::ingress::SqsContactsQueue};
+use document_storage_service_client::DocumentStorageServiceClient;
+use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};
+use foreign_entity::{
+ domain::service::ForeignEntityServiceImpl,
+ outbound::pg_foreign_entity_repo::PgForeignEntityRepo,
+};
+use github::{
+ domain::service::{GithubLinkConfig, GithubLinkServiceImpl},
+ outbound::{
+ github_auth_client::GithubAuthImpl, github_oauth_client::GithubOauthImpl,
+ pg_github_repo::PgGithubRepo,
+ },
+};
+use github_pull_requests::{
+ domain::service::GithubPullRequestServiceImpl,
+ outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo,
+};
+use loops_client::LoopsClient;
+use macro_auth::middleware::decode_jwt::JwtValidationArgs;
+use macro_authorization::{
+ InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationState,
+ PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,
+};
+use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
+use macro_service_urls::{
+ AppServiceUrl, ConnectionGatewayUrl, DocumentStorageServiceUrl, EmailServiceUrl,
+};
+use native_app_service::{
+ domain::{models::PlatformData, service::NativeAppServiceImpl},
+ outbound::DefaultBundleFetcher,
+};
+use notification::outbound::queue::SqsQueue;
+use notification::{
+ domain::service::SqsNotificationIngress, outbound::rate_limit::RedisRateLimitAdapter,
+};
+use rate_limit::domain::service::RateLimitServiceImpl;
+use roles_and_permissions::{
+ domain::service::UserRolesAndPermissionsServiceImpl, outbound::pgpool::MacroDB,
+};
+use secretsmanager_client::SecretManager;
+use sqlx::postgres::PgPoolOptions;
+use teams::{
+ domain::team_service::TeamServiceImpl,
+ outbound::{
+ contacts_enqueuer::ContactsIngressEnqueuer, customer_repo::CustomerRepositoryImpl,
+ team_analytics::AnalyticsClientTeamAnalytics, team_repo::TeamRepositoryImpl,
+ },
+};
+
+use gtm_invite::{
+ domain::service::GtmInviteServiceImpl, outbound::pg_gtm_invite_repo::PgGtmInviteRepo,
+};
+use referral::{
+ domain::service::ReferralServiceImpl,
+ outbound::{pg_referral_repo::PgReferralRepo, stripe_discount_client::StripeDiscountClient},
+};
+
+use crate::{
+ api::{
+ self,
+ context::{
+ ApiContext, AuthorizationService, MacroApiTokenContext, MacroApiTokenExpirySeconds,
+ MacroApiTokenIssuer, MacroApiTokenPrivateSecretKey, StripeWebhookSecretKey,
+ },
+ },
+ config::{Config, Environment},
+ microsoft_token_cipher::{
+ EnvelopeMicrosoftTokenCipher, KmsDataKeyProvider, MicrosoftTokenCipher,
+ },
+};
+use std::{sync::Arc, time::Duration};
+use tokio_util::task::TaskTracker;
+
+/// Build the service from the environment and serve until shutdown.
+pub async fn run() -> anyhow::Result<()> {
+ let env = Environment::new_or_prod();
+
+ // One SDK config is sufficient for every AWS client in this process.
+ let aws_config = macro_aws_config::get_macro_aws_config().await;
+ let secretsmanager_client = secretsmanager_client::SecretsManager::new(
+ aws_sdk_secretsmanager::Client::new(&aws_config),
+ );
+
+ // Parse our configuration from the environment.
+ let config = Config::from_env().context("expected to be able to generate config")?;
+ let signup_policy = Arc::new(
+ config
+ .signup_policy()
+ .context("invalid signup policy configuration")?,
+ );
+ let gtm_invite_config = config
+ .gtm_invite_config()
+ .context("invalid GTM invite link configuration")?;
+ let microsoft_credentials = config
+ .microsoft_credentials()
+ .context("invalid Microsoft OAuth configuration")?;
+ let microsoft_token_cipher = microsoft_credentials.as_ref().map(|credentials| {
+ Arc::new(EnvelopeMicrosoftTokenCipher::new(KmsDataKeyProvider::new(
+ aws_sdk_kms::Client::new(&aws_config),
+ credentials.token_kms_key_id.clone(),
+ ))) as Arc<dyn MicrosoftTokenCipher>
+ });
+
+ // A separate KMS key from the Microsoft one by design: sharing it would
+ // grant whatever decrypts Cursor keys access to the key protecting
+ // everyone's mailbox credentials.
+ let cursor_api_key_cipher = Arc::new(cursor_api_key::cipher::KmsCursorApiKeyCipher::new(
+ cursor_api_key::cipher::AwsKmsCiphertexts::new(
+ aws_sdk_kms::Client::new(&aws_config),
+ config.cursor_api_key_kms_key_id()?,
+ ),
+ )) as Arc<dyn cursor_api_key::cipher::CursorApiKeyCipher>;
+
+ let internal_api_key = config.internal_api_key.clone();
+
+ let stripe_webhook_secret = secretsmanager_client
+ .get_maybe_secret_value(env, StripeWebhookSecretKey::new()?)
+ .await?;
+
+ tracing::trace!("initialized config");
+
+ let (min_connections, max_connections): (u32, u32) = match config.environment {
+ Environment::Production => (5, 25),
+ Environment::Develop => (1, 25),
+ Environment::Local => (1, 10),
+ };
+
+ let db = PgPoolOptions::new()
+ .min_connections(min_connections)
+ .max_connections(max_connections)
+ .connect(&config.database_url)
+ .await
+ .context("could not connect to db")?;
+
+ tracing::trace!(
+ min_connections,
+ max_connections,
+ "initialized db connection"
+ );
+
+ // Macro API token
+ let macro_api_token_private_key = secretsmanager_client
+ .get_maybe_secret_value(config.environment, MacroApiTokenPrivateSecretKey::new()?)
+ .await?;
+
+ let fusionauth_api_key = match config.environment {
+ Environment::Local => config.fusionauth_api_key_secret_key.to_string().clone(),
+ _ => secretsmanager_client
+ .get_secret_value(&config.fusionauth_api_key_secret_key)
+ .await
+ .context("unable to get secret")?
+ .to_string(),
+ };
+
+ let fusionauth_client_secret = match config.environment {
+ Environment::Local => config.fusionauth_client_secret_key.to_string().clone(),
+ _ => secretsmanager_client
+ .get_secret_value(&config.fusionauth_client_secret_key)
+ .await
+ .context("unable to get secret")?
+ .to_string(),
+ };
+
+ let stripe_client_secret = match config.environment {
+ Environment::Local => config.stripe_secret_key.to_string().clone(),
+ _ => secretsmanager_client
+ .get_secret_value(&config.stripe_secret_key)
+ .await
+ .context("unable to get secret")?
+ .to_string(),
+ };
+
+ let google_client_secret = match config.environment {
+ Environment::Local => config.google_client_secret_key.to_string().clone(),
+ _ => secretsmanager_client
+ .get_secret_value(&config.google_client_secret_key)
+ .await
+ .context("unable to get google client secret")?
+ .to_string(),
+ };
+
+ let fusionauth_public_url = config
+ .fusionauth_public_url
+ .value()
+ .unwrap_or(config.fusionauth_base_url.as_ref())
+ .to_owned();
+ let auth_client = fusionauth::FusionAuthClient::new(
+ fusionauth_api_key,
+ config.fusionauth_client_id.to_string().clone(),
+ fusionauth_client_secret,
+ config.fusionauth_base_url.to_string().clone(),
+ config.fusionauth_oauth_redirect_uri.to_string().clone(),
+ config.google_client_id.to_string().clone(),
+ google_client_secret,
+ )
+ .with_public_url(fusionauth_public_url);
+ let auth_client = match microsoft_credentials {
+ Some(credentials) => auth_client.with_microsoft_credentials(
+ credentials.client_id,
+ credentials.client_secret,
+ credentials.tenant_id,
+ ),
+ None => auth_client,
+ };
+ tracing::trace!("initialized auth client");
+
+ let document_storage_service_client = DocumentStorageServiceClient::new(
+ config.service_internal_auth_key.to_string().clone(),
+ DocumentStorageServiceUrl::new()?.to_string(),
+ );
+ tracing::trace!("initialized document storage service client");
+
+ let email_service_client =
+ email::outbound::EmailServiceHttpClient::new(EmailServiceUrl::new()?.to_string());
+ tracing::trace!("initialized email service client");
+
+ let macro_cache_client =
+ macro_cache_client::MacroCache::new(config.redis_uri.to_string().as_str());
+
+ tracing::trace!("initialized redis client");
+
+ let stripe_client = stripe::Client::new(stripe_client_secret);
+ tracing::trace!("initialized stripe client");
+
+ // `from_env` routes to local SMTP (Mailpit) when SMTP_HOST is set, else SES.
+ let ses_client = ses_client::Ses::from_env(
+ aws_sdk_sesv2::Client::new(&aws_config),
+ &config.environment.to_string(),
+ );
+
+ let jwt_args =
+ JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)
+ .await?;
+ let authorization_state = MacroAuthorizationState::new(Arc::new(AuthorizationService::new(
+ MacroAuthJwtValidator::new(jwt_args.clone()),
+ InternalAuthConfig {
+ api_key: internal_api_key.to_string(),
+ default_user_id: None,
+ },
+ macro_authorization::NoBotAuthorizer,
+ PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),
+ )));
+
+ let redis_client = redis::Client::open(config.redis_uri.to_string().as_str())
+ .context("failed to create redis client")?;
+ let redis_multiplexed_conn = redis_client
+ .get_multiplexed_async_connection()
+ .await
+ .context("failed to get multiplexed redis connection")?;
+
+ let notification_queue = macro_queues::NotificationIngressQueue::new();
+ let search_event_queue = macro_queues::SearchEventQueue::new();
+ let link_manager_queue = macro_queues::LinkManagerQueue::new();
+ let email_backfill_queue = macro_queues::EmailBackfillQueue::new();
+ let ingress_queue = SqsQueue::new(
+ aws_sdk_sqs::Client::new(&macro_aws_config::get_macro_aws_config().await),
+ notification_queue.to_string(),
+ );
+ let notification_ingress_service = SqsNotificationIngress {
+ queue: ingress_queue,
+ };
+ tracing::trace!("initialized notification ingress service");
+
+ let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(&aws_config))
+ .search_event_queue(&search_event_queue)
+ .email_link_manager_queue(&link_manager_queue)
+ .email_backfill_queue(&email_backfill_queue);
+ tracing::trace!("initialized sqs client");
+
+ // Initialize analytics client with configured providers
+ let analytics_client = Arc::new(AnalyticsClient::new(AnalyticsClientConfig {
+ google_analytics: config
+ .ga_measurement_id
+ .value()
+ .zip(config.ga_api_secret.value())
+ .map(|(measurement_id, api_secret)| {
+ tracing::info!("configuring Google Analytics");
+ GoogleAnalyticsConfig {
+ measurement_id: measurement_id.to_string(),
+ api_secret: api_secret.to_string(),
+ }
+ }),
+ meta: config
+ .meta_pixel_id
+ .value()
+ .zip(config.meta_access_token.value())
+ .map(|(pixel_id, access_token)| {
+ tracing::info!("configuring Meta Conversions API");
+ MetaConfig {
+ pixel_id: pixel_id.to_string(),
+ access_token: access_token.to_string(),
+ test_event_code: config.meta_test_event_code.value().map(str::to_string),
+ }
+ }),
+ posthog: config.posthog_api_key.value().map(|api_key| {
+ tracing::info!("configuring PostHog");
+ PostHogConfig {
+ api_key: api_key.to_string(),
+ host: config
+ .posthog_host
+ .value()
+ .map(str::to_string)
+ .unwrap_or_else(|| "https://us.i.posthog.com".to_string()),
+ }
+ }),
+ }));
+ tracing::trace!("initialized analytics client");
+
+ // Initialize Loops client. Production and develop sign-ups are added to
+ // Loops; on localhost, or wherever no API key is configured, this is a
+ // no-op. Note there is one Loops audience — a develop sign-up creates a
+ // real contact and can trigger live workflows, so leave the key unset in
+ // any environment that should not send.
+ let loops_client = match (config.environment, config.loops_api_key.value()) {
+ (Environment::Production | Environment::Develop, Some(api_key)) => {
+ tracing::info!("configuring Loops");
+ LoopsClient::new(api_key.to_string())
+ }
+ _ => LoopsClient::noop(),
+ };
+ tracing::trace!("initialized loops client");
+
+ let user_roles_and_permissions_macro_db = MacroDB::new(db.clone());
+
+ let user_roles_and_permissions_service = UserRolesAndPermissionsServiceImpl::new(
+ user_roles_and_permissions_macro_db.clone(),
+ user_roles_and_permissions_macro_db,
+ );
+
+ let stripe_prices = api::StripePrices {
+ premium: config.stripe_price_id.to_string(),
+ max: config
+ .stripe_max_price_id
+ .value()
+ .filter(|id| !id.trim().is_empty())
+ .map(str::to_owned),
+ };
+ if stripe_prices.max.is_none() {
+ tracing::warn!("STRIPE_MAX_PRICE_ID is not set; the Max plan cannot be sold");
+ }
+
+ let teams_repo_impl = TeamRepositoryImpl::new(db.clone());
+ let customer_repo_impl =
+ CustomerRepositoryImpl::new(stripe_client.clone(), stripe_prices.seat_prices());
+ let favorites_service = favorites::domain::service::FavoritesServiceImpl::new(
+ favorites::outbound::pg_favorites_repo::PgFavoritesRepo::new(db.clone()),
+ );
+ let team_crm_settings_repo_impl =
+ teams::outbound::team_crm_settings_repo::TeamCrmSettingsRepositoryImpl::new(db.clone());
+
+ let notification_ingress_service = Arc::new(notification_ingress_service);
+
+ let crm_enqueuer = teams::outbound::crm_enqueuer::SqsCrmEnqueuer::new(sqs_client.clone());
+ let sqs_client = Arc::new(sqs_client);
+ let contacts_ingress = Arc::new(SqsContactsIngress {
+ queue: SqsContactsQueue::new(
+ aws_sdk_sqs::Client::new(&aws_config),
+ macro_queues::ContactsQueue::new().to_string(),
+ ),
+ });
+ let contacts_enqueuer = ContactsIngressEnqueuer::new(contacts_ingress.clone());
+ let team_analytics = AnalyticsClientTeamAnalytics::new(analytics_client.clone());
+ let event_broker_tracker = TaskTracker::new();
+ let macro_event_broker = MacroEventBrokerService::new(
+ KafkaEventPublisher::new(config.kafka_brokers.as_ref())
+ .context("failed to create kafka event publisher")?,
+ event_broker_tracker.clone(),
+ );
+ let entity_access_service_impl = Arc::new(EntityAccessServiceImpl::new(
+ PgAccessRepository::new(db.clone()),
+ ));
+ let connection_gateway_client = Arc::new(ConnectionGatewayClient::new(
+ config.service_internal_auth_key.to_string(),
+ ConnectionGatewayUrl::new()?.to_string(),
+ ));
+ // Authentication creates channels and posts support welcome messages in-process, so its
+ // channel service must dispatch the same realtime, notification, contact, and broker side
+ // effects as the document-storage channel API. Channel broker events drive live search
+ // indexing.
+ let channel_side_effects = ChannelSideEffectService::new(
+ PgChannelSideEffectContext::new(db.clone()),
+ ConnectionGatewayChannelRealtimePublisher::new(connection_gateway_client.clone()),
+ NotificationChannelSender::new(notification_ingress_service.clone()),
+ ContactsChannelDispatcher::new(contacts_ingress),
+ )
+ .with_macro_event_broker(macro_event_broker.clone());
+ let channel_event_dispatcher = SpawnedChannelEventDispatcher::new(channel_side_effects);
+ let channel_service = ChannelServiceImpl::with_dependencies(
+ PgChannelsRepo::new(db.clone()),
+ channel_event_dispatcher.clone(),
+ PgChannelReferenceSharePermissions::new(db.clone(), entity_access_service_impl.clone()),
+ );
+ // The welcome message goes through the shared message service so it gets the
+ // same persistence and delivery as every other channel message.
+ let channel_messages: Arc<dyn messages::domain::api::MessageCommands> = Arc::new(
+ messages::domain::service::MessageService::new(
+ messages::outbound::pg_message_repo::PgMessageRepository::new(db.clone())
+ .with_initiatives(initiative::domain::lookup::InitiativeLookup::new(
+ initiative::outbound::PgInitiativeRepo::new(db.clone()),
+ ))
+ .with_crm(crm::outbound::lookup::PgCrmParentReader::new(db.clone())),
+ messages::domain::effects::MessageEffects::new(
+ messages::outbound::broker::BrokerMessagePublisher::new(macro_event_broker.clone()),
+ messages::domain::ports::NoMessageEventPublisher,
+ channels::domain::message_delivery::ChannelMessageDelivery::new(
+ PgChannelsRepo::new(db.clone()),
+ channel_event_dispatcher,
+ PgChannelReferenceSharePermissions::new(
+ db.clone(),
+ entity_access_service_impl.clone(),
+ ),
+ messages::outbound::connection_gateway::ConnectionGatewayMessages(
+ connection_gateway_client,
+ ),
+ ),
+ ),
+ )
+ .with_group_recipients(channels::domain::group_mentions::ChannelGroupRecipients(
+ PgChannelsRepo::new(db.clone()),
+ ))
+ .with_references(
+ messages::outbound::entity_access_audience::EntityAccessMessageReferences(
+ (*entity_access_service_impl).clone(),
+ ),
+ ),
+ );
+
+ let foreign_entity_service =
+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone()));
+
+ let github_link_service_impl = GithubLinkServiceImpl::new(
+ PgGithubRepo::new(db.clone()),
+ GithubOauthImpl::default(),
+ GithubAuthImpl::new(auth_client.clone(), redis_multiplexed_conn),
+ GithubPullRequestServiceImpl::new(
+ foreign_entity_service,
+ PgGithubPullRequestRepo::new(db.clone()),
+ ),
+ GithubLinkConfig {
+ client_id: config.github_client_id.to_string(),
+ client_secret: config.github_client_secret.to_string(),
+ idp_id: config.github_idp_id.to_string(),
+ },
+ );
+
+ let rate_limit = RateLimitServiceImpl {
+ repo: RedisRateLimitAdapter {
+ redis: redis_client,
+ },
+ };
+ let referral_service = ReferralServiceImpl {
+ repo: PgReferralRepo::new(db.clone()),
+ discount_client: StripeDiscountClient::new(
+ stripe_client.clone(),
+ 10000, /*100$ credit, in cents*/
+ ),
+ notification_ingress: notification_ingress_service.clone(),
+ };
+ let gtm_invite_service = GtmInviteServiceImpl {
+ repo: PgGtmInviteRepo::new(db.clone()),
+ config: gtm_invite_config,
+ };
+
+ let codex_connection = if let Some(key_id) = config.codex_oauth_kms_key_id() {
+ let cipher = Arc::new(codex_connection::outbound::cipher::EnvelopeCipher::new(
+ aws_sdk_kms::Client::new(&aws_config),
+ key_id,
+ )?);
+ let repository = Arc::new(
+ codex_connection::outbound::postgres::PostgresRepository::new(db.clone(), cipher),
+ );
+ let provider = codex_cloud_agents::outbound::openai::OpenAi::new()
+ .map_err(|_| anyhow::anyhow!("failed to initialize Codex OAuth client"))?;
+ Some(
+ Arc::new(codex_connection::domain::ConnectionServiceImpl::new(
+ repository, provider,
+ )) as Arc<dyn codex_connection::domain::ConnectionService>,
+ )
+ } else {
+ None
+ };
+
+ let stripe_client = Arc::new(stripe_client);
+ let gtm_invite_service = Arc::new(gtm_invite_service);
+ let subscription_checkout =
+ Arc::new(crate::service::subscription_checkout::CheckoutService::new(
+ crate::outbound::subscription_checkout::StripeCheckoutGateway::new(
+ db.clone(),
+ stripe_client.clone(),
+ gtm_invite_service.clone(),
+ stripe_prices.seat_prices(),
+ ),
+ ));
+ let ai_payment_gateway = ai_billing::outbound::StripePaymentGateway::new(stripe_client.clone());
+ let ai_pricing = config.ai_pricing();
+ let ai_billing_service = Arc::new(
+ ai_billing::domain::BillingServiceImpl::new(
+ ai_billing::outbound::RolesTeamsEntitlementSource::new(
+ user_roles_and_permissions_service.clone(),
+ teams_repo_impl.clone(),
+ ),
+ ai_billing::outbound::PgUsageReader::new(db.clone()),
+ ai_billing::outbound::PgBillingRepo::new(db.clone(), ai_pricing),
+ ai_payment_gateway.clone(),
+ ai_pricing,
+ )
+ .with_enforcement(config.enable_ai_usage_enforcement)
+ .with_billing(config.enable_ai_usage_billing),
+ );
+ let teams_service_impl = TeamServiceImpl::new_with_analytics(
+ teams_repo_impl.clone(),
+ customer_repo_impl,
+ channel_service.clone(),
+ user_roles_and_permissions_service.clone(),
+ notification_ingress_service.clone(),
+ crm_enqueuer,
+ team_crm_settings_repo_impl,
+ team_analytics,
+ )
+ .with_contacts_enqueuer(contacts_enqueuer)
+ .with_event_broker(macro_event_broker)
+ .with_open_seat_release((*ai_billing_service).clone());
+ let teams_service = Arc::new(teams_service_impl);
+ let document_storage_service_client = Arc::new(document_storage_service_client);
+ // The harness and scheduled-action services validate the fleet-wide
+ // internal key, not this service's own inbound key.
+ let user_deletion = Arc::new(
+ crate::outbound::user_deletion::UserDeletionAdapter::new(
+ db.clone(),
+ document_storage_service_client.clone(),
+ teams_service.clone(),
+ onboarding::outbound::pg_onboarding_repo::PgOnboardingRepo::new(db.clone()),
+ stripe_client.clone(),
+ config.service_internal_auth_key.to_string(),
+ macro_service_urls::AgentHarnessServiceUrl::new()?.to_string(),
+ macro_service_urls::ScheduledActionServiceUrl::new()?.to_string(),
+ )
+ .map_err(|error| anyhow::anyhow!("{error:?}"))?,
+ );
+
+ let server_result = api::setup_and_serve(
+ ApiContext {
+ db,
+ github_link_service: Arc::new(github_link_service_impl),
+ auth_client: Arc::new(auth_client),
+ microsoft_token_cipher,
+ cursor_api_key_cipher,
+ codex_connection,
+ macro_cache_client: Arc::new(macro_cache_client),
+ stripe_client,
+ subscription_checkout,
+ document_storage_service_client,
+ user_deletion,
+ email_service_client: Arc::new(email_service_client),
+ ses_client: Arc::new(ses_client),
+ notification_ingress_service,
+ sqs_client,
+ environment: config.environment,
+ signup_policy,
+ rate_limit_service: rate_limit,
+ calendar_scope_enabled: config.calendar_scope_enabled,
+ jwt_args,
+ authorization_state,
+ token_context: MacroApiTokenContext {
+ issuer: MacroApiTokenIssuer::new()?,
+ macro_api_token_private_key,
+ expiry_seconds: MacroApiTokenExpirySeconds::new()?
+ .as_ref()
+ .parse()
+ .context("failed to parse MACRO_API_TOKEN_EXPIRY_SECONDS as usize")?,
+ },
+ internal_api_key,
+ stripe_webhook_secret,
+ user_roles_and_permissions_service: Arc::new(user_roles_and_permissions_service),
+ teams_service,
+ channel_service: Arc::new(channel_service),
+ channel_messages,
+ favorites_service: Arc::new(favorites_service),
+ entity_access_service: entity_access_service_impl,
+ referral_service: Arc::new(referral_service),
+ gtm_invite_service,
+ native_app_service: Arc::new(NativeAppServiceImpl {
+ bundle_fetcher: DefaultBundleFetcher::new(
+ AppServiceUrl::new_for_environment(config.environment)
+ .context("failed to resolve app service URL")?
+ .parse_url()
+ .context("failed to parse app service URL")?,
+ ),
+ bundle_policy: native_app_service::domain::models::BundleUpdatePolicy::from_env()
+ .map_err(|err| {
+ anyhow!("failed to load bundle update policy: {err}")
+ })?,
+ platform_data: PlatformData {
+ ios_development_team_id: IOS_DEVELOPMENT_TEAM_ID.to_string(),
+ ios_app_bundle_id: IOS_APP_BUNDLE_ID.to_string(),
+ },
+ }),
+ loops_client: Arc::new(loops_client),
+ analytics_client,
+ stripe_prices,
+ ai_billing_service,
+ ai_payment_gateway: Arc::new(ai_payment_gateway),
+ },
+ config.port,
+ )
+ .await;
+
+ tracing::info!("waiting for event broker publishes to drain");
+ event_broker_tracker.close();
+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {
+ Ok(()) => tracing::info!("event broker publishes drained"),
+ Err(error) => {
+ tracing::warn!(
+ error=?error,
+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),
+ "timed out waiting for event broker publishes to drain"
+ );
+ }
+ }
+
+ server_result
+}
+
+const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
+
+// SAFETY: this is not a secret value
+const IOS_DEVELOPMENT_TEAM_ID: &str = "TY74Q77JBD";
+// SAFETY: this is not a secret value
+const IOS_APP_BUNDLE_ID: &str = "com.macro.app.prod";
diff --git a/services/document_cognition_service/src/api/context/test.rs b/services/document_cognition_service/src/api/context/test.rs
index 1b27f37319..01256ec3b2 100644
--- a/services/document_cognition_service/src/api/context/test.rs
+++ b/services/document_cognition_service/src/api/context/test.rs
@@ -68,6 +68,10 @@ pub async fn test_model_access(ctx: &ApiContext) -> DcsChatModelAccess {
pub struct MockConnectionRepo;
impl MockConnectionRepo {
+ #[expect(
+ clippy::new_ret_no_self,
+ reason = "tests take the mock as the trait object ApiContext stores"
+ )]
pub fn new() -> Arc<dyn ConnectionRepo> {
Arc::new(Self)
}
@@ -134,6 +138,10 @@ pub struct MockStreamRepo {
}
impl MockStreamRepo {
+ #[expect(
+ clippy::new_ret_no_self,
+ reason = "tests take the mock as the trait object ApiContext stores"
+ )]
pub fn new() -> Arc<dyn StreamRepo> {
let (tx, _) = broadcast::channel(16);
Arc::new(Self { tx })
diff --git a/services/document_cognition_service/src/lib.rs b/services/document_cognition_service/src/lib.rs
new file mode 100644
index 0000000000..63957096eb
--- /dev/null
+++ b/services/document_cognition_service/src/lib.rs
@@ -0,0 +1,14 @@
+#![recursion_limit = "256"]
+//! Document cognition service. The service binary and the OpenAPI generator
+//! both link this library, so the service compiles once and caches as one rlib.
+
+mod api;
+mod config;
+mod core;
+mod model;
+mod server;
+mod service;
+
+/// The service's OpenAPI document, rendered by the `*_openapi` binary.
+pub use api::swagger::ApiDoc;
+pub use server::run;
diff --git a/services/document_cognition_service/src/main.rs b/services/document_cognition_service/src/main.rs
index f970aa05db..a41e6c8023 100644
--- a/services/document_cognition_service/src/main.rs
+++ b/services/document_cognition_service/src/main.rs
@@ -1,984 +1,12 @@
+// `run()`'s future type is deep enough to need the library's recursion limit
+// wherever it is awaited.
#![recursion_limit = "256"]
-use crate::api::context::ApiContext;
-use ai_tools::{NoOpCallRtcClient, NoOpConnectionService, NoOpNotificationIngress};
-use anyhow::Context;
-use bots::outbound::pg_bots_repo::PgBotsRepo;
-use call::domain::service::{CallRecordQueryServiceImpl, CallServiceImpl};
-use call::inbound::toolset::CallToolContext;
-use call::outbound::pg_call_repo::PgCallRepo;
-use call::outbound::s3_recording_storage::S3RecordingStorage;
-use channels::{
- domain::list_service::ChannelListServiceImpl, outbound::pg_channels_repo::PgChannelsRepo,
-};
-use config::{Config, Environment};
-use document_storage_service_client::DocumentStorageServiceClient;
-use documents::{
- domain::{models::CloudFrontConfig, service::DocumentServiceImpl},
- inbound::toolset::DocumentToolContext,
- outbound::{pg_document_repo::PgDocumentRepo, s3_upload_url::S3UploadUrlAdapter},
-};
-use email::domain::ports::ReadonlyEmailPreviewAdapter;
-use email::domain::service::EmailServiceImpl;
-use email::outbound::EmailPgRepo;
-use email_service_client::{EmailServiceClient, EmailServiceClientExternal};
-use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};
-use entity_registry::OwnerGrantPolicy;
-use entity_registry_db_utils::OwnedEntityRegistrar;
-use foreign_entity::{
- domain::service::ForeignEntityServiceImpl,
- outbound::pg_foreign_entity_repo::PgForeignEntityRepo,
-};
-use frecency::domain::services::FrecencyQueryServiceImpl;
-use frecency::outbound::postgres::FrecencyPgStorage;
-use github_pull_requests::{
- domain::service::GithubPullRequestServiceImpl,
- outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo,
-};
-use macro_auth::middleware::decode_jwt::JwtValidationArgs;
-use macro_authorization::{
- InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationServiceImpl,
- MacroAuthorizationState, PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,
-};
-use macro_entrypoint::MacroEntrypoint;
-use macro_service_urls::{
- AuthServiceUrl, CalendarServiceUrl, ConnectionGatewayUrl, DocumentStorageServiceUrl,
- EmailServiceUrl, LexicalServiceUrl, StaticFileServiceUrl, SyncServiceUrl,
-};
-use notification::domain::service::{
- NotificationReaderService, PlatformArnConfig, SqsNotificationIngress,
-};
-use notification::outbound::queue::SqsQueue;
-use notification::outbound::repository::DbNotificationRepository;
-use notification::outbound::websocket::ConnectionGatewayClient;
-use readonly_pool::ReadOnlyPool;
-use search_service_client::SearchServiceClient;
-use sqlx::postgres::PgPoolOptions;
-use std::{sync::Arc, time::Duration};
-use stream::outbound::redis_pg::RedisPostgresStreamRepo;
-use sync_service_client::SyncServiceClient;
-use tokio_util::task::TaskTracker;
-mod api;
-mod config;
-mod core;
-mod model;
-mod service;
+use macro_entrypoint::MacroEntrypoint;
#[tokio::main]
#[tracing::instrument(err)]
async fn main() -> anyhow::Result<()> {
MacroEntrypoint::default().init();
-
- let aws_config = macro_aws_config::get_macro_aws_config().await;
-
- let secretsmanager_client = secretsmanager_client::SecretsManager::new(
- aws_sdk_secretsmanager::Client::new(&aws_config),
- );
-
- // Parse our configuration from the environment.
- let config = Config::from_env()
- .context("expected to be able to generate config")?
- .resolve_remote_secrets(Environment::new_or_prod(), &secretsmanager_client)
- .await
- .context("expected to be able to resolve config secrets")?;
- let non_user_owners = config.non_user_owners()?;
-
- tracing::info!("initialized config");
-
- let (min_connections, max_connections): (u32, u32) = match config.environment {
- Environment::Production => (5, 30),
- Environment::Develop => (3, 20),
- Environment::Local => (3, 10),
- };
-
- let db = PgPoolOptions::new()
- .min_connections(min_connections)
- .max_connections(max_connections)
- .connect(&config.database_url)
- .await
- .context("failed to connect to macrodb")?;
-
- tracing::info!(
- min_connections,
- max_connections,
- "initialized db connection"
- );
-
- let dynamodb_client = aws_sdk_dynamodb::Client::new(&aws_config);
- let queue_aws_client = aws_sdk_sqs::Client::new(&aws_config);
-
- let document_text_extractor_queue = macro_queues::DocumentTextExtractorQueue::new();
- let chat_delete_queue = macro_queues::ChatDeleteQueue::new();
- let email_scheduled_queue = macro_queues::EmailScheduledQueue::new();
- let gmail_ops_queue = macro_queues::GmailOpsQueue::new();
- let ai_projection_queue = macro_queues::AiProjectionQueue::new();
- let notification_queue = macro_queues::NotificationIngressQueue::new();
- let sqs_client = sqs_client::SQS::new(queue_aws_client)
- .document_text_extractor_queue(&document_text_extractor_queue)
- .chat_delete_queue(&chat_delete_queue)
- .email_scheduled_queue(&email_scheduled_queue)
- .gmail_ops_queue(&gmail_ops_queue)
- .ai_projection_queue(&ai_projection_queue);
-
- let internal_api_key = config.internal_api_key.to_string();
-
- let document_storage_client = DocumentStorageServiceClient::new(
- config
- .document_storage_service_auth_key
- .as_ref()
- .to_string(),
- DocumentStorageServiceUrl::new()?.to_string(),
- );
-
- tracing::info!("initialized dss client");
-
- let sync_service_client = SyncServiceClient::new(
- config.sync_service_auth_key.as_ref().to_string(),
- SyncServiceUrl::new()?.to_string(),
- );
- tracing::info!("initialized sync service client");
-
- let search_service_client = SearchServiceClient::new(
- config
- .document_storage_service_auth_key
- .as_ref()
- .to_string(),
- DocumentStorageServiceUrl::new()?.to_string(),
- );
- tracing::info!("initialized search service client");
-
- let jwt_args =
- JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)
- .await
- .context("failed to create jwt validation args")?;
-
- let authorization_state =
- MacroAuthorizationState::new(Arc::new(MacroAuthorizationServiceImpl::new(
- MacroAuthJwtValidator::new(jwt_args),
- InternalAuthConfig {
- api_key: internal_api_key.clone(),
- default_user_id: None,
- },
- macro_authorization::NoBotAuthorizer,
- PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),
- )));
-
- let lexical_client = Arc::new(lexical_client::LexicalClient::new(
- internal_api_key.clone(),
- LexicalServiceUrl::new()?.to_string(),
- ));
-
- let email_service_client = Arc::new(EmailServiceClient::new(
- internal_api_key.clone(),
- EmailServiceUrl::new()?.to_string(),
- ));
-
- tracing::info!("initialized static file service client");
-
- // Initialize Redis client for stream service
- let redis_client = Arc::new(
- redis::Client::open(config.redis_host.as_ref())
- .inspect(|client| {
- client
- .get_connection()
- .map(|_| tracing::trace!("initialized redis connection"))
- .inspect_err(|e| {
- tracing::error!(error=?e, "failed to connect to redis");
- })
- .expect("redis connetion required");
- })
- .context("failed to connect to redis")?,
- );
- let stream_repo = RedisPostgresStreamRepo::new((*redis_client).clone(), db.clone()).obj();
-
- tracing::info!("initialized stream repo");
-
- let connection_manager =
- connection_gateway::service::dynamodb::create_dynamo_db_connection_manager(dynamodb_client)
- .await
- .context("failed to create connection manager")?;
-
- tracing::info!("initialized connection repo");
- let connection_gateway_client = Arc::new(ConnectionGatewayClient::new(
- internal_api_key.clone(),
- ConnectionGatewayUrl::new()?.to_string(),
- ));
-
- let ingress_queue = SqsQueue::new(
- aws_sdk_sqs::Client::new(&aws_config),
- notification_queue.to_string(),
- );
- let notification_ingress_service = Arc::new(SqsNotificationIngress {
- queue: ingress_queue,
- });
-
- let notification_reader_queue = SqsQueue::new(
- aws_sdk_sqs::Client::new(&aws_config),
- notification_queue.to_string(),
- );
- let notification_reader_service = NotificationReaderService {
- repository: DbNotificationRepository::new(db.clone()),
- queue: ai_tools::ToolNotificationQueue::Sqs(notification_reader_queue),
- sns_endpoint: ai_tools::NoOpSnsEndpointManager,
- platform_config: PlatformArnConfig {
- apns_platform_arn: String::new(),
- fcm_platform_arn: String::new(),
- apns_voip_platform_arn: String::new(),
- },
- realtime: notification::domain::ports::NoopNotificationRealtimePublisher,
- };
- let notification_tool_context =
- notification::inbound::ai_tool::NotificationToolContext::new(notification_reader_service);
-
- tracing::info!("initialized notification ingress service");
- let entity_access_service = Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(
- db.clone(),
- )));
-
- let frecency_storage = FrecencyPgStorage::new(db.clone());
- let frecency_service = FrecencyQueryServiceImpl::new(frecency_storage.clone());
- let crm_service = crm::domain::service::CrmServiceImpl::new(
- crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),
- crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,
- );
- let email_service = EmailServiceImpl::new(
- EmailPgRepo::new(db.clone()),
- frecency_service.clone(),
- email::domain::ports::NoOpEnqueuer,
- crm_service.clone(),
- entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
- entity_access_management::outbound::PgRepository::new(db.clone()),
- ),
- 0,
- );
- let channels_service = ChannelListServiceImpl::new(
- PgChannelsRepo::new(db.clone()),
- PgChannelsRepo::new(db.clone()),
- frecency_storage,
- );
- let email_service_for_tools: Arc<ai_tools::ToolEmailService> = Arc::new(email_service.clone());
- let github_pull_request_service = GithubPullRequestServiceImpl::new(
- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
- PgGithubPullRequestRepo::new(db.clone()),
- );
- let soup_service = Arc::new(soup::domain::service::SoupImpl::new(
- soup::outbound::pg_soup_repo::PgSoupRepo::new(ReadOnlyPool(db.clone())),
- frecency_service,
- ReadonlyEmailPreviewAdapter(email_service),
- channels_service,
- CallRecordQueryServiceImpl::new(PgCallRepo::new(db.clone())),
- crm::domain::service::NoOpCrmService,
- github_pull_request_service,
- ));
-
- tracing::info!("initialized soup service");
-
- let s3_client = macro_aws_config::s3_client().await;
- let document_files = documents::outbound::s3_document_files::S3DocumentFiles::new(
- db.clone(),
- s3_client.clone(),
- config.document_storage_bucket.to_string(),
- );
- let s3_upload_adapter = S3UploadUrlAdapter::new(
- s3_client,
- config.document_storage_bucket.to_string(),
- config.docx_document_upload_bucket.to_string(),
- );
- let owned_entity_registrar =
- OwnedEntityRegistrar::new(OwnerGrantPolicy::new(PgBotsRepo::new(db.clone())));
- let document_repo = PgDocumentRepo::new(db.clone(), owned_entity_registrar.clone());
-
- let cloudfront_config = CloudFrontConfig {
- distribution_url: config
- .document_storage_service_cloudfront_distribution_url
- .to_string(),
- signer_public_key_id: config
- .document_storage_service_cloudfront_signer_public_key_id
- .to_string(),
- signer_private_key: config
- .document_storage_service_cloudfront_signer_private_key
- .as_ref()
- .to_string(),
- presigned_url_expiry_seconds: 3600,
- browser_cache_expiry_seconds: 86400,
- };
- let event_broker_tracker = TaskTracker::new();
- let macro_event_broker = macro_event_broker::MacroEventBrokerService::new(
- macro_event_broker::KafkaEventPublisher::new(config.kafka_brokers.as_ref())
- .context("failed to create kafka event publisher")?,
- event_broker_tracker.clone(),
- );
- let properties_service = ai_tools::build_properties_service_with_broker(
- db.clone(),
- entity_access_service.clone(),
- macro_event_broker.clone(),
- );
- let task_properties_service = ai_tools::build_task_properties_adapter(
- db.clone(),
- properties_service.clone(),
- entity_access_service.clone(),
- );
- // The import pipeline sets the same task system properties on imported
- // Linear issues (status, priority, due date, assignee).
- let task_properties_for_import = task_properties_service.clone();
- let document_properties_for_import =
- import::outbound::document_properties::DocumentPropertiesApplicator::new(
- properties_service.clone(),
- );
- let document_service = DocumentServiceImpl::new(
- document_repo,
- cloudfront_config,
- sync_service_client.clone(),
- s3_upload_adapter,
- task_properties_service,
- NoOpConnectionService,
- entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
- entity_access_management::outbound::PgRepository::new(db.clone()),
- ),
- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
- macro_event_broker.clone(),
- );
- // Messages sent by AI tools (chat, agents) dispatch the same side effects
- // as the document-storage message API, so mentions, replies and document
- // comments notify recipients and stream to connected clients.
- let channels_connection_gateway =
- Arc::new(connection_gateway_client::ConnectionGatewayClient::new(
- internal_api_key.clone(),
- ConnectionGatewayUrl::new()?.to_string(),
- ));
- let side_effect_clients = ai_tools::ChannelSideEffectClients {
- connection_gateway: channels_connection_gateway.clone(),
- sqs: aws_sdk_sqs::Client::new(&aws_config),
- macro_event_broker: macro_event_broker.clone(),
- };
- let lexical_client_for_tools = (*lexical_client).clone();
- let document_tool_context = DocumentToolContext::new(
- document_service,
- (*entity_access_service).clone(),
- lexical_client_for_tools,
- sync_service_client.clone(),
- documents::outbound::editing_worker_client::ReqwestEditingWorkerClient::new(
- config.ai_editing_worker_url.clone(),
- std::sync::Arc::new(reqwest::Client::new()),
- ),
- config.document_permission_jwt.as_ref().to_string(),
- ai_tools::build_message_service_with_side_effects(
- db.clone(),
- lexical_client.clone(),
- &side_effect_clients,
- ),
- )
- .with_presentation_files(Arc::new(document_files.clone()))
- .with_design_files(Arc::new(document_files.clone()))
- .with_photoshop_files(Arc::new(document_files.clone()))
- .with_illustrator_files(Arc::new(document_files));
-
- tracing::info!("initialized document tool context");
-
- let attachment_provider = attachment::provider::AttachmentProvider {
- document: documents::inbound::attachment::DocumentAttachmentService::new(
- document_tool_context.service.clone(),
- document_tool_context.entity_access_service.clone(),
- document_tool_context.lexical_client.clone(),
- ),
- email_thread: email::inbound::attachment::EmailAttachmentService::new(
- email_service_for_tools.clone(),
- entity_access_service.clone(),
- ),
- chat: chat::inbound::attachment::ChatAttachmentService::new(
- Arc::new(chat::outbound::postgres::PgChatRepo::new(
- db.clone(),
- owned_entity_registrar.clone(),
- )),
- entity_access_service.clone(),
- ),
- channel: channels::inbound::attachment::ChannelAttachmentService::new(
- Arc::new(PgChannelsRepo::new(db.clone())),
- entity_access_service.clone(),
- ),
- static_file: static_file::inbound::attachment::StaticFileAttachmentService::new(Arc::new(
- static_file::outbound::CdnStaticFileRepo::new(StaticFileServiceUrl::new()?.to_string()),
- )),
- };
- let message_service = Arc::new(
- chat::domain::service::MessageServiceImpl::new(
- chat::outbound::postgres::PgChatRepo::new(db.clone(), owned_entity_registrar.clone()),
- attachment_provider,
- )
- .with_event_broker(macro_event_broker.clone()),
- );
-
- tracing::info!("initialized attachment provider");
-
- let email_service_client_external = Arc::new(EmailServiceClientExternal::new(
- email_service_client.url().to_owned(),
- ));
-
- let search_service_client = Arc::new(search_service_client);
-
- let properties_tool_context = ai_tools::build_properties_tool_context(
- properties_service.clone(),
- entity_access_service.clone(),
- );
-
- tracing::info!("initialized properties tool context");
-
- let user_email_service = Arc::new(
- EmailServiceImpl::new(
- EmailPgRepo::new(db.clone()),
- FrecencyQueryServiceImpl::new(FrecencyPgStorage::new(db.clone())),
- sqs_client.clone(),
- crm_service.clone(),
- entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
- entity_access_management::outbound::PgRepository::new(db.clone()),
- ),
- 0,
- )
- .with_macro_event_broker(macro_event_broker.clone()),
- );
- let email_tool_context = email::inbound::toolset::EmailToolContext::new(
- user_email_service.clone(),
- Arc::new(email::domain::ports::NoOpGmailTokenProvider),
- Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(
- db.clone(),
- ))),
- lexical_client.clone(),
- );
-
- tracing::info!("initialized email tool context");
-
- let call_service = CallServiceImpl::new(
- PgCallRepo::new(db.clone()),
- NoOpCallRtcClient,
- NoOpConnectionService,
- (*entity_access_service).clone(),
- NoOpNotificationIngress,
- None::<S3RecordingStorage>,
- String::new(),
- );
- let call_tool_context = CallToolContext::new(call_service, (*entity_access_service).clone());
-
- tracing::info!("initialized call tool context");
-
- let chat_tool_context = chat::inbound::toolset::ChatToolContext::new(
- chat::domain::service::ChatServiceImpl::new(
- chat::outbound::postgres::PgChatRepo::new(db.clone(), owned_entity_registrar.clone()),
- Arc::new(ai_toolset::AsyncToolCollection::new()),
- (),
- entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
- entity_access_management::outbound::PgRepository::new(db.clone()),
- ),
- ),
- (*entity_access_service).clone(),
- );
-
- tracing::info!("initialized chat tool context");
-
- let channel_tool_context = ai_tools::build_channel_tool_context_with_side_effects(
- db.clone(),
- lexical_client.clone(),
- &side_effect_clients,
- );
- let user_permissions_service = Arc::new(
- roles_and_permissions::domain::service::UserRolesAndPermissionsServiceImpl::new(
- roles_and_permissions::outbound::pgpool::MacroDB::new(db.clone()),
- roles_and_permissions::outbound::pgpool::MacroDB::new(db.clone()),
- ),
- );
-
- // The AI billing gate reads allowances, credits, and overage state here;
- // collection (Stripe) lives in the authentication service, which the
- // recorder below asks to settle once a payer runs past their allowance
- // and ENABLE_AI_USAGE_BILLING is enabled. This instance never settles.
- let auth_service_client = Arc::new(authentication_service_client::AuthServiceClient::new(
- config
- .authentication_service_secret_key
- .as_ref()
- .to_string(),
- AuthServiceUrl::new()?.to_string(),
- ));
- let ai_pricing = config.ai_pricing();
- let ai_billing = Arc::new(
- ai_billing::domain::BillingServiceImpl::new(
- ai_billing::outbound::RolesTeamsEntitlementSource::new(
- (*user_permissions_service).clone(),
- teams::outbound::team_repo::TeamRepositoryImpl::new(db.clone()),
- ),
- ai_billing::outbound::PgUsageReader::new(db.clone()),
- ai_billing::outbound::PgBillingRepo::new(db.clone(), ai_pricing),
- ai_billing::outbound::HttpPaymentGateway::new(auth_service_client.clone()),
- ai_pricing,
- )
- .with_enforcement(config.enable_ai_usage_enforcement),
- );
- let admission: Arc<dyn ai_billing::AiAdmissionService> =
- Arc::new(ai_billing::BillingAdmissionService::new(
- ai_billing.clone(),
- config.enable_ai_usage_enforcement,
- ));
- let recorder: Arc<dyn ai_usage::UsageRecorder> =
- Arc::new(ai_billing::outbound::SettlingUsageRecorder::new(
- Arc::new(
- ai_usage::domain::service::UsageServiceImpl::new(
- ai_usage::outbound::PgUsageRepo::new(db.clone()),
- )
- .with_enforcement(config.enable_ai_usage_enforcement),
- ),
- ai_billing.clone(),
- ai_billing::outbound::HttpSettlementTrigger::new(auth_service_client),
- config.enable_ai_usage_billing,
- ));
-
- // Per-attempt observations use a separate journal and never feed the
- // legacy settlement trigger, which remains counted-usage-only and gated by
- // ENABLE_AI_USAGE_BILLING.
- let recorder = ai_usage::with_tracking(recorder, ai_usage::pg_tracking(db.clone()));
-
- // The import pipeline: staged/imported external items, gather jobs over
- // the user's connectors, and the Haiku import job. Built before the tool
- // service context so the chat toolset gets a wired import context.
- let mcp_encryption_key = mcp_client::domain::models::AesKey::try_from(
- config.mcp_credentials_key_secret_name.as_ref(),
- )
- .context("invalid MCP credentials encryption key")?;
- let mcp_server_repo =
- mcp_client::outbound::pg_server_repo::PgServerRepo::new(db.clone(), mcp_encryption_key);
-
- // The Pipedream MCP stack, fully separate from the native one above
- // (own endpoints, own table, own toolset). Without credentials its
- // endpoints answer 501 and its toolsets come up empty. Connect-flow
- // webhooks need both URI and secret; a half-set pair is ignored so we
- // never mint a callback we cannot serve.
- let pipedream_client: ai_tools::ToolPipedreamConnection = match (
- config.pipedream_client_id.value(),
- config.pipedream_client_secret.value(),
- config.pipedream_project_id.value(),
- ) {
- (Some(client_id), Some(client_secret), Some(project_id)) => {
- let webhook_uri = match (
- config.pipedream_webhook_uri.value(),
- config.pipedream_webhook_secret.value(),
- ) {
- (Some(uri), Some(_)) => Some(uri.to_owned()),
- (None, None) => None,
- _ => {
- tracing::warn!(
- "ignoring incomplete Pipedream webhook config; set both PIPEDREAM_WEBHOOK_URI and PIPEDREAM_WEBHOOK_SECRET"
- );
- None
- }
- };
- let client = pipedream_mcp::outbound::api::PipedreamClient::new(
- pipedream_mcp::outbound::api::PipedreamConfig {
- client_id: client_id.to_owned(),
- client_secret: client_secret.to_owned(),
- project_id: project_id.to_owned(),
- environment: config
- .pipedream_environment
- .value()
- .unwrap_or(match config.environment {
- Environment::Production => "production",
- _ => "development",
- })
- .to_owned(),
- api_url: config
- .pipedream_api_url
- .value()
- .unwrap_or(pipedream_mcp::outbound::api::DEFAULT_API_URL)
- .to_owned(),
- mcp_url: config
- .pipedream_mcp_url
- .value()
- .unwrap_or(pipedream_mcp::outbound::api::DEFAULT_MCP_URL)
- .to_owned(),
- allowed_origins: match config.pipedream_allowed_origins.value() {
- Some(origins) => origins
- .split(',')
- .map(|origin| origin.trim().to_owned())
- .filter(|origin| !origin.is_empty())
- .collect(),
- None => match config.environment {
- Environment::Production => vec!["https://macro.com".to_owned()],
- Environment::Develop => vec![
- "https://dev.macro.com".to_owned(),
- "http://localhost:3000".to_owned(),
- ],
- Environment::Local => vec!["http://localhost:3000".to_owned()],
- },
- },
- },
- )
- .context("failed to build Pipedream client")?;
- Some(Arc::new(match webhook_uri {
- Some(uri) => client.with_webhook_uri(uri),
- None => client,
- }))
- }
- _ => {
- tracing::info!("Pipedream credentials not set; Pipedream MCP connectors disabled");
- None
- }
- };
- let pipedream_repo =
- pipedream_mcp::outbound::pg_connection_repo::PgConnectionRepo::new(db.clone());
-
- // The one sanctioned meeting point of the two MCP stacks: agents load
- // tools through this selector, which prefers a user's Pipedream
- // connectors and falls back to the native ones (see `mcp_select`).
- // The import service's live Slack source shares this selector.
- let mcp_selector: Arc<ai_tools::ToolMcpSelector> = Arc::new(mcp_select::McpToolSelector::new(
- Arc::new(mcp_server_repo.clone()),
- Arc::new(pipedream_repo.clone()),
- Arc::new(pipedream_client.clone()),
- ));
-
- // Nudges the user's connected clients when import rows flip, so setup
- // sections and chat surfaces update immediately instead of on the next
- // poll (see import::outbound::gateway_notifier).
- let import_notify = import::outbound::gateway_notifier::gateway_import_notify(
- channels_connection_gateway.clone(),
- );
-
- let entity_creator = ai_tools::ToolEntityCreator {
- document_creator: document_tool_context.creator.clone(),
- entity_access_service: entity_access_service.clone(),
- channel_service: channel_tool_context.service.clone(),
- task_properties: task_properties_for_import,
- document_properties: document_properties_for_import,
- team_repository: ai_tools::build_team_repository(db.clone()),
- };
- let import_service = Arc::new(
- import::domain::service::ImportServiceImpl::new(
- import::outbound::pg_import_repo::PgImportRepo::new(db.clone()),
- mcp_selector.clone(),
- Arc::new(entity_creator),
- recorder.clone(),
- )
- .with_slack_source(Arc::new(
- import::outbound::mcp_slack_source::McpSlackSource::new(mcp_selector.clone()),
- ))
- .with_admission(admission.clone())
- .with_notifier(import_notify),
- );
-
- // No boot-time recovery: running batches heartbeat their rows, and the
- // read path reaps rows whose heartbeat stopped — safe with multiple
- // replicas, where a boot-time sweep would clobber other instances' jobs.
- tracing::info!("initialized import service");
-
- let project_tool_context = ai_tools::build_project_tool_context(
- db.clone(),
- macro_event_broker.clone(),
- entity_access_service.clone(),
- document_tool_context.service.clone(),
- chat_tool_context.service.clone(),
- user_email_service.clone(),
- );
-
- let initiative_tool_context = ai_tools::build_initiative_tool_context(
- db.clone(),
- &document_tool_context,
- properties_service.clone(),
- entity_access_service.clone(),
- macro_event_broker.clone(),
- );
-
- let databases_tool_context = ai_tools::build_databases_tool_context(
- db.clone(),
- entity_access_service.clone(),
- ai_tools::ToolTableEventPublisher::Gateway(
- databases::outbound::gateway_event_publisher::GatewayTableEventPublisher::new(
- channels_connection_gateway.as_ref().clone(),
- ),
- ),
- ai_tools::MaybeToolEventBroker::Real(macro_event_broker.clone()),
- );
- let databases_sql_tool_context = ai_tools::build_databases_sql_tool_context(
- &databases_tool_context,
- soup_service.clone(),
- db.clone(),
- );
-
- let tool_service_context = ai_tools::ToolServiceContext {
- search_service_client: search_service_client.clone(),
- email_service_client: email_service_client_external.clone(),
- soup_service: soup_service.clone(),
- email_service: email_service_for_tools.clone(),
- activity_tool_context: ai_tools::build_activity_tool_context(
- db.clone(),
- properties_service,
- entity_access_service.clone(),
- ),
- image_generation_tool_context: ai_tools::build_image_generation_tool_context(
- &document_tool_context,
- ai_tools::build_image_generator_from_env(),
- recorder.clone(),
- )?,
- document_tool_context: document_tool_context.clone(),
- properties_tool_context: properties_tool_context.clone(),
- email_tool_context: email_tool_context.clone(),
- call_tool_context: call_tool_context.clone(),
- booking_link_tool_context: ai_tools::build_booking_link_tool_context(
- db.clone(),
- config.environment,
- ),
- calendar_tool_context: ai_tools::build_calendar_tool_context(
- db.clone(),
- CalendarServiceUrl::new()?,
- internal_api_key.clone(),
- ),
- notification_tool_context: notification_tool_context.clone(),
- databases_tool_context,
- databases_sql_tool_context,
- import_tool_context: import::inbound::toolset::ImportToolContext::wired(
- import_service.clone(),
- ),
- chat_tool_context,
- channel_tool_context,
- bot_tool_context: ai_tools::build_bot_tool_context(
- db.clone(),
- ai_tools::MaybeToolEventBroker::Real(macro_event_broker.clone()),
- entity_access_service.clone(),
- DocumentStorageServiceUrl::new()?.to_string(),
- pipedream_client.clone(),
- ),
- coding_agent_tool_context: ai_tools::build_coding_agent_tool_context(
- macro_service_urls::AgentHarnessServiceUrl::new()?,
- internal_api_key.clone(),
- )?,
- project_tool_context,
- initiative_tool_context,
- team_tool_context: ai_tools::build_team_tool_context(db.clone()),
- crm_tool_context: ai_tools::build_crm_tool_context(db.clone()),
- skill_tool_context: ai_tools::build_skill_tool_context(
- search_service_client.clone(),
- soup_service.clone(),
- &document_tool_context,
- ),
- schedule_tool_context: ai_tools::build_routine_tool_context()?,
- anthropic_tool_context: ai_tools::build_anthropic_tool_context(),
- admission,
- recorder,
- usage_context: ai_usage::UsageContext::system(ai_usage::AiFeature::Chat),
- };
- let all_tools = ai_tools::tools_for(ai_tools::AiHost::Chat);
- let all_tools_toolset = all_tools.toolset.clone();
- let all_tools_prompt: Arc<dyn std::fmt::Display + Send + Sync> =
- Arc::new(all_tools.prompt.to_string());
-
- // Build memory service
- let memory_repo = memory::outbound::pg_memory_repo::PgMemoryRepo::new(db.clone());
- let memory_service = Arc::new(memory::domain::service::MemoryServiceImpl::new(
- memory_repo,
- tool_service_context.clone(),
- all_tools,
- ));
-
- tracing::info!("initialized memory service");
-
- // Build the AI cost service. It backs both the admin query/pricing router
- // and the usage recorder threaded through the tool service context.
- let usage_service = Arc::new(
- ai_usage::domain::service::UsageServiceImpl::new(ai_usage::outbound::PgUsageRepo::new(
- db.clone(),
- ))
- .with_enforcement(config.enable_ai_usage_enforcement),
- );
-
- tracing::info!("initialized ai cost service");
-
- // Generator that materializes projections by running their prompt through
- // the shared AI toolset, attributed to the requesting user.
- let projection_generator =
- ai_projections::outbound::agent_generator::AgentProjectionGenerator::new(
- tool_service_context.clone(),
- ai_tools::tools_for(ai_tools::AiHost::Chat),
- );
- // Notifier that pushes finished materializations to the target's connected
- // clients through the connection gateway.
- let projection_notifier =
- ai_projections::outbound::gateway_notifier::GatewayProjectionNotifier::new(Arc::new(
- connection_gateway_client::ConnectionGatewayClient::new(
- internal_api_key.clone(),
- ConnectionGatewayUrl::new()?.to_string(),
- ),
- ));
- let ai_projections_service_impl =
- ai_projections::domain::ai_projection_service::AiProjectionServiceImpl::new(
- ai_projections::outbound::ai_projection_repo::AiProjectionRepositoryImpl::new(
- db.clone(),
- ),
- sqs_client.clone(),
- projection_generator,
- projection_notifier,
- );
-
- // Spawn the inbound worker that polls ai_projection_queue and materializes
- // projection instances. It owns its own clone of the service.
- let ai_projection_worker = ai_projections::worker::AiProjectionWorker::new(
- sqs_worker::SQSWorker::new(
- aws_sdk_sqs::Client::new(&aws_config),
- ai_projection_queue.to_string(),
- 10,
- 10,
- ),
- ai_projections_service_impl.clone(),
- );
- let ai_projection_worker_task = tokio::spawn(async move {
- ai_projection_worker.poll().await;
- });
-
- let ai_projections_service = Arc::new(ai_projections_service_impl);
-
- tracing::info!("initialized ai projections service");
-
- // The onboarding flow drives the import pipeline: reads/hooks start
- // auto-importing gather runs for authenticated connectors, and
- // completion deletes unreserved onboarding-staged candidates.
- let onboarding_service = Arc::new(onboarding::domain::service::OnboardingServiceImpl::new(
- onboarding::outbound::pg_onboarding_repo::PgOnboardingRepo::new(db.clone()),
- Arc::new(mcp_server_repo.clone()),
- import_service.clone(),
- mcp_selector.clone(),
- ));
-
- tracing::info!("initialized onboarding service");
-
- let mcp_public_url = &config.mcp_public_url;
- let mcp_client_metadata = mcp_client::domain::models::OAuthClientMetadata::new(
- format!("{mcp_public_url}/mcp/servers/auth/client-metadata"),
- format!("{mcp_public_url}/mcp/servers/auth/callback"),
- );
- let mcp_oauth_state_store =
- mcp_client::outbound::redis_state_store::RedisOAuthStateStore::new(redis_client.clone());
- let mcp_pre_registered =
- mcp_client::domain::provider_registry::PreRegisteredProviders::from_env()?;
- let mcp_oauth = mcp_client::outbound::oauth::OAuthService::new(
- mcp_server_repo.clone(),
- mcp_oauth_state_store,
- mcp_client_metadata.clone(),
- mcp_pre_registered,
- );
- // The moment a connector finishes OAuth, reconcile onboarding for that
- // user — gather jobs start before the user even returns to their
- // original tab. Spawned so the callback response never waits on them.
- let onboarding_for_auth_hook = onboarding_service.clone();
- let mcp_auth_hook: mcp_client::inbound::axum_router::McpAuthCompletedHook =
- Arc::new(move |record: mcp_client::domain::models::McpServerRecord| {
- let service = onboarding_for_auth_hook.clone();
- Box::pin(async move {
- tokio::spawn(async move {
- use onboarding::domain::service::OnboardingService;
- if let Err(e) = service.reconcile(record.user_id).await {
- tracing::warn!(error = ?e, "post-auth onboarding reconcile failed");
- }
- });
- })
- });
- let mcp_state = mcp_client::inbound::McpRouterState::new(
- mcp_server_repo,
- mcp_oauth,
- authorization_state.clone(),
- mcp_client_metadata,
- )
- .with_auth_completed_hook(mcp_auth_hook);
-
- // The Pipedream stack gets the same post-connect reconcile hook as the
- // native one: a finished Connect flow starts gather jobs immediately.
- let onboarding_for_pipedream_hook = onboarding_service.clone();
- let pipedream_auth_hook: pipedream_mcp::inbound::axum_router::PipedreamAuthCompletedHook =
- Arc::new(
- move |connection: pipedream_mcp::domain::models::PipedreamConnection| {
- let service = onboarding_for_pipedream_hook.clone();
- Box::pin(async move {
- tokio::spawn(async move {
- use onboarding::domain::service::OnboardingService;
- if let Err(e) = service.reconcile(connection.user_id).await {
- tracing::warn!(error = ?e, "post-connect onboarding reconcile failed");
- }
- });
- })
- },
- );
- let pipedream_state = pipedream_mcp::inbound::PipedreamRouterState::new(
- pipedream_repo,
- pipedream_client,
- authorization_state.clone(),
- )
- .with_auth_completed_hook(pipedream_auth_hook);
-
- let api_result = api::setup_and_serve(ApiContext {
- db: db.clone(),
- email_service_client_external,
- sqs_client: Arc::new(sqs_client),
- document_storage_client: Arc::new(document_storage_client),
- search_service_client,
- authorization_state,
- user_permissions_service,
- non_user_owners,
- ai_admission: tool_service_context.admission.clone(),
- ai_billing,
- internal_api_key: config.internal_api_key.clone(),
- config: Arc::new(config),
- notification_ingress_service,
- connection_repo: connection_manager.persistence,
- connection_gateway_client,
- soup_service,
- email_service: email_service_for_tools,
- stream_repo,
- document_tool_context,
- memory_service,
- usage_service,
- ai_projections_service,
- properties_tool_context,
- email_tool_context: email_tool_context.clone(),
- call_tool_context,
- tool_service_context,
- all_tools: all_tools_toolset,
- all_tools_prompt,
- entity_access_service,
- message_service,
- ai_stream_registry: service::ai_stream_registry::AiStreamRegistry::new(
- redis_client.clone(),
- ),
- mcp_state,
- pipedream_state,
- mcp_selector,
- import_service,
- onboarding_service,
- macro_event_broker: macro_event_broker.clone(),
- })
- .await
- .context("failed to setup and serve api");
-
- ai_projection_worker_task.abort();
- match ai_projection_worker_task.await {
- Err(error) if error.is_cancelled() => {
- tracing::info!("ai projection worker stopped");
- }
- Err(error) => {
- tracing::error!(error = ?error, "ai projection worker exited unexpectedly");
- }
- Ok(()) => {
- tracing::error!(
- error = "worker exited naturally",
- "ai projection worker exited unexpectedly"
- );
- }
- }
-
- tracing::info!("waiting for event broker publishes to drain");
- event_broker_tracker.close();
- match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {
- Ok(()) => tracing::info!("event broker publishes drained"),
- Err(error) => tracing::warn!(
- error=?error,
- timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),
- "timed out waiting for event broker publishes to drain"
- ),
- }
-
- api_result
+ document_cognition_service::run().await
}
-
-const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
diff --git a/services/document_cognition_service/src/openapi.rs b/services/document_cognition_service/src/openapi.rs
index 09b0b1c676..9ecbe00785 100644
--- a/services/document_cognition_service/src/openapi.rs
+++ b/services/document_cognition_service/src/openapi.rs
@@ -1,19 +1,6 @@
-#![recursion_limit = "256"]
-#![allow(unused)]
-
-mod api;
-mod config;
-mod core;
-mod model;
-mod service;
-
-pub use config::Config;
-
+use document_cognition_service::ApiDoc;
use utoipa::OpenApi;
fn main() {
- println!(
- "{}",
- api::swagger::ApiDoc::openapi().to_pretty_json().unwrap()
- );
+ println!("{}", ApiDoc::openapi().to_pretty_json().unwrap());
}
diff --git a/services/document_cognition_service/src/server.rs b/services/document_cognition_service/src/server.rs
new file mode 100644
index 0000000000..3b9edac569
--- /dev/null
+++ b/services/document_cognition_service/src/server.rs
@@ -0,0 +1,979 @@
+//! The service's composition root: reads config and wires every adapter and
+//! domain service into the HTTP server.
+
+use crate::{
+ api::{self, context::ApiContext},
+ config::{Config, Environment},
+ service,
+};
+use ai_tools::{NoOpCallRtcClient, NoOpConnectionService, NoOpNotificationIngress};
+use anyhow::Context;
+use bots::outbound::pg_bots_repo::PgBotsRepo;
+use call::domain::service::{CallRecordQueryServiceImpl, CallServiceImpl};
+use call::inbound::toolset::CallToolContext;
+use call::outbound::pg_call_repo::PgCallRepo;
+use call::outbound::s3_recording_storage::S3RecordingStorage;
+use channels::{
+ domain::list_service::ChannelListServiceImpl, outbound::pg_channels_repo::PgChannelsRepo,
+};
+use document_storage_service_client::DocumentStorageServiceClient;
+use documents::{
+ domain::{models::CloudFrontConfig, service::DocumentServiceImpl},
+ inbound::toolset::DocumentToolContext,
+ outbound::{pg_document_repo::PgDocumentRepo, s3_upload_url::S3UploadUrlAdapter},
+};
+use email::domain::ports::ReadonlyEmailPreviewAdapter;
+use email::domain::service::EmailServiceImpl;
+use email::outbound::EmailPgRepo;
+use email_service_client::{EmailServiceClient, EmailServiceClientExternal};
+use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};
+use entity_registry::OwnerGrantPolicy;
+use entity_registry_db_utils::OwnedEntityRegistrar;
+use foreign_entity::{
+ domain::service::ForeignEntityServiceImpl,
+ outbound::pg_foreign_entity_repo::PgForeignEntityRepo,
+};
+use frecency::domain::services::FrecencyQueryServiceImpl;
+use frecency::outbound::postgres::FrecencyPgStorage;
+use github_pull_requests::{
+ domain::service::GithubPullRequestServiceImpl,
+ outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo,
+};
+use macro_auth::middleware::decode_jwt::JwtValidationArgs;
+use macro_authorization::{
+ InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationServiceImpl,
+ MacroAuthorizationState, PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,
+};
+use macro_service_urls::{
+ AuthServiceUrl, CalendarServiceUrl, ConnectionGatewayUrl, DocumentStorageServiceUrl,
+ EmailServiceUrl, LexicalServiceUrl, StaticFileServiceUrl, SyncServiceUrl,
+};
+use notification::domain::service::{
+ NotificationReaderService, PlatformArnConfig, SqsNotificationIngress,
+};
+use notification::outbound::queue::SqsQueue;
+use notification::outbound::repository::DbNotificationRepository;
+use notification::outbound::websocket::ConnectionGatewayClient;
+use readonly_pool::ReadOnlyPool;
+use search_service_client::SearchServiceClient;
+use sqlx::postgres::PgPoolOptions;
+use std::{sync::Arc, time::Duration};
+use stream::outbound::redis_pg::RedisPostgresStreamRepo;
+use sync_service_client::SyncServiceClient;
+use tokio_util::task::TaskTracker;
+
+/// Build the service from the environment and serve until shutdown.
+pub async fn run() -> anyhow::Result<()> {
+ let aws_config = macro_aws_config::get_macro_aws_config().await;
+
+ let secretsmanager_client = secretsmanager_client::SecretsManager::new(
+ aws_sdk_secretsmanager::Client::new(&aws_config),
+ );
+
+ // Parse our configuration from the environment.
+ let config = Config::from_env()
+ .context("expected to be able to generate config")?
+ .resolve_remote_secrets(Environment::new_or_prod(), &secretsmanager_client)
+ .await
+ .context("expected to be able to resolve config secrets")?;
+ let non_user_owners = config.non_user_owners()?;
+
+ tracing::info!("initialized config");
+
+ let (min_connections, max_connections): (u32, u32) = match config.environment {
+ Environment::Production => (5, 30),
+ Environment::Develop => (3, 20),
+ Environment::Local => (3, 10),
+ };
+
+ let db = PgPoolOptions::new()
+ .min_connections(min_connections)
+ .max_connections(max_connections)
+ .connect(&config.database_url)
+ .await
+ .context("failed to connect to macrodb")?;
+
+ tracing::info!(
+ min_connections,
+ max_connections,
+ "initialized db connection"
+ );
+
+ let dynamodb_client = aws_sdk_dynamodb::Client::new(&aws_config);
+ let queue_aws_client = aws_sdk_sqs::Client::new(&aws_config);
+
+ let document_text_extractor_queue = macro_queues::DocumentTextExtractorQueue::new();
+ let chat_delete_queue = macro_queues::ChatDeleteQueue::new();
+ let email_scheduled_queue = macro_queues::EmailScheduledQueue::new();
+ let gmail_ops_queue = macro_queues::GmailOpsQueue::new();
+ let ai_projection_queue = macro_queues::AiProjectionQueue::new();
+ let notification_queue = macro_queues::NotificationIngressQueue::new();
+ let sqs_client = sqs_client::SQS::new(queue_aws_client)
+ .document_text_extractor_queue(&document_text_extractor_queue)
+ .chat_delete_queue(&chat_delete_queue)
+ .email_scheduled_queue(&email_scheduled_queue)
+ .gmail_ops_queue(&gmail_ops_queue)
+ .ai_projection_queue(&ai_projection_queue);
+
+ let internal_api_key = config.internal_api_key.to_string();
+
+ let document_storage_client = DocumentStorageServiceClient::new(
+ config
+ .document_storage_service_auth_key
+ .as_ref()
+ .to_string(),
+ DocumentStorageServiceUrl::new()?.to_string(),
+ );
+
+ tracing::info!("initialized dss client");
+
+ let sync_service_client = SyncServiceClient::new(
+ config.sync_service_auth_key.as_ref().to_string(),
+ SyncServiceUrl::new()?.to_string(),
+ );
+ tracing::info!("initialized sync service client");
+
+ let search_service_client = SearchServiceClient::new(
+ config
+ .document_storage_service_auth_key
+ .as_ref()
+ .to_string(),
+ DocumentStorageServiceUrl::new()?.to_string(),
+ );
+ tracing::info!("initialized search service client");
+
+ let jwt_args =
+ JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)
+ .await
+ .context("failed to create jwt validation args")?;
+
+ let authorization_state =
+ MacroAuthorizationState::new(Arc::new(MacroAuthorizationServiceImpl::new(
+ MacroAuthJwtValidator::new(jwt_args),
+ InternalAuthConfig {
+ api_key: internal_api_key.clone(),
+ default_user_id: None,
+ },
+ macro_authorization::NoBotAuthorizer,
+ PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),
+ )));
+
+ let lexical_client = Arc::new(lexical_client::LexicalClient::new(
+ internal_api_key.clone(),
+ LexicalServiceUrl::new()?.to_string(),
+ ));
+
+ let email_service_client = Arc::new(EmailServiceClient::new(
+ internal_api_key.clone(),
+ EmailServiceUrl::new()?.to_string(),
+ ));
+
+ tracing::info!("initialized static file service client");
+
+ // Initialize Redis client for stream service
+ let redis_client = Arc::new(
+ redis::Client::open(config.redis_host.as_ref())
+ .inspect(|client| {
+ client
+ .get_connection()
+ .map(|_| tracing::trace!("initialized redis connection"))
+ .inspect_err(|e| {
+ tracing::error!(error=?e, "failed to connect to redis");
+ })
+ .expect("redis connetion required");
+ })
+ .context("failed to connect to redis")?,
+ );
+ let stream_repo = RedisPostgresStreamRepo::new((*redis_client).clone(), db.clone()).obj();
+
+ tracing::info!("initialized stream repo");
+
+ let connection_manager =
+ connection_gateway::service::dynamodb::create_dynamo_db_connection_manager(dynamodb_client)
+ .await
+ .context("failed to create connection manager")?;
+
+ tracing::info!("initialized connection repo");
+ let connection_gateway_client = Arc::new(ConnectionGatewayClient::new(
+ internal_api_key.clone(),
+ ConnectionGatewayUrl::new()?.to_string(),
+ ));
+
+ let ingress_queue = SqsQueue::new(
+ aws_sdk_sqs::Client::new(&aws_config),
+ notification_queue.to_string(),
+ );
+ let notification_ingress_service = Arc::new(SqsNotificationIngress {
+ queue: ingress_queue,
+ });
+
+ let notification_reader_queue = SqsQueue::new(
+ aws_sdk_sqs::Client::new(&aws_config),
+ notification_queue.to_string(),
+ );
+ let notification_reader_service = NotificationReaderService {
+ repository: DbNotificationRepository::new(db.clone()),
+ queue: ai_tools::ToolNotificationQueue::Sqs(notification_reader_queue),
+ sns_endpoint: ai_tools::NoOpSnsEndpointManager,
+ platform_config: PlatformArnConfig {
+ apns_platform_arn: String::new(),
+ fcm_platform_arn: String::new(),
+ apns_voip_platform_arn: String::new(),
+ },
+ realtime: notification::domain::ports::NoopNotificationRealtimePublisher,
+ };
+ let notification_tool_context =
+ notification::inbound::ai_tool::NotificationToolContext::new(notification_reader_service);
+
+ tracing::info!("initialized notification ingress service");
+ let entity_access_service = Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(
+ db.clone(),
+ )));
+
+ let frecency_storage = FrecencyPgStorage::new(db.clone());
+ let frecency_service = FrecencyQueryServiceImpl::new(frecency_storage.clone());
+ let crm_service = crm::domain::service::CrmServiceImpl::new(
+ crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),
+ crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,
+ );
+ let email_service = EmailServiceImpl::new(
+ EmailPgRepo::new(db.clone()),
+ frecency_service.clone(),
+ email::domain::ports::NoOpEnqueuer,
+ crm_service.clone(),
+ entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
+ entity_access_management::outbound::PgRepository::new(db.clone()),
+ ),
+ 0,
+ );
+ let channels_service = ChannelListServiceImpl::new(
+ PgChannelsRepo::new(db.clone()),
+ PgChannelsRepo::new(db.clone()),
+ frecency_storage,
+ );
+ let email_service_for_tools: Arc<ai_tools::ToolEmailService> = Arc::new(email_service.clone());
+ let github_pull_request_service = GithubPullRequestServiceImpl::new(
+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
+ PgGithubPullRequestRepo::new(db.clone()),
+ );
+ let soup_service = Arc::new(soup::domain::service::SoupImpl::new(
+ soup::outbound::pg_soup_repo::PgSoupRepo::new(ReadOnlyPool(db.clone())),
+ frecency_service,
+ ReadonlyEmailPreviewAdapter(email_service),
+ channels_service,
+ CallRecordQueryServiceImpl::new(PgCallRepo::new(db.clone())),
+ crm::domain::service::NoOpCrmService,
+ github_pull_request_service,
+ ));
+
+ tracing::info!("initialized soup service");
+
+ let s3_client = macro_aws_config::s3_client().await;
+ let document_files = documents::outbound::s3_document_files::S3DocumentFiles::new(
+ db.clone(),
+ s3_client.clone(),
+ config.document_storage_bucket.to_string(),
+ );
+ let s3_upload_adapter = S3UploadUrlAdapter::new(
+ s3_client,
+ config.document_storage_bucket.to_string(),
+ config.docx_document_upload_bucket.to_string(),
+ );
+ let owned_entity_registrar =
+ OwnedEntityRegistrar::new(OwnerGrantPolicy::new(PgBotsRepo::new(db.clone())));
+ let document_repo = PgDocumentRepo::new(db.clone(), owned_entity_registrar.clone());
+
+ let cloudfront_config = CloudFrontConfig {
+ distribution_url: config
+ .document_storage_service_cloudfront_distribution_url
+ .to_string(),
+ signer_public_key_id: config
+ .document_storage_service_cloudfront_signer_public_key_id
+ .to_string(),
+ signer_private_key: config
+ .document_storage_service_cloudfront_signer_private_key
+ .as_ref()
+ .to_string(),
+ presigned_url_expiry_seconds: 3600,
+ browser_cache_expiry_seconds: 86400,
+ };
+ let event_broker_tracker = TaskTracker::new();
+ let macro_event_broker = macro_event_broker::MacroEventBrokerService::new(
+ macro_event_broker::KafkaEventPublisher::new(config.kafka_brokers.as_ref())
+ .context("failed to create kafka event publisher")?,
+ event_broker_tracker.clone(),
+ );
+ let properties_service = ai_tools::build_properties_service_with_broker(
+ db.clone(),
+ entity_access_service.clone(),
+ macro_event_broker.clone(),
+ );
+ let task_properties_service = ai_tools::build_task_properties_adapter(
+ db.clone(),
+ properties_service.clone(),
+ entity_access_service.clone(),
+ );
+ // The import pipeline sets the same task system properties on imported
+ // Linear issues (status, priority, due date, assignee).
+ let task_properties_for_import = task_properties_service.clone();
+ let document_properties_for_import =
+ import::outbound::document_properties::DocumentPropertiesApplicator::new(
+ properties_service.clone(),
+ );
+ let document_service = DocumentServiceImpl::new(
+ document_repo,
+ cloudfront_config,
+ sync_service_client.clone(),
+ s3_upload_adapter,
+ task_properties_service,
+ NoOpConnectionService,
+ entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
+ entity_access_management::outbound::PgRepository::new(db.clone()),
+ ),
+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
+ macro_event_broker.clone(),
+ );
+ // Messages sent by AI tools (chat, agents) dispatch the same side effects
+ // as the document-storage message API, so mentions, replies and document
+ // comments notify recipients and stream to connected clients.
+ let channels_connection_gateway =
+ Arc::new(connection_gateway_client::ConnectionGatewayClient::new(
+ internal_api_key.clone(),
+ ConnectionGatewayUrl::new()?.to_string(),
+ ));
+ let side_effect_clients = ai_tools::ChannelSideEffectClients {
+ connection_gateway: channels_connection_gateway.clone(),
+ sqs: aws_sdk_sqs::Client::new(&aws_config),
+ macro_event_broker: macro_event_broker.clone(),
+ };
+ let lexical_client_for_tools = (*lexical_client).clone();
+ let document_tool_context = DocumentToolContext::new(
+ document_service,
+ (*entity_access_service).clone(),
+ lexical_client_for_tools,
+ sync_service_client.clone(),
+ documents::outbound::editing_worker_client::ReqwestEditingWorkerClient::new(
+ config.ai_editing_worker_url.clone(),
+ std::sync::Arc::new(reqwest::Client::new()),
+ ),
+ config.document_permission_jwt.as_ref().to_string(),
+ ai_tools::build_message_service_with_side_effects(
+ db.clone(),
+ lexical_client.clone(),
+ &side_effect_clients,
+ ),
+ )
+ .with_presentation_files(Arc::new(document_files.clone()))
+ .with_design_files(Arc::new(document_files.clone()))
+ .with_photoshop_files(Arc::new(document_files.clone()))
+ .with_illustrator_files(Arc::new(document_files));
+
+ tracing::info!("initialized document tool context");
+
+ let attachment_provider = attachment::provider::AttachmentProvider {
+ document: documents::inbound::attachment::DocumentAttachmentService::new(
+ document_tool_context.service.clone(),
+ document_tool_context.entity_access_service.clone(),
+ document_tool_context.lexical_client.clone(),
+ ),
+ email_thread: email::inbound::attachment::EmailAttachmentService::new(
+ email_service_for_tools.clone(),
+ entity_access_service.clone(),
+ ),
+ chat: chat::inbound::attachment::ChatAttachmentService::new(
+ Arc::new(chat::outbound::postgres::PgChatRepo::new(
+ db.clone(),
+ owned_entity_registrar.clone(),
+ )),
+ entity_access_service.clone(),
+ ),
+ channel: channels::inbound::attachment::ChannelAttachmentService::new(
+ Arc::new(PgChannelsRepo::new(db.clone())),
+ entity_access_service.clone(),
+ ),
+ static_file: static_file::inbound::attachment::StaticFileAttachmentService::new(Arc::new(
+ static_file::outbound::CdnStaticFileRepo::new(StaticFileServiceUrl::new()?.to_string()),
+ )),
+ };
+ let message_service = Arc::new(
+ chat::domain::service::MessageServiceImpl::new(
+ chat::outbound::postgres::PgChatRepo::new(db.clone(), owned_entity_registrar.clone()),
+ attachment_provider,
+ )
+ .with_event_broker(macro_event_broker.clone()),
+ );
+
+ tracing::info!("initialized attachment provider");
+
+ let email_service_client_external = Arc::new(EmailServiceClientExternal::new(
+ email_service_client.url().to_owned(),
+ ));
+
+ let search_service_client = Arc::new(search_service_client);
+
+ let properties_tool_context = ai_tools::build_properties_tool_context(
+ properties_service.clone(),
+ entity_access_service.clone(),
+ );
+
+ tracing::info!("initialized properties tool context");
+
+ let user_email_service = Arc::new(
+ EmailServiceImpl::new(
+ EmailPgRepo::new(db.clone()),
+ FrecencyQueryServiceImpl::new(FrecencyPgStorage::new(db.clone())),
+ sqs_client.clone(),
+ crm_service.clone(),
+ entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
+ entity_access_management::outbound::PgRepository::new(db.clone()),
+ ),
+ 0,
+ )
+ .with_macro_event_broker(macro_event_broker.clone()),
+ );
+ let email_tool_context = email::inbound::toolset::EmailToolContext::new(
+ user_email_service.clone(),
+ Arc::new(email::domain::ports::NoOpGmailTokenProvider),
+ Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(
+ db.clone(),
+ ))),
+ lexical_client.clone(),
+ );
+
+ tracing::info!("initialized email tool context");
+
+ let call_service = CallServiceImpl::new(
+ PgCallRepo::new(db.clone()),
+ NoOpCallRtcClient,
+ NoOpConnectionService,
+ (*entity_access_service).clone(),
+ NoOpNotificationIngress,
+ None::<S3RecordingStorage>,
+ String::new(),
+ );
+ let call_tool_context = CallToolContext::new(call_service, (*entity_access_service).clone());
+
+ tracing::info!("initialized call tool context");
+
+ let chat_tool_context = chat::inbound::toolset::ChatToolContext::new(
+ chat::domain::service::ChatServiceImpl::new(
+ chat::outbound::postgres::PgChatRepo::new(db.clone(), owned_entity_registrar.clone()),
+ Arc::new(ai_toolset::AsyncToolCollection::new()),
+ (),
+ entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
+ entity_access_management::outbound::PgRepository::new(db.clone()),
+ ),
+ ),
+ (*entity_access_service).clone(),
+ );
+
+ tracing::info!("initialized chat tool context");
+
+ let channel_tool_context = ai_tools::build_channel_tool_context_with_side_effects(
+ db.clone(),
+ lexical_client.clone(),
+ &side_effect_clients,
+ );
+ let user_permissions_service = Arc::new(
+ roles_and_permissions::domain::service::UserRolesAndPermissionsServiceImpl::new(
+ roles_and_permissions::outbound::pgpool::MacroDB::new(db.clone()),
+ roles_and_permissions::outbound::pgpool::MacroDB::new(db.clone()),
+ ),
+ );
+
+ // The AI billing gate reads allowances, credits, and overage state here;
+ // collection (Stripe) lives in the authentication service, which the
+ // recorder below asks to settle once a payer runs past their allowance
+ // and ENABLE_AI_USAGE_BILLING is enabled. This instance never settles.
+ let auth_service_client = Arc::new(authentication_service_client::AuthServiceClient::new(
+ config
+ .authentication_service_secret_key
+ .as_ref()
+ .to_string(),
+ AuthServiceUrl::new()?.to_string(),
+ ));
+ let ai_pricing = config.ai_pricing();
+ let ai_billing = Arc::new(
+ ai_billing::domain::BillingServiceImpl::new(
+ ai_billing::outbound::RolesTeamsEntitlementSource::new(
+ (*user_permissions_service).clone(),
+ teams::outbound::team_repo::TeamRepositoryImpl::new(db.clone()),
+ ),
+ ai_billing::outbound::PgUsageReader::new(db.clone()),
+ ai_billing::outbound::PgBillingRepo::new(db.clone(), ai_pricing),
+ ai_billing::outbound::HttpPaymentGateway::new(auth_service_client.clone()),
+ ai_pricing,
+ )
+ .with_enforcement(config.enable_ai_usage_enforcement),
+ );
+ let admission: Arc<dyn ai_billing::AiAdmissionService> =
+ Arc::new(ai_billing::BillingAdmissionService::new(
+ ai_billing.clone(),
+ config.enable_ai_usage_enforcement,
+ ));
+ let recorder: Arc<dyn ai_usage::UsageRecorder> =
+ Arc::new(ai_billing::outbound::SettlingUsageRecorder::new(
+ Arc::new(
+ ai_usage::domain::service::UsageServiceImpl::new(
+ ai_usage::outbound::PgUsageRepo::new(db.clone()),
+ )
+ .with_enforcement(config.enable_ai_usage_enforcement),
+ ),
+ ai_billing.clone(),
+ ai_billing::outbound::HttpSettlementTrigger::new(auth_service_client),
+ config.enable_ai_usage_billing,
+ ));
+
+ // Per-attempt observations use a separate journal and never feed the
+ // legacy settlement trigger, which remains counted-usage-only and gated by
+ // ENABLE_AI_USAGE_BILLING.
+ let recorder = ai_usage::with_tracking(recorder, ai_usage::pg_tracking(db.clone()));
+
+ // The import pipeline: staged/imported external items, gather jobs over
+ // the user's connectors, and the Haiku import job. Built before the tool
+ // service context so the chat toolset gets a wired import context.
+ let mcp_encryption_key = mcp_client::domain::models::AesKey::try_from(
+ config.mcp_credentials_key_secret_name.as_ref(),
+ )
+ .context("invalid MCP credentials encryption key")?;
+ let mcp_server_repo =
+ mcp_client::outbound::pg_server_repo::PgServerRepo::new(db.clone(), mcp_encryption_key);
+
+ // The Pipedream MCP stack, fully separate from the native one above
+ // (own endpoints, own table, own toolset). Without credentials its
+ // endpoints answer 501 and its toolsets come up empty. Connect-flow
+ // webhooks need both URI and secret; a half-set pair is ignored so we
+ // never mint a callback we cannot serve.
+ let pipedream_client: ai_tools::ToolPipedreamConnection = match (
+ config.pipedream_client_id.value(),
+ config.pipedream_client_secret.value(),
+ config.pipedream_project_id.value(),
+ ) {
+ (Some(client_id), Some(client_secret), Some(project_id)) => {
+ let webhook_uri = match (
+ config.pipedream_webhook_uri.value(),
+ config.pipedream_webhook_secret.value(),
+ ) {
+ (Some(uri), Some(_)) => Some(uri.to_owned()),
+ (None, None) => None,
+ _ => {
+ tracing::warn!(
+ "ignoring incomplete Pipedream webhook config; set both PIPEDREAM_WEBHOOK_URI and PIPEDREAM_WEBHOOK_SECRET"
+ );
+ None
+ }
+ };
+ let client = pipedream_mcp::outbound::api::PipedreamClient::new(
+ pipedream_mcp::outbound::api::PipedreamConfig {
+ client_id: client_id.to_owned(),
+ client_secret: client_secret.to_owned(),
+ project_id: project_id.to_owned(),
+ environment: config
+ .pipedream_environment
+ .value()
+ .unwrap_or(match config.environment {
+ Environment::Production => "production",
+ _ => "development",
+ })
+ .to_owned(),
+ api_url: config
+ .pipedream_api_url
+ .value()
+ .unwrap_or(pipedream_mcp::outbound::api::DEFAULT_API_URL)
+ .to_owned(),
+ mcp_url: config
+ .pipedream_mcp_url
+ .value()
+ .unwrap_or(pipedream_mcp::outbound::api::DEFAULT_MCP_URL)
+ .to_owned(),
+ allowed_origins: match config.pipedream_allowed_origins.value() {
+ Some(origins) => origins
+ .split(',')
+ .map(|origin| origin.trim().to_owned())
+ .filter(|origin| !origin.is_empty())
+ .collect(),
+ None => match config.environment {
+ Environment::Production => vec!["https://macro.com".to_owned()],
+ Environment::Develop => vec![
+ "https://dev.macro.com".to_owned(),
+ "http://localhost:3000".to_owned(),
+ ],
+ Environment::Local => vec!["http://localhost:3000".to_owned()],
+ },
+ },
+ },
+ )
+ .context("failed to build Pipedream client")?;
+ Some(Arc::new(match webhook_uri {
+ Some(uri) => client.with_webhook_uri(uri),
+ None => client,
+ }))
+ }
+ _ => {
+ tracing::info!("Pipedream credentials not set; Pipedream MCP connectors disabled");
+ None
+ }
+ };
+ let pipedream_repo =
+ pipedream_mcp::outbound::pg_connection_repo::PgConnectionRepo::new(db.clone());
+
+ // The one sanctioned meeting point of the two MCP stacks: agents load
+ // tools through this selector, which prefers a user's Pipedream
+ // connectors and falls back to the native ones (see `mcp_select`).
+ // The import service's live Slack source shares this selector.
+ let mcp_selector: Arc<ai_tools::ToolMcpSelector> = Arc::new(mcp_select::McpToolSelector::new(
+ Arc::new(mcp_server_repo.clone()),
+ Arc::new(pipedream_repo.clone()),
+ Arc::new(pipedream_client.clone()),
+ ));
+
+ // Nudges the user's connected clients when import rows flip, so setup
+ // sections and chat surfaces update immediately instead of on the next
+ // poll (see import::outbound::gateway_notifier).
+ let import_notify = import::outbound::gateway_notifier::gateway_import_notify(
+ channels_connection_gateway.clone(),
+ );
+
+ let entity_creator = ai_tools::ToolEntityCreator {
+ document_creator: document_tool_context.creator.clone(),
+ entity_access_service: entity_access_service.clone(),
+ channel_service: channel_tool_context.service.clone(),
+ task_properties: task_properties_for_import,
+ document_properties: document_properties_for_import,
+ team_repository: ai_tools::build_team_repository(db.clone()),
+ };
+ let import_service = Arc::new(
+ import::domain::service::ImportServiceImpl::new(
+ import::outbound::pg_import_repo::PgImportRepo::new(db.clone()),
+ mcp_selector.clone(),
+ Arc::new(entity_creator),
+ recorder.clone(),
+ )
+ .with_slack_source(Arc::new(
+ import::outbound::mcp_slack_source::McpSlackSource::new(mcp_selector.clone()),
+ ))
+ .with_admission(admission.clone())
+ .with_notifier(import_notify),
+ );
+
+ // No boot-time recovery: running batches heartbeat their rows, and the
+ // read path reaps rows whose heartbeat stopped — safe with multiple
+ // replicas, where a boot-time sweep would clobber other instances' jobs.
+ tracing::info!("initialized import service");
+
+ let project_tool_context = ai_tools::build_project_tool_context(
+ db.clone(),
+ macro_event_broker.clone(),
+ entity_access_service.clone(),
+ document_tool_context.service.clone(),
+ chat_tool_context.service.clone(),
+ user_email_service.clone(),
+ );
+
+ let initiative_tool_context = ai_tools::build_initiative_tool_context(
+ db.clone(),
+ &document_tool_context,
+ properties_service.clone(),
+ entity_access_service.clone(),
+ macro_event_broker.clone(),
+ );
+
+ let databases_tool_context = ai_tools::build_databases_tool_context(
+ db.clone(),
+ entity_access_service.clone(),
+ ai_tools::ToolTableEventPublisher::Gateway(
+ databases::outbound::gateway_event_publisher::GatewayTableEventPublisher::new(
+ channels_connection_gateway.as_ref().clone(),
+ ),
+ ),
+ ai_tools::MaybeToolEventBroker::Real(macro_event_broker.clone()),
+ );
+ let databases_sql_tool_context = ai_tools::build_databases_sql_tool_context(
+ &databases_tool_context,
+ soup_service.clone(),
+ db.clone(),
+ );
+
+ let tool_service_context = ai_tools::ToolServiceContext {
+ search_service_client: search_service_client.clone(),
+ email_service_client: email_service_client_external.clone(),
+ soup_service: soup_service.clone(),
+ email_service: email_service_for_tools.clone(),
+ activity_tool_context: ai_tools::build_activity_tool_context(
+ db.clone(),
+ properties_service,
+ entity_access_service.clone(),
+ ),
+ image_generation_tool_context: ai_tools::build_image_generation_tool_context(
+ &document_tool_context,
+ ai_tools::build_image_generator_from_env(),
+ recorder.clone(),
+ )?,
+ document_tool_context: document_tool_context.clone(),
+ properties_tool_context: properties_tool_context.clone(),
+ email_tool_context: email_tool_context.clone(),
+ call_tool_context: call_tool_context.clone(),
+ booking_link_tool_context: ai_tools::build_booking_link_tool_context(
+ db.clone(),
+ config.environment,
+ ),
+ calendar_tool_context: ai_tools::build_calendar_tool_context(
+ db.clone(),
+ CalendarServiceUrl::new()?,
+ internal_api_key.clone(),
+ ),
+ notification_tool_context: notification_tool_context.clone(),
+ databases_tool_context,
+ databases_sql_tool_context,
+ import_tool_context: import::inbound::toolset::ImportToolContext::wired(
+ import_service.clone(),
+ ),
+ chat_tool_context,
+ channel_tool_context,
+ bot_tool_context: ai_tools::build_bot_tool_context(
+ db.clone(),
+ ai_tools::MaybeToolEventBroker::Real(macro_event_broker.clone()),
+ entity_access_service.clone(),
+ DocumentStorageServiceUrl::new()?.to_string(),
+ pipedream_client.clone(),
+ ),
+ coding_agent_tool_context: ai_tools::build_coding_agent_tool_context(
+ macro_service_urls::AgentHarnessServiceUrl::new()?,
+ internal_api_key.clone(),
+ )?,
+ project_tool_context,
+ initiative_tool_context,
+ team_tool_context: ai_tools::build_team_tool_context(db.clone()),
+ crm_tool_context: ai_tools::build_crm_tool_context(db.clone()),
+ skill_tool_context: ai_tools::build_skill_tool_context(
+ search_service_client.clone(),
+ soup_service.clone(),
+ &document_tool_context,
+ ),
+ schedule_tool_context: ai_tools::build_routine_tool_context()?,
+ anthropic_tool_context: ai_tools::build_anthropic_tool_context(),
+ admission,
+ recorder,
+ usage_context: ai_usage::UsageContext::system(ai_usage::AiFeature::Chat),
+ };
+ let all_tools = ai_tools::tools_for(ai_tools::AiHost::Chat);
+ let all_tools_toolset = all_tools.toolset.clone();
+ let all_tools_prompt: Arc<dyn std::fmt::Display + Send + Sync> =
+ Arc::new(all_tools.prompt.to_string());
+
+ // Build memory service
+ let memory_repo = memory::outbound::pg_memory_repo::PgMemoryRepo::new(db.clone());
+ let memory_service = Arc::new(memory::domain::service::MemoryServiceImpl::new(
+ memory_repo,
+ tool_service_context.clone(),
+ all_tools,
+ ));
+
+ tracing::info!("initialized memory service");
+
+ // Build the AI cost service. It backs both the admin query/pricing router
+ // and the usage recorder threaded through the tool service context.
+ let usage_service = Arc::new(
+ ai_usage::domain::service::UsageServiceImpl::new(ai_usage::outbound::PgUsageRepo::new(
+ db.clone(),
+ ))
+ .with_enforcement(config.enable_ai_usage_enforcement),
+ );
+
+ tracing::info!("initialized ai cost service");
+
+ // Generator that materializes projections by running their prompt through
+ // the shared AI toolset, attributed to the requesting user.
+ let projection_generator =
+ ai_projections::outbound::agent_generator::AgentProjectionGenerator::new(
+ tool_service_context.clone(),
+ ai_tools::tools_for(ai_tools::AiHost::Chat),
+ );
+ // Notifier that pushes finished materializations to the target's connected
+ // clients through the connection gateway.
+ let projection_notifier =
+ ai_projections::outbound::gateway_notifier::GatewayProjectionNotifier::new(Arc::new(
+ connection_gateway_client::ConnectionGatewayClient::new(
+ internal_api_key.clone(),
+ ConnectionGatewayUrl::new()?.to_string(),
+ ),
+ ));
+ let ai_projections_service_impl =
+ ai_projections::domain::ai_projection_service::AiProjectionServiceImpl::new(
+ ai_projections::outbound::ai_projection_repo::AiProjectionRepositoryImpl::new(
+ db.clone(),
+ ),
+ sqs_client.clone(),
+ projection_generator,
+ projection_notifier,
+ );
+
+ // Spawn the inbound worker that polls ai_projection_queue and materializes
+ // projection instances. It owns its own clone of the service.
+ let ai_projection_worker = ai_projections::worker::AiProjectionWorker::new(
+ sqs_worker::SQSWorker::new(
+ aws_sdk_sqs::Client::new(&aws_config),
+ ai_projection_queue.to_string(),
+ 10,
+ 10,
+ ),
+ ai_projections_service_impl.clone(),
+ );
+ let ai_projection_worker_task = tokio::spawn(async move {
+ ai_projection_worker.poll().await;
+ });
+
+ let ai_projections_service = Arc::new(ai_projections_service_impl);
+
+ tracing::info!("initialized ai projections service");
+
+ // The onboarding flow drives the import pipeline: reads/hooks start
+ // auto-importing gather runs for authenticated connectors, and
+ // completion deletes unreserved onboarding-staged candidates.
+ let onboarding_service = Arc::new(onboarding::domain::service::OnboardingServiceImpl::new(
+ onboarding::outbound::pg_onboarding_repo::PgOnboardingRepo::new(db.clone()),
+ Arc::new(mcp_server_repo.clone()),
+ import_service.clone(),
+ mcp_selector.clone(),
+ ));
+
+ tracing::info!("initialized onboarding service");
+
+ let mcp_public_url = &config.mcp_public_url;
+ let mcp_client_metadata = mcp_client::domain::models::OAuthClientMetadata::new(
+ format!("{mcp_public_url}/mcp/servers/auth/client-metadata"),
+ format!("{mcp_public_url}/mcp/servers/auth/callback"),
+ );
+ let mcp_oauth_state_store =
+ mcp_client::outbound::redis_state_store::RedisOAuthStateStore::new(redis_client.clone());
+ let mcp_pre_registered =
+ mcp_client::domain::provider_registry::PreRegisteredProviders::from_env()?;
+ let mcp_oauth = mcp_client::outbound::oauth::OAuthService::new(
+ mcp_server_repo.clone(),
+ mcp_oauth_state_store,
+ mcp_client_metadata.clone(),
+ mcp_pre_registered,
+ );
+ // The moment a connector finishes OAuth, reconcile onboarding for that
+ // user — gather jobs start before the user even returns to their
+ // original tab. Spawned so the callback response never waits on them.
+ let onboarding_for_auth_hook = onboarding_service.clone();
+ let mcp_auth_hook: mcp_client::inbound::axum_router::McpAuthCompletedHook =
+ Arc::new(move |record: mcp_client::domain::models::McpServerRecord| {
+ let service = onboarding_for_auth_hook.clone();
+ Box::pin(async move {
+ tokio::spawn(async move {
+ use onboarding::domain::service::OnboardingService;
+ if let Err(e) = service.reconcile(record.user_id).await {
+ tracing::warn!(error = ?e, "post-auth onboarding reconcile failed");
+ }
+ });
+ })
+ });
+ let mcp_state = mcp_client::inbound::McpRouterState::new(
+ mcp_server_repo,
+ mcp_oauth,
+ authorization_state.clone(),
+ mcp_client_metadata,
+ )
+ .with_auth_completed_hook(mcp_auth_hook);
+
+ // The Pipedream stack gets the same post-connect reconcile hook as the
+ // native one: a finished Connect flow starts gather jobs immediately.
+ let onboarding_for_pipedream_hook = onboarding_service.clone();
+ let pipedream_auth_hook: pipedream_mcp::inbound::axum_router::PipedreamAuthCompletedHook =
+ Arc::new(
+ move |connection: pipedream_mcp::domain::models::PipedreamConnection| {
+ let service = onboarding_for_pipedream_hook.clone();
+ Box::pin(async move {
+ tokio::spawn(async move {
+ use onboarding::domain::service::OnboardingService;
+ if let Err(e) = service.reconcile(connection.user_id).await {
+ tracing::warn!(error = ?e, "post-connect onboarding reconcile failed");
+ }
+ });
+ })
+ },
+ );
+ let pipedream_state = pipedream_mcp::inbound::PipedreamRouterState::new(
+ pipedream_repo,
+ pipedream_client,
+ authorization_state.clone(),
+ )
+ .with_auth_completed_hook(pipedream_auth_hook);
+
+ let api_result = api::setup_and_serve(ApiContext {
+ db: db.clone(),
+ email_service_client_external,
+ sqs_client: Arc::new(sqs_client),
+ document_storage_client: Arc::new(document_storage_client),
+ search_service_client,
+ authorization_state,
+ user_permissions_service,
+ non_user_owners,
+ ai_admission: tool_service_context.admission.clone(),
+ ai_billing,
+ internal_api_key: config.internal_api_key.clone(),
+ config: Arc::new(config),
+ notification_ingress_service,
+ connection_repo: connection_manager.persistence,
+ connection_gateway_client,
+ soup_service,
+ email_service: email_service_for_tools,
+ stream_repo,
+ document_tool_context,
+ memory_service,
+ usage_service,
+ ai_projections_service,
+ properties_tool_context,
+ email_tool_context: email_tool_context.clone(),
+ call_tool_context,
+ tool_service_context,
+ all_tools: all_tools_toolset,
+ all_tools_prompt,
+ entity_access_service,
+ message_service,
+ ai_stream_registry: service::ai_stream_registry::AiStreamRegistry::new(
+ redis_client.clone(),
+ ),
+ mcp_state,
+ pipedream_state,
+ mcp_selector,
+ import_service,
+ onboarding_service,
+ macro_event_broker: macro_event_broker.clone(),
+ })
+ .await
+ .context("failed to setup and serve api");
+
+ ai_projection_worker_task.abort();
+ match ai_projection_worker_task.await {
+ Err(error) if error.is_cancelled() => {
+ tracing::info!("ai projection worker stopped");
+ }
+ Err(error) => {
+ tracing::error!(error = ?error, "ai projection worker exited unexpectedly");
+ }
+ Ok(()) => {
+ tracing::error!(
+ error = "worker exited naturally",
+ "ai projection worker exited unexpectedly"
+ );
+ }
+ }
+
+ tracing::info!("waiting for event broker publishes to drain");
+ event_broker_tracker.close();
+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {
+ Ok(()) => tracing::info!("event broker publishes drained"),
+ Err(error) => tracing::warn!(
+ error=?error,
+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),
+ "timed out waiting for event broker publishes to drain"
+ ),
+ }
+
+ api_result
+}
+
+const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
diff --git a/services/document_storage_service/src/lib.rs b/services/document_storage_service/src/lib.rs
index 65880be0e3..24899104fd 100644
--- a/services/document_storage_service/src/lib.rs
+++ b/services/document_storage_service/src/lib.rs
@@ -1 +1,14 @@
+#![recursion_limit = "256"]
+//! Document storage service. The service binary and the OpenAPI generator both
+//! link this library, so the service compiles once and caches as one rlib.
+
+mod api;
+mod config;
pub mod model;
+mod outbound;
+mod server;
+mod service;
+
+/// The service's OpenAPI document, rendered by the `*_openapi` binary.
+pub use api::swagger::ApiDoc;
+pub use server::run;
diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rs
index bcd09e2c0d..f6ac13d731 100644
--- a/services/document_storage_service/src/main.rs
+++ b/services/document_storage_service/src/main.rs
@@ -1,2077 +1,13 @@
+// `run()`'s future type is deep enough to need the library's recursion limit
+// wherever it is awaited.
#![recursion_limit = "256"]
-use crate::{
- api::{
- MACRO_INTERNAL_USER_ID,
- context::{
- ApiContext, AuthorizationService, DocumentStorageServiceAuthKey, TaskPropertiesAdapter,
- },
- },
- config::{
- CalEventTypeContentNamesKey, CalWebhookSecretKey, MetaAccessToken, MetaPixelId,
- MetaTestEventCode,
- },
- service::s3::S3,
-};
-use agent_session_realtime::{
- domain::service::AgentSessionLogConsumerService, outbound::log_topic_consumer::LogTopicConsumer,
-};
-use analytics_client::{AnalyticsClient, AnalyticsClientConfig, MetaConfig};
-use anyhow::Context;
-use bots::{domain::service::BotServiceImpl, outbound::pg_bots_repo::PgBotsRepo};
-use cal::{
- domain::service::{CalConfig, CalEventMeta, CalWebhookServiceImpl},
- inbound::cal_webhook_router::CalWebhookRouterState,
- outbound::analytics_client::AnalyticsClientSink,
-};
-use calendar_events::domain::reminder_dispatch::CalendarReminderDispatchService;
-use calendar_events::inbound::axum_router::CalendarRouterState;
-use calendar_events::inbound::dispatch_worker::CalendarReminderDispatchWorker;
-use calendar_events::outbound::notification_notifier::NotificationCalendarReminderNotifier;
-use calendar_events::outbound::sqs_dispatch_queue::SqsCalendarDispatchQueue;
-use call::{
- domain::service::CallServiceImpl,
- inbound::axum_router::{CallRouterState, InternalCallRouterState, WebhookRouterState},
- outbound::{
- ai_call_summarizer::AiCallSummarizer,
- livekit_rtc_client::LivekitRtcClient,
- pg_call_repo::PgCallRepo,
- pg_voice_repo::PgVoiceRepo,
- s3_recording_storage::{RecordingCloudFrontConfig, S3RecordingStorage},
- },
-};
-use channel_labels::{
- domain::service::ChannelLabelsServiceImpl, inbound::axum_router::ChannelLabelsRouterState,
- outbound::pg_channel_labels_repo::PgChannelLabelsRepo,
-};
-use channels::{
- domain::{
- list_service::ChannelListServiceImpl,
- service::ChannelServiceImpl,
- side_effects::{ChannelSideEffectService, SpawnedChannelEventDispatcher},
- },
- inbound::{axum_router::ChannelsRouterState, list_router::ChannelListRouterState},
- outbound::{
- connection_gateway_realtime::ConnectionGatewayChannelRealtimePublisher,
- contacts_dispatcher::ContactsChannelDispatcher,
- notification_sender::NotificationChannelSender,
- pg_channel_reference_share_permissions::PgChannelReferenceSharePermissions,
- pg_channels_repo::PgChannelsRepo, pg_side_effect_context::PgChannelSideEffectContext,
- },
-};
-use collab_surface::inbound::axum_router::CollabSurfaceRouterState;
-use config::{Config, Environment};
-use connection::{
- domain::service::ConnectionServiceImpl,
- outbound::connection_gateway_client::ConnectionGatewayImpl,
-};
-use connection_gateway_client::client::ConnectionGatewayClient;
-use documents_hex::domain::models::CloudFrontConfig;
-use documents_hex::domain::service::DocumentServiceImpl;
-use documents_hex::inbound::axum_router::DocumentRouterState;
-use documents_hex::outbound::pg_document_repo::PgDocumentRepo;
-use documents_hex::outbound::s3_upload_url::S3UploadUrlAdapter;
-use dynamodb_client::DynamodbClient;
-use email::{
- domain::{ports::ReadonlyEmailPreviewAdapter, service::EmailServiceImpl},
- outbound::EmailPgRepo,
-};
-use embedding::embedding_provider::openai::TextEmbedding3Small;
-use entity_registry::OwnerGrantPolicy;
-use entity_registry_db_utils::OwnedEntityRegistrar;
-use favorites::{
- domain::{mutation_service::FavoritesMutationServiceImpl, service::FavoritesServiceImpl},
- inbound::axum_router::FavoritesRouterState,
- outbound::pg_favorites_repo::PgFavoritesRepo,
-};
-use foreign_entity::{
- domain::service::ForeignEntityServiceImpl, inbound::axum_router::ForeignEntityRouterState,
- outbound::pg_foreign_entity_repo::PgForeignEntityRepo,
-};
-use frecency::{domain::services::FrecencyQueryServiceImpl, outbound::postgres::FrecencyPgStorage};
-use github::domain::service::{
- GithubSyncConfig, GithubSyncServiceImpl, InstallationTokenConfig, InstallationTokenService,
- PullRequestIndexService,
-};
-use github::inbound::pull_request_index_router::PullRequestIndexRouterState;
-use github::outbound::connection_gateway_realtime::ConnectionGatewayGithubRealtime;
-use github::outbound::github_sync_client::GithubSyncClientImpl;
-use github::outbound::pg_github_sync_repo::PgGithubSyncRepo;
-use github::outbound::pull_request_diff::GithubPullRequestDiffClient;
-use github_pull_requests::{
- domain::service::{
- GithubPullRequestChangesServiceImpl, GithubPullRequestChangesetStore,
- GithubPullRequestServiceImpl,
- },
- inbound::{
- axum_router::GithubPullRequestRouterState,
- changes_router::GithubPullRequestChangesRouterState,
- },
- outbound::{
- pg_github_pull_request_repo::PgGithubPullRequestRepo,
- s3_patch_store::S3GithubPullRequestPatchStore,
- },
-};
-use graphql_scheduled_action::ScheduledActionGraphqlContext;
-use harnesses::outbound::pg_harness_repo::PgHarnessRepo;
-use initiative::{
- domain::service::InitiativeServiceImpl, inbound::axum_router::InitiativeRouterState,
- outbound::PgInitiativeRepo,
-};
-use lexical_client::LexicalClient;
-use macro_auth::middleware::decode_jwt::JwtValidationArgs;
-use macro_authorization::{
- InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationServiceImpl,
- MacroAuthorizationState, PgBotAuthorizationRepo, PgBotAuthorizer, PgHarnessAuthorizationRepo,
- PgHarnessAuthorizer, PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,
-};
-use macro_entrypoint::MacroEntrypoint;
-use macro_env_var::maybe_env_vars;
-use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
-#[cfg(feature = "delete_document_worker")]
-use macro_service_urls::AiEditingWorkerUrl;
-use macro_service_urls::{
- ConnectionGatewayUrl, LexicalServiceUrl, StaticFileServiceUrl, SyncServiceUrl,
-};
-use macro_sha_count_client::Redis;
-use notification::domain::service::{
- NotificationReaderService, PlatformArnConfig, SqsNotificationIngress,
- WebSocketNotificationConsumerService,
-};
-use notification::outbound::{notification_consumer::NotificationTopicConsumer, queue::SqsQueue};
-use opensearch_client::OpensearchClient;
-use projects_hex::{
- domain::service::ProjectServiceImpl,
- inbound::axum_router::ProjectRouterState,
- outbound::{
- DynamoBulkUploadAdapter, PgProjectRepo, S3ProjectUploadAdapter, ShaCountAdapter,
- SqsProjectSearchIndexer,
- },
-};
-use properties::{
- NotificationServiceImpl, PermissionServiceImpl, PropertiesPgRepo, PropertiesServiceImpl,
-};
-use rate_limit::{RateLimitServiceImpl, RedisRateLimitAdapter};
-use reminders::{
- domain::service::dispatch::ReminderDispatchService,
- inbound::{axum_router::RemindersRouterState, dispatch_worker::DispatchWorker},
- outbound::{
- notification_notifier::NotificationReminderNotifier, pg_reminders_repo::PgRemindersRepo,
- sqs_dispatch_queue::SqsDispatchQueue,
- },
-};
-use scheduled_action::domain::read_service::ScheduledActionReadServiceImpl;
-use scheduled_action::outbound::pg_scheduled_action_repo::PgScheduledActionRepo;
-use secretsmanager_client::SecretManager;
-use soup::{
- domain::service::SoupImpl, inbound::axum_router::SoupRouterState,
- outbound::pg_soup_repo::PgSoupRepo,
-};
-use soup_realtime::{
- domain::service::{SoupRealtimeConsumerService, SoupRealtimeServiceImpl},
- outbound::{
- entity_access::EntityAccessExpander, kafka_publisher::KafkaSoupRealtimePublisher,
- soup_consumer::SoupTopicConsumer,
- },
-};
-use sqlx::postgres::PgPoolOptions;
-use std::{sync::Arc, time::Duration};
-use sync_service_client::SyncServiceClient;
-use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};
-use task_dedup::{
- TaskDedupService,
- outbound::{
- cohere::CohereReranker,
- connection_gateway::ConnectionGatewayTaskDedupNotifier,
- judge::AgentDuplicateJudge,
- postgres::{PgTaskMatchRepo, PgTaskVectorDb},
- },
-};
-use tokio_util::{sync::CancellationToken, task::TaskTracker};
-use user_api_key::{
- domain::service::UserApiKeyServiceImpl, inbound::axum_router::UserApiKeyRouterState,
- outbound::pg_user_api_keys_repo::PgUserApiKeysRepo,
-};
-
-mod api;
-mod config;
-mod model;
-mod outbound;
-mod service;
-
-const SOUP_CONSUMER_RESTART_MAX_DELAY_SECS: u64 = 60;
-const SOUP_CONSUMER_RESTART_ALERT_THRESHOLD: u32 = 5;
-
-fn soup_consumer_restart_delay(consecutive_failures: u32) -> Duration {
- let exponent = consecutive_failures.saturating_sub(1).min(6);
- Duration::from_secs((1_u64 << exponent).min(SOUP_CONSUMER_RESTART_MAX_DELAY_SECS))
-}
-maybe_env_vars! {
- struct AppleBundleId;
- struct SnsApnsVoipPlatformArn;
-}
+use macro_entrypoint::MacroEntrypoint;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let entrypoint = MacroEntrypoint::default().init();
- let result = run().await;
+ let result = document_storage_service::run().await;
entrypoint.shutdown();
result
}
-
-async fn run() -> anyhow::Result<()> {
- let env = Environment::new_or_prod();
-
- let aws_config = macro_aws_config::get_macro_aws_config().await;
-
- let secretsmanager_client = secretsmanager_client::SecretsManager::new(
- aws_sdk_secretsmanager::Client::new(&aws_config),
- );
-
- // Parse our configuration from the environment, then resolve any secret-manager backed values.
- let config = Config::from_env()
- .context("expected to be able to generate config")?
- .resolve_remote_secrets(env, &secretsmanager_client)
- .await
- .context("expected to be able to resolve config secrets")?;
- let non_user_owners = config.non_user_owners()?;
-
- tracing::trace!("initialized config");
-
- let (min_connections, max_connections): (u32, u32) = match config.environment {
- Environment::Production => (50, 150),
- Environment::Develop => (25, 100),
- Environment::Local => (15, 50),
- };
-
- let db = PgPoolOptions::new()
- .min_connections(min_connections)
- .max_connections(max_connections)
- .connect(&config.database_url)
- .await
- .context("could not connect to db")?;
-
- tracing::trace!(
- min_connections,
- max_connections,
- "initialized db connection"
- );
-
- let readonly_db = match PgPoolOptions::new()
- .min_connections(min_connections)
- .max_connections(max_connections)
- .connect(&config.database_url_readonly)
- .await
- {
- Ok(pool) => {
- tracing::trace!("initialized readonly db connection");
- pool
- }
- Err(e) => {
- tracing::warn!(error=?e, "failed to connect to readonly db, falling back to primary");
- db.clone()
- }
- };
-
- let dynamo_db = aws_sdk_dynamodb::Client::new(&aws_config);
-
- let dynamodb_client = DynamodbClient::new_from_client(
- dynamo_db.clone(),
- Some(config.bulk_upload_requests_table.as_ref().to_string()),
- );
- tracing::trace!("initialized dynamodb client");
-
- let s3_client = macro_aws_config::s3_client().await;
-
- tracing::trace!("initialized s3 client");
-
- let search_event_queue = macro_queues::SearchEventQueue::new();
- let document_delete_queue = macro_queues::DocumentDeleteQueue::new();
- let contacts_queue = macro_queues::ContactsQueue::new();
- let notification_queue = macro_queues::NotificationIngressQueue::new();
- let gmail_ops_queue = macro_queues::GmailOpsQueue::new();
- let reminder_dispatch_queue = macro_queues::ReminderDispatchQueue::new();
- let calendar_reminder_dispatch_queue = macro_queues::CalendarReminderDispatchQueue::new();
- let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(&aws_config))
- .search_event_queue(&search_event_queue)
- .document_delete_queue(&document_delete_queue)
- .gmail_ops_queue(&gmail_ops_queue);
- let webhook_event_queue = webhook::outbound::SqsWebhookQueue::new(
- Arc::new(sqs_client.clone()),
- macro_queues::WebhookEventQueue::new().to_string(),
- config.webhook_queue_max_messages,
- config.webhook_queue_wait_time_seconds,
- );
-
- let contacts_ingress = Arc::new(contacts::domain::service::SqsContactsIngress {
- queue: contacts::outbound::ingress::SqsContactsQueue::new(
- aws_sdk_sqs::Client::new(&aws_config),
- contacts_queue.to_string(),
- ),
- });
-
- tracing::trace!("initialized sqs client");
-
- // Redis handles it own connection pool internally. Each time we use redis
- // we should be using redis_client.get_connection() to grab a specific connection
- let redis_client = redis::Client::open(config.redis_uri.to_string().as_ref())
- .expect("could not connect to redis client");
-
- match redis_client.get_connection().is_err() {
- true => {
- tracing::error!("unable to connect to redis");
- }
- false => {
- tracing::trace!("initialized redis connection");
- }
- }
-
- let dss_auth_key = DocumentStorageServiceAuthKey::new()?;
-
- let conn_gateway_client = ConnectionGatewayClient::new(
- config.internal_api_key.to_string(),
- ConnectionGatewayUrl::new()?.to_string(),
- );
-
- let sync_service_client = Arc::new(SyncServiceClient::new(
- config.sync_service_auth_key.as_ref().to_string(),
- SyncServiceUrl::new()?.to_string(),
- ));
-
- let lexical_client = Arc::new(LexicalClient::new(
- config.internal_api_key.to_string(),
- LexicalServiceUrl::new()?.to_string(),
- ));
-
- let jwt_validation_args =
- JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)
- .await?;
-
- let consumer_cancellation_token = CancellationToken::new();
- let consumer_tracker = TaskTracker::new();
- let event_broker_tracker = TaskTracker::new();
- let macro_event_broker = MacroEventBrokerService::new(
- KafkaEventPublisher::new(config.kafka_brokers.as_ref())
- .context("failed to create kafka event publisher")?,
- event_broker_tracker.clone(),
- );
- let bots_repo = PgBotsRepo::new(db.clone());
- // The agent API checks a newly selected MCP app against Pipedream's
- // directory, the same check the ConfigureAgent tool makes.
- let bots_service = BotServiceImpl::new(bots_repo, macro_event_broker.clone()).with_mcp_apps(
- ai_tools::PipedreamMcpAppCatalog::new(ai_tools::pipedream_client_from_env()?),
- );
-
- let authorization_service: AuthorizationService = MacroAuthorizationServiceImpl::new(
- MacroAuthJwtValidator::new(jwt_validation_args.clone()),
- InternalAuthConfig {
- api_key: dss_auth_key.as_ref().to_string(),
- default_user_id: Some(MACRO_INTERNAL_USER_ID.to_string()),
- },
- PgBotAuthorizer::new(PgBotAuthorizationRepo::new(db.clone())),
- PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),
- )
- .with_harness_authorizer(PgHarnessAuthorizer::new(PgHarnessAuthorizationRepo::new(
- db.clone(),
- )));
- let authorization_state = MacroAuthorizationState::new(Arc::new(authorization_service));
- let harnesses_service =
- harnesses::domain::service::HarnessServiceImpl::new(PgHarnessRepo::new(db.clone()));
-
- // Initialize OpenSearch client
- let opensearch_client = OpensearchClient::new(
- config.opensearch_url.to_string(),
- config.opensearch_username.to_string(),
- config.opensearch_password.to_string(),
- )
- .context("unable to create opensearch client")?;
-
- if let Err(e) = opensearch_client.health().await {
- tracing::error!(error=?e, "error connecting to opensearch");
- return Err(e);
- }
- tracing::trace!("initialized opensearch client");
-
- let frecency_storage = FrecencyPgStorage::new(db.clone());
- let frecency_service = FrecencyQueryServiceImpl::new(frecency_storage.clone());
- // DSS only reads CRM rows — no-op resolver keeps reqwest/scraper
- // out of this binary.
- let crm_service = crm::domain::service::CrmServiceImpl::new(
- crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),
- crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,
- );
- let email_service = EmailServiceImpl::new(
- EmailPgRepo::new(db.clone()),
- frecency_service.clone(),
- sqs_client.clone(),
- crm_service.clone(),
- entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
- entity_access_management::outbound::PgRepository::new(db.clone()),
- ),
- 0,
- )
- .with_macro_event_broker(macro_event_broker.clone());
- let readonly_email_service = ReadonlyEmailPreviewAdapter(
- EmailServiceImpl::new(
- EmailPgRepo::new(readonly_db.clone()),
- frecency_service.clone(),
- sqs_client.clone(),
- crm_service.clone(),
- entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
- entity_access_management::outbound::PgRepository::new(readonly_db.clone()),
- ),
- 0,
- )
- .with_macro_event_broker(macro_event_broker.clone()),
- );
- let system_properties_service =
- SystemPropertiesServiceImpl::new(PgSystemPropertiesRepository::new(db.clone()));
- let ingress_queue = SqsQueue::new(
- aws_sdk_sqs::Client::new(&aws_config),
- notification_queue.to_string(),
- );
- let notification_ingress_service = Arc::new(SqsNotificationIngress {
- queue: ingress_queue.clone(),
- });
- let graphql_notification_reader: Arc<ai_tools::ToolNotificationService> =
- Arc::new(NotificationReaderService {
- repository: notification::outbound::repository::DbNotificationRepository::new(
- db.clone(),
- ),
- queue: ai_tools::ToolNotificationQueue::Sqs(ingress_queue.clone()),
- sns_endpoint: ai_tools::NoOpSnsEndpointManager,
- platform_config: PlatformArnConfig {
- apns_platform_arn: String::new(),
- fcm_platform_arn: String::new(),
- apns_voip_platform_arn: String::new(),
- },
- realtime: notification::domain::ports::NoopNotificationRealtimePublisher,
- });
- tracing::trace!("initialized notification ingress service");
-
- let entity_access_service = Arc::new(
- entity_access::domain::service::EntityAccessServiceImpl::new(
- entity_access::outbound::PgAccessRepository::new(db.clone()),
- ),
- );
-
- let permission_checker = PermissionServiceImpl::new(db.clone(), entity_access_service.clone());
- let notification_service = NotificationServiceImpl::new(
- SqsNotificationIngress {
- queue: ingress_queue,
- },
- db.clone(),
- );
- let properties_service = Arc::new(
- PropertiesServiceImpl::new(
- PropertiesPgRepo::new(db.clone()),
- Some(permission_checker),
- Some(notification_service),
- )
- .with_event_broker(macro_event_broker.clone())
- .with_initiative_assignees(Arc::new(
- initiative::domain::assignees::InitiativeAssignees::new(PgInitiativeRepo::new(
- db.clone(),
- )),
- )),
- );
-
- // Create the channel list service used by soup.
- // Create the legacy channel list router state for routes mounted under /comms.
- let channel_list_state = ChannelListRouterState::new(
- ChannelListServiceImpl::new(
- PgChannelsRepo::new(db.clone()),
- PgChannelsRepo::new(db.clone()),
- frecency_storage.clone(),
- ),
- authorization_state.clone(),
- );
-
- let s3 = Arc::new(S3::new(
- s3_client,
- config.document_storage_bucket.as_ref(),
- config.docx_document_upload_bucket.as_ref(),
- config.upload_staging_bucket.as_ref(),
- ));
- let system_properties_service = Arc::new(system_properties_service);
-
- let owned_entity_registrar =
- OwnedEntityRegistrar::new(OwnerGrantPolicy::new(PgBotsRepo::new(db.clone())));
- let document_repo = PgDocumentRepo::new(db.clone(), owned_entity_registrar.clone());
- let cloudfront_config = CloudFrontConfig {
- distribution_url: config
- .document_storage_service_cloudfront_distribution_url
- .as_ref()
- .to_string(),
- signer_public_key_id: config
- .document_storage_service_cloudfront_signer_public_key_id
- .as_ref()
- .to_string(),
- signer_private_key: config
- .document_storage_service_cloudfront_signer_private_key
- .as_ref()
- .to_string(),
- presigned_url_expiry_seconds: config.document_storage_service_presigned_url_expiry_seconds,
- browser_cache_expiry_seconds: config
- .document_storage_service_presigned_url_browser_cache_expiry_seconds,
- };
- let s3_upload_adapter = S3UploadUrlAdapter::new(
- macro_aws_config::s3_client().await,
- config.document_storage_bucket.as_ref(),
- config.docx_document_upload_bucket.as_ref(),
- );
- let markdown_initializer =
- documents_hex::outbound::markdown_init::LexicalSyncMarkdownInitializer::new(
- lexical_client.as_ref().clone(),
- sync_service_client.as_ref().clone(),
- );
-
- let connection_gateway = Arc::new(ConnectionGatewayImpl::new(conn_gateway_client.clone()));
-
- let connection_service =
- ConnectionServiceImpl::new(entity_access_service.clone(), connection_gateway.clone());
-
- let entity_access_management_service =
- entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
- entity_access_management::outbound::PgRepository::new(db.clone()),
- );
-
- let chat_mutation_service =
- Arc::new(chat::domain::service::ChatServiceImpl::new_without_tools(
- chat::outbound::postgres::PgChatRepo::new(db.clone(), owned_entity_registrar.clone()),
- entity_access_management_service.clone(),
- ));
-
- let project_service = Arc::new(ProjectServiceImpl::new(
- PgProjectRepo::new(db.clone(), owned_entity_registrar.clone()),
- S3ProjectUploadAdapter::new(
- macro_aws_config::s3_client().await,
- config.document_storage_bucket.as_ref(),
- config.docx_document_upload_bucket.as_ref(),
- config.upload_staging_bucket.as_ref(),
- ),
- DynamoBulkUploadAdapter::new(dynamodb_client.clone()),
- ShaCountAdapter::new(Redis::new(redis_client.clone())),
- entity_access_management_service.clone(),
- SqsProjectSearchIndexer::new(Arc::new(sqs_client.clone()), macro_event_broker.clone()),
- if cfg!(feature = "local") {
- Some(uuid::uuid!("d50676e2-0a12-4c62-bc07-4b1cb6d8e9bc"))
- } else {
- None
- },
- macro_event_broker.clone(),
- ));
-
- let document_service = Arc::new(DocumentServiceImpl::new(
- document_repo,
- cloudfront_config,
- sync_service_client.as_ref().clone(),
- s3_upload_adapter,
- TaskPropertiesAdapter {
- system_properties: system_properties_service.clone(),
- properties: properties_service.clone(),
- entity_access_service: entity_access_service.clone(),
- },
- connection_service,
- entity_access_management_service.clone(),
- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
- macro_event_broker.clone(),
- ));
-
- let foreign_entity_service = Arc::new(ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(
- db.clone(),
- )));
-
- let github_sync_service_impl = GithubSyncServiceImpl::new(
- GithubSyncConfig {
- webhook_secret: config.github_webhook_secret_key.as_ref().to_string(),
- github_sync_app_url: config.github_sync_app_url.to_string(),
- sync_app_pem: config.github_sync_app_pem_secret_key.as_ref().to_string(),
- sync_app_client_id: config.github_sync_app_client_id.to_string(),
- sync_app_client_secret: config.github_sync_app_client_secret.to_string(),
- installation_state_secret: config.github_installation_state_secret.to_string(),
- },
- document_service.clone(),
- Arc::new(GithubPullRequestServiceImpl::new(
- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
- PgGithubPullRequestRepo::new(db.clone()),
- )),
- (*notification_ingress_service).clone(),
- PgGithubSyncRepo::new(db.clone()),
- GithubSyncClientImpl::default(),
- ConnectionGatewayGithubRealtime::new(conn_gateway_client.clone()),
- );
-
- let github_pull_request_index_state = PullRequestIndexRouterState {
- service: Arc::new(PullRequestIndexService::new(
- InstallationTokenConfig {
- client_id: config.github_sync_app_client_id.to_string(),
- private_key_pem: config.github_sync_app_pem_secret_key.as_ref().to_string(),
- },
- PgGithubSyncRepo::new(db.clone()),
- GithubSyncClientImpl::default(),
- GithubPullRequestServiceImpl::new(
- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
- PgGithubPullRequestRepo::new(db.clone()),
- ),
- )),
- authorization_state: authorization_state.clone(),
- };
-
- let github_pull_request_state = GithubPullRequestRouterState::new(
- Arc::new(GithubPullRequestServiceImpl::new(
- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
- PgGithubPullRequestRepo::new(db.clone()),
- )),
- entity_access_service.clone(),
- authorization_state.clone(),
- );
-
- let github_pull_request_changes_state = GithubPullRequestChangesRouterState::new(
- Arc::new(GithubPullRequestChangesServiceImpl::new(
- GithubPullRequestServiceImpl::new(
- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
- PgGithubPullRequestRepo::new(db.clone()),
- ),
- GithubPullRequestChangesetStore::new(
- GithubPullRequestDiffClient::new(InstallationTokenService::new(
- InstallationTokenConfig {
- client_id: config.github_sync_app_client_id.to_string(),
- private_key_pem: config.github_sync_app_pem_secret_key.as_ref().to_string(),
- },
- PgGithubSyncRepo::new(db.clone()),
- GithubSyncClientImpl::default(),
- )),
- PgGithubPullRequestRepo::new(db.clone()),
- S3GithubPullRequestPatchStore::new(
- macro_aws_config::s3_client().await,
- config.github_pull_request_patch_bucket.to_string(),
- ),
- ),
- )),
- entity_access_service.clone(),
- authorization_state.clone(),
- );
-
- let foreign_entity_state = ForeignEntityRouterState::new(
- foreign_entity_service.clone(),
- entity_access_service.clone(),
- authorization_state.clone(),
- );
-
- // Cal.com webhooks → Meta Lead events. Both secrets are loaded here
- // (rather than on Config) to keep cal/Meta wiring colocated.
- let cal_webhook_secret = secretsmanager_client
- .get_maybe_secret_value(env, CalWebhookSecretKey::new()?)
- .await?;
- let cal_event_type_content_names_secret = secretsmanager_client
- .get_maybe_secret_value(env, CalEventTypeContentNamesKey::new()?)
- .await?;
-
- let analytics_client = Arc::new(AnalyticsClient::new(AnalyticsClientConfig {
- google_analytics: None,
- meta: Some(MetaConfig {
- pixel_id: MetaPixelId::new()?.as_ref().to_string(),
- access_token: MetaAccessToken::new()?.as_ref().to_string(),
- test_event_code: MetaTestEventCode::new().map(|v| v.as_ref().to_string()),
- }),
- posthog: None,
- }));
-
- let cal_event_type_meta: std::collections::HashMap<u64, CalEventMeta> =
- serde_json::from_str(cal_event_type_content_names_secret.as_ref())
- .context("CalEventTypeContentNames secret must be a JSON object mapping eventTypeId (u64) to { content_name: string, value: number (USD) }")?;
- let cal_webhook_service = CalWebhookServiceImpl::new(
- CalConfig {
- webhook_secret: cal_webhook_secret.as_ref().to_string(),
- event_type_meta: cal_event_type_meta,
- },
- AnalyticsClientSink::new(analytics_client.clone()),
- );
- let cal_webhook_state = CalWebhookRouterState::new(cal_webhook_service);
-
- // Call service (LiveKit)
- let transcription_agent_name = config
- .livekit_transcription_agent_name
- .value()
- .map(|v| v.to_owned());
-
- let internal_call_secret = config.internal_call_secret.value().map(|v| v.to_owned());
-
- anyhow::ensure!(
- transcription_agent_name.is_none() || internal_call_secret.is_some(),
- "LIVEKIT_TRANSCRIPTION_AGENT_NAME is set but INTERNAL_CALL_SECRET is missing — \
- the transcription agent will not be able to submit transcripts"
- );
- let livekit_rtc_client = LivekitRtcClient::new(
- config.livekit_server_url.as_ref(),
- config.livekit_api_key.as_ref(),
- config.livekit_api_secret.as_ref(),
- transcription_agent_name,
- );
- let call_connection_service =
- ConnectionServiceImpl::new(entity_access_service.clone(), connection_gateway.clone());
- let call_repo = PgCallRepo::new(db.clone());
- let egress_config = match (
- config::CallRecordingS3Bucket::new(),
- config::CallRecordingS3Region::new(),
- config::CallRecordingS3AccessKey::new(),
- config::CallRecordingS3Secret::new(),
- ) {
- (Some(bucket), Some(region), Some(access_key), Some(secret)) => {
- tracing::info!(bucket = bucket.as_ref(), "call recording enabled");
- Some(call::domain::models::EgressS3Config {
- bucket: bucket.as_ref().to_string(),
- region: region.as_ref().to_string(),
- access_key: access_key.as_ref().to_string(),
- secret: secret.as_ref().to_string(),
- })
- }
- _ => None,
- };
- let recording_storage = match &egress_config {
- Some(egress_config) => {
- let cloudfront_config = RecordingCloudFrontConfig {
- distribution_url: config
- .document_storage_service_cloudfront_distribution_url
- .as_ref()
- .to_string(),
- signer_public_key_id: config
- .document_storage_service_cloudfront_signer_public_key_id
- .as_ref()
- .to_string(),
- signer_private_key: config
- .document_storage_service_cloudfront_signer_private_key
- .as_ref()
- .to_string(),
- presigned_url_expiry_seconds: config::CALL_RECORDING_PRESIGNED_URL_EXPIRY_SECONDS,
- };
- Some(S3RecordingStorage::new(egress_config.bucket.clone(), cloudfront_config).await)
- }
- None => None,
- };
- let mut call_service_builder = CallServiceImpl::<_, _, _, _, _, _, AiCallSummarizer>::new(
- call_repo,
- livekit_rtc_client,
- call_connection_service,
- (*entity_access_service).clone(),
- (*notification_ingress_service).clone(),
- recording_storage,
- config.livekit_server_url.as_ref(),
- )
- .with_summarizer(AiCallSummarizer::new(
- ai_usage::pg_recorder_with_enforcement(db.clone(), config.enable_ai_usage_enforcement),
- ));
- if let Some(secret) = internal_call_secret {
- call_service_builder = call_service_builder.with_internal_call_secret(secret);
- }
- if let Some(config) = egress_config {
- call_service_builder = call_service_builder.with_egress(config);
- }
- if let Some(base_url) = config::CallRingStatusBaseUrl::new() {
- call_service_builder =
- call_service_builder.with_ring_status_base_url(base_url.as_ref().to_owned());
- }
-
- // VoIP push is optional: enabled only when both env vars are present.
- // APPLE_BUNDLE_ID: the app bundle ID (e.g. "com.macro.app.prod")
- // SNS_APNS_VOIP_PLATFORM_ARN: SNS platform app ARN for APNS_VOIP
- //
- // Option<VoipPushServiceImpl<...>> is used as the type parameter so the
- // type stays stable regardless of whether VoIP push is configured.
- let voip_sender = if let (Some(bundle_id), Some(voip_arn)) =
- (AppleBundleId::new(), SnsApnsVoipPlatformArn::new())
- {
- if voip_arn.is_empty() {
- tracing::warn!("voip push disabled: SNS_APNS_VOIP_PLATFORM_ARN is set but empty");
- None
- } else {
- let voip_repo =
- notification::outbound::repository::DbNotificationRepository::new(db.clone());
- let voip_mobile = notification::outbound::mobile::MobilePushAdapter {
- push_service: aws_sdk_sns::Client::new(&aws_config),
- apns_bundle_id: bundle_id.to_string(),
- voip_bundle_id: Some(format!("{}.voip", bundle_id.as_ref())),
- };
- tracing::info!(
- bundle_id = bundle_id.as_ref(),
- voip_arn = voip_arn.as_ref(),
- "voip push enabled"
- );
- Some(notification::domain::service::VoipPushServiceImpl::new(
- voip_repo,
- voip_mobile,
- ))
- }
- } else {
- tracing::info!("voip push disabled: APPLE_BUNDLE_ID or SNS_APNS_VOIP_PLATFORM_ARN not set");
- None
- };
- let call_service_builder = call_service_builder.with_voip_push_sender(voip_sender);
-
- let call_service = Arc::new(
- call_service_builder
- .with_voice_repo(PgVoiceRepo::new(db.clone()))
- .with_event_broker(macro_event_broker.clone()),
- );
-
- consumer_tracker.spawn({
- let service = call_service.clone();
- let cancellation_token = consumer_cancellation_token.clone();
- async move {
- cancellation_token
- .run_until_cancelled(call::inbound::stale_call_sweeper::run_stale_call_sweeper(
- service,
- call::inbound::stale_call_sweeper::SWEEP_INTERVAL,
- ))
- .await;
- }
- });
-
- let call_state = CallRouterState::new(
- call_service.clone(),
- entity_access_service.clone(),
- authorization_state.clone(),
- );
- let call_webhook_state = WebhookRouterState::new(call_service.clone());
- let call_public_rate_limiter = RateLimitServiceImpl {
- repo: RedisRateLimitAdapter {
- redis: redis_client.clone(),
- },
- };
-
- let webhook_repository = webhook::outbound::PgRepository::new(db.clone());
- let webhook_endpoint_scheme_policy = if matches!(env, Environment::Local) {
- webhook::domain::models::WebhookEndpointSchemePolicy::HttpAndHttps
- } else {
- webhook::domain::models::WebhookEndpointSchemePolicy::HttpsOnly
- };
- let webhook_http_client =
- webhook::outbound::ReqwestWebhookDeliveryClient::new_with_endpoint_scheme_policy(
- webhook_endpoint_scheme_policy,
- )
- .context("failed to create webhook HTTP client")?;
- let webhook_service =
- webhook::domain::service::WebhookServiceImpl::new_with_endpoint_scheme_policy(
- webhook_repository.clone(),
- webhook_http_client.clone(),
- webhook_endpoint_scheme_policy,
- macro_event_broker.clone(),
- );
- let webhook_rate_limiter = RateLimitServiceImpl {
- repo: RedisRateLimitAdapter {
- redis: redis_client.clone(),
- },
- };
- let webhook_state = webhook::inbound::axum_router::WebhookRouterState::new(
- webhook_service,
- webhook_rate_limiter,
- authorization_state.clone(),
- );
-
- let (webhook_stream_sender, _) =
- tokio::sync::broadcast::channel(webhook::domain::stream::WEBHOOK_STREAM_CHANNEL_CAPACITY);
- let sse_stream_service = webhook::domain::stream::WebhookEventStreamServiceImpl::new(
- webhook_stream_sender.clone(),
- entity_access_service.clone(),
- webhook_repository.clone(),
- );
- let sse_stream_state = webhook::inbound::stream_router::WebhookStreamRouterState::new(
- sse_stream_service,
- authorization_state.clone(),
- );
- consumer_tracker.spawn({
- let brokers = config.kafka_brokers.as_ref().to_string();
- let sender = webhook_stream_sender;
- let cancellation_token = consumer_cancellation_token.clone();
- async move {
- loop {
- let result = tokio::select! {
- biased;
- _ = cancellation_token.cancelled() => break,
- result = webhook::inbound::kafka_stream_consumer::run_webhook_stream_consumer(
- &brokers,
- &sender,
- ) => result,
- };
-
- if cancellation_token.is_cancelled() {
- break;
- }
- if let Err(error) = result {
- tracing::error!(
- error = ?error,
- "webhook SSE Kafka consumer stopped"
- );
- tokio::select! {
- biased;
- _ = cancellation_token.cancelled() => break,
- _ = tokio::time::sleep(Duration::from_secs(1)) => {}
- }
- }
- }
- }
- });
-
- let webhook_ingestion_service =
- webhook::domain::ingestion::WebhookEventIngestionServiceImpl::new(
- entity_access_service.clone(),
- webhook_repository,
- webhook_event_queue.clone(),
- );
- let webhook_delivery_repository =
- webhook::outbound::PgWebhookDeliveryRepository::new(db.clone());
- let webhook_delivery_service = webhook::domain::delivery::WebhookEventDeliveryServiceImpl::new(
- webhook_delivery_repository,
- webhook_http_client,
- );
- let webhook_worker = webhook::inbound::worker::WebhookEventWorker::new(
- webhook_event_queue,
- webhook_delivery_service,
- );
- let webhook_worker_task = tokio::spawn(async move {
- tracing::info!("starting webhook event worker");
- webhook_worker.run().await;
- });
- tokio::spawn(async move {
- match webhook_worker_task.await {
- Ok(()) => tracing::error!("webhook event worker exited unexpectedly"),
- Err(error) => {
- tracing::error!(error = ?error, "webhook event worker exited unexpectedly");
- }
- }
- });
-
- let webhook_consumer_brokers = config.kafka_brokers.as_ref().to_string();
- consumer_tracker.spawn({
- let cancellation_token = consumer_cancellation_token.clone();
- async move {
- loop {
- if cancellation_token.is_cancelled() {
- break;
- }
-
- tracing::info!("starting webhook event consumer");
- let result = webhook::inbound::kafka_consumer::run_webhook_event_consumer(
- &webhook_consumer_brokers,
- webhook_ingestion_service.clone(),
- cancellation_token.cancelled(),
- )
- .await;
-
- if cancellation_token.is_cancelled() {
- break;
- }
-
- match result {
- Ok(()) => tracing::error!("webhook event consumer exited unexpectedly"),
- Err(error) => {
- tracing::error!(error = ?error, "webhook event consumer exited unexpectedly");
- }
- }
-
- tokio::select! {
- biased;
- _ = cancellation_token.cancelled() => break,
- _ = tokio::time::sleep(Duration::from_secs(5)) => {}
- }
- }
- }
- });
-
- let activity_consumer_brokers = config.kafka_brokers.as_ref().to_string();
- let editing_activity = Arc::new(
- documents_hex::outbound::editing_activity::RedisEditingActivityStore::new(
- redis_client.clone(),
- ),
- );
- consumer_tracker.spawn({
- let cancellation_token = consumer_cancellation_token.clone();
- let activity_repo = activity::outbound::pg_activity_repo::PgActivityRepo::new(db.clone());
- let activity_realtime = activity::domain::announcements::ActivityAnnouncements::new(
- activity::KafkaActivityRealtimePublisher::new(macro_event_broker.clone()),
- crate::service::activity::EntityAccessActivityAudience::new(
- entity_access_service.clone(),
- ),
- );
- async move {
- let consumer = activity::inbound::kafka_consumer::ActivityConsumer::<
- _,
- crate::service::activity::ActivitySourceEvent,
- _,
- _,
- >::new(
- activity_repo,
- move |event| {
- let editing_activity = editing_activity.clone();
- async move {
- crate::service::activity::ingest(event, editing_activity.as_ref()).await
- }
- },
- activity_realtime,
- );
- loop {
- if cancellation_token.is_cancelled() {
- break;
- }
-
- tracing::info!("starting activity consumer");
- let result = consumer
- .run(&activity_consumer_brokers, cancellation_token.cancelled())
- .await;
-
- if cancellation_token.is_cancelled() {
- break;
- }
-
- match result {
- Ok(()) => tracing::error!("activity consumer exited unexpectedly"),
- Err(error) => {
- tracing::error!(error = ?error, "activity consumer exited unexpectedly");
- }
- }
-
- tokio::select! {
- biased;
- _ = cancellation_token.cancelled() => break,
- _ = tokio::time::sleep(Duration::from_secs(5)) => {}
- }
- }
- }
- });
-
- // Links archived calls to the CRM records of the people on them.
- let crm_call_link_brokers = config.kafka_brokers.as_ref().to_string();
- consumer_tracker.spawn({
- let cancellation_token = consumer_cancellation_token.clone();
- let consumer = crm::inbound::call_archived::CallArchivedConsumer::new(
- crm::domain::call_links::CallRecordLinker::new(
- call::domain::service::CallRecordQueryServiceImpl::new(PgCallRepo::new(
- db.clone(),
- )),
- crm::outbound::call_link::PgCallCrmLinker::new(
- db.clone(),
- (*system_properties_service).clone(),
- ),
- ),
- );
- async move {
- loop {
- if cancellation_token.is_cancelled() {
- break;
- }
-
- tracing::info!("starting CRM call-link consumer");
- let result = consumer
- .run(&crm_call_link_brokers, cancellation_token.cancelled())
- .await;
-
- if cancellation_token.is_cancelled() {
- break;
- }
-
- match result {
- Ok(()) => tracing::error!("CRM call-link consumer exited unexpectedly"),
- Err(error) => {
- tracing::error!(error = ?error, "CRM call-link consumer exited unexpectedly");
- }
- }
-
- tokio::select! {
- biased;
- _ = cancellation_token.cancelled() => break,
- _ = tokio::time::sleep(Duration::from_secs(5)) => {}
- }
- }
- }
- });
-
- let call_internal_state = InternalCallRouterState::new(call_service.clone());
-
- // Create the SQS worker for delete document processing before config is moved.
- #[cfg(feature = "delete_document_worker")]
- let delete_document_worker = sqs_worker::SQSWorker::new(
- aws_sdk_sqs::Client::new(&aws_config),
- document_delete_queue.to_string(),
- config.queue_max_messages,
- config.queue_wait_time_seconds,
- );
-
- let sqs_client = Arc::new(sqs_client);
- let conn_gateway_client = Arc::new(conn_gateway_client);
-
- // MacroConfig reads the shared OPENAI_API_KEY from Doppler's APP_SECRETS_JSON
- // or the local environment. Validate once before constructing consumers.
- let openai_api_key = config.openai_api_key.as_ref().to_owned();
- anyhow::ensure!(
- !openai_api_key.trim().is_empty(),
- "OpenAI API key is required for task dedup embeddings and dictation",
- );
- let cohere_api_key = config.cohere_api_key.as_ref().to_owned();
- anyhow::ensure!(
- !cohere_api_key.trim().is_empty(),
- "Cohere API key is required for task dedup reranking",
- );
- let dictation_state = dictation::inbound::axum_router::DictationRouterState::new(
- dictation::domain::DictationServiceImpl::new(
- dictation::outbound::WhisperTranscriber::new(&config.openai_api_key)?,
- dictation::outbound::SymphoniaRecordingInspector,
- ai_usage::pg_recorder_with_enforcement(db.clone(), config.enable_ai_usage_enforcement),
- ),
- RateLimitServiceImpl {
- repo: RedisRateLimitAdapter {
- redis: redis_client.clone(),
- },
- },
- authorization_state.clone(),
- );
- let task_dedup_service = Arc::new(TaskDedupService::new(
- TextEmbedding3Small::new(openai_api_key),
- PgTaskVectorDb::new(db.clone()),
- CohereReranker::new(cohere_api_key),
- Arc::new(AgentDuplicateJudge::new(
- ai_usage::pg_recorder_with_enforcement(db.clone(), config.enable_ai_usage_enforcement),
- )),
- Arc::new(ConnectionGatewayTaskDedupNotifier::new(
- conn_gateway_client.clone(),
- )),
- Arc::new(PgTaskMatchRepo::new(db.clone())),
- ));
- let channels_repo = PgChannelsRepo::new(db.clone());
- // Built-in agents read the same committed-post stream as hosted and
- // external agents, for channel and document posts alike, so the channel
- // side effects no longer carry their own trigger queue.
- let (bot_trigger_sender, bot_trigger_receiver) = tokio::sync::mpsc::unbounded_channel();
-
- let channel_side_effects = ChannelSideEffectService::new(
- PgChannelSideEffectContext::new(db.clone()),
- ConnectionGatewayChannelRealtimePublisher::new(conn_gateway_client.clone()),
- NotificationChannelSender::new(notification_ingress_service.clone()),
- ContactsChannelDispatcher::new(contacts_ingress.clone()),
- )
- .with_macro_event_broker(macro_event_broker.clone());
-
- let channels_service = Arc::new(
- ChannelServiceImpl::with_dependencies(
- channels_repo,
- SpawnedChannelEventDispatcher::new(channel_side_effects.clone()),
- PgChannelReferenceSharePermissions::new(db.clone(), entity_access_service.clone()),
- )
- .with_picture_files(
- channels::outbound::static_file_pictures::StaticFileChannelPictures::new(
- static_file_service_client::StaticFileServiceClient::new(
- config.internal_api_key.to_string(),
- StaticFileServiceUrl::new()?.to_string(),
- ),
- ),
- ),
- );
-
- // One message implementation for channels and documents. Committed changes fan
- // out to the broker, the local agent queue, and the parent's own delivery: channel
- // messages keep their notifications, activity, and sharing side effects and send
- // the common `message_update` payload; document discussions get the common
- // payload and document comment notifications.
- let message_realtime = messages::outbound::connection_gateway::ConnectionGatewayMessages(
- conn_gateway_client.clone(),
- );
- let discussion_delivery = messages::domain::delivery::DiscussionDelivery::new(
- messages::outbound::pg_discussion_context::PgDiscussionContext(db.clone())
- .with_initiatives(
- initiative::domain::lookup::InitiativeLookup::new(
- initiative::outbound::PgInitiativeRepo::new(db.clone()),
- ),
- properties_service.clone(),
- )
- .with_crm(crm::outbound::lookup::PgCrmParentReader::new(db.clone())),
- messages::outbound::entity_access_audience::EntityAccessMessageAudience(
- (*entity_access_service).clone(),
- ),
- message_realtime.clone(),
- messages::outbound::notification_sender::MessageNotificationSender(
- notification_ingress_service.clone(),
- ),
- )
- .with_sharing(messages::outbound::pg_discussion_context::PgDiscussionContext(db.clone()));
- let annotation_service = Arc::new(messages::domain::annotations::AnnotationService::new(
- macro_db_client::annotations::repository::PgAnnotationRepository(db.clone()),
- (*entity_access_service).clone(),
- discussion_delivery.clone(),
- ));
- let channel_delivery = channels::domain::message_delivery::ChannelMessageDelivery::new(
- PgChannelsRepo::new(db.clone()),
- SpawnedChannelEventDispatcher::new(channel_side_effects.clone()),
- PgChannelReferenceSharePermissions::new(db.clone(), entity_access_service.clone()),
- message_realtime,
- );
- let message_service = Arc::new(
- messages::domain::service::MessageService::new(
- messages::outbound::pg_message_repo::PgMessageRepository::new(db.clone())
- .with_initiatives(initiative::domain::lookup::InitiativeLookup::new(
- initiative::outbound::PgInitiativeRepo::new(db.clone()),
- ))
- .with_crm(crm::outbound::lookup::PgCrmParentReader::new(db.clone())),
- messages::domain::effects::MessageEffects::new(
- messages::outbound::broker::BrokerMessagePublisher::new(macro_event_broker.clone()),
- channel_bots::outbound::conversation::LocalBotPublisher::new(bot_trigger_sender),
- messages::domain::delivery::ParentMessagePublisher::new(
- channel_delivery,
- discussion_delivery,
- ),
- ),
- )
- .with_group_recipients(channels::domain::group_mentions::ChannelGroupRecipients(
- PgChannelsRepo::new(db.clone()),
- ))
- .with_mention_extractor(lexical_mention_extractor::LexicalMentionExtractor::new(
- lexical_client.clone(),
- ))
- .with_references(
- messages::outbound::entity_access_audience::EntityAccessMessageReferences(
- (*entity_access_service).clone(),
- ),
- ),
- );
- let message_commands: Arc<dyn messages::domain::api::MessageCommands> = message_service.clone();
- let messages_state = messages::inbound::axum_router::MessagesRouterState {
- service: message_service.clone(),
- access: entity_access_service.clone(),
- authorization: authorization_state.clone(),
- };
-
- let teammate_dms_brokers = config.kafka_brokers.as_ref().to_string();
- consumer_tracker.spawn({
- let cancellation_token = consumer_cancellation_token.clone();
- let channels = (*channels_service).clone();
- async move {
- loop {
- if cancellation_token.is_cancelled() {
- break;
- }
-
- tracing::info!("starting teammate DM consumer");
- let result = channels::inbound::teammate_dms_consumer::run_teammate_dms_consumer(
- &teammate_dms_brokers,
- channels.clone(),
- cancellation_token.cancelled(),
- )
- .await;
-
- if cancellation_token.is_cancelled() {
- break;
- }
-
- match result {
- Ok(()) => tracing::error!("teammate DM consumer exited unexpectedly"),
- Err(error) => {
- tracing::error!(
- error = ?error,
- "teammate DM consumer exited unexpectedly"
- );
- }
- }
-
- tokio::select! {
- biased;
- _ = cancellation_token.cancelled() => break,
- _ = tokio::time::sleep(Duration::from_secs(5)) => {}
- }
- }
- }
- });
-
- // Wire Macro AI to react to mentions with the classic in-channel chat
- // reply. The router posts replies through the channel service we just
- // built and runs the agent loop in-process with the channel-bot toolset:
- // a channel has no composer to finish a chat-deferred user tool in, so
- // calendar event creation executes directly and SendEmail is unavailable.
- // Agent sessions belong to a different bot entirely
- // (`bot_id::MACRO_NEW_BOT_ID`, served by the harness), so the two paths
- // can never answer the same mention.
- let mut macro_agent_tool_context = ai_tools::build_tool_service_context_from_env(
- db.clone(),
- event_broker_tracker.clone(),
- config.enable_ai_usage_enforcement,
- config.ai_pricing(),
- )
- .await
- .context("failed to build Macro agent tool context")?;
- // Wire the agent's SendChannelMessage tool to this service's own
- // side-effect pipeline so agent-posted messages share the exact instance
- // used by the HTTP API, including the in-process bot trigger sender (the
- // env builder wires an equivalent pipeline, but without bot triggers).
- macro_agent_tool_context.channel_tool_context =
- ai_tools::build_channel_tool_context_with_dispatcher(
- db.clone(),
- std::sync::Arc::new(SpawnedChannelEventDispatcher::new(channel_side_effects)),
- message_service.clone(),
- );
- let macro_agent_tools = ai_tools::tools_for(ai_tools::AiHost::ChannelBot);
- let conversation_access = Arc::new(
- channel_bots::outbound::conversation::EntityAccessConversation(
- entity_access_service.clone(),
- ),
- );
- let admission = macro_agent_tool_context.admission.clone();
- let bot_trigger_router = channel_bots::inbound::BotTriggerRouter::new_with_admission(
- message_service.clone(),
- conversation_access.clone(),
- Arc::new(channel_bots::outbound::AgentLoopResponder::new(
- macro_agent_tool_context,
- macro_agent_tools,
- )),
- Arc::new(
- channel_bots::domain::trigger_detector::MentionOrInferredDetector::new(
- message_service.clone(),
- conversation_access,
- Arc::new(channel_bots::outbound::FastModelTriggerClassifier::new(
- ai_usage::pg_recorder_with_enforcement(
- db.clone(),
- config.enable_ai_usage_enforcement,
- ),
- )),
- )
- .with_admission(admission.clone()),
- ),
- Arc::new(channel_bots::outbound::PrimaryCalendarTimeZones::new(
- Arc::new(calendar_events::domain::service::CalendarService::new(
- calendar_events::outbound::pg::PgCalendarRepository::new(readonly_db.clone()),
- )),
- )),
- Arc::new(channel_bots::outbound::LexicalCommentMarks::new(
- (*lexical_client).clone(),
- )),
- admission,
- );
- bot_trigger_router.spawn(bot_trigger_receiver);
-
- let channel_bot_webhook_state =
- bots::inbound::channel_webhook_router::ChannelBotWebhookRouterState::new(
- bots_service.clone(),
- message_commands.clone(),
- (*entity_access_service).clone(),
- authorization_state.clone(),
- );
-
- let slack_limits = slack_integration::domain::models::ImportLimits::default();
- let slack_enabled = config.slack_import_enabled;
- let slack_service = api::context::SlackService::new(
- slack_integration::outbound::pg_slack_import_repo::PgSlackImportRepo::new(
- db.clone(),
- slack_limits,
- ),
- slack_integration::outbound::s3_storage::S3ImportStorage::new(
- macro_aws_config::s3_client().await,
- config.upload_staging_bucket.as_ref().to_owned(),
- slack_limits,
- )
- .map_err(|_| anyhow::anyhow!("invalid Slack import storage configuration"))?,
- slack_integration::outbound::import_ledger::CanonicalImportLedger::new(
- import::outbound::pg_import_repo::PgImportRepo::new(db.clone()),
- ),
- slack_integration::outbound::gateway_notifier::GatewayImportNotifier::new(
- conn_gateway_client.clone(),
- ),
- api::context::SlackAdminAuthorizer(
- entity_access_service.clone(),
- entity_access::outbound::PgAccessRepository::new(db.clone()),
- ),
- Box::new(move |_| slack_enabled),
- slack_limits,
- )?;
- let slack_state = slack_integration::inbound::axum_router::SlackRouterState {
- service: Arc::new(slack_service),
- entity_access_service: entity_access_service.clone(),
- authorization_state: authorization_state.clone(),
- };
-
- // Held by value here and behind an `Arc` in the router state: the impl is a
- // pool handle, so cloning is cheap and `SoupImpl` needs an owned service.
- // Additional connections for guards spanning email I/O. HTTP requests and
- // dispatch share this budget, independently of the main data pool.
- let followup_lock_capacity = match config.environment {
- Environment::Production => 32,
- Environment::Develop => 16,
- Environment::Local => 8,
- };
- let email_followups = reminders::domain::email_followup::service::EmailFollowupService::new(
- PgRemindersRepo::with_followup_lock_capacity(db.clone(), followup_lock_capacity),
- email_service.clone(),
- );
- let reminders_service =
- reminders::domain::email_followup::reminder_service::EmailRemindersService::new(
- email_followups.clone(),
- )
- .with_entity_access((*entity_access_service).clone());
-
- let document_creator = documents_hex::domain::create::DocumentCreator::new(
- document_service.clone(),
- markdown_initializer,
- documents_hex::outbound::document_bytes_upload::ReqwestDocumentBytesUploader::default(),
- documents_hex::outbound::mention_tracker::LexicalCommsMentionTracker::new(
- db.clone(),
- lexical_client.clone(),
- ),
- );
- // Forms' ids are their surfaces' ids, so the public surface API asks the
- // forms domain which ids are taken.
- let collab_surface_service = Arc::new(
- collab_surface::outbound::pg_collab_surface_service(
- db.clone(),
- lexical_client.as_ref().clone(),
- sync_service_client.as_ref().clone(),
- config.document_permission_jwt.as_ref().to_string(),
- )
- .with_form_ids(Arc::new(
- forms::outbound::collaborative_layout::RepositoryFormIds::new(Arc::new(
- forms::outbound::pg_forms_repo::PgFormsRepo::new(db.clone()),
- )),
- )),
- );
-
- let initiative_service = Arc::new(
- InitiativeServiceImpl::new(
- PgInitiativeRepo::new(db.clone()),
- initiative_description::InitiativeDescriptionSurfacesAdapter::new(
- collab_surface_service.clone(),
- ),
- Arc::new(initiative::outbound::resources::ProjectResources::new(
- properties_service.clone(),
- system_properties_service.clone(),
- entity_access_service.clone(),
- )),
- )
- .with_event_publisher(Arc::new(
- initiative::outbound::event_publisher::BrokerInitiativeEventPublisher::new(
- macro_event_broker.clone(),
- ),
- )),
- );
-
- // Shared by the databases router and the unified entity-mutation router.
- let databases_service = Arc::new(databases::wiring::build_service(
- db.clone(),
- entity_access_service.clone(),
- databases::outbound::gateway_event_publisher::GatewayTableEventPublisher::new(
- conn_gateway_client.as_ref().clone(),
- ),
- macro_event_broker.clone(),
- ));
- let forms_service = Arc::new(forms::wiring::build_service(
- db.clone(),
- databases_service.clone(),
- entity_access_service.clone(),
- forms::outbound::gateway_event_publisher::GatewayFormEventPublisher::new(
- conn_gateway_client.as_ref().clone(),
- ),
- macro_event_broker.clone(),
- collab_surface_service.clone(),
- ));
-
- // Individual initiative reads preserve read-after-write consistency when a
- // newly created project opens immediately. Lists retain the replica reader.
- // Reuse Soup hydration so detail and mutation metadata include viewer history.
- let initiative_entity_soup = Arc::new(
- SoupImpl::new(
- PgSoupRepo::new(readonly_pool::ReadOnlyPool(db.clone())),
- frecency_service.clone(),
- ReadonlyEmailPreviewAdapter(email_service.clone()),
- ChannelListServiceImpl::new(
- PgChannelsRepo::new(db.clone()),
- PgChannelsRepo::new(db.clone()),
- frecency_storage.clone(),
- ),
- call::domain::service::CallRecordQueryServiceImpl::new(PgCallRepo::new(db.clone())),
- crm_service.clone(),
- GithubPullRequestServiceImpl::new(
- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
- PgGithubPullRequestRepo::new(db.clone()),
- ),
- )
- .with_agent_branches(agent_changes::outbound::postgres::PgChangesetRepo::new(
- db.clone(),
- )),
- );
-
- let favorites_service = Arc::new(FavoritesServiceImpl::new(PgFavoritesRepo::new(db.clone())));
-
- // Keep the replica-backed Soup reader alongside the primary-backed email
- // writer in SoupRouterState. REST, GraphQL lists, and realtime hydration
- // share this reader; only email mutations and their reply loader use the
- // writer service, so normal list traffic never switches to the primary.
- let soup_service = Arc::new(
- SoupImpl::new(
- PgSoupRepo::new(readonly_pool::ReadOnlyPool(readonly_db.clone())),
- frecency_service,
- readonly_email_service,
- ChannelListServiceImpl::new(
- PgChannelsRepo::new(readonly_db.clone()),
- PgChannelsRepo::new(readonly_db.clone()),
- frecency_storage.clone(),
- ),
- call::domain::service::CallRecordQueryServiceImpl::new(PgCallRepo::new(
- readonly_db.clone(),
- )),
- crm_service.clone(),
- GithubPullRequestServiceImpl::new(
- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(readonly_db.clone())),
- PgGithubPullRequestRepo::new(readonly_db.clone()),
- ),
- )
- .with_favorites(favorites_service.clone())
- .with_agent_branches(agent_changes::outbound::postgres::PgChangesetRepo::new(
- readonly_db.clone(),
- )),
- );
-
- let websocket_notification_consumer_service =
- Arc::new(WebSocketNotificationConsumerService::new(
- NotificationTopicConsumer::<model_notifications::NotifEvent>::from_env(
- config.kafka_brokers.as_ref(),
- )
- .map_err(|error| {
- anyhow::anyhow!("failed to create WebSocket notification topic consumer: {error:?}")
- })?,
- ));
- consumer_tracker.spawn({
- let service = Arc::clone(&websocket_notification_consumer_service);
- let cancellation_token = consumer_cancellation_token.clone();
- async move {
- loop {
- let result = tokio::select! {
- biased;
- _ = cancellation_token.cancelled() => break,
- result = service.run() => result,
- };
-
- if cancellation_token.is_cancelled() {
- break;
- }
-
- let _ = result.inspect_err(|error| {
- tracing::error!(
- error = ?error,
- "WebSocket notification consumer stopped"
- );
- });
-
- tokio::select! {
- biased;
- _ = cancellation_token.cancelled() => break,
- _ = tokio::time::sleep(Duration::from_secs(1)) => {}
- }
- }
- }
- });
-
- let activity_realtime_service = Arc::new(activity::ActivityRealtimeConsumerService::new(
- activity::ActivityTopicConsumer::from_env(config.kafka_brokers.as_ref()).map_err(
- |error| anyhow::anyhow!("failed to create realtime activity topic consumer: {error:?}"),
- )?,
- ));
- consumer_tracker.spawn({
- let service = Arc::clone(&activity_realtime_service);
- let cancellation_token = consumer_cancellation_token.clone();
- async move {
- loop {
- let result = tokio::select! {
- biased;
- _ = cancellation_token.cancelled() => break,
- result = service.run() => result,
- };
-
- if cancellation_token.is_cancelled() {
- break;
- }
-
- let _ = result.inspect_err(|error| {
- tracing::error!(
- error = ?error,
- "realtime activity subscription consumer stopped"
- );
- });
-
- tokio::select! {
- biased;
- _ = cancellation_token.cancelled() => break,
- _ = tokio::time::sleep(Duration::from_secs(1)) => {}
- }
- }
- }
- });
-
- let soup_realtime_service = Arc::new(SoupRealtimeConsumerService::new(
- SoupTopicConsumer::from_env(config.kafka_brokers.as_ref()).map_err(|error| {
- anyhow::anyhow!("failed to create realtime Soup topic consumer: {error:?}")
- })?,
- ));
- consumer_tracker.spawn({
- let soup_realtime_service = Arc::clone(&soup_realtime_service);
- let cancellation_token = consumer_cancellation_token.clone();
- async move {
- loop {
- let result = tokio::select! {
- biased;
- _ = cancellation_token.cancelled() => break,
- result = soup_realtime_service.run() => result,
- };
-
- if cancellation_token.is_cancelled() {
- break;
- }
-
- let _ = result.inspect_err(|error| {
- tracing::error!(
- error = ?error,
- "realtime Soup subscription consumer stopped"
- );
- });
-
- tokio::select! {
- biased;
- _ = cancellation_token.cancelled() => break,
- _ = tokio::time::sleep(Duration::from_secs(1)) => {}
- }
- }
- }
- });
-
- let agent_session_log_realtime = Arc::new(AgentSessionLogConsumerService::new(
- LogTopicConsumer::from_env(config.kafka_brokers.as_ref()).map_err(|error| {
- anyhow::anyhow!("failed to create agent session log topic consumer: {error:?}")
- })?,
- ));
- consumer_tracker.spawn({
- let service = Arc::clone(&agent_session_log_realtime);
- let cancellation_token = consumer_cancellation_token.clone();
- async move {
- loop {
- let result = tokio::select! {
- biased;
- _ = cancellation_token.cancelled() => break,
- result = service.run() => result,
- };
-
- if cancellation_token.is_cancelled() {
- break;
- }
-
- let _ = result.inspect_err(|error| {
- tracing::error!(
- error = ?error,
- "agent session log subscription consumer stopped"
- );
- });
-
- tokio::select! {
- biased;
- _ = cancellation_token.cancelled() => break,
- _ = tokio::time::sleep(Duration::from_secs(1)) => {}
- }
- }
- }
- });
-
- consumer_tracker.spawn({
- let brokers = config.kafka_brokers.as_ref().to_string();
- let entity_access_service = entity_access_service.as_ref().clone();
- let macro_event_broker = macro_event_broker.clone();
- let cancellation_token = consumer_cancellation_token.clone();
- async move {
- let mut consecutive_failures = 0_u32;
- loop {
- if cancellation_token.is_cancelled() {
- break;
- }
-
- let fanout_service = SoupRealtimeServiceImpl::new(
- EntityAccessExpander::new(entity_access_service.clone()),
- KafkaSoupRealtimePublisher::new(macro_event_broker.clone()),
- );
- tracing::info!("starting realtime Soup entity consumer");
- let result = fanout_service
- .run_entity_update_consumer(&brokers, cancellation_token.cancelled())
- .await;
-
- if cancellation_token.is_cancelled() {
- break;
- }
-
- let restart_delay = match result {
- Ok(()) => {
- consecutive_failures = 0;
- tracing::error!("realtime Soup entity consumer exited unexpectedly");
- soup_consumer_restart_delay(1)
- }
- Err(error) => {
- consecutive_failures = consecutive_failures.saturating_add(1);
- let restart_delay = soup_consumer_restart_delay(consecutive_failures);
- if consecutive_failures >= SOUP_CONSUMER_RESTART_ALERT_THRESHOLD {
- tracing::error!(
- error = ?error,
- consecutive_failures,
- restart_delay_secs = restart_delay.as_secs(),
- "realtime Soup entity consumer repeatedly failed"
- );
- } else {
- tracing::warn!(
- error = ?error,
- consecutive_failures,
- restart_delay_secs = restart_delay.as_secs(),
- "realtime Soup entity consumer failed; scheduling restart"
- );
- }
- restart_delay
- }
- };
-
- tokio::select! {
- biased;
- _ = cancellation_token.cancelled() => break,
- _ = tokio::time::sleep(restart_delay) => {}
- }
- }
- }
- });
-
- let favorites_mutation_service = Arc::new(FavoritesMutationServiceImpl::new(
- favorites_service.clone(),
- entity_access_service.clone(),
- ));
- let user_api_key_service = Arc::new(UserApiKeyServiceImpl::new(PgUserApiKeysRepo::new(
- db.clone(),
- )));
- let calendar_state = CalendarRouterState::new(
- Arc::new(calendar_events::domain::service::CalendarService::new(
- calendar_events::outbound::pg::PgCalendarRepository::new(readonly_db.clone()),
- )),
- authorization_state.clone(),
- );
-
- // Reminder dispatch. An EventBridge rule drops a sweep tick on this queue
- // every minute; the sweep fans one message out per due firing, onto the
- // same queue, and every task in the pool delivers them in parallel.
- let reminder_dispatch_worker = {
- let queue = SqsDispatchQueue::new(
- aws_sdk_sqs::Client::new(&aws_config),
- reminder_dispatch_queue.to_string(),
- );
- let dispatch_service = ReminderDispatchService::new(
- PgRemindersRepo::new(db.clone()),
- NotificationReminderNotifier::new((*notification_ingress_service).clone()),
- queue.clone(),
- );
- DispatchWorker::new(
- reminders::domain::email_followup::dispatch::EmailReminderDispatch::new(
- dispatch_service,
- email_followups.clone(),
- ),
- queue,
- )
- };
-
- consumer_tracker.spawn({
- let cancellation_token = consumer_cancellation_token.clone();
- async move {
- tracing::info!("starting reminder dispatch worker");
- reminder_dispatch_worker.run(cancellation_token).await;
- tracing::info!("reminder dispatch worker stopped");
- }
- });
-
- // Calendar event reminder dispatch: same sweep/deliver shape as reminders,
- // on its own queue. Behind a master switch — when off, the worker drains
- // the minutely tick so the queue neither backs up nor dead-letters.
- let calendar_reminder_dispatch_worker = {
- let queue = SqsCalendarDispatchQueue::new(
- aws_sdk_sqs::Client::new(&aws_config),
- calendar_reminder_dispatch_queue.to_string(),
- );
- let dispatch_service = CalendarReminderDispatchService::new(
- calendar_events::outbound::pg::PgCalendarRepository::new(db.clone()),
- NotificationCalendarReminderNotifier::new((*notification_ingress_service).clone()),
- queue.clone(),
- );
- CalendarReminderDispatchWorker::new(dispatch_service, queue)
- };
-
- consumer_tracker.spawn({
- let cancellation_token = consumer_cancellation_token.clone();
- let enabled = config.calendar_reminder_dispatch_enabled;
- async move {
- if enabled {
- tracing::info!("starting calendar reminder dispatch worker");
- calendar_reminder_dispatch_worker
- .run(cancellation_token)
- .await;
- tracing::info!("calendar reminder dispatch worker stopped");
- } else {
- tracing::info!("calendar reminder dispatch disabled; draining its queue");
- calendar_reminder_dispatch_worker
- .drain(cancellation_token)
- .await;
- }
- }
- });
-
- let redis_sha_client = Arc::new(Redis::new(redis_client));
-
- let graphql_entity_mutation_service =
- Arc::new(service::entity_mutation::DssEntityMutationService::new(
- document_service.clone(),
- chat_mutation_service,
- channels_service.clone(),
- call_service.clone(),
- Arc::new(email_service.clone()),
- project_service.clone(),
- databases_service.clone(),
- forms_service.clone(),
- entity_access_service.clone(),
- Arc::new(outbound::entity_mutation::DssEntityLifecycleAdapter::new(
- db.clone(),
- redis_sha_client.clone(),
- sqs_client.clone(),
- macro_event_broker.clone(),
- )),
- ));
-
- // Routine writes still use the scheduled-action service. Read from the
- // primary pool here so the GraphQL list cannot restore stale state after
- // a REST write.
- let scheduled_action_read_service = Arc::new(ScheduledActionReadServiceImpl::new(
- Arc::new(PgScheduledActionRepo::new(
- db.clone(),
- owned_entity_registrar.clone(),
- )),
- entity_access_service.clone(),
- ));
-
- let api_context = ApiContext {
- dictation_state,
- contacts_ingress: contacts_ingress.clone(),
- soup_router_state: SoupRouterState::from_arc(
- soup_service.clone(),
- email_service.clone(),
- entity_access_service.clone(),
- authorization_state.clone(),
- )
- .with_favorites_reader(Arc::new(
- service::soup_favorites_reader::DssSoupFavoritesReader(favorites_service.clone()),
- )),
- favorites_state: FavoritesRouterState::new(
- favorites_service.clone(),
- entity_access_service.clone(),
- authorization_state.clone(),
- ),
- favorites_service,
- favorites_mutation_service,
- channel_labels_state: ChannelLabelsRouterState::new(
- Arc::new(ChannelLabelsServiceImpl::new(PgChannelLabelsRepo::new(
- db.clone(),
- ))),
- entity_access_service.clone(),
- authorization_state.clone(),
- ),
- user_api_key_state: UserApiKeyRouterState::new(
- user_api_key_service,
- authorization_state.clone(),
- ),
- slack_state,
- reminders_state: RemindersRouterState::new(
- Arc::new(reminders_service),
- entity_access_service.clone(),
- authorization_state.clone(),
- ),
- graphql_initiative_entity_loader: graphql_initiative::InitiativeEntityLoader(
- graphql_soup::soup_item_loader(initiative_entity_soup, Arc::new(email_service.clone())),
- ),
- graphql_initiative_context: graphql_initiative::InitiativeGraphqlContext::new(
- initiative_service.clone(),
- entity_access_service.clone(),
- ),
- graphql_scheduled_action_context: ScheduledActionGraphqlContext::new(
- scheduled_action_read_service,
- ),
- initiative_state: InitiativeRouterState::new(
- initiative_service,
- entity_access_service.clone(),
- authorization_state.clone(),
- ),
- databases_state: databases::inbound::axum_router::DatabasesRouterState::new(
- databases_service,
- entity_access_service.clone(),
- authorization_state.clone(),
- ),
- forms_state: forms::inbound::axum_router::FormsRouterState::new(
- forms_service,
- entity_access_service.clone(),
- authorization_state.clone(),
- ),
- database_starter_state: databases::inbound::starter_router::DatabaseStarterRouterState::new(
- Arc::new(databases::domain::starter::DatabaseStarterServiceImpl::new(
- databases::outbound::pg_starter::PgDatabaseStarterRepo::new(
- db.clone(),
- properties::outbound::properties_pg_repo::PropertiesPgRepo::new(db.clone()),
- ),
- macro_event_broker.clone(),
- )),
- authorization_state.clone(),
- ),
- collab_surface_state: CollabSurfaceRouterState::new(
- collab_surface_service,
- entity_access_service.clone(),
- authorization_state.clone(),
- ),
- graphql_soup_schema: complete_graph::build_schema_from_arcs(
- soup_service,
- soup_realtime_service,
- websocket_notification_consumer_service,
- activity_realtime_service,
- ),
- agent_session_log_subscriptions: complete_graph::agent_session_log_subscriptions(
- agent_session_log_realtime,
- ),
- graphql_notification_reader,
- // GraphQL reads the activity log through the readonly pool; the
- // Kafka consumer's writer-pool repo above is separate on purpose.
- activity_reader: complete_graph::ActivityPortReader::new(Arc::new(
- initiative::domain::personal_activity::ProjectVisibleActivityReads::new(
- activity::outbound::pg_activity_repo::PgActivityRepo::new(readonly_db.clone()),
- entity_access_service.clone(),
- ),
- )),
- graphql_entity_mutation_service,
- github_sync_service: Arc::new(github_sync_service_impl),
- github_pull_request_index_state,
- github_pull_request_state,
- github_pull_request_changes_state,
- foreign_entity_state,
- db: db.clone(),
- readonly_db: readonly_pool::ReadOnlyPool(readonly_db.clone()),
- redis_client: redis_sha_client,
- s3_client: s3,
- dynamodb_client: Arc::new(dynamodb_client),
- dynamo_db,
- macro_event_broker: macro_event_broker.clone(),
- sqs_client: sqs_client.clone(),
- notification_ingress_service: notification_ingress_service.clone(),
- conn_gateway_client: conn_gateway_client.clone(),
- sync_service_client: sync_service_client.clone(),
- system_properties_service: system_properties_service.clone(),
- properties_service: properties_service.clone(),
- opensearch_client: Arc::new(opensearch_client),
- authorization_state: authorization_state.clone(),
- jwt_validation_args,
- dss_auth_key,
- // Shared frecency storage and legacy channel list routes.
- frecency_storage,
- channel_list_state,
- entity_access_service: entity_access_service.clone(),
- calendar_state,
- projects_state: ProjectRouterState {
- service: project_service,
- access_service: entity_access_service.clone(),
- authorization_state: authorization_state.clone(),
- non_user_owners,
- },
- documents_state: DocumentRouterState {
- service: document_service,
- access_service: entity_access_service.clone(),
- authorization_state: authorization_state.clone(),
- pool: db.clone(),
- task_dedup_service,
- lexical_client: lexical_client.clone(),
- creator: document_creator,
- document_permission_jwt_secret: config.document_permission_jwt.as_ref().to_string(),
- non_user_owners,
- },
- config: Arc::new(config),
- channel_service: channels_service.clone(),
- channels_state: ChannelsRouterState::from_arc(
- channels_service,
- (*entity_access_service).clone(),
- authorization_state.clone(),
- ),
- messages_state,
- annotation_service,
- bots_state: bots::inbound::axum_router::BotsRouterState::new(
- bots_service.clone(),
- (*entity_access_service).clone(),
- authorization_state.clone(),
- ),
- harnesses_state: harnesses::inbound::axum_router::HarnessesRouterState::new(
- harnesses_service,
- authorization_state.clone(),
- ),
- channel_bot_webhook_state,
- call_state,
- call_webhook_state,
- call_public_rate_limiter,
- webhook_state,
- sse_stream_state,
- call_internal_state,
- cal_webhook_state,
- entity_access_management_service,
- crm_state: crm::inbound::axum_router::CrmRouterState {
- service: Arc::new(crm_service),
- stage_service: Arc::new(crm::domain::stages::CrmStageServiceImpl::new(
- crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),
- crm::outbound::stage_definitions::PropertiesStageDefinitionStore::new(
- properties_service.clone(),
- ),
- )),
- entity_access_service: entity_access_service.clone(),
- authorization_state: authorization_state.clone(),
- },
- };
-
- #[cfg(feature = "delete_document_worker")]
- {
- let editing_worker_client = Arc::new(
- documents_hex::outbound::editing_worker_client::ReqwestEditingWorkerClient::from_url(
- AiEditingWorkerUrl::new()?.to_string(),
- )
- .with_internal_auth_key(Some(api_context.config.internal_api_key.to_string())),
- );
-
- // Spawn the delete document worker.
- let delete_worker_ctx = service::delete_document_worker::DeleteDocumentWorkerContext {
- worker: Arc::new(delete_document_worker),
- db: db.clone(),
- s3_client: api_context.s3_client.clone(),
- redis_client: api_context.redis_client.clone(),
- sync_service_client: api_context.sync_service_client.clone(),
- editing_worker_client,
- properties_service: api_context.properties_service.clone(),
- };
-
- tokio::spawn(async move {
- service::delete_document_worker::run_worker(delete_worker_ctx).await;
- });
- }
-
- let server_result = api::setup_and_serve(api_context).await;
-
- tracing::info!("stopping event consumers");
- consumer_cancellation_token.cancel();
- consumer_tracker.close();
- consumer_tracker.wait().await;
- tracing::info!("event consumers stopped");
-
- tracing::info!("waiting for event broker publishes to drain");
- event_broker_tracker.close();
- match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {
- Ok(()) => tracing::info!("event broker publishes drained"),
- Err(error) => {
- tracing::warn!(
- error=?error,
- timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),
- "timed out waiting for event broker publishes to drain"
- );
- }
- }
-
- server_result
-}
-
-const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
diff --git a/services/document_storage_service/src/openapi.rs b/services/document_storage_service/src/openapi.rs
index 6818634a03..5f9112c126 100644
--- a/services/document_storage_service/src/openapi.rs
+++ b/services/document_storage_service/src/openapi.rs
@@ -1,17 +1,6 @@
-#![allow(unused)]
-#![recursion_limit = "256"]
-
-mod api;
-mod config;
-mod model;
-mod outbound;
-mod service;
-
+use document_storage_service::ApiDoc;
use utoipa::OpenApi;
fn main() {
- println!(
- "{}",
- api::swagger::ApiDoc::openapi().to_pretty_json().unwrap()
- );
+ println!("{}", ApiDoc::openapi().to_pretty_json().unwrap());
}
diff --git a/services/document_storage_service/src/server.rs b/services/document_storage_service/src/server.rs
new file mode 100644
index 0000000000..c24717d2ee
--- /dev/null
+++ b/services/document_storage_service/src/server.rs
@@ -0,0 +1,2065 @@
+//! The service's composition root: reads config and wires every adapter and
+//! domain service into the HTTP server and its background workers.
+
+use crate::{
+ api::{
+ self, MACRO_INTERNAL_USER_ID,
+ context::{
+ ApiContext, AuthorizationService, DocumentStorageServiceAuthKey, TaskPropertiesAdapter,
+ },
+ },
+ config::{
+ self, CalEventTypeContentNamesKey, CalWebhookSecretKey, Config, Environment,
+ MetaAccessToken, MetaPixelId, MetaTestEventCode,
+ },
+ outbound,
+ service::{self, s3::S3},
+};
+use agent_session_realtime::{
+ domain::service::AgentSessionLogConsumerService, outbound::log_topic_consumer::LogTopicConsumer,
+};
+use analytics_client::{AnalyticsClient, AnalyticsClientConfig, MetaConfig};
+use anyhow::Context;
+use bots::{domain::service::BotServiceImpl, outbound::pg_bots_repo::PgBotsRepo};
+use cal::{
+ domain::service::{CalConfig, CalEventMeta, CalWebhookServiceImpl},
+ inbound::cal_webhook_router::CalWebhookRouterState,
+ outbound::analytics_client::AnalyticsClientSink,
+};
+use calendar_events::domain::reminder_dispatch::CalendarReminderDispatchService;
+use calendar_events::inbound::axum_router::CalendarRouterState;
+use calendar_events::inbound::dispatch_worker::CalendarReminderDispatchWorker;
+use calendar_events::outbound::notification_notifier::NotificationCalendarReminderNotifier;
+use calendar_events::outbound::sqs_dispatch_queue::SqsCalendarDispatchQueue;
+use call::{
+ domain::service::CallServiceImpl,
+ inbound::axum_router::{CallRouterState, InternalCallRouterState, WebhookRouterState},
+ outbound::{
+ ai_call_summarizer::AiCallSummarizer,
+ livekit_rtc_client::LivekitRtcClient,
+ pg_call_repo::PgCallRepo,
+ pg_voice_repo::PgVoiceRepo,
+ s3_recording_storage::{RecordingCloudFrontConfig, S3RecordingStorage},
+ },
+};
+use channel_labels::{
+ domain::service::ChannelLabelsServiceImpl, inbound::axum_router::ChannelLabelsRouterState,
+ outbound::pg_channel_labels_repo::PgChannelLabelsRepo,
+};
+use channels::{
+ domain::{
+ list_service::ChannelListServiceImpl,
+ service::ChannelServiceImpl,
+ side_effects::{ChannelSideEffectService, SpawnedChannelEventDispatcher},
+ },
+ inbound::{axum_router::ChannelsRouterState, list_router::ChannelListRouterState},
+ outbound::{
+ connection_gateway_realtime::ConnectionGatewayChannelRealtimePublisher,
+ contacts_dispatcher::ContactsChannelDispatcher,
+ notification_sender::NotificationChannelSender,
+ pg_channel_reference_share_permissions::PgChannelReferenceSharePermissions,
+ pg_channels_repo::PgChannelsRepo, pg_side_effect_context::PgChannelSideEffectContext,
+ },
+};
+use collab_surface::inbound::axum_router::CollabSurfaceRouterState;
+use connection::{
+ domain::service::ConnectionServiceImpl,
+ outbound::connection_gateway_client::ConnectionGatewayImpl,
+};
+use connection_gateway_client::client::ConnectionGatewayClient;
+use documents_hex::domain::models::CloudFrontConfig;
+use documents_hex::domain::service::DocumentServiceImpl;
+use documents_hex::inbound::axum_router::DocumentRouterState;
+use documents_hex::outbound::pg_document_repo::PgDocumentRepo;
+use documents_hex::outbound::s3_upload_url::S3UploadUrlAdapter;
+use dynamodb_client::DynamodbClient;
+use email::{
+ domain::{ports::ReadonlyEmailPreviewAdapter, service::EmailServiceImpl},
+ outbound::EmailPgRepo,
+};
+use embedding::embedding_provider::openai::TextEmbedding3Small;
+use entity_registry::OwnerGrantPolicy;
+use entity_registry_db_utils::OwnedEntityRegistrar;
+use favorites::{
+ domain::{mutation_service::FavoritesMutationServiceImpl, service::FavoritesServiceImpl},
+ inbound::axum_router::FavoritesRouterState,
+ outbound::pg_favorites_repo::PgFavoritesRepo,
+};
+use foreign_entity::{
+ domain::service::ForeignEntityServiceImpl, inbound::axum_router::ForeignEntityRouterState,
+ outbound::pg_foreign_entity_repo::PgForeignEntityRepo,
+};
+use frecency::{domain::services::FrecencyQueryServiceImpl, outbound::postgres::FrecencyPgStorage};
+use github::domain::service::{
+ GithubSyncConfig, GithubSyncServiceImpl, InstallationTokenConfig, InstallationTokenService,
+ PullRequestIndexService,
+};
+use github::inbound::pull_request_index_router::PullRequestIndexRouterState;
+use github::outbound::connection_gateway_realtime::ConnectionGatewayGithubRealtime;
+use github::outbound::github_sync_client::GithubSyncClientImpl;
+use github::outbound::pg_github_sync_repo::PgGithubSyncRepo;
+use github::outbound::pull_request_diff::GithubPullRequestDiffClient;
+use github_pull_requests::{
+ domain::service::{
+ GithubPullRequestChangesServiceImpl, GithubPullRequestChangesetStore,
+ GithubPullRequestServiceImpl,
+ },
+ inbound::{
+ axum_router::GithubPullRequestRouterState,
+ changes_router::GithubPullRequestChangesRouterState,
+ },
+ outbound::{
+ pg_github_pull_request_repo::PgGithubPullRequestRepo,
+ s3_patch_store::S3GithubPullRequestPatchStore,
+ },
+};
+use graphql_scheduled_action::ScheduledActionGraphqlContext;
+use harnesses::outbound::pg_harness_repo::PgHarnessRepo;
+use initiative::{
+ domain::service::InitiativeServiceImpl, inbound::axum_router::InitiativeRouterState,
+ outbound::PgInitiativeRepo,
+};
+use lexical_client::LexicalClient;
+use macro_auth::middleware::decode_jwt::JwtValidationArgs;
+use macro_authorization::{
+ InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationServiceImpl,
+ MacroAuthorizationState, PgBotAuthorizationRepo, PgBotAuthorizer, PgHarnessAuthorizationRepo,
+ PgHarnessAuthorizer, PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,
+};
+use macro_env_var::maybe_env_vars;
+use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
+#[cfg(feature = "delete_document_worker")]
+use macro_service_urls::AiEditingWorkerUrl;
+use macro_service_urls::{
+ ConnectionGatewayUrl, LexicalServiceUrl, StaticFileServiceUrl, SyncServiceUrl,
+};
+use macro_sha_count_client::Redis;
+use notification::domain::service::{
+ NotificationReaderService, PlatformArnConfig, SqsNotificationIngress,
+ WebSocketNotificationConsumerService,
+};
+use notification::outbound::{notification_consumer::NotificationTopicConsumer, queue::SqsQueue};
+use opensearch_client::OpensearchClient;
+use projects_hex::{
+ domain::service::ProjectServiceImpl,
+ inbound::axum_router::ProjectRouterState,
+ outbound::{
+ DynamoBulkUploadAdapter, PgProjectRepo, S3ProjectUploadAdapter, ShaCountAdapter,
+ SqsProjectSearchIndexer,
+ },
+};
+use properties::{
+ NotificationServiceImpl, PermissionServiceImpl, PropertiesPgRepo, PropertiesServiceImpl,
+};
+use rate_limit::{RateLimitServiceImpl, RedisRateLimitAdapter};
+use reminders::{
+ domain::service::dispatch::ReminderDispatchService,
+ inbound::{axum_router::RemindersRouterState, dispatch_worker::DispatchWorker},
+ outbound::{
+ notification_notifier::NotificationReminderNotifier, pg_reminders_repo::PgRemindersRepo,
+ sqs_dispatch_queue::SqsDispatchQueue,
+ },
+};
+use scheduled_action::domain::read_service::ScheduledActionReadServiceImpl;
+use scheduled_action::outbound::pg_scheduled_action_repo::PgScheduledActionRepo;
+use secretsmanager_client::SecretManager;
+use soup::{
+ domain::service::SoupImpl, inbound::axum_router::SoupRouterState,
+ outbound::pg_soup_repo::PgSoupRepo,
+};
+use soup_realtime::{
+ domain::service::{SoupRealtimeConsumerService, SoupRealtimeServiceImpl},
+ outbound::{
+ entity_access::EntityAccessExpander, kafka_publisher::KafkaSoupRealtimePublisher,
+ soup_consumer::SoupTopicConsumer,
+ },
+};
+use sqlx::postgres::PgPoolOptions;
+use std::{sync::Arc, time::Duration};
+use sync_service_client::SyncServiceClient;
+use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};
+use task_dedup::{
+ TaskDedupService,
+ outbound::{
+ cohere::CohereReranker,
+ connection_gateway::ConnectionGatewayTaskDedupNotifier,
+ judge::AgentDuplicateJudge,
+ postgres::{PgTaskMatchRepo, PgTaskVectorDb},
+ },
+};
+use tokio_util::{sync::CancellationToken, task::TaskTracker};
+use user_api_key::{
+ domain::service::UserApiKeyServiceImpl, inbound::axum_router::UserApiKeyRouterState,
+ outbound::pg_user_api_keys_repo::PgUserApiKeysRepo,
+};
+
+const SOUP_CONSUMER_RESTART_MAX_DELAY_SECS: u64 = 60;
+const SOUP_CONSUMER_RESTART_ALERT_THRESHOLD: u32 = 5;
+
+fn soup_consumer_restart_delay(consecutive_failures: u32) -> Duration {
+ let exponent = consecutive_failures.saturating_sub(1).min(6);
+ Duration::from_secs((1_u64 << exponent).min(SOUP_CONSUMER_RESTART_MAX_DELAY_SECS))
+}
+
+maybe_env_vars! {
+ struct AppleBundleId;
+ struct SnsApnsVoipPlatformArn;
+}
+
+/// Build the service from the environment and serve until shutdown.
+pub async fn run() -> anyhow::Result<()> {
+ let env = Environment::new_or_prod();
+
+ let aws_config = macro_aws_config::get_macro_aws_config().await;
+
+ let secretsmanager_client = secretsmanager_client::SecretsManager::new(
+ aws_sdk_secretsmanager::Client::new(&aws_config),
+ );
+
+ // Parse our configuration from the environment, then resolve any secret-manager backed values.
+ let config = Config::from_env()
+ .context("expected to be able to generate config")?
+ .resolve_remote_secrets(env, &secretsmanager_client)
+ .await
+ .context("expected to be able to resolve config secrets")?;
+ let non_user_owners = config.non_user_owners()?;
+
+ tracing::trace!("initialized config");
+
+ let (min_connections, max_connections): (u32, u32) = match config.environment {
+ Environment::Production => (50, 150),
+ Environment::Develop => (25, 100),
+ Environment::Local => (15, 50),
+ };
+
+ let db = PgPoolOptions::new()
+ .min_connections(min_connections)
+ .max_connections(max_connections)
+ .connect(&config.database_url)
+ .await
+ .context("could not connect to db")?;
+
+ tracing::trace!(
+ min_connections,
+ max_connections,
+ "initialized db connection"
+ );
+
+ let readonly_db = match PgPoolOptions::new()
+ .min_connections(min_connections)
+ .max_connections(max_connections)
+ .connect(&config.database_url_readonly)
+ .await
+ {
+ Ok(pool) => {
+ tracing::trace!("initialized readonly db connection");
+ pool
+ }
+ Err(e) => {
+ tracing::warn!(error=?e, "failed to connect to readonly db, falling back to primary");
+ db.clone()
+ }
+ };
+
+ let dynamo_db = aws_sdk_dynamodb::Client::new(&aws_config);
+
+ let dynamodb_client = DynamodbClient::new_from_client(
+ dynamo_db.clone(),
+ Some(config.bulk_upload_requests_table.as_ref().to_string()),
+ );
+ tracing::trace!("initialized dynamodb client");
+
+ let s3_client = macro_aws_config::s3_client().await;
+
+ tracing::trace!("initialized s3 client");
+
+ let search_event_queue = macro_queues::SearchEventQueue::new();
+ let document_delete_queue = macro_queues::DocumentDeleteQueue::new();
+ let contacts_queue = macro_queues::ContactsQueue::new();
+ let notification_queue = macro_queues::NotificationIngressQueue::new();
+ let gmail_ops_queue = macro_queues::GmailOpsQueue::new();
+ let reminder_dispatch_queue = macro_queues::ReminderDispatchQueue::new();
+ let calendar_reminder_dispatch_queue = macro_queues::CalendarReminderDispatchQueue::new();
+ let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(&aws_config))
+ .search_event_queue(&search_event_queue)
+ .document_delete_queue(&document_delete_queue)
+ .gmail_ops_queue(&gmail_ops_queue);
+ let webhook_event_queue = webhook::outbound::SqsWebhookQueue::new(
+ Arc::new(sqs_client.clone()),
+ macro_queues::WebhookEventQueue::new().to_string(),
+ config.webhook_queue_max_messages,
+ config.webhook_queue_wait_time_seconds,
+ );
+
+ let contacts_ingress = Arc::new(contacts::domain::service::SqsContactsIngress {
+ queue: contacts::outbound::ingress::SqsContactsQueue::new(
+ aws_sdk_sqs::Client::new(&aws_config),
+ contacts_queue.to_string(),
+ ),
+ });
+
+ tracing::trace!("initialized sqs client");
+
+ // Redis handles it own connection pool internally. Each time we use redis
+ // we should be using redis_client.get_connection() to grab a specific connection
+ let redis_client = redis::Client::open(config.redis_uri.to_string().as_ref())
+ .expect("could not connect to redis client");
+
+ match redis_client.get_connection().is_err() {
+ true => {
+ tracing::error!("unable to connect to redis");
+ }
+ false => {
+ tracing::trace!("initialized redis connection");
+ }
+ }
+
+ let dss_auth_key = DocumentStorageServiceAuthKey::new()?;
+
+ let conn_gateway_client = ConnectionGatewayClient::new(
+ config.internal_api_key.to_string(),
+ ConnectionGatewayUrl::new()?.to_string(),
+ );
+
+ let sync_service_client = Arc::new(SyncServiceClient::new(
+ config.sync_service_auth_key.as_ref().to_string(),
+ SyncServiceUrl::new()?.to_string(),
+ ));
+
+ let lexical_client = Arc::new(LexicalClient::new(
+ config.internal_api_key.to_string(),
+ LexicalServiceUrl::new()?.to_string(),
+ ));
+
+ let jwt_validation_args =
+ JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)
+ .await?;
+
+ let consumer_cancellation_token = CancellationToken::new();
+ let consumer_tracker = TaskTracker::new();
+ let event_broker_tracker = TaskTracker::new();
+ let macro_event_broker = MacroEventBrokerService::new(
+ KafkaEventPublisher::new(config.kafka_brokers.as_ref())
+ .context("failed to create kafka event publisher")?,
+ event_broker_tracker.clone(),
+ );
+ let bots_repo = PgBotsRepo::new(db.clone());
+ // The agent API checks a newly selected MCP app against Pipedream's
+ // directory, the same check the ConfigureAgent tool makes.
+ let bots_service = BotServiceImpl::new(bots_repo, macro_event_broker.clone()).with_mcp_apps(
+ ai_tools::PipedreamMcpAppCatalog::new(ai_tools::pipedream_client_from_env()?),
+ );
+
+ let authorization_service: AuthorizationService = MacroAuthorizationServiceImpl::new(
+ MacroAuthJwtValidator::new(jwt_validation_args.clone()),
+ InternalAuthConfig {
+ api_key: dss_auth_key.as_ref().to_string(),
+ default_user_id: Some(MACRO_INTERNAL_USER_ID.to_string()),
+ },
+ PgBotAuthorizer::new(PgBotAuthorizationRepo::new(db.clone())),
+ PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),
+ )
+ .with_harness_authorizer(PgHarnessAuthorizer::new(PgHarnessAuthorizationRepo::new(
+ db.clone(),
+ )));
+ let authorization_state = MacroAuthorizationState::new(Arc::new(authorization_service));
+ let harnesses_service =
+ harnesses::domain::service::HarnessServiceImpl::new(PgHarnessRepo::new(db.clone()));
+
+ // Initialize OpenSearch client
+ let opensearch_client = OpensearchClient::new(
+ config.opensearch_url.to_string(),
+ config.opensearch_username.to_string(),
+ config.opensearch_password.to_string(),
+ )
+ .context("unable to create opensearch client")?;
+
+ if let Err(e) = opensearch_client.health().await {
+ tracing::error!(error=?e, "error connecting to opensearch");
+ return Err(e);
+ }
+ tracing::trace!("initialized opensearch client");
+
+ let frecency_storage = FrecencyPgStorage::new(db.clone());
+ let frecency_service = FrecencyQueryServiceImpl::new(frecency_storage.clone());
+ // DSS only reads CRM rows — no-op resolver keeps reqwest/scraper
+ // out of this binary.
+ let crm_service = crm::domain::service::CrmServiceImpl::new(
+ crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),
+ crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,
+ );
+ let email_service = EmailServiceImpl::new(
+ EmailPgRepo::new(db.clone()),
+ frecency_service.clone(),
+ sqs_client.clone(),
+ crm_service.clone(),
+ entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
+ entity_access_management::outbound::PgRepository::new(db.clone()),
+ ),
+ 0,
+ )
+ .with_macro_event_broker(macro_event_broker.clone());
+ let readonly_email_service = ReadonlyEmailPreviewAdapter(
+ EmailServiceImpl::new(
+ EmailPgRepo::new(readonly_db.clone()),
+ frecency_service.clone(),
+ sqs_client.clone(),
+ crm_service.clone(),
+ entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
+ entity_access_management::outbound::PgRepository::new(readonly_db.clone()),
+ ),
+ 0,
+ )
+ .with_macro_event_broker(macro_event_broker.clone()),
+ );
+ let system_properties_service =
+ SystemPropertiesServiceImpl::new(PgSystemPropertiesRepository::new(db.clone()));
+ let ingress_queue = SqsQueue::new(
+ aws_sdk_sqs::Client::new(&aws_config),
+ notification_queue.to_string(),
+ );
+ let notification_ingress_service = Arc::new(SqsNotificationIngress {
+ queue: ingress_queue.clone(),
+ });
+ let graphql_notification_reader: Arc<ai_tools::ToolNotificationService> =
+ Arc::new(NotificationReaderService {
+ repository: notification::outbound::repository::DbNotificationRepository::new(
+ db.clone(),
+ ),
+ queue: ai_tools::ToolNotificationQueue::Sqs(ingress_queue.clone()),
+ sns_endpoint: ai_tools::NoOpSnsEndpointManager,
+ platform_config: PlatformArnConfig {
+ apns_platform_arn: String::new(),
+ fcm_platform_arn: String::new(),
+ apns_voip_platform_arn: String::new(),
+ },
+ realtime: notification::domain::ports::NoopNotificationRealtimePublisher,
+ });
+ tracing::trace!("initialized notification ingress service");
+
+ let entity_access_service = Arc::new(
+ entity_access::domain::service::EntityAccessServiceImpl::new(
+ entity_access::outbound::PgAccessRepository::new(db.clone()),
+ ),
+ );
+
+ let permission_checker = PermissionServiceImpl::new(db.clone(), entity_access_service.clone());
+ let notification_service = NotificationServiceImpl::new(
+ SqsNotificationIngress {
+ queue: ingress_queue,
+ },
+ db.clone(),
+ );
+ let properties_service = Arc::new(
+ PropertiesServiceImpl::new(
+ PropertiesPgRepo::new(db.clone()),
+ Some(permission_checker),
+ Some(notification_service),
+ )
+ .with_event_broker(macro_event_broker.clone())
+ .with_initiative_assignees(Arc::new(
+ initiative::domain::assignees::InitiativeAssignees::new(PgInitiativeRepo::new(
+ db.clone(),
+ )),
+ )),
+ );
+
+ // Create the channel list service used by soup.
+ // Create the legacy channel list router state for routes mounted under /comms.
+ let channel_list_state = ChannelListRouterState::new(
+ ChannelListServiceImpl::new(
+ PgChannelsRepo::new(db.clone()),
+ PgChannelsRepo::new(db.clone()),
+ frecency_storage.clone(),
+ ),
+ authorization_state.clone(),
+ );
+
+ let s3 = Arc::new(S3::new(
+ s3_client,
+ config.document_storage_bucket.as_ref(),
+ config.docx_document_upload_bucket.as_ref(),
+ config.upload_staging_bucket.as_ref(),
+ ));
+ let system_properties_service = Arc::new(system_properties_service);
+
+ let owned_entity_registrar =
+ OwnedEntityRegistrar::new(OwnerGrantPolicy::new(PgBotsRepo::new(db.clone())));
+ let document_repo = PgDocumentRepo::new(db.clone(), owned_entity_registrar.clone());
+ let cloudfront_config = CloudFrontConfig {
+ distribution_url: config
+ .document_storage_service_cloudfront_distribution_url
+ .as_ref()
+ .to_string(),
+ signer_public_key_id: config
+ .document_storage_service_cloudfront_signer_public_key_id
+ .as_ref()
+ .to_string(),
+ signer_private_key: config
+ .document_storage_service_cloudfront_signer_private_key
+ .as_ref()
+ .to_string(),
+ presigned_url_expiry_seconds: config.document_storage_service_presigned_url_expiry_seconds,
+ browser_cache_expiry_seconds: config
+ .document_storage_service_presigned_url_browser_cache_expiry_seconds,
+ };
+ let s3_upload_adapter = S3UploadUrlAdapter::new(
+ macro_aws_config::s3_client().await,
+ config.document_storage_bucket.as_ref(),
+ config.docx_document_upload_bucket.as_ref(),
+ );
+ let markdown_initializer =
+ documents_hex::outbound::markdown_init::LexicalSyncMarkdownInitializer::new(
+ lexical_client.as_ref().clone(),
+ sync_service_client.as_ref().clone(),
+ );
+
+ let connection_gateway = Arc::new(ConnectionGatewayImpl::new(conn_gateway_client.clone()));
+
+ let connection_service =
+ ConnectionServiceImpl::new(entity_access_service.clone(), connection_gateway.clone());
+
+ let entity_access_management_service =
+ entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
+ entity_access_management::outbound::PgRepository::new(db.clone()),
+ );
+
+ let chat_mutation_service =
+ Arc::new(chat::domain::service::ChatServiceImpl::new_without_tools(
+ chat::outbound::postgres::PgChatRepo::new(db.clone(), owned_entity_registrar.clone()),
+ entity_access_management_service.clone(),
+ ));
+
+ let project_service = Arc::new(ProjectServiceImpl::new(
+ PgProjectRepo::new(db.clone(), owned_entity_registrar.clone()),
+ S3ProjectUploadAdapter::new(
+ macro_aws_config::s3_client().await,
+ config.document_storage_bucket.as_ref(),
+ config.docx_document_upload_bucket.as_ref(),
+ config.upload_staging_bucket.as_ref(),
+ ),
+ DynamoBulkUploadAdapter::new(dynamodb_client.clone()),
+ ShaCountAdapter::new(Redis::new(redis_client.clone())),
+ entity_access_management_service.clone(),
+ SqsProjectSearchIndexer::new(Arc::new(sqs_client.clone()), macro_event_broker.clone()),
+ if cfg!(feature = "local") {
+ Some(uuid::uuid!("d50676e2-0a12-4c62-bc07-4b1cb6d8e9bc"))
+ } else {
+ None
+ },
+ macro_event_broker.clone(),
+ ));
+
+ let document_service = Arc::new(DocumentServiceImpl::new(
+ document_repo,
+ cloudfront_config,
+ sync_service_client.as_ref().clone(),
+ s3_upload_adapter,
+ TaskPropertiesAdapter {
+ system_properties: system_properties_service.clone(),
+ properties: properties_service.clone(),
+ entity_access_service: entity_access_service.clone(),
+ },
+ connection_service,
+ entity_access_management_service.clone(),
+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
+ macro_event_broker.clone(),
+ ));
+
+ let foreign_entity_service = Arc::new(ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(
+ db.clone(),
+ )));
+
+ let github_sync_service_impl = GithubSyncServiceImpl::new(
+ GithubSyncConfig {
+ webhook_secret: config.github_webhook_secret_key.as_ref().to_string(),
+ github_sync_app_url: config.github_sync_app_url.to_string(),
+ sync_app_pem: config.github_sync_app_pem_secret_key.as_ref().to_string(),
+ sync_app_client_id: config.github_sync_app_client_id.to_string(),
+ sync_app_client_secret: config.github_sync_app_client_secret.to_string(),
+ installation_state_secret: config.github_installation_state_secret.to_string(),
+ },
+ document_service.clone(),
+ Arc::new(GithubPullRequestServiceImpl::new(
+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
+ PgGithubPullRequestRepo::new(db.clone()),
+ )),
+ (*notification_ingress_service).clone(),
+ PgGithubSyncRepo::new(db.clone()),
+ GithubSyncClientImpl::default(),
+ ConnectionGatewayGithubRealtime::new(conn_gateway_client.clone()),
+ );
+
+ let github_pull_request_index_state = PullRequestIndexRouterState {
+ service: Arc::new(PullRequestIndexService::new(
+ InstallationTokenConfig {
+ client_id: config.github_sync_app_client_id.to_string(),
+ private_key_pem: config.github_sync_app_pem_secret_key.as_ref().to_string(),
+ },
+ PgGithubSyncRepo::new(db.clone()),
+ GithubSyncClientImpl::default(),
+ GithubPullRequestServiceImpl::new(
+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
+ PgGithubPullRequestRepo::new(db.clone()),
+ ),
+ )),
+ authorization_state: authorization_state.clone(),
+ };
+
+ let github_pull_request_state = GithubPullRequestRouterState::new(
+ Arc::new(GithubPullRequestServiceImpl::new(
+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
+ PgGithubPullRequestRepo::new(db.clone()),
+ )),
+ entity_access_service.clone(),
+ authorization_state.clone(),
+ );
+
+ let github_pull_request_changes_state = GithubPullRequestChangesRouterState::new(
+ Arc::new(GithubPullRequestChangesServiceImpl::new(
+ GithubPullRequestServiceImpl::new(
+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
+ PgGithubPullRequestRepo::new(db.clone()),
+ ),
+ GithubPullRequestChangesetStore::new(
+ GithubPullRequestDiffClient::new(InstallationTokenService::new(
+ InstallationTokenConfig {
+ client_id: config.github_sync_app_client_id.to_string(),
+ private_key_pem: config.github_sync_app_pem_secret_key.as_ref().to_string(),
+ },
+ PgGithubSyncRepo::new(db.clone()),
+ GithubSyncClientImpl::default(),
+ )),
+ PgGithubPullRequestRepo::new(db.clone()),
+ S3GithubPullRequestPatchStore::new(
+ macro_aws_config::s3_client().await,
+ config.github_pull_request_patch_bucket.to_string(),
+ ),
+ ),
+ )),
+ entity_access_service.clone(),
+ authorization_state.clone(),
+ );
+
+ let foreign_entity_state = ForeignEntityRouterState::new(
+ foreign_entity_service.clone(),
+ entity_access_service.clone(),
+ authorization_state.clone(),
+ );
+
+ // Cal.com webhooks → Meta Lead events. Both secrets are loaded here
+ // (rather than on Config) to keep cal/Meta wiring colocated.
+ let cal_webhook_secret = secretsmanager_client
+ .get_maybe_secret_value(env, CalWebhookSecretKey::new()?)
+ .await?;
+ let cal_event_type_content_names_secret = secretsmanager_client
+ .get_maybe_secret_value(env, CalEventTypeContentNamesKey::new()?)
+ .await?;
+
+ let analytics_client = Arc::new(AnalyticsClient::new(AnalyticsClientConfig {
+ google_analytics: None,
+ meta: Some(MetaConfig {
+ pixel_id: MetaPixelId::new()?.as_ref().to_string(),
+ access_token: MetaAccessToken::new()?.as_ref().to_string(),
+ test_event_code: MetaTestEventCode::new().map(|v| v.as_ref().to_string()),
+ }),
+ posthog: None,
+ }));
+
+ let cal_event_type_meta: std::collections::HashMap<u64, CalEventMeta> =
+ serde_json::from_str(cal_event_type_content_names_secret.as_ref())
+ .context("CalEventTypeContentNames secret must be a JSON object mapping eventTypeId (u64) to { content_name: string, value: number (USD) }")?;
+ let cal_webhook_service = CalWebhookServiceImpl::new(
+ CalConfig {
+ webhook_secret: cal_webhook_secret.as_ref().to_string(),
+ event_type_meta: cal_event_type_meta,
+ },
+ AnalyticsClientSink::new(analytics_client.clone()),
+ );
+ let cal_webhook_state = CalWebhookRouterState::new(cal_webhook_service);
+
+ // Call service (LiveKit)
+ let transcription_agent_name = config
+ .livekit_transcription_agent_name
+ .value()
+ .map(|v| v.to_owned());
+
+ let internal_call_secret = config.internal_call_secret.value().map(|v| v.to_owned());
+
+ anyhow::ensure!(
+ transcription_agent_name.is_none() || internal_call_secret.is_some(),
+ "LIVEKIT_TRANSCRIPTION_AGENT_NAME is set but INTERNAL_CALL_SECRET is missing — \
+ the transcription agent will not be able to submit transcripts"
+ );
+ let livekit_rtc_client = LivekitRtcClient::new(
+ config.livekit_server_url.as_ref(),
+ config.livekit_api_key.as_ref(),
+ config.livekit_api_secret.as_ref(),
+ transcription_agent_name,
+ );
+ let call_connection_service =
+ ConnectionServiceImpl::new(entity_access_service.clone(), connection_gateway.clone());
+ let call_repo = PgCallRepo::new(db.clone());
+ let egress_config = match (
+ config::CallRecordingS3Bucket::new(),
+ config::CallRecordingS3Region::new(),
+ config::CallRecordingS3AccessKey::new(),
+ config::CallRecordingS3Secret::new(),
+ ) {
+ (Some(bucket), Some(region), Some(access_key), Some(secret)) => {
+ tracing::info!(bucket = bucket.as_ref(), "call recording enabled");
+ Some(call::domain::models::EgressS3Config {
+ bucket: bucket.as_ref().to_string(),
+ region: region.as_ref().to_string(),
+ access_key: access_key.as_ref().to_string(),
+ secret: secret.as_ref().to_string(),
+ })
+ }
+ _ => None,
+ };
+ let recording_storage = match &egress_config {
+ Some(egress_config) => {
+ let cloudfront_config = RecordingCloudFrontConfig {
+ distribution_url: config
+ .document_storage_service_cloudfront_distribution_url
+ .as_ref()
+ .to_string(),
+ signer_public_key_id: config
+ .document_storage_service_cloudfront_signer_public_key_id
+ .as_ref()
+ .to_string(),
+ signer_private_key: config
+ .document_storage_service_cloudfront_signer_private_key
+ .as_ref()
+ .to_string(),
+ presigned_url_expiry_seconds: config::CALL_RECORDING_PRESIGNED_URL_EXPIRY_SECONDS,
+ };
+ Some(S3RecordingStorage::new(egress_config.bucket.clone(), cloudfront_config).await)
+ }
+ None => None,
+ };
+ let mut call_service_builder = CallServiceImpl::<_, _, _, _, _, _, AiCallSummarizer>::new(
+ call_repo,
+ livekit_rtc_client,
+ call_connection_service,
+ (*entity_access_service).clone(),
+ (*notification_ingress_service).clone(),
+ recording_storage,
+ config.livekit_server_url.as_ref(),
+ )
+ .with_summarizer(AiCallSummarizer::new(
+ ai_usage::pg_recorder_with_enforcement(db.clone(), config.enable_ai_usage_enforcement),
+ ));
+ if let Some(secret) = internal_call_secret {
+ call_service_builder = call_service_builder.with_internal_call_secret(secret);
+ }
+ if let Some(config) = egress_config {
+ call_service_builder = call_service_builder.with_egress(config);
+ }
+ if let Some(base_url) = config::CallRingStatusBaseUrl::new() {
+ call_service_builder =
+ call_service_builder.with_ring_status_base_url(base_url.as_ref().to_owned());
+ }
+
+ // VoIP push is optional: enabled only when both env vars are present.
+ // APPLE_BUNDLE_ID: the app bundle ID (e.g. "com.macro.app.prod")
+ // SNS_APNS_VOIP_PLATFORM_ARN: SNS platform app ARN for APNS_VOIP
+ //
+ // Option<VoipPushServiceImpl<...>> is used as the type parameter so the
+ // type stays stable regardless of whether VoIP push is configured.
+ let voip_sender = if let (Some(bundle_id), Some(voip_arn)) =
+ (AppleBundleId::new(), SnsApnsVoipPlatformArn::new())
+ {
+ if voip_arn.is_empty() {
+ tracing::warn!("voip push disabled: SNS_APNS_VOIP_PLATFORM_ARN is set but empty");
+ None
+ } else {
+ let voip_repo =
+ notification::outbound::repository::DbNotificationRepository::new(db.clone());
+ let voip_mobile = notification::outbound::mobile::MobilePushAdapter {
+ push_service: aws_sdk_sns::Client::new(&aws_config),
+ apns_bundle_id: bundle_id.to_string(),
+ voip_bundle_id: Some(format!("{}.voip", bundle_id.as_ref())),
+ };
+ tracing::info!(
+ bundle_id = bundle_id.as_ref(),
+ voip_arn = voip_arn.as_ref(),
+ "voip push enabled"
+ );
+ Some(notification::domain::service::VoipPushServiceImpl::new(
+ voip_repo,
+ voip_mobile,
+ ))
+ }
+ } else {
+ tracing::info!("voip push disabled: APPLE_BUNDLE_ID or SNS_APNS_VOIP_PLATFORM_ARN not set");
+ None
+ };
+ let call_service_builder = call_service_builder.with_voip_push_sender(voip_sender);
+
+ let call_service = Arc::new(
+ call_service_builder
+ .with_voice_repo(PgVoiceRepo::new(db.clone()))
+ .with_event_broker(macro_event_broker.clone()),
+ );
+
+ consumer_tracker.spawn({
+ let service = call_service.clone();
+ let cancellation_token = consumer_cancellation_token.clone();
+ async move {
+ cancellation_token
+ .run_until_cancelled(call::inbound::stale_call_sweeper::run_stale_call_sweeper(
+ service,
+ call::inbound::stale_call_sweeper::SWEEP_INTERVAL,
+ ))
+ .await;
+ }
+ });
+
+ let call_state = CallRouterState::new(
+ call_service.clone(),
+ entity_access_service.clone(),
+ authorization_state.clone(),
+ );
+ let call_webhook_state = WebhookRouterState::new(call_service.clone());
+ let call_public_rate_limiter = RateLimitServiceImpl {
+ repo: RedisRateLimitAdapter {
+ redis: redis_client.clone(),
+ },
+ };
+
+ let webhook_repository = webhook::outbound::PgRepository::new(db.clone());
+ let webhook_endpoint_scheme_policy = if matches!(env, Environment::Local) {
+ webhook::domain::models::WebhookEndpointSchemePolicy::HttpAndHttps
+ } else {
+ webhook::domain::models::WebhookEndpointSchemePolicy::HttpsOnly
+ };
+ let webhook_http_client =
+ webhook::outbound::ReqwestWebhookDeliveryClient::new_with_endpoint_scheme_policy(
+ webhook_endpoint_scheme_policy,
+ )
+ .context("failed to create webhook HTTP client")?;
+ let webhook_service =
+ webhook::domain::service::WebhookServiceImpl::new_with_endpoint_scheme_policy(
+ webhook_repository.clone(),
+ webhook_http_client.clone(),
+ webhook_endpoint_scheme_policy,
+ macro_event_broker.clone(),
+ );
+ let webhook_rate_limiter = RateLimitServiceImpl {
+ repo: RedisRateLimitAdapter {
+ redis: redis_client.clone(),
+ },
+ };
+ let webhook_state = webhook::inbound::axum_router::WebhookRouterState::new(
+ webhook_service,
+ webhook_rate_limiter,
+ authorization_state.clone(),
+ );
+
+ let (webhook_stream_sender, _) =
+ tokio::sync::broadcast::channel(webhook::domain::stream::WEBHOOK_STREAM_CHANNEL_CAPACITY);
+ let sse_stream_service = webhook::domain::stream::WebhookEventStreamServiceImpl::new(
+ webhook_stream_sender.clone(),
+ entity_access_service.clone(),
+ webhook_repository.clone(),
+ );
+ let sse_stream_state = webhook::inbound::stream_router::WebhookStreamRouterState::new(
+ sse_stream_service,
+ authorization_state.clone(),
+ );
+ consumer_tracker.spawn({
+ let brokers = config.kafka_brokers.as_ref().to_string();
+ let sender = webhook_stream_sender;
+ let cancellation_token = consumer_cancellation_token.clone();
+ async move {
+ loop {
+ let result = tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => break,
+ result = webhook::inbound::kafka_stream_consumer::run_webhook_stream_consumer(
+ &brokers,
+ &sender,
+ ) => result,
+ };
+
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+ if let Err(error) = result {
+ tracing::error!(
+ error = ?error,
+ "webhook SSE Kafka consumer stopped"
+ );
+ tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => break,
+ _ = tokio::time::sleep(Duration::from_secs(1)) => {}
+ }
+ }
+ }
+ }
+ });
+
+ let webhook_ingestion_service =
+ webhook::domain::ingestion::WebhookEventIngestionServiceImpl::new(
+ entity_access_service.clone(),
+ webhook_repository,
+ webhook_event_queue.clone(),
+ );
+ let webhook_delivery_repository =
+ webhook::outbound::PgWebhookDeliveryRepository::new(db.clone());
+ let webhook_delivery_service = webhook::domain::delivery::WebhookEventDeliveryServiceImpl::new(
+ webhook_delivery_repository,
+ webhook_http_client,
+ );
+ let webhook_worker = webhook::inbound::worker::WebhookEventWorker::new(
+ webhook_event_queue,
+ webhook_delivery_service,
+ );
+ let webhook_worker_task = tokio::spawn(async move {
+ tracing::info!("starting webhook event worker");
+ webhook_worker.run().await;
+ });
+ tokio::spawn(async move {
+ match webhook_worker_task.await {
+ Ok(()) => tracing::error!("webhook event worker exited unexpectedly"),
+ Err(error) => {
+ tracing::error!(error = ?error, "webhook event worker exited unexpectedly");
+ }
+ }
+ });
+
+ let webhook_consumer_brokers = config.kafka_brokers.as_ref().to_string();
+ consumer_tracker.spawn({
+ let cancellation_token = consumer_cancellation_token.clone();
+ async move {
+ loop {
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+
+ tracing::info!("starting webhook event consumer");
+ let result = webhook::inbound::kafka_consumer::run_webhook_event_consumer(
+ &webhook_consumer_brokers,
+ webhook_ingestion_service.clone(),
+ cancellation_token.cancelled(),
+ )
+ .await;
+
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+
+ match result {
+ Ok(()) => tracing::error!("webhook event consumer exited unexpectedly"),
+ Err(error) => {
+ tracing::error!(error = ?error, "webhook event consumer exited unexpectedly");
+ }
+ }
+
+ tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => break,
+ _ = tokio::time::sleep(Duration::from_secs(5)) => {}
+ }
+ }
+ }
+ });
+
+ let activity_consumer_brokers = config.kafka_brokers.as_ref().to_string();
+ let editing_activity = Arc::new(
+ documents_hex::outbound::editing_activity::RedisEditingActivityStore::new(
+ redis_client.clone(),
+ ),
+ );
+ consumer_tracker.spawn({
+ let cancellation_token = consumer_cancellation_token.clone();
+ let activity_repo = activity::outbound::pg_activity_repo::PgActivityRepo::new(db.clone());
+ let activity_realtime = activity::domain::announcements::ActivityAnnouncements::new(
+ activity::KafkaActivityRealtimePublisher::new(macro_event_broker.clone()),
+ crate::service::activity::EntityAccessActivityAudience::new(
+ entity_access_service.clone(),
+ ),
+ );
+ async move {
+ let consumer = activity::inbound::kafka_consumer::ActivityConsumer::<
+ _,
+ crate::service::activity::ActivitySourceEvent,
+ _,
+ _,
+ >::new(
+ activity_repo,
+ move |event| {
+ let editing_activity = editing_activity.clone();
+ async move {
+ crate::service::activity::ingest(event, editing_activity.as_ref()).await
+ }
+ },
+ activity_realtime,
+ );
+ loop {
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+
+ tracing::info!("starting activity consumer");
+ let result = consumer
+ .run(&activity_consumer_brokers, cancellation_token.cancelled())
+ .await;
+
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+
+ match result {
+ Ok(()) => tracing::error!("activity consumer exited unexpectedly"),
+ Err(error) => {
+ tracing::error!(error = ?error, "activity consumer exited unexpectedly");
+ }
+ }
+
+ tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => break,
+ _ = tokio::time::sleep(Duration::from_secs(5)) => {}
+ }
+ }
+ }
+ });
+
+ // Links archived calls to the CRM records of the people on them.
+ let crm_call_link_brokers = config.kafka_brokers.as_ref().to_string();
+ consumer_tracker.spawn({
+ let cancellation_token = consumer_cancellation_token.clone();
+ let consumer = crm::inbound::call_archived::CallArchivedConsumer::new(
+ crm::domain::call_links::CallRecordLinker::new(
+ call::domain::service::CallRecordQueryServiceImpl::new(PgCallRepo::new(
+ db.clone(),
+ )),
+ crm::outbound::call_link::PgCallCrmLinker::new(
+ db.clone(),
+ (*system_properties_service).clone(),
+ ),
+ ),
+ );
+ async move {
+ loop {
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+
+ tracing::info!("starting CRM call-link consumer");
+ let result = consumer
+ .run(&crm_call_link_brokers, cancellation_token.cancelled())
+ .await;
+
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+
+ match result {
+ Ok(()) => tracing::error!("CRM call-link consumer exited unexpectedly"),
+ Err(error) => {
+ tracing::error!(error = ?error, "CRM call-link consumer exited unexpectedly");
+ }
+ }
+
+ tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => break,
+ _ = tokio::time::sleep(Duration::from_secs(5)) => {}
+ }
+ }
+ }
+ });
+
+ let call_internal_state = InternalCallRouterState::new(call_service.clone());
+
+ // Create the SQS worker for delete document processing before config is moved.
+ #[cfg(feature = "delete_document_worker")]
+ let delete_document_worker = sqs_worker::SQSWorker::new(
+ aws_sdk_sqs::Client::new(&aws_config),
+ document_delete_queue.to_string(),
+ config.queue_max_messages,
+ config.queue_wait_time_seconds,
+ );
+
+ let sqs_client = Arc::new(sqs_client);
+ let conn_gateway_client = Arc::new(conn_gateway_client);
+
+ // MacroConfig reads the shared OPENAI_API_KEY from Doppler's APP_SECRETS_JSON
+ // or the local environment. Validate once before constructing consumers.
+ let openai_api_key = config.openai_api_key.as_ref().to_owned();
+ anyhow::ensure!(
+ !openai_api_key.trim().is_empty(),
+ "OpenAI API key is required for task dedup embeddings and dictation",
+ );
+ let cohere_api_key = config.cohere_api_key.as_ref().to_owned();
+ anyhow::ensure!(
+ !cohere_api_key.trim().is_empty(),
+ "Cohere API key is required for task dedup reranking",
+ );
+ let dictation_state = dictation::inbound::axum_router::DictationRouterState::new(
+ dictation::domain::DictationServiceImpl::new(
+ dictation::outbound::WhisperTranscriber::new(&config.openai_api_key)?,
+ dictation::outbound::SymphoniaRecordingInspector,
+ ai_usage::pg_recorder_with_enforcement(db.clone(), config.enable_ai_usage_enforcement),
+ ),
+ RateLimitServiceImpl {
+ repo: RedisRateLimitAdapter {
+ redis: redis_client.clone(),
+ },
+ },
+ authorization_state.clone(),
+ );
+ let task_dedup_service = Arc::new(TaskDedupService::new(
+ TextEmbedding3Small::new(openai_api_key),
+ PgTaskVectorDb::new(db.clone()),
+ CohereReranker::new(cohere_api_key),
+ Arc::new(AgentDuplicateJudge::new(
+ ai_usage::pg_recorder_with_enforcement(db.clone(), config.enable_ai_usage_enforcement),
+ )),
+ Arc::new(ConnectionGatewayTaskDedupNotifier::new(
+ conn_gateway_client.clone(),
+ )),
+ Arc::new(PgTaskMatchRepo::new(db.clone())),
+ ));
+ let channels_repo = PgChannelsRepo::new(db.clone());
+ // Built-in agents read the same committed-post stream as hosted and
+ // external agents, for channel and document posts alike, so the channel
+ // side effects no longer carry their own trigger queue.
+ let (bot_trigger_sender, bot_trigger_receiver) = tokio::sync::mpsc::unbounded_channel();
+
+ let channel_side_effects = ChannelSideEffectService::new(
+ PgChannelSideEffectContext::new(db.clone()),
+ ConnectionGatewayChannelRealtimePublisher::new(conn_gateway_client.clone()),
+ NotificationChannelSender::new(notification_ingress_service.clone()),
+ ContactsChannelDispatcher::new(contacts_ingress.clone()),
+ )
+ .with_macro_event_broker(macro_event_broker.clone());
+
+ let channels_service = Arc::new(
+ ChannelServiceImpl::with_dependencies(
+ channels_repo,
+ SpawnedChannelEventDispatcher::new(channel_side_effects.clone()),
+ PgChannelReferenceSharePermissions::new(db.clone(), entity_access_service.clone()),
+ )
+ .with_picture_files(
+ channels::outbound::static_file_pictures::StaticFileChannelPictures::new(
+ static_file_service_client::StaticFileServiceClient::new(
+ config.internal_api_key.to_string(),
+ StaticFileServiceUrl::new()?.to_string(),
+ ),
+ ),
+ ),
+ );
+
+ // One message implementation for channels and documents. Committed changes fan
+ // out to the broker, the local agent queue, and the parent's own delivery: channel
+ // messages keep their notifications, activity, and sharing side effects and send
+ // the common `message_update` payload; document discussions get the common
+ // payload and document comment notifications.
+ let message_realtime = messages::outbound::connection_gateway::ConnectionGatewayMessages(
+ conn_gateway_client.clone(),
+ );
+ let discussion_delivery = messages::domain::delivery::DiscussionDelivery::new(
+ messages::outbound::pg_discussion_context::PgDiscussionContext(db.clone())
+ .with_initiatives(
+ initiative::domain::lookup::InitiativeLookup::new(
+ initiative::outbound::PgInitiativeRepo::new(db.clone()),
+ ),
+ properties_service.clone(),
+ )
+ .with_crm(crm::outbound::lookup::PgCrmParentReader::new(db.clone())),
+ messages::outbound::entity_access_audience::EntityAccessMessageAudience(
+ (*entity_access_service).clone(),
+ ),
+ message_realtime.clone(),
+ messages::outbound::notification_sender::MessageNotificationSender(
+ notification_ingress_service.clone(),
+ ),
+ )
+ .with_sharing(messages::outbound::pg_discussion_context::PgDiscussionContext(db.clone()));
+ let annotation_service = Arc::new(messages::domain::annotations::AnnotationService::new(
+ macro_db_client::annotations::repository::PgAnnotationRepository(db.clone()),
+ (*entity_access_service).clone(),
+ discussion_delivery.clone(),
+ ));
+ let channel_delivery = channels::domain::message_delivery::ChannelMessageDelivery::new(
+ PgChannelsRepo::new(db.clone()),
+ SpawnedChannelEventDispatcher::new(channel_side_effects.clone()),
+ PgChannelReferenceSharePermissions::new(db.clone(), entity_access_service.clone()),
+ message_realtime,
+ );
+ let message_service = Arc::new(
+ messages::domain::service::MessageService::new(
+ messages::outbound::pg_message_repo::PgMessageRepository::new(db.clone())
+ .with_initiatives(initiative::domain::lookup::InitiativeLookup::new(
+ initiative::outbound::PgInitiativeRepo::new(db.clone()),
+ ))
+ .with_crm(crm::outbound::lookup::PgCrmParentReader::new(db.clone())),
+ messages::domain::effects::MessageEffects::new(
+ messages::outbound::broker::BrokerMessagePublisher::new(macro_event_broker.clone()),
+ channel_bots::outbound::conversation::LocalBotPublisher::new(bot_trigger_sender),
+ messages::domain::delivery::ParentMessagePublisher::new(
+ channel_delivery,
+ discussion_delivery,
+ ),
+ ),
+ )
+ .with_group_recipients(channels::domain::group_mentions::ChannelGroupRecipients(
+ PgChannelsRepo::new(db.clone()),
+ ))
+ .with_mention_extractor(lexical_mention_extractor::LexicalMentionExtractor::new(
+ lexical_client.clone(),
+ ))
+ .with_references(
+ messages::outbound::entity_access_audience::EntityAccessMessageReferences(
+ (*entity_access_service).clone(),
+ ),
+ ),
+ );
+ let message_commands: Arc<dyn messages::domain::api::MessageCommands> = message_service.clone();
+ let messages_state = messages::inbound::axum_router::MessagesRouterState {
+ service: message_service.clone(),
+ access: entity_access_service.clone(),
+ authorization: authorization_state.clone(),
+ };
+
+ let teammate_dms_brokers = config.kafka_brokers.as_ref().to_string();
+ consumer_tracker.spawn({
+ let cancellation_token = consumer_cancellation_token.clone();
+ let channels = (*channels_service).clone();
+ async move {
+ loop {
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+
+ tracing::info!("starting teammate DM consumer");
+ let result = channels::inbound::teammate_dms_consumer::run_teammate_dms_consumer(
+ &teammate_dms_brokers,
+ channels.clone(),
+ cancellation_token.cancelled(),
+ )
+ .await;
+
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+
+ match result {
+ Ok(()) => tracing::error!("teammate DM consumer exited unexpectedly"),
+ Err(error) => {
+ tracing::error!(
+ error = ?error,
+ "teammate DM consumer exited unexpectedly"
+ );
+ }
+ }
+
+ tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => break,
+ _ = tokio::time::sleep(Duration::from_secs(5)) => {}
+ }
+ }
+ }
+ });
+
+ // Wire Macro AI to react to mentions with the classic in-channel chat
+ // reply. The router posts replies through the channel service we just
+ // built and runs the agent loop in-process with the channel-bot toolset:
+ // a channel has no composer to finish a chat-deferred user tool in, so
+ // calendar event creation executes directly and SendEmail is unavailable.
+ // Agent sessions belong to a different bot entirely
+ // (`bot_id::MACRO_NEW_BOT_ID`, served by the harness), so the two paths
+ // can never answer the same mention.
+ let mut macro_agent_tool_context = ai_tools::build_tool_service_context_from_env(
+ db.clone(),
+ event_broker_tracker.clone(),
+ config.enable_ai_usage_enforcement,
+ config.ai_pricing(),
+ )
+ .await
+ .context("failed to build Macro agent tool context")?;
+ // Wire the agent's SendChannelMessage tool to this service's own
+ // side-effect pipeline so agent-posted messages share the exact instance
+ // used by the HTTP API, including the in-process bot trigger sender (the
+ // env builder wires an equivalent pipeline, but without bot triggers).
+ macro_agent_tool_context.channel_tool_context =
+ ai_tools::build_channel_tool_context_with_dispatcher(
+ db.clone(),
+ std::sync::Arc::new(SpawnedChannelEventDispatcher::new(channel_side_effects)),
+ message_service.clone(),
+ );
+ let macro_agent_tools = ai_tools::tools_for(ai_tools::AiHost::ChannelBot);
+ let conversation_access = Arc::new(
+ channel_bots::outbound::conversation::EntityAccessConversation(
+ entity_access_service.clone(),
+ ),
+ );
+ let admission = macro_agent_tool_context.admission.clone();
+ let bot_trigger_router = channel_bots::inbound::BotTriggerRouter::new_with_admission(
+ message_service.clone(),
+ conversation_access.clone(),
+ Arc::new(channel_bots::outbound::AgentLoopResponder::new(
+ macro_agent_tool_context,
+ macro_agent_tools,
+ )),
+ Arc::new(
+ channel_bots::domain::trigger_detector::MentionOrInferredDetector::new(
+ message_service.clone(),
+ conversation_access,
+ Arc::new(channel_bots::outbound::FastModelTriggerClassifier::new(
+ ai_usage::pg_recorder_with_enforcement(
+ db.clone(),
+ config.enable_ai_usage_enforcement,
+ ),
+ )),
+ )
+ .with_admission(admission.clone()),
+ ),
+ Arc::new(channel_bots::outbound::PrimaryCalendarTimeZones::new(
+ Arc::new(calendar_events::domain::service::CalendarService::new(
+ calendar_events::outbound::pg::PgCalendarRepository::new(readonly_db.clone()),
+ )),
+ )),
+ Arc::new(channel_bots::outbound::LexicalCommentMarks::new(
+ (*lexical_client).clone(),
+ )),
+ admission,
+ );
+ bot_trigger_router.spawn(bot_trigger_receiver);
+
+ let channel_bot_webhook_state =
+ bots::inbound::channel_webhook_router::ChannelBotWebhookRouterState::new(
+ bots_service.clone(),
+ message_commands.clone(),
+ (*entity_access_service).clone(),
+ authorization_state.clone(),
+ );
+
+ let slack_limits = slack_integration::domain::models::ImportLimits::default();
+ let slack_enabled = config.slack_import_enabled;
+ let slack_service = api::context::SlackService::new(
+ slack_integration::outbound::pg_slack_import_repo::PgSlackImportRepo::new(
+ db.clone(),
+ slack_limits,
+ ),
+ slack_integration::outbound::s3_storage::S3ImportStorage::new(
+ macro_aws_config::s3_client().await,
+ config.upload_staging_bucket.as_ref().to_owned(),
+ slack_limits,
+ )
+ .map_err(|_| anyhow::anyhow!("invalid Slack import storage configuration"))?,
+ slack_integration::outbound::import_ledger::CanonicalImportLedger::new(
+ import::outbound::pg_import_repo::PgImportRepo::new(db.clone()),
+ ),
+ slack_integration::outbound::gateway_notifier::GatewayImportNotifier::new(
+ conn_gateway_client.clone(),
+ ),
+ api::context::SlackAdminAuthorizer(
+ entity_access_service.clone(),
+ entity_access::outbound::PgAccessRepository::new(db.clone()),
+ ),
+ Box::new(move |_| slack_enabled),
+ slack_limits,
+ )?;
+ let slack_state = slack_integration::inbound::axum_router::SlackRouterState {
+ service: Arc::new(slack_service),
+ entity_access_service: entity_access_service.clone(),
+ authorization_state: authorization_state.clone(),
+ };
+
+ // Held by value here and behind an `Arc` in the router state: the impl is a
+ // pool handle, so cloning is cheap and `SoupImpl` needs an owned service.
+ // Additional connections for guards spanning email I/O. HTTP requests and
+ // dispatch share this budget, independently of the main data pool.
+ let followup_lock_capacity = match config.environment {
+ Environment::Production => 32,
+ Environment::Develop => 16,
+ Environment::Local => 8,
+ };
+ let email_followups = reminders::domain::email_followup::service::EmailFollowupService::new(
+ PgRemindersRepo::with_followup_lock_capacity(db.clone(), followup_lock_capacity),
+ email_service.clone(),
+ );
+ let reminders_service =
+ reminders::domain::email_followup::reminder_service::EmailRemindersService::new(
+ email_followups.clone(),
+ )
+ .with_entity_access((*entity_access_service).clone());
+
+ let document_creator = documents_hex::domain::create::DocumentCreator::new(
+ document_service.clone(),
+ markdown_initializer,
+ documents_hex::outbound::document_bytes_upload::ReqwestDocumentBytesUploader::default(),
+ documents_hex::outbound::mention_tracker::LexicalCommsMentionTracker::new(
+ db.clone(),
+ lexical_client.clone(),
+ ),
+ );
+ // Forms' ids are their surfaces' ids, so the public surface API asks the
+ // forms domain which ids are taken.
+ let collab_surface_service = Arc::new(
+ collab_surface::outbound::pg_collab_surface_service(
+ db.clone(),
+ lexical_client.as_ref().clone(),
+ sync_service_client.as_ref().clone(),
+ config.document_permission_jwt.as_ref().to_string(),
+ )
+ .with_form_ids(Arc::new(
+ forms::outbound::collaborative_layout::RepositoryFormIds::new(Arc::new(
+ forms::outbound::pg_forms_repo::PgFormsRepo::new(db.clone()),
+ )),
+ )),
+ );
+
+ let initiative_service = Arc::new(
+ InitiativeServiceImpl::new(
+ PgInitiativeRepo::new(db.clone()),
+ initiative_description::InitiativeDescriptionSurfacesAdapter::new(
+ collab_surface_service.clone(),
+ ),
+ Arc::new(initiative::outbound::resources::ProjectResources::new(
+ properties_service.clone(),
+ system_properties_service.clone(),
+ entity_access_service.clone(),
+ )),
+ )
+ .with_event_publisher(Arc::new(
+ initiative::outbound::event_publisher::BrokerInitiativeEventPublisher::new(
+ macro_event_broker.clone(),
+ ),
+ )),
+ );
+
+ // Shared by the databases router and the unified entity-mutation router.
+ let databases_service = Arc::new(databases::wiring::build_service(
+ db.clone(),
+ entity_access_service.clone(),
+ databases::outbound::gateway_event_publisher::GatewayTableEventPublisher::new(
+ conn_gateway_client.as_ref().clone(),
+ ),
+ macro_event_broker.clone(),
+ ));
+ let forms_service = Arc::new(forms::wiring::build_service(
+ db.clone(),
+ databases_service.clone(),
+ entity_access_service.clone(),
+ forms::outbound::gateway_event_publisher::GatewayFormEventPublisher::new(
+ conn_gateway_client.as_ref().clone(),
+ ),
+ macro_event_broker.clone(),
+ collab_surface_service.clone(),
+ ));
+
+ // Individual initiative reads preserve read-after-write consistency when a
+ // newly created project opens immediately. Lists retain the replica reader.
+ // Reuse Soup hydration so detail and mutation metadata include viewer history.
+ let initiative_entity_soup = Arc::new(
+ SoupImpl::new(
+ PgSoupRepo::new(readonly_pool::ReadOnlyPool(db.clone())),
+ frecency_service.clone(),
+ ReadonlyEmailPreviewAdapter(email_service.clone()),
+ ChannelListServiceImpl::new(
+ PgChannelsRepo::new(db.clone()),
+ PgChannelsRepo::new(db.clone()),
+ frecency_storage.clone(),
+ ),
+ call::domain::service::CallRecordQueryServiceImpl::new(PgCallRepo::new(db.clone())),
+ crm_service.clone(),
+ GithubPullRequestServiceImpl::new(
+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
+ PgGithubPullRequestRepo::new(db.clone()),
+ ),
+ )
+ .with_agent_branches(agent_changes::outbound::postgres::PgChangesetRepo::new(
+ db.clone(),
+ )),
+ );
+
+ let favorites_service = Arc::new(FavoritesServiceImpl::new(PgFavoritesRepo::new(db.clone())));
+
+ // Keep the replica-backed Soup reader alongside the primary-backed email
+ // writer in SoupRouterState. REST, GraphQL lists, and realtime hydration
+ // share this reader; only email mutations and their reply loader use the
+ // writer service, so normal list traffic never switches to the primary.
+ let soup_service = Arc::new(
+ SoupImpl::new(
+ PgSoupRepo::new(readonly_pool::ReadOnlyPool(readonly_db.clone())),
+ frecency_service,
+ readonly_email_service,
+ ChannelListServiceImpl::new(
+ PgChannelsRepo::new(readonly_db.clone()),
+ PgChannelsRepo::new(readonly_db.clone()),
+ frecency_storage.clone(),
+ ),
+ call::domain::service::CallRecordQueryServiceImpl::new(PgCallRepo::new(
+ readonly_db.clone(),
+ )),
+ crm_service.clone(),
+ GithubPullRequestServiceImpl::new(
+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(readonly_db.clone())),
+ PgGithubPullRequestRepo::new(readonly_db.clone()),
+ ),
+ )
+ .with_favorites(favorites_service.clone())
+ .with_agent_branches(agent_changes::outbound::postgres::PgChangesetRepo::new(
+ readonly_db.clone(),
+ )),
+ );
+
+ let websocket_notification_consumer_service =
+ Arc::new(WebSocketNotificationConsumerService::new(
+ NotificationTopicConsumer::<model_notifications::NotifEvent>::from_env(
+ config.kafka_brokers.as_ref(),
+ )
+ .map_err(|error| {
+ anyhow::anyhow!("failed to create WebSocket notification topic consumer: {error:?}")
+ })?,
+ ));
+ consumer_tracker.spawn({
+ let service = Arc::clone(&websocket_notification_consumer_service);
+ let cancellation_token = consumer_cancellation_token.clone();
+ async move {
+ loop {
+ let result = tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => break,
+ result = service.run() => result,
+ };
+
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+
+ let _ = result.inspect_err(|error| {
+ tracing::error!(
+ error = ?error,
+ "WebSocket notification consumer stopped"
+ );
+ });
+
+ tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => break,
+ _ = tokio::time::sleep(Duration::from_secs(1)) => {}
+ }
+ }
+ }
+ });
+
+ let activity_realtime_service = Arc::new(activity::ActivityRealtimeConsumerService::new(
+ activity::ActivityTopicConsumer::from_env(config.kafka_brokers.as_ref()).map_err(
+ |error| anyhow::anyhow!("failed to create realtime activity topic consumer: {error:?}"),
+ )?,
+ ));
+ consumer_tracker.spawn({
+ let service = Arc::clone(&activity_realtime_service);
+ let cancellation_token = consumer_cancellation_token.clone();
+ async move {
+ loop {
+ let result = tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => break,
+ result = service.run() => result,
+ };
+
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+
+ let _ = result.inspect_err(|error| {
+ tracing::error!(
+ error = ?error,
+ "realtime activity subscription consumer stopped"
+ );
+ });
+
+ tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => break,
+ _ = tokio::time::sleep(Duration::from_secs(1)) => {}
+ }
+ }
+ }
+ });
+
+ let soup_realtime_service = Arc::new(SoupRealtimeConsumerService::new(
+ SoupTopicConsumer::from_env(config.kafka_brokers.as_ref()).map_err(|error| {
+ anyhow::anyhow!("failed to create realtime Soup topic consumer: {error:?}")
+ })?,
+ ));
+ consumer_tracker.spawn({
+ let soup_realtime_service = Arc::clone(&soup_realtime_service);
+ let cancellation_token = consumer_cancellation_token.clone();
+ async move {
+ loop {
+ let result = tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => break,
+ result = soup_realtime_service.run() => result,
+ };
+
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+
+ let _ = result.inspect_err(|error| {
+ tracing::error!(
+ error = ?error,
+ "realtime Soup subscription consumer stopped"
+ );
+ });
+
+ tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => break,
+ _ = tokio::time::sleep(Duration::from_secs(1)) => {}
+ }
+ }
+ }
+ });
+
+ let agent_session_log_realtime = Arc::new(AgentSessionLogConsumerService::new(
+ LogTopicConsumer::from_env(config.kafka_brokers.as_ref()).map_err(|error| {
+ anyhow::anyhow!("failed to create agent session log topic consumer: {error:?}")
+ })?,
+ ));
+ consumer_tracker.spawn({
+ let service = Arc::clone(&agent_session_log_realtime);
+ let cancellation_token = consumer_cancellation_token.clone();
+ async move {
+ loop {
+ let result = tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => break,
+ result = service.run() => result,
+ };
+
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+
+ let _ = result.inspect_err(|error| {
+ tracing::error!(
+ error = ?error,
+ "agent session log subscription consumer stopped"
+ );
+ });
+
+ tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => break,
+ _ = tokio::time::sleep(Duration::from_secs(1)) => {}
+ }
+ }
+ }
+ });
+
+ consumer_tracker.spawn({
+ let brokers = config.kafka_brokers.as_ref().to_string();
+ let entity_access_service = entity_access_service.as_ref().clone();
+ let macro_event_broker = macro_event_broker.clone();
+ let cancellation_token = consumer_cancellation_token.clone();
+ async move {
+ let mut consecutive_failures = 0_u32;
+ loop {
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+
+ let fanout_service = SoupRealtimeServiceImpl::new(
+ EntityAccessExpander::new(entity_access_service.clone()),
+ KafkaSoupRealtimePublisher::new(macro_event_broker.clone()),
+ );
+ tracing::info!("starting realtime Soup entity consumer");
+ let result = fanout_service
+ .run_entity_update_consumer(&brokers, cancellation_token.cancelled())
+ .await;
+
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+
+ let restart_delay = match result {
+ Ok(()) => {
+ consecutive_failures = 0;
+ tracing::error!("realtime Soup entity consumer exited unexpectedly");
+ soup_consumer_restart_delay(1)
+ }
+ Err(error) => {
+ consecutive_failures = consecutive_failures.saturating_add(1);
+ let restart_delay = soup_consumer_restart_delay(consecutive_failures);
+ if consecutive_failures >= SOUP_CONSUMER_RESTART_ALERT_THRESHOLD {
+ tracing::error!(
+ error = ?error,
+ consecutive_failures,
+ restart_delay_secs = restart_delay.as_secs(),
+ "realtime Soup entity consumer repeatedly failed"
+ );
+ } else {
+ tracing::warn!(
+ error = ?error,
+ consecutive_failures,
+ restart_delay_secs = restart_delay.as_secs(),
+ "realtime Soup entity consumer failed; scheduling restart"
+ );
+ }
+ restart_delay
+ }
+ };
+
+ tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => break,
+ _ = tokio::time::sleep(restart_delay) => {}
+ }
+ }
+ }
+ });
+
+ let favorites_mutation_service = Arc::new(FavoritesMutationServiceImpl::new(
+ favorites_service.clone(),
+ entity_access_service.clone(),
+ ));
+ let user_api_key_service = Arc::new(UserApiKeyServiceImpl::new(PgUserApiKeysRepo::new(
+ db.clone(),
+ )));
+ let calendar_state = CalendarRouterState::new(
+ Arc::new(calendar_events::domain::service::CalendarService::new(
+ calendar_events::outbound::pg::PgCalendarRepository::new(readonly_db.clone()),
+ )),
+ authorization_state.clone(),
+ );
+
+ // Reminder dispatch. An EventBridge rule drops a sweep tick on this queue
+ // every minute; the sweep fans one message out per due firing, onto the
+ // same queue, and every task in the pool delivers them in parallel.
+ let reminder_dispatch_worker = {
+ let queue = SqsDispatchQueue::new(
+ aws_sdk_sqs::Client::new(&aws_config),
+ reminder_dispatch_queue.to_string(),
+ );
+ let dispatch_service = ReminderDispatchService::new(
+ PgRemindersRepo::new(db.clone()),
+ NotificationReminderNotifier::new((*notification_ingress_service).clone()),
+ queue.clone(),
+ );
+ DispatchWorker::new(
+ reminders::domain::email_followup::dispatch::EmailReminderDispatch::new(
+ dispatch_service,
+ email_followups.clone(),
+ ),
+ queue,
+ )
+ };
+
+ consumer_tracker.spawn({
+ let cancellation_token = consumer_cancellation_token.clone();
+ async move {
+ tracing::info!("starting reminder dispatch worker");
+ reminder_dispatch_worker.run(cancellation_token).await;
+ tracing::info!("reminder dispatch worker stopped");
+ }
+ });
+
+ // Calendar event reminder dispatch: same sweep/deliver shape as reminders,
+ // on its own queue. Behind a master switch — when off, the worker drains
+ // the minutely tick so the queue neither backs up nor dead-letters.
+ let calendar_reminder_dispatch_worker = {
+ let queue = SqsCalendarDispatchQueue::new(
+ aws_sdk_sqs::Client::new(&aws_config),
+ calendar_reminder_dispatch_queue.to_string(),
+ );
+ let dispatch_service = CalendarReminderDispatchService::new(
+ calendar_events::outbound::pg::PgCalendarRepository::new(db.clone()),
+ NotificationCalendarReminderNotifier::new((*notification_ingress_service).clone()),
+ queue.clone(),
+ );
+ CalendarReminderDispatchWorker::new(dispatch_service, queue)
+ };
+
+ consumer_tracker.spawn({
+ let cancellation_token = consumer_cancellation_token.clone();
+ let enabled = config.calendar_reminder_dispatch_enabled;
+ async move {
+ if enabled {
+ tracing::info!("starting calendar reminder dispatch worker");
+ calendar_reminder_dispatch_worker
+ .run(cancellation_token)
+ .await;
+ tracing::info!("calendar reminder dispatch worker stopped");
+ } else {
+ tracing::info!("calendar reminder dispatch disabled; draining its queue");
+ calendar_reminder_dispatch_worker
+ .drain(cancellation_token)
+ .await;
+ }
+ }
+ });
+
+ let redis_sha_client = Arc::new(Redis::new(redis_client));
+
+ let graphql_entity_mutation_service =
+ Arc::new(service::entity_mutation::DssEntityMutationService::new(
+ document_service.clone(),
+ chat_mutation_service,
+ channels_service.clone(),
+ call_service.clone(),
+ Arc::new(email_service.clone()),
+ project_service.clone(),
+ databases_service.clone(),
+ forms_service.clone(),
+ entity_access_service.clone(),
+ Arc::new(outbound::entity_mutation::DssEntityLifecycleAdapter::new(
+ db.clone(),
+ redis_sha_client.clone(),
+ sqs_client.clone(),
+ macro_event_broker.clone(),
+ )),
+ ));
+
+ // Routine writes still use the scheduled-action service. Read from the
+ // primary pool here so the GraphQL list cannot restore stale state after
+ // a REST write.
+ let scheduled_action_read_service = Arc::new(ScheduledActionReadServiceImpl::new(
+ Arc::new(PgScheduledActionRepo::new(
+ db.clone(),
+ owned_entity_registrar.clone(),
+ )),
+ entity_access_service.clone(),
+ ));
+
+ let api_context = ApiContext {
+ dictation_state,
+ contacts_ingress: contacts_ingress.clone(),
+ soup_router_state: SoupRouterState::from_arc(
+ soup_service.clone(),
+ email_service.clone(),
+ entity_access_service.clone(),
+ authorization_state.clone(),
+ )
+ .with_favorites_reader(Arc::new(
+ service::soup_favorites_reader::DssSoupFavoritesReader(favorites_service.clone()),
+ )),
+ favorites_state: FavoritesRouterState::new(
+ favorites_service.clone(),
+ entity_access_service.clone(),
+ authorization_state.clone(),
+ ),
+ favorites_service,
+ favorites_mutation_service,
+ channel_labels_state: ChannelLabelsRouterState::new(
+ Arc::new(ChannelLabelsServiceImpl::new(PgChannelLabelsRepo::new(
+ db.clone(),
+ ))),
+ entity_access_service.clone(),
+ authorization_state.clone(),
+ ),
+ user_api_key_state: UserApiKeyRouterState::new(
+ user_api_key_service,
+ authorization_state.clone(),
+ ),
+ slack_state,
+ reminders_state: RemindersRouterState::new(
+ Arc::new(reminders_service),
+ entity_access_service.clone(),
+ authorization_state.clone(),
+ ),
+ graphql_initiative_entity_loader: graphql_initiative::InitiativeEntityLoader(
+ graphql_soup::soup_item_loader(initiative_entity_soup, Arc::new(email_service.clone())),
+ ),
+ graphql_initiative_context: graphql_initiative::InitiativeGraphqlContext::new(
+ initiative_service.clone(),
+ entity_access_service.clone(),
+ ),
+ graphql_scheduled_action_context: ScheduledActionGraphqlContext::new(
+ scheduled_action_read_service,
+ ),
+ initiative_state: InitiativeRouterState::new(
+ initiative_service,
+ entity_access_service.clone(),
+ authorization_state.clone(),
+ ),
+ databases_state: databases::inbound::axum_router::DatabasesRouterState::new(
+ databases_service,
+ entity_access_service.clone(),
+ authorization_state.clone(),
+ ),
+ forms_state: forms::inbound::axum_router::FormsRouterState::new(
+ forms_service,
+ entity_access_service.clone(),
+ authorization_state.clone(),
+ ),
+ database_starter_state: databases::inbound::starter_router::DatabaseStarterRouterState::new(
+ Arc::new(databases::domain::starter::DatabaseStarterServiceImpl::new(
+ databases::outbound::pg_starter::PgDatabaseStarterRepo::new(
+ db.clone(),
+ properties::outbound::properties_pg_repo::PropertiesPgRepo::new(db.clone()),
+ ),
+ macro_event_broker.clone(),
+ )),
+ authorization_state.clone(),
+ ),
+ collab_surface_state: CollabSurfaceRouterState::new(
+ collab_surface_service,
+ entity_access_service.clone(),
+ authorization_state.clone(),
+ ),
+ graphql_soup_schema: complete_graph::build_schema_from_arcs(
+ soup_service,
+ soup_realtime_service,
+ websocket_notification_consumer_service,
+ activity_realtime_service,
+ ),
+ agent_session_log_subscriptions: complete_graph::agent_session_log_subscriptions(
+ agent_session_log_realtime,
+ ),
+ graphql_notification_reader,
+ // GraphQL reads the activity log through the readonly pool; the
+ // Kafka consumer's writer-pool repo above is separate on purpose.
+ activity_reader: complete_graph::ActivityPortReader::new(Arc::new(
+ initiative::domain::personal_activity::ProjectVisibleActivityReads::new(
+ activity::outbound::pg_activity_repo::PgActivityRepo::new(readonly_db.clone()),
+ entity_access_service.clone(),
+ ),
+ )),
+ graphql_entity_mutation_service,
+ github_sync_service: Arc::new(github_sync_service_impl),
+ github_pull_request_index_state,
+ github_pull_request_state,
+ github_pull_request_changes_state,
+ foreign_entity_state,
+ db: db.clone(),
+ readonly_db: readonly_pool::ReadOnlyPool(readonly_db.clone()),
+ redis_client: redis_sha_client,
+ s3_client: s3,
+ dynamodb_client: Arc::new(dynamodb_client),
+ dynamo_db,
+ macro_event_broker: macro_event_broker.clone(),
+ sqs_client: sqs_client.clone(),
+ notification_ingress_service: notification_ingress_service.clone(),
+ conn_gateway_client: conn_gateway_client.clone(),
+ sync_service_client: sync_service_client.clone(),
+ system_properties_service: system_properties_service.clone(),
+ properties_service: properties_service.clone(),
+ opensearch_client: Arc::new(opensearch_client),
+ authorization_state: authorization_state.clone(),
+ jwt_validation_args,
+ dss_auth_key,
+ // Shared frecency storage and legacy channel list routes.
+ frecency_storage,
+ channel_list_state,
+ entity_access_service: entity_access_service.clone(),
+ calendar_state,
+ projects_state: ProjectRouterState {
+ service: project_service,
+ access_service: entity_access_service.clone(),
+ authorization_state: authorization_state.clone(),
+ non_user_owners,
+ },
+ documents_state: DocumentRouterState {
+ service: document_service,
+ access_service: entity_access_service.clone(),
+ authorization_state: authorization_state.clone(),
+ pool: db.clone(),
+ task_dedup_service,
+ lexical_client: lexical_client.clone(),
+ creator: document_creator,
+ document_permission_jwt_secret: config.document_permission_jwt.as_ref().to_string(),
+ non_user_owners,
+ },
+ config: Arc::new(config),
+ channel_service: channels_service.clone(),
+ channels_state: ChannelsRouterState::from_arc(
+ channels_service,
+ (*entity_access_service).clone(),
+ authorization_state.clone(),
+ ),
+ messages_state,
+ annotation_service,
+ bots_state: bots::inbound::axum_router::BotsRouterState::new(
+ bots_service.clone(),
+ (*entity_access_service).clone(),
+ authorization_state.clone(),
+ ),
+ harnesses_state: harnesses::inbound::axum_router::HarnessesRouterState::new(
+ harnesses_service,
+ authorization_state.clone(),
+ ),
+ channel_bot_webhook_state,
+ call_state,
+ call_webhook_state,
+ call_public_rate_limiter,
+ webhook_state,
+ sse_stream_state,
+ call_internal_state,
+ cal_webhook_state,
+ entity_access_management_service,
+ crm_state: crm::inbound::axum_router::CrmRouterState {
+ service: Arc::new(crm_service),
+ stage_service: Arc::new(crm::domain::stages::CrmStageServiceImpl::new(
+ crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),
+ crm::outbound::stage_definitions::PropertiesStageDefinitionStore::new(
+ properties_service.clone(),
+ ),
+ )),
+ entity_access_service: entity_access_service.clone(),
+ authorization_state: authorization_state.clone(),
+ },
+ };
+
+ #[cfg(feature = "delete_document_worker")]
+ {
+ let editing_worker_client = Arc::new(
+ documents_hex::outbound::editing_worker_client::ReqwestEditingWorkerClient::from_url(
+ AiEditingWorkerUrl::new()?.to_string(),
+ )
+ .with_internal_auth_key(Some(api_context.config.internal_api_key.to_string())),
+ );
+
+ // Spawn the delete document worker.
+ let delete_worker_ctx = service::delete_document_worker::DeleteDocumentWorkerContext {
+ worker: Arc::new(delete_document_worker),
+ db: db.clone(),
+ s3_client: api_context.s3_client.clone(),
+ redis_client: api_context.redis_client.clone(),
+ sync_service_client: api_context.sync_service_client.clone(),
+ editing_worker_client,
+ properties_service: api_context.properties_service.clone(),
+ };
+
+ tokio::spawn(async move {
+ service::delete_document_worker::run_worker(delete_worker_ctx).await;
+ });
+ }
+
+ let server_result = api::setup_and_serve(api_context).await;
+
+ tracing::info!("stopping event consumers");
+ consumer_cancellation_token.cancel();
+ consumer_tracker.close();
+ consumer_tracker.wait().await;
+ tracing::info!("event consumers stopped");
+
+ tracing::info!("waiting for event broker publishes to drain");
+ event_broker_tracker.close();
+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {
+ Ok(()) => tracing::info!("event broker publishes drained"),
+ Err(error) => {
+ tracing::warn!(
+ error=?error,
+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),
+ "timed out waiting for event broker publishes to drain"
+ );
+ }
+ }
+
+ server_result
+}
+
+const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
diff --git a/services/email_service/src/api/context.rs b/services/email_service/src/api/context.rs
index bb98d02125..309476b4fe 100644
--- a/services/email_service/src/api/context.rs
+++ b/services/email_service/src/api/context.rs
@@ -1,3 +1,6 @@
+use crate::calendar_refresh::ConnectionGatewayCalendarRefresh;
+use crate::calendar_request_gate::RedisCalendarRequestGate;
+use crate::calendar_tokens::CalendarTokenProviderAdapter;
use axum::extract::FromRef;
use calendar_events::{
domain::{mutations::CalendarMutationServiceImpl, service::CalendarService},
@@ -12,13 +15,10 @@ use email::{
},
outbound::{EmailPgRepo, GmailTokenProviderImpl},
};
-use email_service::calendar_refresh::ConnectionGatewayCalendarRefresh;
-use email_service::calendar_request_gate::RedisCalendarRequestGate;
-use email_service::calendar_tokens::CalendarTokenProviderAdapter;
-use email_service::config::Config;
-use email_service::outbound::email_api::GmailApi;
-use email_service::util::redis::RedisClient;
+use crate::config::Config;
+use crate::outbound::email_api::GmailApi;
+use crate::util::redis::RedisClient;
use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};
use entity_access_management::domain::service::EntityAccessManagementServiceImpl;
use frecency::{domain::services::FrecencyQueryServiceImpl, outbound::postgres::FrecencyPgStorage};
diff --git a/services/email_service/src/api/email/attachments/get_document_id.rs b/services/email_service/src/api/email/attachments/get_document_id.rs
index 664b517022..63eaec1d7f 100644
--- a/services/email_service/src/api/email/attachments/get_document_id.rs
+++ b/services/email_service/src/api/email/attachments/get_document_id.rs
@@ -1,11 +1,11 @@
use crate::api::context::ApiContext;
+use crate::util::upload_attachment::{
+ UploadAttachmentContext, UploadAttachmentError, upload_attachment,
+};
use axum::extract::{Path, State};
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use axum::{Extension, Json};
-use email_service::util::upload_attachment::{
- UploadAttachmentContext, UploadAttachmentError, upload_attachment,
-};
use entity_access::domain::ports::EntityAccessService;
use macro_user_id::user_id::MacroUserIdStr;
use model::response::ErrorResponse;
diff --git a/services/email_service/src/api/email/drafts/delete.rs b/services/email_service/src/api/email/drafts/delete.rs
index a6e80c957e..14bdf941e5 100644
--- a/services/email_service/src/api/email/drafts/delete.rs
+++ b/services/email_service/src/api/email/drafts/delete.rs
@@ -1,9 +1,9 @@
use crate::api::context::ApiContext;
+use crate::util::gmail::send::cleanup_draft_attachments;
use axum::Extension;
use axum::extract::{Path, State};
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
-use email_service::util::gmail::send::cleanup_draft_attachments;
use model::response::{EmptyResponse, ErrorResponse};
use models_email::service::link::Link;
use strum_macros::AsRefStr;
diff --git a/services/email_service/src/api/email/init.rs b/services/email_service/src/api/email/init.rs
index 43b94e63bc..b0cc2e4eb8 100644
--- a/services/email_service/src/api/email/init.rs
+++ b/services/email_service/src/api/email/init.rs
@@ -1,5 +1,6 @@
use crate::api::ApiContext;
use crate::api::context::{AuthorizationService, CalendarGrantService};
+use crate::pubsub::publish_email_event;
use crate::utils::extract_email_with_response;
use anyhow::Context;
use axum::{
@@ -13,7 +14,6 @@ use email::domain::models::UserProvider;
use email::domain::ports::EmailRepo;
use email::outbound::EmailPgRepo;
use email_api_client::domain::models::{EmailApiError, TokenFreshness};
-use email_service::pubsub::publish_email_event;
use macro_authorization::{MacroAuthorizationExtractor, UserOrInternal};
use macro_db_client::in_progress_user_link::InProgressUserLink;
use macro_user_id::email::EmailStr;
diff --git a/services/email_service/src/api/email/messages/labels.rs b/services/email_service/src/api/email/messages/labels.rs
index 68d235437d..60976e0b25 100644
--- a/services/email_service/src/api/email/messages/labels.rs
+++ b/services/email_service/src/api/email/messages/labels.rs
@@ -1,12 +1,12 @@
use crate::api::context::{ApiContext, AuthorizationService};
use crate::api::email::messages::BATCH_UPDATE_MESSAGE_LIMIT;
+use crate::pubsub::publish_email_event;
use anyhow::Context;
use axum::Json;
use axum::extract::State;
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use email::domain::events::{EmailEventOrigin, EmailMacroEvent, LabelRef};
-use email_service::pubsub::publish_email_event;
use macro_authorization::{MacroAuthorizationExtractor, UserOrInternal};
use model::response::ErrorResponse;
use models_email::service;
diff --git a/services/email_service/src/api/email/settings/import_signature.rs b/services/email_service/src/api/email/settings/import_signature.rs
index 082a4472ba..c471e455a0 100644
--- a/services/email_service/src/api/email/settings/import_signature.rs
+++ b/services/email_service/src/api/email/settings/import_signature.rs
@@ -1,11 +1,11 @@
use crate::api::context::ApiContext;
use crate::api::email::settings::patch::UnresolvedSignatureImagesError;
+use crate::util::process_pre_insert::sfs_map;
use axum::extract::State;
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use axum::{Extension, Json};
use email_api_client::domain::models::{EmailApiError, TokenFreshness};
-use email_service::util::process_pre_insert::sfs_map;
use email_utils::sanitize_html_fragment;
use model::response::ErrorResponse;
use models_email::service::link::Link;
diff --git a/services/email_service/src/api/email/settings/patch.rs b/services/email_service/src/api/email/settings/patch.rs
index a74385852d..42eb8a5968 100644
--- a/services/email_service/src/api/email/settings/patch.rs
+++ b/services/email_service/src/api/email/settings/patch.rs
@@ -1,9 +1,9 @@
use crate::api::context::ApiContext;
+use crate::util::process_pre_insert::sfs_map;
use axum::extract::State;
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use axum::{Extension, Json};
-use email_service::util::process_pre_insert::sfs_map;
use email_utils::sanitize_html_fragment;
use model::response::ErrorResponse;
use models_email::service::link::Link;
diff --git a/services/email_service/src/api/email/threads/archived.rs b/services/email_service/src/api/email/threads/archived.rs
index 5097b94a33..e70e501898 100644
--- a/services/email_service/src/api/email/threads/archived.rs
+++ b/services/email_service/src/api/email/threads/archived.rs
@@ -1,4 +1,5 @@
use crate::api::context::{ApiContext, AuthorizationService};
+use crate::pubsub::publish_email_event;
use anyhow::Context;
use axum::Json;
use axum::extract::{Path, State};
@@ -6,7 +7,6 @@ use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use email::domain::events::{EmailEventOrigin, EmailMacroEvent, ThreadArchivedMetadata};
use email_db_client::threads::update::update_inbox_visible_status;
-use email_service::pubsub::publish_email_event;
use macro_authorization::{MacroAuthorizationExtractor, UserOrInternal};
use model::response::{EmptyResponse, ErrorResponse};
use models_email::service::label::system_labels;
diff --git a/services/email_service/src/api/email/threads/seen.rs b/services/email_service/src/api/email/threads/seen.rs
index 8e7cae3098..91e6a0ce62 100644
--- a/services/email_service/src/api/email/threads/seen.rs
+++ b/services/email_service/src/api/email/threads/seen.rs
@@ -1,11 +1,11 @@
use crate::api::context::{ApiContext, AuthorizationService};
+use crate::pubsub::publish_email_event;
use anyhow::Context;
use axum::Json;
use axum::extract::{Path, State};
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use email::domain::events::{EmailEventOrigin, EmailMacroEvent, ThreadReadMetadata};
-use email_service::pubsub::publish_email_event;
use macro_authorization::{MacroAuthorizationExtractor, UserOrInternal};
use model::response::{EmptyResponse, ErrorResponse};
use models_email::service::label::system_labels;
diff --git a/services/email_service/src/api/gmail/webhook.rs b/services/email_service/src/api/gmail/webhook.rs
index eb0cffa500..45876c3c54 100644
--- a/services/email_service/src/api/gmail/webhook.rs
+++ b/services/email_service/src/api/gmail/webhook.rs
@@ -1,4 +1,5 @@
use crate::api::context::ApiContext;
+use crate::util;
use axum::extract::State;
use axum::http::HeaderMap;
use axum::{
@@ -6,7 +7,6 @@ use axum::{
http::StatusCode,
response::{IntoResponse, Json, Response},
};
-use email_service::util;
use model::response::ErrorResponse;
use models_email::gmail::inbox_sync::{
GmailInboxSyncPayload, GmailMessagePayload, InboxSyncOperation, InboxSyncPubsubMessage,
diff --git a/services/email_service/src/lib.rs b/services/email_service/src/lib.rs
index 18dc220f69..6a28ed9745 100644
--- a/services/email_service/src/lib.rs
+++ b/services/email_service/src/lib.rs
@@ -1,3 +1,9 @@
+#![recursion_limit = "256"]
+//! Email service. The service binary, the OpenAPI generator, and the backfill
+//! and worker binaries all link this library, so the service compiles once and
+//! caches as one rlib.
+
+mod api;
/// Durable email-backfill completion orchestration.
pub mod backfill_completion_service;
/// Fenced email-backfill initialization orchestration.
@@ -14,4 +20,10 @@ pub mod config;
/// Outbound infrastructure adapters for email provider capabilities.
pub mod outbound;
pub mod pubsub;
+mod server;
pub mod util;
+mod utils;
+
+/// The service's OpenAPI document, rendered by the `*_openapi` binary.
+pub use api::swagger::ApiDoc;
+pub use server::run;
diff --git a/services/email_service/src/main.rs b/services/email_service/src/main.rs
index 7f5d818b4f..cd01fd815c 100644
--- a/services/email_service/src/main.rs
+++ b/services/email_service/src/main.rs
@@ -1,278 +1,12 @@
+// `run()`'s future type is deep enough to need the library's recursion limit
+// wherever it is awaited.
#![recursion_limit = "256"]
-use crate::api::context::{ApiContext, AuthorizationService};
-use anyhow::Context;
-use calendar_events::{
- domain::{
- invitations::CalendarInvitationResolver, mutations::CalendarMutationServiceImpl,
- service::CalendarService,
- },
- outbound::{google::GoogleCalendarClient, pg::PgCalendarRepository},
-};
-use document_storage_service_client::DocumentStorageServiceClient;
-use email::{
- domain::service::EmailServiceImpl,
- inbound::axum::{
- axum_impls::GmailTokenState, get_thread_router::EmailThreadRouterState,
- previews_router::EmailRouterState,
- },
- outbound::{EmailPgRepo, GmailTokenProviderImpl},
-};
-use email_api_client::GmailApiClientRepository;
-use email_service::calendar_refresh::ConnectionGatewayCalendarRefresh;
-use email_service::calendar_request_gate::RedisCalendarRequestGate;
-use email_service::calendar_tokens::CalendarTokenProviderAdapter;
-use email_service::outbound::email_api::{
- EmailServiceTokenSource, GmailApi, RateBudget, RedisProviderRateLimiter,
-};
-use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};
-use frecency::{domain::services::FrecencyQueryServiceImpl, outbound::postgres::FrecencyPgStorage};
-use macro_auth::middleware::decode_jwt::JwtValidationArgs;
-use macro_authorization::{
- InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationState,
- PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,
-};
-use macro_entrypoint::MacroEntrypoint;
-use macro_env::Environment;
-use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
-use macro_service_urls::{
- AuthServiceUrl, ConnectionGatewayUrl, DocumentStorageServiceUrl, StaticFileServiceUrl,
-};
-use sqlx::postgres::PgPoolOptions;
-use static_file_service_client::StaticFileServiceClient;
-use std::{sync::Arc, time::Duration};
-use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};
-use tokio_util::task::TaskTracker;
-mod api;
-mod utils;
+use macro_entrypoint::MacroEntrypoint;
#[tokio::main]
#[tracing::instrument(err)]
async fn main() -> anyhow::Result<()> {
MacroEntrypoint::default().init();
- let env = Environment::new_or_prod();
-
- let aws_config = macro_aws_config::get_macro_aws_config().await;
-
- let s3_client = s3_client::S3::new(macro_aws_config::s3_client().await);
-
- let secretsmanager_client = secretsmanager_client::SecretsManager::new(
- aws_sdk_secretsmanager::Client::new(&aws_config),
- );
-
- // Parse our configuration from the environment, then resolve any secret-manager backed values.
- let config = email_service::config::Config::from_env()
- .context("expected to be able to generate config")?
- .resolve_remote_secrets(env, &secretsmanager_client)
- .await
- .context("expected to be able to resolve config secrets")?;
-
- // limiting to max of 200 connections (12.5% of macrodb total) in prod.
- let (min_connections, max_connections): (u32, u32) = match config.environment {
- Environment::Production => (3, 20),
- Environment::Develop => (1, 10),
- Environment::Local => (1, 10),
- };
-
- let db = PgPoolOptions::new()
- .min_connections(min_connections)
- .max_connections(max_connections)
- .connect(&config.macro_db_url)
- .await
- .context("could not connect to db")?;
-
- let gmail_inbox_sync_queue = macro_queues::GmailInboxSyncQueue::new();
- let gmail_inbox_sync_retry_queue = macro_queues::GmailInboxSyncRetryQueue::new();
- let gmail_ops_queue = macro_queues::GmailOpsQueue::new();
- let backfill_queue = macro_queues::EmailBackfillQueue::new();
- let email_scheduled_queue = macro_queues::EmailScheduledQueue::new();
- let sfs_uploader_queue = macro_queues::SfsUploaderQueue::new();
- let link_manager_queue = macro_queues::LinkManagerQueue::new();
- let sqs_client = sqs_client::SQS::new(macro_aws_config::sqs_client().await)
- .gmail_inbox_sync_queue(&gmail_inbox_sync_queue)
- .gmail_inbox_sync_retry_queue(&gmail_inbox_sync_retry_queue)
- .gmail_ops_queue(&gmail_ops_queue)
- .email_backfill_queue(&backfill_queue)
- .email_scheduled_queue(&email_scheduled_queue)
- .sfs_uploader_queue(&sfs_uploader_queue)
- .email_link_manager_queue(&link_manager_queue);
-
- let auth_service_client = authentication_service_client::AuthServiceClient::new(
- config
- .authentication_service_secret_key
- .as_ref()
- .to_string(),
- AuthServiceUrl::new()?.to_string(),
- );
-
- let gmail_client = gmail_client::GmailClient::new(config.gmail_gcp_queue.as_ref().to_string());
- let gmail_api_repository = GmailApiClientRepository::new(gmail_client.clone());
-
- let redis_inner_client = redis::Client::open(config.redis_uri.as_ref())
- .inspect(|client| {
- client
- .get_connection()
- .map(|_| tracing::info!("initialized redis connection"))
- .inspect_err(|e| {
- tracing::error!(error=?e, "failed to connect to redis");
- })
- .ok();
- })
- .context("failed to connect to redis")?;
-
- let redis_client = email_service::util::redis::RedisClient::new(
- redis_inner_client,
- config.redis_rate_limit_reqs,
- config.redis_rate_limit_reqs_backfill,
- config.redis_rate_limit_window_secs,
- );
-
- let sfs_client = StaticFileServiceClient::new(
- config.internal_api_key.to_string(),
- StaticFileServiceUrl::new()?.to_string(),
- );
-
- let dss_client = DocumentStorageServiceClient::new(
- config.internal_api_key.to_string(),
- DocumentStorageServiceUrl::new()?.to_string(),
- );
-
- let system_properties_service = Arc::new(SystemPropertiesServiceImpl::new(
- PgSystemPropertiesRepository::new(db.clone()),
- ));
-
- let jwt_args =
- JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)
- .await?;
- let authorization_state = MacroAuthorizationState::new(Arc::new(AuthorizationService::new(
- MacroAuthJwtValidator::new(jwt_args.clone()),
- InternalAuthConfig {
- api_key: config.internal_api_key.to_string(),
- default_user_id: Some("macro|[email protected]".to_string()),
- },
- macro_authorization::NoBotAuthorizer,
- PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),
- )));
-
- let sqs_client = Arc::new(sqs_client);
- let gmail_client = Arc::new(gmail_client);
- // HTTP API only reads CRM rows — populate runs in the pubsub
- // worker. The no-op resolver makes the unused branch explicit and
- // keeps reqwest/scraper out of this binary.
- let crm_service = crm::domain::service::CrmServiceImpl::new(
- crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),
- crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,
- );
- let event_broker_tracker = TaskTracker::new();
- let macro_event_broker = MacroEventBrokerService::new(
- KafkaEventPublisher::new(config.kafka_brokers.as_ref())
- .context("failed to create kafka event publisher")?,
- event_broker_tracker.clone(),
- );
- let email_service = EmailRouterState::new(
- EmailServiceImpl::new(
- EmailPgRepo::new(db.clone()),
- FrecencyQueryServiceImpl::new(FrecencyPgStorage::new(db.clone())),
- (*sqs_client).clone(),
- crm_service,
- entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
- entity_access_management::outbound::PgRepository::new(db.clone()),
- ),
- config.sent_undo_delay_secs,
- )
- .with_macro_event_broker(macro_event_broker.clone()),
- );
- let entity_access_service = Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(
- db.clone(),
- )));
- let email_thread_state = EmailThreadRouterState {
- service: email_service.service(),
- access_service: entity_access_service.clone(),
- authorization_state: authorization_state.clone(),
- };
- let auth_service_client = Arc::new(auth_service_client);
- let redis_conn = redis_client
- .inner
- .get_multiplexed_async_connection()
- .await
- .context("failed to get multiplexed redis connection for gmail token provider")?;
- let email_api = GmailApi::new(
- gmail_api_repository,
- EmailServiceTokenSource::new(
- db.clone(),
- redis_conn.clone(),
- auth_service_client.as_ref().clone(),
- sqs_client.as_ref().clone(),
- ),
- RedisProviderRateLimiter::new(redis_client.clone(), RateBudget::Live),
- );
- let redis_client = Arc::new(redis_client);
- let gmail_token_state = GmailTokenState::new(GmailTokenProviderImpl::new(
- redis_conn.clone(),
- auth_service_client.clone(),
- ));
- let calendar_service = Arc::new(CalendarService::new(PgCalendarRepository::new(db.clone())));
- let connection_gateway_client = connection_gateway_client::client::ConnectionGatewayClient::new(
- config.internal_api_key.to_string(),
- ConnectionGatewayUrl::new()?.to_string(),
- );
- let calendar_mutation_service = Arc::new(CalendarMutationServiceImpl::new(
- PgCalendarRepository::new(db.clone()),
- GoogleCalendarClient::with_gate(
- reqwest::Client::builder()
- .timeout(std::time::Duration::from_secs(30))
- .build()
- .context("failed to build the google calendar mutation http client")?,
- RedisCalendarRequestGate::new((*redis_client).clone()),
- ),
- CalendarTokenProviderAdapter::new(redis_conn.clone(), auth_service_client.clone()),
- macro_event_broker.clone(),
- ConnectionGatewayCalendarRefresh::new(connection_gateway_client, db.clone()),
- ));
- let api_result = api::setup_and_serve(ApiContext {
- invitation_snapshots: email::outbound::invitation_pg::InvitationPgRepository(db.clone()),
- // calendar_service's sync kill switch rejects RSVP writes itself.
- invitation_resolver: Arc::new(CalendarInvitationResolver::new(PgCalendarRepository::new(
- db.clone(),
- ))),
- db,
- internal_api_key: config.internal_api_key.clone(),
- config: Arc::new(config),
- auth_service_client,
- redis_client,
- sqs_client,
- sfs_client: Arc::new(sfs_client),
- gmail_client: gmail_client.clone(),
- email_api,
- s3_client: Arc::new(s3_client),
- dss_client: Arc::new(dss_client),
- system_properties_service,
- authorization_state: authorization_state.clone(),
- jwt_args,
- email_service,
- entity_access_service,
- email_thread_state,
- gmail_token_state,
- macro_event_broker: Arc::new(macro_event_broker),
- calendar_service,
- calendar_mutation_service,
- })
- .await;
-
- tracing::info!("waiting for event broker publishes to drain");
- event_broker_tracker.close();
- match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {
- Ok(()) => tracing::info!("event broker publishes drained"),
- Err(error) => {
- tracing::warn!(
- error=?error,
- timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),
- "timed out waiting for event broker publishes to drain"
- );
- }
- }
-
- api_result
+ email_service::run().await
}
-
-const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
diff --git a/services/email_service/src/openapi.rs b/services/email_service/src/openapi.rs
index 029b481120..f9e57b4325 100644
--- a/services/email_service/src/openapi.rs
+++ b/services/email_service/src/openapi.rs
@@ -1,20 +1,6 @@
-#![allow(unused)]
-
-mod api;
-mod backfill_completion_service;
-mod backfill_init_service;
-mod backfill_outbox;
-mod config;
-mod outbound;
-mod pubsub;
-mod util;
-mod utils;
-
+use email_service::ApiDoc;
use utoipa::OpenApi;
fn main() {
- println!(
- "{}",
- api::swagger::ApiDoc::openapi().to_pretty_json().unwrap()
- );
+ println!("{}", ApiDoc::openapi().to_pretty_json().unwrap());
}
diff --git a/services/email_service/src/server.rs b/services/email_service/src/server.rs
new file mode 100644
index 0000000000..6a745bc14e
--- /dev/null
+++ b/services/email_service/src/server.rs
@@ -0,0 +1,277 @@
+//! The service's composition root: reads config and wires every adapter and
+//! domain service into the HTTP server.
+
+use crate::api::{
+ self,
+ context::{ApiContext, AuthorizationService},
+};
+use crate::calendar_refresh::ConnectionGatewayCalendarRefresh;
+use crate::calendar_request_gate::RedisCalendarRequestGate;
+use crate::calendar_tokens::CalendarTokenProviderAdapter;
+use crate::outbound::email_api::{
+ EmailServiceTokenSource, GmailApi, RateBudget, RedisProviderRateLimiter,
+};
+use anyhow::Context;
+use calendar_events::{
+ domain::{
+ invitations::CalendarInvitationResolver, mutations::CalendarMutationServiceImpl,
+ service::CalendarService,
+ },
+ outbound::{google::GoogleCalendarClient, pg::PgCalendarRepository},
+};
+use document_storage_service_client::DocumentStorageServiceClient;
+use email::{
+ domain::service::EmailServiceImpl,
+ inbound::axum::{
+ axum_impls::GmailTokenState, get_thread_router::EmailThreadRouterState,
+ previews_router::EmailRouterState,
+ },
+ outbound::{EmailPgRepo, GmailTokenProviderImpl},
+};
+use email_api_client::GmailApiClientRepository;
+use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};
+use frecency::{domain::services::FrecencyQueryServiceImpl, outbound::postgres::FrecencyPgStorage};
+use macro_auth::middleware::decode_jwt::JwtValidationArgs;
+use macro_authorization::{
+ InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationState,
+ PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,
+};
+use macro_env::Environment;
+use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
+use macro_service_urls::{
+ AuthServiceUrl, ConnectionGatewayUrl, DocumentStorageServiceUrl, StaticFileServiceUrl,
+};
+use sqlx::postgres::PgPoolOptions;
+use static_file_service_client::StaticFileServiceClient;
+use std::{sync::Arc, time::Duration};
+use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};
+use tokio_util::task::TaskTracker;
+
+/// Build the service from the environment and serve until shutdown.
+pub async fn run() -> anyhow::Result<()> {
+ let env = Environment::new_or_prod();
+
+ let aws_config = macro_aws_config::get_macro_aws_config().await;
+
+ let s3_client = s3_client::S3::new(macro_aws_config::s3_client().await);
+
+ let secretsmanager_client = secretsmanager_client::SecretsManager::new(
+ aws_sdk_secretsmanager::Client::new(&aws_config),
+ );
+
+ // Parse our configuration from the environment, then resolve any secret-manager backed values.
+ let config = crate::config::Config::from_env()
+ .context("expected to be able to generate config")?
+ .resolve_remote_secrets(env, &secretsmanager_client)
+ .await
+ .context("expected to be able to resolve config secrets")?;
+
+ // limiting to max of 200 connections (12.5% of macrodb total) in prod.
+ let (min_connections, max_connections): (u32, u32) = match config.environment {
+ Environment::Production => (3, 20),
+ Environment::Develop => (1, 10),
+ Environment::Local => (1, 10),
+ };
+
+ let db = PgPoolOptions::new()
+ .min_connections(min_connections)
+ .max_connections(max_connections)
+ .connect(&config.macro_db_url)
+ .await
+ .context("could not connect to db")?;
+
+ let gmail_inbox_sync_queue = macro_queues::GmailInboxSyncQueue::new();
+ let gmail_inbox_sync_retry_queue = macro_queues::GmailInboxSyncRetryQueue::new();
+ let gmail_ops_queue = macro_queues::GmailOpsQueue::new();
+ let backfill_queue = macro_queues::EmailBackfillQueue::new();
+ let email_scheduled_queue = macro_queues::EmailScheduledQueue::new();
+ let sfs_uploader_queue = macro_queues::SfsUploaderQueue::new();
+ let link_manager_queue = macro_queues::LinkManagerQueue::new();
+ let sqs_client = sqs_client::SQS::new(macro_aws_config::sqs_client().await)
+ .gmail_inbox_sync_queue(&gmail_inbox_sync_queue)
+ .gmail_inbox_sync_retry_queue(&gmail_inbox_sync_retry_queue)
+ .gmail_ops_queue(&gmail_ops_queue)
+ .email_backfill_queue(&backfill_queue)
+ .email_scheduled_queue(&email_scheduled_queue)
+ .sfs_uploader_queue(&sfs_uploader_queue)
+ .email_link_manager_queue(&link_manager_queue);
+
+ let auth_service_client = authentication_service_client::AuthServiceClient::new(
+ config
+ .authentication_service_secret_key
+ .as_ref()
+ .to_string(),
+ AuthServiceUrl::new()?.to_string(),
+ );
+
+ let gmail_client = gmail_client::GmailClient::new(config.gmail_gcp_queue.as_ref().to_string());
+ let gmail_api_repository = GmailApiClientRepository::new(gmail_client.clone());
+
+ let redis_inner_client = redis::Client::open(config.redis_uri.as_ref())
+ .inspect(|client| {
+ client
+ .get_connection()
+ .map(|_| tracing::info!("initialized redis connection"))
+ .inspect_err(|e| {
+ tracing::error!(error=?e, "failed to connect to redis");
+ })
+ .ok();
+ })
+ .context("failed to connect to redis")?;
+
+ let redis_client = crate::util::redis::RedisClient::new(
+ redis_inner_client,
+ config.redis_rate_limit_reqs,
+ config.redis_rate_limit_reqs_backfill,
+ config.redis_rate_limit_window_secs,
+ );
+
+ let sfs_client = StaticFileServiceClient::new(
+ config.internal_api_key.to_string(),
+ StaticFileServiceUrl::new()?.to_string(),
+ );
+
+ let dss_client = DocumentStorageServiceClient::new(
+ config.internal_api_key.to_string(),
+ DocumentStorageServiceUrl::new()?.to_string(),
+ );
+
+ let system_properties_service = Arc::new(SystemPropertiesServiceImpl::new(
+ PgSystemPropertiesRepository::new(db.clone()),
+ ));
+
+ let jwt_args =
+ JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)
+ .await?;
+ let authorization_state = MacroAuthorizationState::new(Arc::new(AuthorizationService::new(
+ MacroAuthJwtValidator::new(jwt_args.clone()),
+ InternalAuthConfig {
+ api_key: config.internal_api_key.to_string(),
+ default_user_id: Some("macro|[email protected]".to_string()),
+ },
+ macro_authorization::NoBotAuthorizer,
+ PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),
+ )));
+
+ let sqs_client = Arc::new(sqs_client);
+ let gmail_client = Arc::new(gmail_client);
+ // HTTP API only reads CRM rows — populate runs in the pubsub
+ // worker. The no-op resolver makes the unused branch explicit and
+ // keeps reqwest/scraper out of this binary.
+ let crm_service = crm::domain::service::CrmServiceImpl::new(
+ crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),
+ crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,
+ );
+ let event_broker_tracker = TaskTracker::new();
+ let macro_event_broker = MacroEventBrokerService::new(
+ KafkaEventPublisher::new(config.kafka_brokers.as_ref())
+ .context("failed to create kafka event publisher")?,
+ event_broker_tracker.clone(),
+ );
+ let email_service = EmailRouterState::new(
+ EmailServiceImpl::new(
+ EmailPgRepo::new(db.clone()),
+ FrecencyQueryServiceImpl::new(FrecencyPgStorage::new(db.clone())),
+ (*sqs_client).clone(),
+ crm_service,
+ entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
+ entity_access_management::outbound::PgRepository::new(db.clone()),
+ ),
+ config.sent_undo_delay_secs,
+ )
+ .with_macro_event_broker(macro_event_broker.clone()),
+ );
+ let entity_access_service = Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(
+ db.clone(),
+ )));
+ let email_thread_state = EmailThreadRouterState {
+ service: email_service.service(),
+ access_service: entity_access_service.clone(),
+ authorization_state: authorization_state.clone(),
+ };
+ let auth_service_client = Arc::new(auth_service_client);
+ let redis_conn = redis_client
+ .inner
+ .get_multiplexed_async_connection()
+ .await
+ .context("failed to get multiplexed redis connection for gmail token provider")?;
+ let email_api = GmailApi::new(
+ gmail_api_repository,
+ EmailServiceTokenSource::new(
+ db.clone(),
+ redis_conn.clone(),
+ auth_service_client.as_ref().clone(),
+ sqs_client.as_ref().clone(),
+ ),
+ RedisProviderRateLimiter::new(redis_client.clone(), RateBudget::Live),
+ );
+ let redis_client = Arc::new(redis_client);
+ let gmail_token_state = GmailTokenState::new(GmailTokenProviderImpl::new(
+ redis_conn.clone(),
+ auth_service_client.clone(),
+ ));
+ let calendar_service = Arc::new(CalendarService::new(PgCalendarRepository::new(db.clone())));
+ let connection_gateway_client = connection_gateway_client::client::ConnectionGatewayClient::new(
+ config.internal_api_key.to_string(),
+ ConnectionGatewayUrl::new()?.to_string(),
+ );
+ let calendar_mutation_service = Arc::new(CalendarMutationServiceImpl::new(
+ PgCalendarRepository::new(db.clone()),
+ GoogleCalendarClient::with_gate(
+ reqwest::Client::builder()
+ .timeout(std::time::Duration::from_secs(30))
+ .build()
+ .context("failed to build the google calendar mutation http client")?,
+ RedisCalendarRequestGate::new((*redis_client).clone()),
+ ),
+ CalendarTokenProviderAdapter::new(redis_conn.clone(), auth_service_client.clone()),
+ macro_event_broker.clone(),
+ ConnectionGatewayCalendarRefresh::new(connection_gateway_client, db.clone()),
+ ));
+ let api_result = api::setup_and_serve(ApiContext {
+ invitation_snapshots: email::outbound::invitation_pg::InvitationPgRepository(db.clone()),
+ // calendar_service's sync kill switch rejects RSVP writes itself.
+ invitation_resolver: Arc::new(CalendarInvitationResolver::new(PgCalendarRepository::new(
+ db.clone(),
+ ))),
+ db,
+ internal_api_key: config.internal_api_key.clone(),
+ config: Arc::new(config),
+ auth_service_client,
+ redis_client,
+ sqs_client,
+ sfs_client: Arc::new(sfs_client),
+ gmail_client: gmail_client.clone(),
+ email_api,
+ s3_client: Arc::new(s3_client),
+ dss_client: Arc::new(dss_client),
+ system_properties_service,
+ authorization_state: authorization_state.clone(),
+ jwt_args,
+ email_service,
+ entity_access_service,
+ email_thread_state,
+ gmail_token_state,
+ macro_event_broker: Arc::new(macro_event_broker),
+ calendar_service,
+ calendar_mutation_service,
+ })
+ .await;
+
+ tracing::info!("waiting for event broker publishes to drain");
+ event_broker_tracker.close();
+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {
+ Ok(()) => tracing::info!("event broker publishes drained"),
+ Err(error) => {
+ tracing::warn!(
+ error=?error,
+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),
+ "timed out waiting for event broker publishes to drain"
+ );
+ }
+ }
+
+ api_result
+}
+
+const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
diff --git a/services/organization_retention_handler/migrations/20240325143515_basic_schema_for_testing.sql b/services/organization_retention_handler/fixtures/schema.sql
similarity index 97%
rename from services/organization_retention_handler/migrations/20240325143515_basic_schema_for_testing.sql
rename to services/organization_retention_handler/fixtures/schema.sql
index 71a713df92..db05debb52 100644
--- a/services/organization_retention_handler/migrations/20240325143515_basic_schema_for_testing.sql
+++ b/services/organization_retention_handler/fixtures/schema.sql
@@ -1,3 +1,9 @@
+-- Test-only schema, loaded as the first fixture with `migrations = false`.
+-- CI clones test databases from a pre-migrated template1 (`just setup_test_template`),
+-- so start from an empty public schema.
+DROP SCHEMA public CASCADE;
+CREATE SCHEMA public;
+
-- NOTE: This is used purely to setup the testing DB
-- Changes from `crates/macro_db_client/migrations` need to be reflected here as raw SQL
-- Changes here WILL NOT ever make it into the dev/production db.
diff --git a/services/organization_retention_handler/src/service/db/get_violating_items.rs b/services/organization_retention_handler/src/service/db/get_violating_items.rs
index e0153fbabb..abdef5e3d2 100644
--- a/services/organization_retention_handler/src/service/db/get_violating_items.rs
+++ b/services/organization_retention_handler/src/service/db/get_violating_items.rs
@@ -85,7 +85,10 @@ pub async fn get_violating_chats(
mod tests {
use super::*;
- #[sqlx::test(fixtures(path = "../../../fixtures", scripts("testing")))]
+ #[sqlx::test(
+ migrations = false,
+ fixtures(path = "../../../fixtures", scripts("schema", "testing"))
+ )]
async fn test_get_violating_items(db: sqlx::Pool<sqlx::Postgres>) -> anyhow::Result<()> {
let mut result = get_violating_items(&db, 1, 2).await?;
result.sort();
diff --git a/services/organization_retention_trigger/migrations/20240325143515_basic_schema_for_testing.sql b/services/organization_retention_trigger/fixtures/schema.sql
similarity index 97%
rename from services/organization_retention_trigger/migrations/20240325143515_basic_schema_for_testing.sql
rename to services/organization_retention_trigger/fixtures/schema.sql
index b810ed1e48..5126745f7e 100644
--- a/services/organization_retention_trigger/migrations/20240325143515_basic_schema_for_testing.sql
+++ b/services/organization_retention_trigger/fixtures/schema.sql
@@ -1,3 +1,9 @@
+-- Test-only schema, loaded as the first fixture with `migrations = false`.
+-- CI clones test databases from a pre-migrated template1 (`just setup_test_template`),
+-- so start from an empty public schema.
+DROP SCHEMA public CASCADE;
+CREATE SCHEMA public;
+
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
CREATE TABLE "Organization"
diff --git a/services/organization_retention_trigger/src/service/db/get_organization_retention.rs b/services/organization_retention_trigger/src/service/db/get_organization_retention.rs
index 885c708401..ccb7f60231 100644
--- a/services/organization_retention_trigger/src/service/db/get_organization_retention.rs
+++ b/services/organization_retention_trigger/src/service/db/get_organization_retention.rs
@@ -28,7 +28,10 @@ pub async fn get_organization_retention(
mod tests {
use super::*;
- #[sqlx::test(fixtures(path = "../../../fixtures", scripts("testing")))]
+ #[sqlx::test(
+ migrations = false,
+ fixtures(path = "../../../fixtures", scripts("schema", "testing"))
+ )]
async fn test_soft_delete_documents(pool: sqlx::Pool<sqlx::Postgres>) -> anyhow::Result<()> {
let result = get_organization_retention(pool.clone()).await?;
diff --git a/services/sha_cleanup_worker/migrations/20240325143515_basic_schema_for_testing.sql b/services/sha_cleanup_worker/fixtures/schema.sql
similarity index 95%
rename from services/sha_cleanup_worker/migrations/20240325143515_basic_schema_for_testing.sql
rename to services/sha_cleanup_worker/fixtures/schema.sql
index c7a1eadb4b..352fdcada9 100644
--- a/services/sha_cleanup_worker/migrations/20240325143515_basic_schema_for_testing.sql
+++ b/services/sha_cleanup_worker/fixtures/schema.sql
@@ -1,3 +1,9 @@
+-- Test-only schema, loaded as the first fixture with `migrations = false`.
+-- CI clones test databases from a pre-migrated template1 (`just setup_test_template`),
+-- so start from an empty public schema.
+DROP SCHEMA public CASCADE;
+CREATE SCHEMA public;
+
-- NOTE: This is used purely to setup the testing DB
-- Changes from `crates/macro_db_client/migrations` need to be reflected here as raw SQL
-- Changes here WILL NOT ever make it into the dev/production db.
diff --git a/services/sha_cleanup_worker/src/service/db/get_sha_count.rs b/services/sha_cleanup_worker/src/service/db/get_sha_count.rs
index eb2cc4521b..57f982ba70 100644
--- a/services/sha_cleanup_worker/src/service/db/get_sha_count.rs
+++ b/services/sha_cleanup_worker/src/service/db/get_sha_count.rs
@@ -23,7 +23,10 @@ pub(in crate::service::db) async fn get_sha_count(
mod tests {
use super::*;
- #[sqlx::test(fixtures(path = "../../../fixtures", scripts("shas")))]
+ #[sqlx::test(
+ migrations = false,
+ fixtures(path = "../../../fixtures", scripts("schema", "shas"))
+ )]
async fn test_get_sha_count(pool: sqlx::Pool<sqlx::Postgres>) -> anyhow::Result<()> {
let result = get_sha_count(pool.clone(), "sha-1").await?;
assert_eq!(result, 2);
diff --git a/tooling/just/rust.just b/tooling/just/rust.just
index 26108b0a2d..bd63226278 100644
--- a/tooling/just/rust.just
+++ b/tooling/just/rust.just
@@ -133,3 +133,10 @@ setup_macrodb:
just crates/macro_db_client/migrate_db
initialize_dbs: setup_macrodb
+
+# Migrate template1 so each `#[sqlx::test]` database starts with the macrodb schema:
+# Postgres clones template1 for every CREATE DATABASE, which leaves the test migrator
+# nothing to apply. Every database created on this server afterwards gets the schema
+# too, so only run it against a Postgres dedicated to one checkout, such as CI's.
+setup_test_template:
+ cd crates/macro_db_client && sqlx migrate run --database-url {{ replace(DATABASE_URL, "/macrodb", "/template1") }}
diff --git a/tooling/xtask/crates/xtask_nextest_filter/Cargo.toml b/tooling/xtask/crates/xtask_nextest_filter/Cargo.toml
index aaf2508aaf..4b1f37be1f 100644
--- a/tooling/xtask/crates/xtask_nextest_filter/Cargo.toml
+++ b/tooling/xtask/crates/xtask_nextest_filter/Cargo.toml
@@ -14,5 +14,6 @@ anyhow = { workspace = true }
determinator = { workspace = true }
guppy = { workspace = true }
tempfile = { workspace = true }
+toml = { workspace = true }
xtask_graph = { path = "../xtask_graph" }
xtask_paths = { path = "../xtask_paths" }
diff --git a/tooling/xtask/crates/xtask_nextest_filter/src/main.rs b/tooling/xtask/crates/xtask_nextest_filter/src/main.rs
index ab9fb572b1..9a20ec27a3 100644
--- a/tooling/xtask/crates/xtask_nextest_filter/src/main.rs
+++ b/tooling/xtask/crates/xtask_nextest_filter/src/main.rs
@@ -4,10 +4,18 @@
//! for Rust CI using [`determinator`].
//!
//! Input is a newline-delimited file of paths relative to the repository root
-//! (typically `git diff --name-only ...`) and a Git base revision. The command
-//! builds Cargo metadata for the base and current revisions, then lets
-//! determinator account for path, dependency, and feature changes. Shared
-//! inputs that Cargo does not know about are declared in `determinator.toml`.
+//! (typically `git diff --name-only ...`) and a Git base revision. When no
+//! `Cargo.toml` or `Cargo.lock` changed, the package graph is identical on
+//! both sides, so the command maps paths to packages (determinator's path
+//! rules) and adds their reverse dependencies from the current graph alone.
+//! Otherwise it builds Cargo metadata for the base and current revisions and
+//! lets determinator account for dependency and feature changes as well.
+//! Shared inputs that Cargo does not know about are declared in
+//! `determinator.toml`.
+//!
+//! A root `Cargo.toml` edit forces `all` unless it only touches
+//! `workspace.members` / `workspace.dependencies`; see
+//! [`root_manifest_change_forces_all`].
//!
//! Stdout is one of:
//! - `all` — every workspace package is affected.
@@ -24,7 +32,10 @@ use std::process::{Command, Stdio};
use anyhow::{Context, Result, bail};
use determinator::Determinator;
use determinator::rules::{DeterminatorRules, PathMatch};
-use guppy::graph::{DependencyDirection, PackageGraph};
+use guppy::PackageId;
+use guppy::graph::cargo::{CargoOptions, CargoSet};
+use guppy::graph::feature::{FeatureSet, StandardFeatures};
+use guppy::graph::{DependencyDirection, PackageGraph, PackageMetadata};
use tempfile::TempDir;
#[cfg(test)]
use xtask_graph::build_graph;
@@ -106,13 +117,73 @@ fn run(changed_files_path: &Path, base_revision: &str) -> Result<()> {
.collect();
let workspace_dir = xtask_paths::workspace_root();
+ if !changed_files.iter().any(|path| changes_cargo_graph(path)) {
+ let graph = build_graph_at(&workspace_dir, true)?;
+ println!(
+ "{}",
+ select_by_paths(&graph, &changed_files, &repository_rules()?)?
+ );
+ return Ok(());
+ }
+
let base_worktree = BaseWorktree::new(&workspace_dir, base_revision)?;
+ if changed_files
+ .iter()
+ .any(|path| path == Path::new(ROOT_MANIFEST))
+ {
+ let old_manifest = read_manifest(base_worktree.path())?;
+ let new_manifest = read_manifest(&workspace_dir)?;
+ if root_manifest_change_forces_all(&old_manifest, &new_manifest)? {
+ println!("{}", PackageSelection::All);
+ return Ok(());
+ }
+ }
let (old_graph, new_graph) = build_graphs(base_worktree.path(), &workspace_dir)?;
- let packages = compute_packages(&old_graph, &new_graph, &changed_files)?;
+ let packages = compute_packages(&old_graph, &new_graph, &changed_files, &repository_rules()?)?;
println!("{packages}");
Ok(())
}
+const ROOT_MANIFEST: &str = "Cargo.toml";
+
+fn read_manifest(dir: &Path) -> Result<String> {
+ let path = dir.join(ROOT_MANIFEST);
+ std::fs::read_to_string(&path).with_context(|| format!("reading {}", path.display()))
+}
+
+/// Root-manifest keys whose edits reach every affected package through the
+/// old/new Cargo graphs, so determinator can scope them:
+/// - `workspace.members`: an added member is a new package (determinator marks
+/// it changed); a removed one changes the manifests that depended on it.
+/// - `workspace.dependencies`: members inherit these specs, so a version,
+/// source, or feature change shows up in the simulated builds of exactly
+/// the members whose resolved dependencies changed. An unused or
+/// resolution-neutral edit selects nothing, which is correct.
+///
+/// Everything else (`[profile.*]`, `[workspace.lints]`, `[workspace.package]`,
+/// `[patch]`, `resolver`, …) changes how every package compiles without
+/// changing the package graph, so determinator cannot see it.
+const GRAPH_VISIBLE_WORKSPACE_KEYS: [&str; 2] = ["members", "dependencies"];
+
+/// Whether a root `Cargo.toml` edit must run the whole workspace: true unless
+/// the old and new manifests are equal once the graph-visible workspace keys
+/// are removed. Compares parsed TOML, so comment and formatting edits are
+/// no-ops.
+fn root_manifest_change_forces_all(old: &str, new: &str) -> Result<bool> {
+ let strip = |manifest: &str, revision: &str| -> Result<toml::Table> {
+ let mut table: toml::Table = manifest
+ .parse()
+ .with_context(|| format!("parsing the {revision} root {ROOT_MANIFEST}"))?;
+ if let Some(toml::Value::Table(workspace)) = table.get_mut("workspace") {
+ for key in GRAPH_VISIBLE_WORKSPACE_KEYS {
+ workspace.remove(key);
+ }
+ }
+ Ok(table)
+ };
+ Ok(strip(old, "base")? != strip(new, "current")?)
+}
+
fn build_graphs(base_dir: &Path, current_dir: &Path) -> Result<(PackageGraph, PackageGraph)> {
Ok((
build_graph_at(base_dir, true)?,
@@ -120,6 +191,114 @@ fn build_graphs(base_dir: &Path, current_dir: &Path) -> Result<(PackageGraph, Pa
))
}
+/// This repository's determinator rules (`determinator.toml`).
+fn repository_rules() -> Result<DeterminatorRules> {
+ DeterminatorRules::parse(DETERMINATOR_RULES).context("parsing nextest determinator rules")
+}
+
+/// Whether a changed path can alter `cargo metadata`: any manifest or
+/// lockfile. Without one, the base and current package graphs are identical
+/// and determinator's build-summary comparison (several seconds of feature
+/// resolution per workspace member) cannot find anything.
+fn changes_cargo_graph(path: &Path) -> bool {
+ matches!(
+ path.file_name().and_then(|name| name.to_str()),
+ Some("Cargo.toml" | "Cargo.lock")
+ )
+}
+
+/// Selection for a change that leaves the package graph untouched: packages
+/// owning a changed path (or named by a determinator path rule) plus every
+/// workspace package whose own build compiles one of them. That is the set
+/// [`compute_packages`] returns when old and new graphs are identical, but
+/// determinator gets there by simulating none/default/all-feature builds of
+/// every member on both sides (~600 simulations, several seconds). Here only
+/// the reverse dependencies of the changed packages are simulated, once.
+fn select_by_paths(
+ graph: &PackageGraph,
+ changed_files: &[PathBuf],
+ rules: &DeterminatorRules,
+) -> Result<PackageSelection> {
+ let mut determinator = Determinator::new(graph, graph);
+ determinator
+ .set_rules(rules)
+ .context("applying nextest determinator rules")?;
+
+ let mut changed = BTreeSet::new();
+ for changed_file in changed_files {
+ let changed_file = changed_file
+ .to_str()
+ .context("changed file path is not valid UTF-8")?;
+ let matched = determinator.match_path(changed_file, |id| {
+ changed.insert(id);
+ });
+ if matches!(matched, PathMatch::RuleMatchedAll) {
+ return Ok(PackageSelection::All);
+ }
+ }
+
+ // Every package that could build a changed one, following all edges. Cheap,
+ // and a superset of the answer.
+ let candidates = graph
+ .query_reverse(changed.iter().copied())
+ .context("querying reverse dependencies")?
+ .resolve();
+ let cargo_options = Determinator::default_cargo_options();
+ let features_only = graph.feature_graph().resolve_none();
+ let mut selected = BTreeSet::new();
+ for package in candidates
+ .packages(DependencyDirection::Forward)
+ .filter(|package| package.in_workspace())
+ {
+ if changed.contains(package.id())
+ || builds_any(package, &changed, &cargo_options, &features_only)?
+ {
+ selected.insert(package.name().to_owned());
+ }
+ }
+ workspace_selection(graph, selected)
+}
+
+/// Whether `package`'s own test build with all its features (dev-dependencies
+/// included, current platform) compiles any of `changed`. This is the one-hop
+/// check determinator's reverse index makes for path changes; enabling more
+/// features only adds dependencies, so the all-features build covers the
+/// none and default ones it also simulates.
+fn builds_any(
+ package: PackageMetadata<'_>,
+ changed: &BTreeSet<&PackageId>,
+ cargo_options: &CargoOptions<'_>,
+ features_only: &FeatureSet<'_>,
+) -> Result<bool> {
+ let initials = package
+ .to_package_set()
+ .to_feature_set(StandardFeatures::All);
+ let build = CargoSet::new(initials, features_only.clone(), cargo_options)
+ .with_context(|| format!("simulating the build of {}", package.name()))?;
+ let target = build.target_features().to_package_set();
+ let host = build.host_features().to_package_set();
+ for id in changed {
+ if target.contains(id)? || host.contains(id)? {
+ return Ok(true);
+ }
+ }
+ Ok(false)
+}
+
+/// `None` for an empty set, `All` when it covers the whole workspace.
+fn workspace_selection(
+ graph: &PackageGraph,
+ packages: BTreeSet<String>,
+) -> Result<PackageSelection> {
+ if packages.is_empty() {
+ return Ok(PackageSelection::None);
+ }
+ if packages.len() == graph.workspace().iter().count() {
+ return Ok(PackageSelection::All);
+ }
+ Ok(PackageSelection::Packages(packages))
+}
+
/// Use determinator to compare both Cargo graphs and return affected package
/// names. Paths outside Cargo packages contribute nothing, preserving the CI
/// behavior for Nix-only and other non-Rust changes.
@@ -127,12 +306,11 @@ fn compute_packages(
old_graph: &PackageGraph,
new_graph: &PackageGraph,
changed_files: &[PathBuf],
+ rules: &DeterminatorRules,
) -> Result<PackageSelection> {
- let rules = DeterminatorRules::parse(DETERMINATOR_RULES)
- .context("parsing nextest determinator rules")?;
let mut determinator = Determinator::new(old_graph, new_graph);
determinator
- .set_rules(&rules)
+ .set_rules(rules)
.context("applying nextest determinator rules")?;
for changed_file in changed_files {
@@ -148,21 +326,14 @@ fn compute_packages(
}
let affected = determinator.compute().affected_set;
- let packages: BTreeSet<_> = affected
- .packages(DependencyDirection::Forward)
- .filter(|package| package.in_workspace())
- .map(|package| package.name().to_owned())
- .collect();
-
- if packages.is_empty() {
- return Ok(PackageSelection::None);
- }
-
- if packages.len() == new_graph.workspace().iter().count() {
- return Ok(PackageSelection::All);
- }
-
- Ok(PackageSelection::Packages(packages))
+ workspace_selection(
+ new_graph,
+ affected
+ .packages(DependencyDirection::Forward)
+ .filter(|package| package.in_workspace())
+ .map(|package| package.name().to_owned())
+ .collect(),
+ )
}
struct BaseWorktree {
diff --git a/tooling/xtask/crates/xtask_nextest_filter/src/test.rs b/tooling/xtask/crates/xtask_nextest_filter/src/test.rs
index ded39134aa..18ffb641be 100644
--- a/tooling/xtask/crates/xtask_nextest_filter/src/test.rs
+++ b/tooling/xtask/crates/xtask_nextest_filter/src/test.rs
@@ -4,6 +4,11 @@ fn changed_files(paths: &[&str]) -> Vec<PathBuf> {
paths.iter().map(PathBuf::from).collect()
}
+/// Path-only selection on the real workspace with the repository's rules.
+fn select_in_repo(graph: &PackageGraph, paths: &[&str]) -> PackageSelection {
+ select_by_paths(graph, &changed_files(paths), &repository_rules().unwrap()).unwrap()
+}
+
fn selected_packages(selection: &PackageSelection) -> &BTreeSet<String> {
selection
.packages()
@@ -26,17 +31,17 @@ fn display_is_the_cli_seam() {
);
}
+/// Path-only changes go through [`select_by_paths`], which skips
+/// determinator's per-member build simulation (seconds of feature resolution
+/// per call). [`compute_packages`] is covered on small synthetic workspaces
+/// below, where that simulation is cheap.
+///
/// A change in a single crate selects that crate and its reverse dependencies,
/// never the whole workspace.
#[test]
fn crate_change_maps_to_rdeps_of_that_crate() {
let graph = build_graph(false).expect("cargo metadata");
- let selection = compute_packages(
- &graph,
- &graph,
- &changed_files(&["crates/email_validator/src/lib.rs"]),
- )
- .unwrap();
+ let selection = select_in_repo(&graph, &["crates/email_validator/src/lib.rs"]);
assert_ne!(selection, PackageSelection::None);
assert_ne!(selection, PackageSelection::All);
assert!(
@@ -50,12 +55,10 @@ fn crate_change_maps_to_rdeps_of_that_crate() {
#[test]
fn unmapped_files_alone_yield_none() {
let graph = build_graph(false).expect("cargo metadata");
- let selection = compute_packages(
- &graph,
+ let selection = select_in_repo(
&graph,
- &changed_files(&["random.json", "package.json", "docs/README.md", "justfile"]),
- )
- .unwrap();
+ &["random.json", "package.json", "docs/README.md", "justfile"],
+ );
assert_eq!(selection, PackageSelection::None);
}
@@ -64,12 +67,7 @@ fn unmapped_files_alone_yield_none() {
#[test]
fn package_readme_selects_its_package() {
let graph = build_graph(false).expect("cargo metadata");
- let selection = compute_packages(
- &graph,
- &graph,
- &changed_files(&["crates/webhook/README.md"]),
- )
- .unwrap();
+ let selection = select_in_repo(&graph, &["crates/webhook/README.md"]);
assert!(
selected_packages(&selection).contains("webhook"),
"package README must select its owner: {selection:?}"
@@ -82,12 +80,7 @@ fn package_readme_selects_its_package() {
fn embedded_assets_select_their_consumers() {
let graph = build_graph(false).expect("cargo metadata");
- let selection = compute_packages(
- &graph,
- &graph,
- &changed_files(&["static_assets/schema.graphql"]),
- )
- .unwrap();
+ let selection = select_in_repo(&graph, &["static_assets/schema.graphql"]);
let set = selected_packages(&selection);
for expected in [
"cache-core",
@@ -103,15 +96,13 @@ fn embedded_assets_select_their_consumers() {
);
}
- let mixed = compute_packages(
- &graph,
+ let mixed = select_in_repo(
&graph,
- &changed_files(&[
+ &[
"crates/email_validator/src/lib.rs",
"static_assets/markdown-golden.1.bin",
- ]),
- )
- .unwrap();
+ ],
+ );
let mixed_set = selected_packages(&mixed);
assert!(mixed_set.contains("email_validator"));
assert!(mixed_set.contains("documents"));
@@ -180,6 +171,279 @@ fn embedded_asset_packages_match_source_references() {
);
}
+const BASE_MANIFEST: &str = r#"
+[workspace]
+resolver = "2"
+members = ["crates/a", "crates/b"]
+
+[workspace.dependencies]
+anyhow = "1.0.100"
+serde = { version = "1.0.214", features = ["derive"] }
+
+[profile.fig-engine-wasm]
+inherits = "release"
+lto = "fat"
+"#;
+
+fn manifest_forces_all(new: &str) -> bool {
+ root_manifest_change_forces_all(BASE_MANIFEST, new).expect("valid manifests")
+}
+
+/// Adding or removing a member is a package-graph change determinator scopes
+/// to the new package (or to the manifests that depended on a removed one).
+#[test]
+fn members_only_change_defers_to_determinator() {
+ let new = BASE_MANIFEST.replace(
+ r#"members = ["crates/a", "crates/b"]"#,
+ r#"members = ["crates/a", "crates/b", "crates/c"]"#,
+ );
+ assert_ne!(new, BASE_MANIFEST);
+ assert!(!manifest_forces_all(&new));
+}
+
+/// Workspace dependency specs reach members through inheritance, so the
+/// graph diff selects exactly the members whose resolved deps changed.
+#[test]
+fn workspace_dependencies_only_change_defers_to_determinator() {
+ let new = BASE_MANIFEST
+ .replace(r#"anyhow = "1.0.100""#, r#"anyhow = "1.0.101""#)
+ .replace(r#"features = ["derive"]"#, r#"features = ["derive", "rc"]"#)
+ .replace(
+ "[workspace.dependencies]\n",
+ "[workspace.dependencies]\nhex = \"0.4.3\"\n",
+ );
+ assert_ne!(new, BASE_MANIFEST);
+ assert!(!manifest_forces_all(&new));
+}
+
+/// Comments and formatting are not semantic edits.
+#[test]
+fn comment_and_formatting_change_is_not_a_change() {
+ let new = BASE_MANIFEST.replace(
+ "[profile.fig-engine-wasm]",
+ "# Browser build for the fig engine.\n[profile.fig-engine-wasm]",
+ );
+ assert!(!manifest_forces_all(&new));
+}
+
+/// Profiles change compile flags for every package without touching the graph.
+#[test]
+fn profile_change_forces_all() {
+ assert!(manifest_forces_all(
+ &BASE_MANIFEST.replace(r#"lto = "fat""#, r#"lto = "thin""#)
+ ));
+ let added = format!("{BASE_MANIFEST}\n[profile.dev]\ndebug = \"limited\"\n");
+ assert!(manifest_forces_all(&added));
+}
+
+/// Workspace lints change every member's clippy/rustc output.
+#[test]
+fn workspace_lints_change_forces_all() {
+ let new = format!("{BASE_MANIFEST}\n[workspace.lints.rust]\nunsafe_code = \"forbid\"\n");
+ assert!(manifest_forces_all(&new));
+}
+
+/// Other workspace-level settings are invisible to the graph diff as well.
+#[test]
+fn resolver_patch_and_package_changes_force_all() {
+ assert!(manifest_forces_all(
+ &BASE_MANIFEST.replace(r#"resolver = "2""#, r#"resolver = "3""#)
+ ));
+ assert!(manifest_forces_all(&format!(
+ "{BASE_MANIFEST}\n[patch.crates-io]\nserde = {{ path = \"vendor/serde\" }}\n"
+ )));
+ assert!(manifest_forces_all(&format!(
+ "{BASE_MANIFEST}\n[workspace.package]\nedition = \"2024\"\n"
+ )));
+}
+
+/// A members edit combined with anything else still forces the full suite.
+#[test]
+fn mixed_members_and_profile_change_forces_all() {
+ let new = BASE_MANIFEST
+ .replace(
+ r#"members = ["crates/a", "crates/b"]"#,
+ r#"members = ["crates/a"]"#,
+ )
+ .replace(r#"lto = "fat""#, "lto = false");
+ assert!(manifest_forces_all(&new));
+}
+
+/// The real root manifest parses, and comparing it with itself is a no-op.
+#[test]
+fn repository_root_manifest_is_classified() {
+ let manifest = read_manifest(&xtask_paths::workspace_root()).expect("root manifest");
+ assert!(!root_manifest_change_forces_all(&manifest, &manifest).unwrap());
+}
+
+/// A tiny two-crate workspace (`b` depends on `a` through
+/// `workspace.dependencies`) written to a temp dir, with optional overrides.
+struct SyntheticWorkspace {
+ dir: TempDir,
+}
+
+impl SyntheticWorkspace {
+ /// `a` plus `others`, each depending on `a` through
+ /// `workspace.dependencies` with `a_features` enabled.
+ fn new(members: &[&str], a_features: &str) -> Self {
+ let manifests: Vec<(&str, &str)> = members
+ .iter()
+ .map(|member| {
+ let deps = if *member == "a" {
+ "[features]\nextra = []\n"
+ } else {
+ "[dependencies]\na = { workspace = true }\n"
+ };
+ (*member, deps)
+ })
+ .collect();
+ Self::with_members(
+ &manifests,
+ &format!("a = {{ path = \"a\", features = [{a_features}] }}\n"),
+ )
+ }
+
+ /// Members given as `(name, manifest body after [package])`.
+ fn with_members(members: &[(&str, &str)], workspace_dependencies: &str) -> Self {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let write = |path: &str, content: &str| {
+ let path = dir.path().join(path);
+ std::fs::create_dir_all(path.parent().unwrap()).unwrap();
+ std::fs::write(path, content).unwrap();
+ };
+ let quoted: Vec<String> = members
+ .iter()
+ .map(|(name, _)| format!("{name:?}"))
+ .collect();
+ write(
+ "Cargo.toml",
+ &format!(
+ "[workspace]\nresolver = \"2\"\nmembers = [{}]\n\n[workspace.dependencies]\n\
+ {workspace_dependencies}",
+ quoted.join(", ")
+ ),
+ );
+ for (name, body) in members {
+ write(
+ &format!("{name}/Cargo.toml"),
+ &format!(
+ "[package]\nname = \"{name}\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n{body}"
+ ),
+ );
+ write(&format!("{name}/src/lib.rs"), "");
+ }
+ Self { dir }
+ }
+
+ fn graph(&self) -> PackageGraph {
+ build_graph_at(self.dir.path(), false).expect("cargo metadata for synthetic workspace")
+ }
+}
+
+/// The repository's path rules name real packages, so the synthetic
+/// workspaces run determinator without them.
+fn no_path_rules() -> DeterminatorRules {
+ DeterminatorRules::parse("use-default-rules = false").expect("empty rules")
+}
+
+/// A new member reaches determinator as a new package; nothing that existed
+/// before is rebuilt.
+#[test]
+fn graph_diff_scopes_a_new_member_to_itself() {
+ let old = SyntheticWorkspace::new(&["a", "b"], "");
+ let new = SyntheticWorkspace::new(&["a", "b", "c"], "");
+ let selection = compute_packages(
+ &old.graph(),
+ &new.graph(),
+ &changed_files(&["Cargo.toml", "c/Cargo.toml", "c/src/lib.rs"]),
+ &no_path_rules(),
+ )
+ .unwrap();
+ assert_eq!(
+ selection,
+ PackageSelection::Packages(BTreeSet::from(["c".to_owned()]))
+ );
+}
+
+/// A feature added in `workspace.dependencies` shows up in the simulated
+/// build of the member that inherits it, so the root-manifest edit selects
+/// that member without anyone touching its own files.
+#[test]
+fn graph_diff_scopes_a_workspace_dependency_change_to_inheritors() {
+ let old = SyntheticWorkspace::new(&["a", "b", "c"], "");
+ let new = SyntheticWorkspace::new(&["a", "b", "c"], "\"extra\"");
+ let selection = compute_packages(
+ &old.graph(),
+ &new.graph(),
+ &changed_files(&["Cargo.toml"]),
+ &no_path_rules(),
+ )
+ .unwrap();
+ assert_eq!(
+ selection,
+ PackageSelection::Packages(BTreeSet::from(["b".to_owned(), "c".to_owned()]))
+ );
+}
+
+/// The fast path agrees with determinator on what a build contains: a
+/// dev-dependency reaches only its declarer's tests (`b` dev-depends on `a`,
+/// `c` uses `b`'s library), normal dependencies are transitive (`e` → `d` →
+/// `a`), and an optional dependency nobody enables builds nothing (`g` uses
+/// `h` without the feature that pulls in `a`).
+#[test]
+fn fast_path_matches_determinator_on_build_edges() {
+ let workspace = SyntheticWorkspace::with_members(
+ &[
+ ("a", ""),
+ ("b", "[dev-dependencies]\na = { path = \"../a\" }\n"),
+ ("c", "[dependencies]\nb = { path = \"../b\" }\n"),
+ ("d", "[dependencies]\na = { path = \"../a\" }\n"),
+ ("e", "[dependencies]\nd = { path = \"../d\" }\n"),
+ ("f", ""),
+ ("g", "[dependencies]\nh = { path = \"../h\" }\n"),
+ (
+ "h",
+ "[dependencies]\na = { path = \"../a\", optional = true }\n\n\
+ [features]\nwith_a = [\"dep:a\"]\n",
+ ),
+ ],
+ "",
+ );
+ let graph = workspace.graph();
+ let expected =
+ PackageSelection::Packages(BTreeSet::from(["a", "b", "d", "e", "h"].map(str::to_owned)));
+ let changed = changed_files(&["a/src/lib.rs"]);
+ assert_eq!(
+ select_by_paths(&graph, &changed, &no_path_rules()).unwrap(),
+ expected
+ );
+ assert_eq!(
+ compute_packages(&graph, &graph, &changed, &no_path_rules()).unwrap(),
+ expected
+ );
+}
+
+/// Only manifests and lockfiles can change `cargo metadata`; everything else
+/// takes the path-only fast path.
+#[test]
+fn only_manifests_and_lockfiles_need_the_graph_diff() {
+ for path in [
+ "Cargo.toml",
+ "Cargo.lock",
+ "crates/email_validator/Cargo.toml",
+ ] {
+ assert!(changes_cargo_graph(Path::new(path)), "{path}");
+ }
+ for path in [
+ "crates/email_validator/src/lib.rs",
+ "Cargo.toml.orig",
+ "docs/Cargo.md",
+ "static_assets/schema.graphql",
+ ] {
+ assert!(!changes_cargo_graph(Path::new(path)), "{path}");
+ }
+}
+
/// Recursively collect `.rs` files under `dir`, skipping hidden and build
/// output directories.
fn collect_rust_files(dir: &Path, out: &mut BTreeSet<PathBuf>) {
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows.rs b/tooling/xtask/crates/xtask_workflows/src/workflows.rs
index caf9710bb3..52cfb6a761 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows.rs
@@ -34,7 +34,6 @@ mod deploy_preview;
mod deploy_sync_service;
mod deploy_web_app;
mod deploy_website;
-mod docs_check;
mod ensure_daytona_snapshot;
mod ios_preview;
mod path_validation;
@@ -325,11 +324,6 @@ const WORKFLOWS: &[WorkflowFile] = &[
file_name: "sdk-check.yml",
render_yaml: || render_gh_workflow(sdk_check::sdk_check)(),
},
- WorkflowFile {
- slug: "docs_check",
- file_name: "docs-check.yml",
- render_yaml: || render_gh_workflow(docs_check::docs_check)(),
- },
];
/// Write every workflow to disk.
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs
index 3a6ba50ad1..e38d76e7f8 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs
@@ -5,10 +5,14 @@
//! moved to Namespace profiles, and compiled objects moved off the S3/local
//! volume backends onto Namespace's official remote sccache (so there are no
//! AWS credentials anywhere).
+//!
+//! Critical path: `path-check` (no Rust compile on a warm binary cache) →
+//! `test`. `check` (fmt + clippy) and `doppler-config` run alongside `test`.
use gh_workflow::{
Container, Env, Event, Expression, Job, Port, PullRequest, PullRequestType, Run, Step, Workflow,
};
+use xtask_paths::RuntimePath;
use crate::workflows::{
runners,
@@ -34,15 +38,31 @@ pub fn code_check_cloud_storage() -> Workflow {
.concurrency(vars::concurrency("code-check-cloud-storage"))
.add_job("path-check", path_check())
.add_job("check", check())
+ .add_job("doppler-config", doppler_config())
.add_job("test", test())
.add_job("status-check", status_check())
}
-/// Decide whether the rest of the workflow runs, and compute the nextest filter.
+/// Same-repo PRs only: the binary cache below is shared, so an untrusted fork
+/// must not be able to plant a binary under a key a later PR would execute.
+const TRUSTED_CONTEXT: &str = steps::TRUSTED_NAMESPACE_SCCACHE_CONTEXT;
+
+/// Where `path-check` keeps the release `xtask_nextest_filter` binary (keyed by
+/// its source trees and `Cargo.lock`) and the target dir that builds it.
+const FILTER_CACHE_DIR: RuntimePath<'static> =
+ xtask_paths::runtime_path!("/home/runner/.cache/macro-nextest-filter");
+
+/// Decide whether the rest of the workflow runs and compute the nextest
+/// filter. Deliberately light: no Nix, no sccache, and a cached release
+/// binary for the filter.
fn path_check() -> Job {
Job::default()
- .runs_on(runners::Runner::Small.to_string())
+ .runs_on(runners::Runner::Small.with_cache_tag(vars::NEXTEST_FILTER_CACHE_TAG))
.add_output("should_run", "${{ steps.filter.outputs.should_run }}")
+ .add_output(
+ "doppler_candidates",
+ "${{ steps.filter.outputs.doppler_candidates }}",
+ )
.add_output(
"rust_packages",
"${{ steps.nextest-filter.outputs.rust_packages }}",
@@ -51,42 +71,75 @@ fn path_check() -> Job {
"skip_tests",
"${{ steps.nextest-filter.outputs.skip_tests }}",
)
- .add_output(
- "doppler_config_bins",
- "${{ steps.doppler-bins.outputs.doppler_config_bins }}",
- )
// PR merge commits need both parents for `git merge-base`; fetching
// the entire repo (every branch) is not required.
.add_step(steps::checkout(false, false).add_with(("fetch-depth", 2)))
.add_step(steps::setup_rust_light())
.add_step(paths_filter())
- .add_step(compute_changed_files())
- .add_step(compute_doppler_bins())
+ .add_step(
+ compute_changed_files()
+ .if_condition(Expression::new("steps.filter.outputs.should_run == 'true'")),
+ )
+ .add_step(
+ steps::mount_path_cache_volume("Mount nextest-filter binary cache", FILTER_CACHE_DIR)
+ .if_condition(Expression::new(format!(
+ "steps.filter.outputs.should_run == 'true' && ({TRUSTED_CONTEXT})"
+ ))),
+ )
.add_step(compute_nextest_filter())
}
-/// fmt + clippy (and Doppler-config validation).
+/// fmt + clippy.
fn check() -> Job {
steps::gated_job()
.runs_on(runners::Runner::RustCi.with_cache_tag(vars::CI_CACHE_TAG))
- .add_env((
- "RUSTFLAGS",
- "-Dwarnings -Dclippy::disallowed_methods -C link-arg=-fuse-ld=mold",
- ))
- .add_env(("RUSTDOCFLAGS", "-Dwarnings"))
- .add_env(("SQLX_OFFLINE", "true"))
+ .map(with_check_env)
.add_step(steps::checkout(false, false))
.add_step(steps::mount_cache_volume())
.add_step(steps::setup_nix())
.add_step(steps::setup_dev_shell())
.add_step(steps::configure_namespace_sccache(vars::CI_SCCACHE_NAME))
- .add_step(validate_doppler_configs())
.add_step(cargo_fmt())
.add_step(cargo_clippy())
.add_step(steps::show_sccache_stats())
.add_step(steps::teardown_nix())
}
+/// The env the clippy-side jobs share, so their sccache keys stay identical.
+fn with_check_env(job: Job) -> Job {
+ job.add_env((
+ "RUSTFLAGS",
+ "-Dwarnings -Dclippy::disallowed_methods -C link-arg=-fuse-ld=mold",
+ ))
+ .add_env(("RUSTDOCFLAGS", "-Dwarnings"))
+ .add_env(("SQLX_OFFLINE", "true"))
+}
+
+/// Live Doppler config-contract check for services whose config may have
+/// changed. Runs beside `check`/`test` instead of inside `path-check`, where
+/// computing the bin list compiled an xtask from scratch on every PR. A cheap
+/// path filter gates it; the xtask then picks the exact binaries.
+fn doppler_config() -> Job {
+ steps::gated_job()
+ .cond(Expression::new(
+ "needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false && needs.path-check.outputs.doppler_candidates == 'true'",
+ ))
+ .runs_on(runners::Runner::RustCi.with_cache_tag(vars::CI_CACHE_TAG))
+ .map(with_check_env)
+ .add_step(steps::checkout(false, false).add_with(("fetch-depth", 2)))
+ // Before the dev shell: its LD_LIBRARY_PATH points the runner's
+ // git-remote-https at Nix's glibc, which aborts `git fetch`.
+ .add_step(compute_changed_files())
+ .add_step(steps::mount_cache_volume())
+ .add_step(steps::setup_nix())
+ .add_step(steps::setup_dev_shell())
+ .add_step(steps::configure_namespace_sccache(vars::CI_SCCACHE_NAME))
+ .add_step(compute_doppler_bins())
+ .add_step(validate_doppler_configs())
+ .add_step(steps::show_sccache_stats())
+ .add_step(steps::teardown_nix())
+}
+
/// cargo nextest against postgres + redis service containers. Skipped when
/// the path filter selected no Rust packages (Nix-only / unmapped files) or
/// when `skip_tests` is set (`.sqlx`-only diffs still clippy the snapshot).
@@ -130,6 +183,7 @@ fn status_check() -> Job {
.needs(vec![
"path-check".to_string(),
"check".to_string(),
+ "doppler-config".to_string(),
"test".to_string(),
])
.add_step(check_job_results())
@@ -137,7 +191,9 @@ fn status_check() -> Job {
// --- workflow-specific steps -------------------------------------------------
-/// Detect whether cloud-storage-relevant paths changed.
+/// Detect whether cloud-storage-relevant paths changed, and whether any
+/// change could affect a service's Doppler config contract (a superset of
+/// what `xtask doppler-bins` checks, so the gate never hides a bin).
fn paths_filter() -> Step<gh_workflow::Use> {
Step::new("Filter changed paths")
.uses(
@@ -172,18 +228,25 @@ fn paths_filter() -> Step<gh_workflow::Use> {
- '.github/actions/setup-sccache/**'
- '.github/services-config.json'
- .github/workflows/code_check_cloud_storage.yml
+ doppler_candidates:
+ - '.github/services-config.json'
+ - '**/src/config.rs'
+ - '**/src/doppler_config.rs'
+ - '**/Cargo.toml'
+ - 'tooling/xtask/crates/xtask_doppler_bins/**'
+ - .github/workflows/code_check_cloud_storage.yml
"#},
))
}
-/// Compute the changed-file set once (shared by the nextest filter and the
-/// Doppler-bin detection) and write it to `/tmp/changed-files`. On a missing
-/// merge-base we leave the list empty, which makes both downstream steps fall
-/// back to "everything": run all tests, validate no Doppler bins.
+/// Compute the changed-file set and write it to `/tmp/changed-files`. Stacked
+/// PRs whose merge commit sits on another branch get their history deepened
+/// until it reaches the base branch. If no merge-base turns up, we leave the
+/// list empty, which makes downstream steps fall back to "everything": run all
+/// tests, validate no Doppler bins.
fn compute_changed_files() -> Step<Run> {
Step::new("compute changed files")
.run(include_str!("scripts/compute_changed_files.sh"))
- .if_condition(Expression::new("steps.filter.outputs.should_run == 'true'"))
.shell("bash")
}
@@ -193,19 +256,21 @@ fn compute_doppler_bins() -> Step<Run> {
Step::new("compute affected Doppler config bins")
.run(include_str!("scripts/compute_doppler_bins.sh"))
.id("doppler-bins")
- .if_condition(Expression::new("steps.filter.outputs.should_run == 'true'"))
.shell("bash")
}
/// Compute the cargo-nextest / clippy package set from the changed files, via
-/// the `xtask nextest-filter` subcommand. Root Cargo/toolchain changes
-/// short-circuit to the full suite; unmapped files (a top-level JSON, a Nix
-/// shell tweak) select no packages.
+/// the `xtask nextest-filter` subcommand.
+/// Toolchain and cargo config changes short-circuit to the full suite; root
+/// `Cargo.toml` edits run the full suite unless they only touch workspace
+/// members or dependencies; unmapped files (a top-level JSON, a Nix shell
+/// tweak) select no packages.
fn compute_nextest_filter() -> Step<Run> {
Step::new("compute nextest package filter")
.run(include_str!("scripts/compute_nextest_filter.sh"))
.id("nextest-filter")
.if_condition(Expression::new("steps.filter.outputs.should_run == 'true'"))
+ .add_env(("FILTER_CACHE_DIR", FILTER_CACHE_DIR.as_str()))
.shell("bash")
}
@@ -216,11 +281,11 @@ fn validate_doppler_configs() -> Step<Run> {
Step::new("validate Doppler configs")
.run(include_str!("scripts/validate_doppler_configs.sh"))
.if_condition(Expression::new(
- "needs.path-check.outputs.doppler_config_bins != ''",
+ "steps.doppler-bins.outputs.doppler_config_bins != ''",
))
.add_env((
"DOPPLER_CONFIG_BINS",
- "${{ needs.path-check.outputs.doppler_config_bins }}",
+ "${{ steps.doppler-bins.outputs.doppler_config_bins }}",
))
.add_env(("DOPPLER_TOKEN", vars::DOPPLER_TOKEN))
}
@@ -273,9 +338,11 @@ fn configure_postgres() -> Step<Run> {
.run(include_str!("scripts/configure_postgres.sh"))
}
-/// Set up test env files and databases.
+/// Set up test env files and databases. The pre-migrated template1 spares each
+/// `#[sqlx::test]` database from replaying every migration.
fn prepare_tests() -> Step<Run> {
- Step::new("prepare tests").run("just setup_test_envs && just initialize_dbs")
+ Step::new("prepare tests")
+ .run("just setup_test_envs && just initialize_dbs && just setup_test_template")
}
/// Run the test suite (no AWS credentials; sccache uses Namespace's remote cache).
@@ -288,11 +355,13 @@ fn check_job_results() -> Step<Run> {
Step::new("Check job results").run(indoc::indoc! {r#"
echo "path-check: ${{ needs.path-check.result }}"
echo "check: ${{ needs.check.result }}"
+ echo "doppler-config: ${{ needs.doppler-config.result }}"
echo "test: ${{ needs.test.result }}"
# Fail if any job failed (skipped and success are both OK)
if [[ "${{ needs.path-check.result }}" == "failure" ]] || \
[[ "${{ needs.check.result }}" == "failure" ]] || \
+ [[ "${{ needs.doppler-config.result }}" == "failure" ]] || \
[[ "${{ needs.test.result }}" == "failure" ]]; then
echo "❌ One or more jobs failed"
exit 1
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs
index 9adc05b093..b307e4be73 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs
@@ -2,10 +2,83 @@ use super::*;
#[test]
fn inventory_only_changes_run_live_config_validation() {
+ let with = serde_json::to_value(&paths_filter().value.with).unwrap();
+ let filters = with["filters"].as_str().unwrap().to_owned();
+ let doppler_filter = filters
+ .split("doppler_candidates:")
+ .nth(1)
+ .expect("doppler gate filter");
+ assert!(doppler_filter.contains(".github/services-config.json"));
+ let job = serde_json::to_value(doppler_config()).unwrap();
+ let condition = job["if"].as_str().unwrap();
+ assert!(!condition.contains("rust_packages"), "{condition}");
+ assert!(
+ condition.contains("doppler_candidates == 'true'"),
+ "{condition}"
+ );
+}
+
+/// Every trigger `xtask doppler-bins` checks must open the gate, or a config
+/// change could skip live validation.
+#[test]
+fn doppler_gate_covers_the_bin_triggers() {
let filters = serde_json::to_string(&paths_filter().value.with).unwrap();
- assert!(filters.contains(".github/services-config.json"));
- let job = serde_json::to_value(check()).unwrap();
- assert!(!job["if"].as_str().unwrap().contains("rust_packages"));
+ for glob in [
+ "**/src/config.rs",
+ "**/src/doppler_config.rs",
+ "**/Cargo.toml",
+ ] {
+ assert!(filters.contains(glob), "missing {glob}: {filters}");
+ }
+}
+
+/// path-check stays compile-free on a warm cache: the Doppler bins moved to
+/// their own job, and validation no longer rides on clippy.
+#[test]
+fn doppler_work_is_off_path_check_and_check() {
+ for job in [path_check(), check()] {
+ let job = serde_json::to_string(&job).unwrap();
+ assert!(!job.contains("doppler-bins"), "{job}");
+ assert!(!job.contains("validate Doppler configs"), "{job}");
+ }
+ let job = serde_json::to_string(&doppler_config()).unwrap();
+ assert!(job.contains("doppler-bins") && job.contains("validate Doppler configs"));
+ assert!(job.contains("steps.doppler-bins.outputs.doppler_config_bins"));
+}
+
+/// The shared binary cache is only mounted for same-repo PRs.
+#[test]
+fn filter_binary_cache_is_trusted_only() {
+ let job = serde_json::to_value(path_check()).unwrap();
+ let mount = job["steps"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .find(|step| step["name"] == "Mount nextest-filter binary cache")
+ .expect("binary cache mount");
+ let condition = mount["if"].as_str().unwrap();
+ assert!(
+ condition.contains("github.event.pull_request.head.repo.full_name == github.repository"),
+ "{condition}"
+ );
+}
+
+/// The required status check fails when any job it gathers failed, including
+/// the new Doppler job.
+#[test]
+fn status_check_covers_every_job() {
+ let status = serde_json::to_value(status_check()).unwrap();
+ let script = status["steps"][0]["run"].as_str().unwrap();
+ for needed in ["path-check", "check", "doppler-config", "test"] {
+ let check = format!(r#"[[ "${{{{ needs.{needed}.result }}}}" == "failure" ]]"#);
+ assert!(script.contains(&check), "{check}: {script}");
+ assert!(
+ status["needs"]
+ .as_array()
+ .unwrap()
+ .contains(&serde_json::json!(needed))
+ );
+ }
}
#[test]
@@ -34,27 +107,43 @@ fn tests_continue_to_exclude_sync_service() {
);
}
-#[test]
-fn sqlx_only_clippies_and_skips_live_tests() {
+/// The `if … fi` block starting at `marker` in the filter script.
+fn filter_script_block(marker: &str) -> &'static str {
let script = include_str!("../scripts/compute_nextest_filter.sh");
let start = script
- .find("if ! grep -qvE '^\\.sqlx/'")
- .expect("sqlx-only short-circuit must remain");
- let sqlx_block = script[start..]
- .split("packages=\"$(cargo run")
- .next()
- .unwrap();
+ .find(marker)
+ .unwrap_or_else(|| panic!("{marker} must remain"));
+ let end = script[start..].find("\nfi\n").unwrap() + start;
+ &script[start..end]
+}
+
+#[test]
+fn sqlx_only_clippies_and_skips_live_tests() {
+ let sqlx_block = filter_script_block("if ! grep -qvE '^\\.sqlx/'");
assert!(
- sqlx_block.contains(r#"rust_packages=all"#),
- "sqlx-only must emit all for clippy, not none: {sqlx_block}"
+ sqlx_block.contains("emit all true"),
+ "sqlx-only must clippy all and skip the live-Postgres suite: {sqlx_block}"
);
assert!(
- !sqlx_block.contains(r#"rust_packages=none"#),
+ !sqlx_block.contains("emit none"),
"sqlx-only must not skip clippy: {sqlx_block}"
);
+}
+
+/// Every exit goes through `emit`, so both outputs are always written once.
+#[test]
+fn every_exit_goes_through_emit() {
+ let script = include_str!("../scripts/compute_nextest_filter.sh");
+ let body = &script[script.find("emit() {").expect("emit must remain")..];
+ let emit = &body[..body.find("\n}\n").unwrap()];
+ assert!(
+ emit.contains("rust_packages=$1") && emit.contains("skip_tests=$2"),
+ "{emit}"
+ );
+ let after_emit = &body[body.find("\n}\n").unwrap()..];
assert!(
- sqlx_block.contains(r#"skip_tests=true"#),
- "sqlx-only diffs must not run the live-Postgres suite: {sqlx_block}"
+ !after_emit.contains(">> \"$GITHUB_OUTPUT\""),
+ "outputs must only be written by emit: {after_emit}"
);
}
@@ -73,6 +162,51 @@ fn test_job_honors_skip_tests() {
);
}
+/// Root Cargo.toml edits are classified by `xtask nextest-filter` (members and
+/// workspace dependencies scope through the graph diff); only toolchain and
+/// cargo config still short-circuit to the full suite in bash.
+#[test]
+fn root_manifest_is_classified_by_the_filter_not_forced_all() {
+ let script = include_str!("../scripts/compute_nextest_filter.sh");
+ let force_all = script
+ .lines()
+ .find(|line| line.contains(r"grep -qE '^(rust-toolchain"))
+ .expect("toolchain changes must still force the full suite");
+ for path in [
+ "Cargo\\.toml",
+ "Cross\\.toml",
+ "deny\\.toml",
+ "clippy\\.toml",
+ ] {
+ assert!(
+ !force_all.contains(path),
+ "{path} must not force the full suite: {force_all}"
+ );
+ }
+ assert!(force_all.contains(r"\.cargo/"));
+}
+
+/// A clippy.toml edit re-lints everything but only skips tests when no
+/// package was selected for other reasons.
+#[test]
+fn clippy_config_only_change_skips_tests() {
+ let block = filter_script_block(r#"if [ "$clippy_config_changed" = true ]; then"#);
+ assert!(block.contains("emit all true"), "{block}");
+ assert!(block.contains("emit all false"), "{block}");
+}
+
+/// Stacked PRs can get a merge commit whose first parent is another branch;
+/// the changed-files step deepens history before giving up on a merge-base.
+#[test]
+fn missing_merge_base_deepens_before_falling_back() {
+ let script = include_str!("../scripts/compute_changed_files.sh");
+ let deepen = script.find("--deepen").expect("must deepen the PR history");
+ let fallback = script
+ .find("falling back to full test suite")
+ .expect("must keep the full-suite fallback");
+ assert!(deepen < fallback);
+}
+
#[test]
fn tests_build_dependencies_without_debug_info() {
let script = include_str!("../scripts/run_tests.sh");
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/docs_check.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/docs_check.rs
deleted file mode 100644
index 38f16c93e0..0000000000
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/docs_check.rs
+++ /dev/null
@@ -1,67 +0,0 @@
-//! `Docs Check` — fails a PR if the generated MCP tool reference under
-//! `apps/docs/` has drifted from the Rust tool registry. The pages, their
-//! nav list, and the `Tool Reference` group in `docs.json` are all written by
-//! `apps/docs/scripts/generate-mcp-tool-pages.ts`, which builds the
-//! `gen_tool_schemas` binary and renders from its JSON. Without this check the
-//! docs site silently rots every time a tool's schema or description changes.
-
-use gh_workflow::{Concurrency, Event, Expression, Job, PullRequest, Run, Step, Workflow};
-
-use crate::workflows::{runners, steps, vars};
-
-/// Build the workflow.
-pub fn docs_check() -> Workflow {
- Workflow::new("Docs Check")
- .on(Event::default().pull_request(
- PullRequest::default()
- .add_branch("main")
- .add_path(xtask_paths::repo_glob!("apps/docs/**"))
- // Tool schemas are derived from types spread across the
- // workspace, not just `crates/ai_tools`, so watch all Rust.
- .add_path(xtask_paths::repo_glob!("crates/**/*.rs"))
- .add_path(xtask_paths::repo_glob!("Cargo.toml"))
- .add_path(xtask_paths::repo_glob!("Cargo.lock"))
- .add_path(".github/workflows/docs-check.yml"),
- ))
- .concurrency(
- Concurrency::new(Expression::new(
- "${{ github.workflow }}-${{ github.ref }}-check",
- ))
- .cancel_in_progress(true),
- )
- .add_job("check-docs", check_docs())
-}
-
-/// Regenerate the tool reference and fail on drift. Shares the web CI cache
-/// volume and sccache with `sdk-check`: both jobs build one small binary out of
-/// the same workspace, so they warm the same cache.
-fn check_docs() -> Job {
- Job::default()
- .name("Docs Generated Code Check")
- .runs_on(runners::Runner::Mid.with_cache_tag(vars::WEB_CI_CACHE_TAG))
- .add_step(steps::checkout(false, false))
- .add_step(steps::mount_web_cache_volume(true))
- .add_step(steps::setup_rust_sccache())
- .add_step(steps::setup_bun())
- .add_step(steps::configure_namespace_sccache(vars::WEB_SCCACHE_NAME))
- .add_step(generate_tool_pages())
- .add_step(steps::show_sccache_stats())
- .add_step(verify_fresh())
-}
-
-fn generate_tool_pages() -> Step<Run> {
- Step::new("Regenerate MCP tool pages")
- .run("bun run generate:tools")
- .working_directory("apps/docs")
-}
-
-fn verify_fresh() -> Step<Run> {
- Step::new("Verify generated docs are fresh").run(indoc::indoc! {r#"
- if [ -n "$(git status --porcelain -- apps/docs)" ]; then
- echo "apps/docs tool reference is stale. Run 'bun run generate:tools' in apps/docs and commit the result."
- git status --porcelain -- apps/docs
- git diff -- apps/docs | head -200
- exit 1
- fi
- "#})
-}
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_changed_files.sh b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_changed_files.sh
index a8b2ec9d7a..5b90372c6f 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_changed_files.sh
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_changed_files.sh
@@ -7,8 +7,26 @@ set -euo pipefail
if [ -z "${GITHUB_BASE_REF:-}" ]; then
compare_rev="$(git rev-parse HEAD~1)"
else
- git fetch origin "$GITHUB_BASE_REF:refs/remotes/origin/$GITHUB_BASE_REF"
- if ! compare_rev="$(git merge-base "origin/${GITHUB_BASE_REF}" HEAD)"; then
+ git fetch --no-tags origin "$GITHUB_BASE_REF:refs/remotes/origin/$GITHUB_BASE_REF"
+
+ # The checkout holds only the PR merge commit and its two parents. Normally
+ # the first parent is on the base branch, so the merge-base is immediate.
+ # Stacked PRs can arrive with a merge commit built on another branch's tip;
+ # deepen the PR side until its history reaches the base branch instead of
+ # falling back to the full suite.
+ compare_rev=""
+ for deepen in 0 200 2000; do
+ if [ "$deepen" -gt 0 ]; then
+ echo "No merge-base with origin/${GITHUB_BASE_REF} yet; deepening PR history by ${deepen}"
+ git fetch --no-tags --deepen="$deepen" origin "$GITHUB_SHA" || break
+ fi
+ if compare_rev="$(git merge-base "origin/${GITHUB_BASE_REF}" HEAD)"; then
+ break
+ fi
+ compare_rev=""
+ done
+
+ if [ -z "$compare_rev" ]; then
echo "Unable to find merge-base for origin/${GITHUB_BASE_REF}; falling back to full test suite" >&2
: > /tmp/changed-files
exit 0
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_nextest_filter.sh b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_nextest_filter.sh
index 9fdd7c72a5..7f269f76f6 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_nextest_filter.sh
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_nextest_filter.sh
@@ -1,30 +1,74 @@
set -euo pipefail
-# Writes `rust_packages` for the check/test jobs:
-# all — compile and run the whole workspace
-# none — no Rust packages to lint/test (Nix-only, top-level JSON, docs, …)
-# <names> — space-separated workspace packages (changed + reverse deps)
+# Writes for the check/test jobs:
+# rust_packages — all | none | space-separated packages (changed + reverse deps)
+# skip_tests — true keeps clippy on rust_packages while the live-Postgres
+# test job stays skipped (`.sqlx`-only or clippy.toml-only diffs)
#
-# `skip_tests=true` keeps clippy on a package set while the live-Postgres
-# test job stays skipped (used for `.sqlx`-only diffs).
-#
-# Root Cargo/toolchain/config edits force `all`. Cargo.lock is handled by
-# determinator's old/new graph comparison. flake.nix, the Nix shell action,
-# and this workflow file used to live on the full-suite list and turned every
-# dev-shell tweak into a full-suite rebuild.
+# Toolchain and cargo/nextest config edits force `all`. Root Cargo.toml and
+# Cargo.lock go to the xtask: determinator's old/new graph comparison scopes
+# lockfile, member, and workspace-dependency changes, and the xtask returns
+# `all` for any other root-manifest edit (profiles, lints, patches, …).
+# Cross.toml and deny.toml configure other tools and select nothing here.
+# clippy.toml changes lint results but not test outcomes (handled below).
+# flake.nix, the Nix shell action, and this workflow file used to live on the
+# full-suite list and turned every dev-shell tweak into a full-suite rebuild.
+
+# The filter runs as a release binary cached per source tree: a debug build
+# spends ~20 s in guppy on this runner, and compiling it took ~40 s more. The
+# key covers every input of the build, so a hit is exactly what `cargo build`
+# would produce. FILTER_CACHE_DIR is a Namespace cache volume for same-repo
+# PRs and a plain scratch directory for forks.
+filter_bin=""
+ensure_filter_bin() {
+ [ -n "$filter_bin" ] && return 0
+ local key bin_dir staging
+ key="$(git rev-parse \
+ HEAD:tooling/xtask/crates/xtask_nextest_filter \
+ HEAD:tooling/xtask/crates/xtask_graph \
+ HEAD:tooling/xtask/crates/xtask_paths \
+ HEAD:Cargo.lock \
+ HEAD:rust-toolchain.toml | sha256sum | cut -c1-16)"
+ bin_dir="$FILTER_CACHE_DIR/bin/$key"
+ filter_bin="$bin_dir/xtask_nextest_filter"
+ if [ -x "$filter_bin" ]; then
+ echo "Using cached nextest-filter build $key"
+ touch "$bin_dir"
+ return 0
+ fi
+
+ echo "Building nextest-filter $key"
+ CARGO_TARGET_DIR="$FILTER_CACHE_DIR/target" \
+ cargo build --release --locked --quiet -p xtask_nextest_filter
+ staging="$(mktemp -d "$FILTER_CACHE_DIR/bin/.staging.XXXXXX")"
+ cp "$FILTER_CACHE_DIR/target/release/xtask_nextest_filter" "$staging/"
+ # Another job may have published the same key meanwhile; either copy works.
+ mv -T "$staging" "$bin_dir" 2>/dev/null || rm -rf "$staging"
+ # Drop builds no PR has used for a week (every Cargo.lock change adds one).
+ find "$FILTER_CACHE_DIR/bin" -mindepth 1 -maxdepth 1 -type d -mtime +7 \
+ -exec rm -rf {} + || true
+}
+mkdir -p "$FILTER_CACHE_DIR/bin"
-echo "skip_tests=false" >> "$GITHUB_OUTPUT"
+emit() {
+ echo "rust_packages=$1" >> "$GITHUB_OUTPUT"
+ echo "skip_tests=$2" >> "$GITHUB_OUTPUT"
+ exit 0
+}
if [ ! -s /tmp/changed-files ]; then
echo "Unknown or empty change set; running all tests"
- echo "rust_packages=all" >> "$GITHUB_OUTPUT"
- exit 0
+ emit all false
fi
-if grep -qE '^(Cargo\.toml|rust-toolchain\.toml|Cross\.toml|clippy\.toml|deny\.toml|\.cargo/.*|\.config/.*)$' /tmp/changed-files; then
- echo "Workspace-level Cargo/toolchain change detected; running all tests"
- echo "rust_packages=all" >> "$GITHUB_OUTPUT"
- exit 0
+if grep -qE '^(rust-toolchain\.toml|\.cargo/.*|\.config/.*)$' /tmp/changed-files; then
+ echo "Toolchain or cargo config change detected; running all tests"
+ emit all false
+fi
+
+clippy_config_changed=false
+if grep -qx 'clippy\.toml' /tmp/changed-files; then
+ clippy_config_changed=true
fi
# The test job compiles queries against live Postgres (SQLX_OFFLINE is unset
@@ -33,18 +77,27 @@ fi
# still compile against the cache rather than skip the check job.
if ! grep -qvE '^\.sqlx/' /tmp/changed-files; then
echo "Only .sqlx changes detected; clippy all, skip live-Postgres tests"
- echo "rust_packages=all" >> "$GITHUB_OUTPUT"
- echo "skip_tests=true" >> "$GITHUB_OUTPUT"
- exit 0
+ emit all true
fi
-packages="$(cargo run --manifest-path tooling/xtask/Cargo.toml -- nextest-filter /tmp/changed-files "$(< /tmp/base-revision)")"
+ensure_filter_bin
+packages="$("$filter_bin" /tmp/changed-files "$(< /tmp/base-revision)")"
+
+# One package list drives both clippy and tests, so a clippy.toml edit can
+# only skip tests when nothing else selected a package.
+if [ "$clippy_config_changed" = true ]; then
+ if [ -z "$packages" ] || [ "$packages" = "none" ]; then
+ echo "Only clippy.toml affects Rust; clippy all, skip live-Postgres tests"
+ emit all true
+ fi
+ echo "clippy.toml changed alongside package changes; running all tests"
+ emit all false
+fi
if [ -z "$packages" ] || [ "$packages" = "none" ]; then
echo "No package-specific Rust changes detected; running no tests"
- echo "rust_packages=none" >> "$GITHUB_OUTPUT"
- exit 0
+ emit none false
fi
echo "rust packages: $packages"
-echo "rust_packages=$packages" >> "$GITHUB_OUTPUT"
+emit "$packages" false
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/sdk_check.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/sdk_check.rs
index 9312f1c7a1..6bf09fc914 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/sdk_check.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/sdk_check.rs
@@ -1,11 +1,13 @@
use gh_workflow::{Concurrency, Event, Expression, Job, PullRequest, Push, Run, Step, Workflow};
-use crate::workflows::{runners, steps, vars};
+use crate::workflows::steps;
#[cfg(test)]
mod test;
-/// Build the workflow.
+/// Build the workflow. Whether `packages/sdk`'s generated layer is fresh is
+/// checked by the web app workflow's `Generated Code Check`, which builds the
+/// spec binaries once for every Rust-generated artifact.
pub fn sdk_check() -> Workflow {
Workflow::new("SDK Check")
.on(Event::default()
@@ -13,11 +15,6 @@ pub fn sdk_check() -> Workflow {
PullRequest::default()
.add_branch("main")
.add_path("packages/sdk/**")
- .add_path("crates/**/*.rs")
- .add_path("Cargo.toml")
- .add_path("Cargo.lock")
- .add_path("apps/web/scripts/generate-api-schema.ts")
- .add_path("apps/web/scripts/services.ts")
.add_path(".github/workflows/sdk-check.yml"),
)
// `Release SDK` publishes from `main` without re-checking the
@@ -35,12 +32,10 @@ pub fn sdk_check() -> Workflow {
.cancel_in_progress(true),
)
.add_job("check-package", check_package())
- .add_job("check-sdk", check_sdk())
}
/// Typecheck, test, and coverage-check the package as committed. Needs nothing
-/// but Bun, so it stays a cheap gate that can run on every SDK change — unlike
-/// [`check_sdk`], which rebuilds the specs from Rust.
+/// but Bun, so it stays a cheap gate that can run on every SDK change.
fn check_package() -> Job {
Job::default()
.name("SDK Package Check")
@@ -59,45 +54,6 @@ fn install_dependencies() -> Step<Run> {
.working_directory(xtask_paths::repo_dir!("packages/sdk"))
}
-/// Regenerate the SDK's generated layer end-to-end and fail on drift. Only
-/// pull requests pay for this: it rebuilds the specs from Rust, and it shares
-/// the web CI cache volume so that build hits the same sccache as the web app
-/// checks.
-fn check_sdk() -> Job {
- Job::default()
- .name("SDK Generated Code Check")
- .cond(Expression::new(
- "${{ github.event_name == 'pull_request' }}",
- ))
- .runs_on(runners::Runner::Mid.with_cache_tag(vars::WEB_CI_CACHE_TAG))
- .add_step(steps::checkout(false, false))
- .add_step(steps::mount_web_cache_volume(true))
- .add_step(steps::setup_nix())
- .add_step(steps::setup_reqs_web("Setup Prereqs", false))
- .add_step(steps::configure_namespace_sccache(vars::WEB_SCCACHE_NAME))
- .add_step(update_generated())
- .add_step(steps::show_sccache_stats())
- .add_step(verify_fresh())
- .add_step(steps::teardown_nix())
-}
-
-fn update_generated() -> Step<Run> {
- Step::new("Regenerate SDK code")
- .run("just update-generated")
- .working_directory("packages/sdk")
-}
-
-fn verify_fresh() -> Step<Run> {
- Step::new("Verify generated code is fresh").run(indoc::indoc! {r#"
- if [ -n "$(git status --porcelain -- packages/sdk)" ]; then
- echo "packages/sdk generated code is stale. Run 'just update-generated' in packages/sdk and commit the result."
- git status --porcelain -- packages/sdk
- git diff -- packages/sdk | head -200
- exit 1
- fi
- "#})
-}
-
/// Every generated endpoint must either have a call site under `src/` or be
/// hand-listed in `src/coverage/skipped.ts`; fails naming the offenders.
fn check_coverage() -> Step<Run> {
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/sdk_check/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/sdk_check/test.rs
index 49f89af878..1a1484ad29 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/sdk_check/test.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/sdk_check/test.rs
@@ -6,26 +6,25 @@ fn rendered_yaml() -> String {
.expect("SDK check workflow should serialize")
}
-/// Merges to `main` publish, so the package suite has to guard them too — but
-/// the expensive spec rebuild stays on pull requests.
+/// Merges to `main` publish, so the package suite has to guard them too.
#[test]
-fn checks_the_package_on_main_and_regenerates_only_on_pull_requests() {
+fn checks_the_package_on_pull_requests_and_main() {
let yaml = rendered_yaml();
assert!(yaml.contains("check-package"));
+ assert!(yaml.contains("pull_request:"));
assert!(yaml.contains("push:"));
- assert!(yaml.contains("${{ github.event_name == 'pull_request' }}"));
}
#[test]
fn the_package_job_needs_no_rust_toolchain() {
let yaml = rendered_yaml();
- let package_job = yaml
- .split("check-sdk:")
- .next()
- .expect("check-package is rendered before check-sdk");
- assert!(package_job.contains("bun install --frozen-lockfile"));
- assert!(!package_job.contains("setup-nix"));
- assert!(!package_job.contains("sccache"));
+ assert!(yaml.contains("bun install --frozen-lockfile"));
+ assert!(!yaml.contains("setup-nix"));
+ assert!(!yaml.contains("sccache"));
+ assert!(
+ !yaml.contains("crates/**"),
+ "Rust changes reach the SDK through the web app's Generated Code Check: {yaml}"
+ );
}
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs
index 7b8be3b368..491ab982c5 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs
@@ -15,7 +15,7 @@ mod test;
/// Namespace's sccache setup mints a short-lived workspace credential. GitHub
/// withholds repository secrets from fork PRs, but this runner-minted token is
/// not a `secrets.*` value, so enforce the equivalent trust boundary here.
-const TRUSTED_NAMESPACE_SCCACHE_CONTEXT: &str = concat!(
+pub(crate) const TRUSTED_NAMESPACE_SCCACHE_CONTEXT: &str = concat!(
"(github.event_name != 'pull_request' && ",
"github.event_name != 'pull_request_target') || ",
"github.event.pull_request.head.repo.full_name == github.repository"
@@ -93,18 +93,6 @@ pub fn setup_rust_light() -> Step<Use> {
.add_with(("rust-cache", "false"))
}
-/// [`setup_rust_light`] plus sccache, for jobs that actually compile something
-/// but do not need the Nix dev shell. Pair with
-/// [`configure_namespace_sccache`] to point the wrapper at the remote cache.
-pub fn setup_rust_sccache() -> Step<Use> {
- uses_local(
- "Setup Rust",
- xtask_paths::repo_dir!(".github/actions/setup-rust"),
- )
- .add_with(("sccache", "true"))
- .add_with(("rust-cache", "false"))
-}
-
/// Install Nix on a macOS runner. The Linux [`setup_nix`] action doesn't apply
/// there, so Apple jobs (DMG signing, the iOS simulator preview) install it
/// with the same script instead.
@@ -203,6 +191,13 @@ pub fn mount_cache_volume() -> Step<Use> {
.add_with(("path", xtask_paths::runtime_path!("/nix").as_str()))
}
+/// Mount only `path` from the profile's cache volume, without a framework
+/// mode. Unlike [`mount_cache_volume`] (`cache: rust` plus `/nix`, ~11 s on
+/// `linux-rust-ci`), a plain path mounts in under a second.
+pub fn mount_path_cache_volume(name: &str, path: RuntimePath<'_>) -> Step<Use> {
+ nscloud_cache_action(name).add_with(("path", path.as_str()))
+}
+
/// The pinned `nscloud-cache-action`, shared by every mount helper below.
/// `continue-on-error` because a cache volume is always a pure optimization — a
/// failure just means cold state, never a wrong build.
@@ -244,14 +239,6 @@ pub fn configure_namespace_sccache(cache_name: &str) -> Step<Run> {
.if_condition(Expression::new(TRUSTED_NAMESPACE_SCCACHE_CONTEXT))
}
-/// Configure Namespace's remote sccache in a trusted context when
-/// `additional_condition` is also true.
-pub fn configure_namespace_sccache_when(cache_name: &str, additional_condition: &str) -> Step<Run> {
- namespace_sccache_step(cache_name).if_condition(Expression::new(format!(
- "({TRUSTED_NAMESPACE_SCCACHE_CONTEXT}) && ({additional_condition})"
- )))
-}
-
fn namespace_sccache_step(cache_name: &str) -> Step<Run> {
Step::new("Configure Namespace remote sccache").run(format!(
r#"set -euo pipefail
@@ -306,6 +293,21 @@ pub fn mount_web_cache_volume(with_rust: bool) -> Step<Use> {
})
}
+/// [`mount_web_cache_volume`] plus Vitest's module cache, so the web `Test`
+/// job reuses transforms from earlier runs instead of redoing them all.
+pub fn mount_web_test_cache_volume() -> Step<Use> {
+ nscloud_cache_action("Mount Namespace cache volume")
+ .add_with(("cache", "nix"))
+ .add_with((
+ "path",
+ format!(
+ "{}\n{}",
+ vars::BUN_CACHE_VOLUME_DIR,
+ vars::VITEST_MODULE_CACHE_VOLUME_DIR,
+ ),
+ ))
+}
+
/// [`mount_web_cache_volume`] for jobs that compile the browser wasm packages
/// (`just build-*`). Cargo registry/git feed wasm-pack; `.wasm-pack` is the
/// downloaded `wasm-opt` the same way [`mount_wasm_cache_volume`] keeps the
@@ -323,6 +325,22 @@ pub fn mount_web_build_cache_volume() -> Step<Use> {
))
}
+/// [`mount_web_build_cache_volume`] plus [`vars::WASM_PKG_CACHE_DIR`], for the PR
+/// build that reuses unchanged wasm packages. Deploy builds always rebuild them.
+pub fn mount_web_build_check_cache_volume() -> Step<Use> {
+ nscloud_cache_action("Mount Namespace cache volume")
+ .add_with(("cache", "nix"))
+ .add_with((
+ "path",
+ format!(
+ "{}\n/home/runner/.cargo/registry\n/home/runner/.cargo/git\n{}\n{}",
+ vars::BUN_CACHE_VOLUME_DIR,
+ xtask_paths::runtime_path!("/home/runner/.cache/.wasm-pack").as_str(),
+ vars::WASM_PKG_CACHE_DIR,
+ ),
+ ))
+}
+
/// The web-app composite: Nix dev shell (bun, biome, just) + `bun install`.
/// Jobs that run `gen-api` follow this with [`configure_namespace_sccache`].
/// Requires [`setup_nix`] first.
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/steps/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/steps/test.rs
index ac950b2555..358b2c690b 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/steps/test.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/steps/test.rs
@@ -33,22 +33,6 @@ fn namespace_sccache_masks_credentials_before_exporting_them() {
assert!(condition.0.contains("github.repository"));
}
-#[test]
-fn namespace_sccache_combines_job_specific_and_trust_conditions() {
- let step = configure_namespace_sccache_when("test-cache", "steps.filter.outputs.hit == 'true'");
- let condition = step
- .value
- .if_condition
- .expect("conditional Namespace sccache setup should have an if expression");
-
- assert!(
- condition
- .0
- .contains("github.event.pull_request.head.repo.full_name")
- );
- assert!(condition.0.contains("steps.filter.outputs.hit == 'true'"));
-}
-
#[test]
fn named_dev_shell_passes_the_flake_attribute() {
let step = setup_dev_shell_named("agent-daemon");
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs
index 76daf86c2e..a9ae08c271 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs
@@ -73,6 +73,12 @@ pub const WEB_CI_CACHE_TAG: &str = "web-ci";
/// a different workload and runner profile.
pub const WEB_SCCACHE_NAME: &str = "web-ci";
+/// Namespace cache tag for `path-check`, which keeps its release
+/// `xtask_nextest_filter` binary on the volume. Without a fixed tag the small
+/// profile's volume is scoped per branch and PR runs never persisted, so every
+/// run rebuilt the filter.
+pub const NEXTEST_FILTER_CACHE_TAG: &str = "nextest-filter";
+
/// Namespace cache tag for the sync-service worker deploy. Its own pool: this
/// job compiles for `wasm32-unknown-unknown`, so nothing in the host-target
/// volumes ([`CI_CACHE_TAG`]) would serve it anyway.
@@ -82,6 +88,15 @@ pub const SYNC_SERVICE_CACHE_TAG: &str = "sync-service-deploy";
/// the Nix dev shell and is not available to Namespace's cache planner yet.
pub const BUN_CACHE_VOLUME_DIR: &str = "/home/runner/.bun/install/cache";
+/// Vitest's filesystem module cache for the web `Test` job. `apps/web`'s
+/// Vitest config reads it from `VITEST_MODULE_CACHE_DIR` and keys entries by
+/// the lockfile and tsconfig, so PR jobs can share one volume.
+pub const VITEST_MODULE_CACHE_VOLUME_DIR: &str = "/home/runner/.cache/vitest-modules";
+
+/// Browser wasm packages keyed by their inputs, so the PR build reuses a package
+/// whose crates did not change (`apps/web/scripts/cached-wasm-build.sh`).
+pub const WASM_PKG_CACHE_DIR: &str = "/home/runner/.cache/wasm-pkgs";
+
/// GHCR repository for the agent-harness sandbox image (the same Dockerfile
/// Daytona snapshots). Pushed as `:$SHA` on PRs and `:$SHA` + `:latest` on main.
pub const AGENT_HARNESS_GHCR_IMAGE: &str = "ghcr.io/macro-inc/macro-agent-harness";
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs
index 5820f740ca..2c7c322e0b 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs
@@ -20,10 +20,12 @@ fn web_runner() -> String {
runners::Runner::Small.with_cache_tag(vars::WEB_CI_CACHE_TAG)
}
-/// Typechecking can compile the Rust binaries used by `gen-api`, and the app
-/// build compiles the two browser wasm packages, so retain the mid-size
-/// profile while sharing the web CI cache volume and remote sccache.
-fn typecheck_runner() -> String {
+/// The mid-size (16x32) profile on the web CI cache volume and remote sccache,
+/// for the jobs that scale with cores: the generated-code job compiles the
+/// schema binaries, the app build compiles the browser wasm packages and runs
+/// Vite, typechecking runs the TS 7 compiler, and vitest runs about 1.9x faster
+/// on 16 cores than on the small profile's 8.
+fn mid_web_runner() -> String {
runners::Runner::Mid.with_cache_tag(vars::WEB_CI_CACHE_TAG)
}
@@ -46,6 +48,7 @@ pub fn web_app_check_main() -> Workflow {
)
.add_job("path-check", path_check())
.add_job("typescript", typescript())
+ .add_job("generated-code", generated_code())
.add_job("biome-check", biome_check())
.add_job("test", test())
.add_job("cycles", cycles())
@@ -57,30 +60,29 @@ fn path_check() -> Job {
Job::default()
.runs_on(runners::Runner::TinyNoCache.to_string())
.add_output("should_run", "${{ steps.filter.outputs.should_run }}")
- .add_output("api_changed", "${{ steps.filter.outputs.api_changed }}")
+ .add_output(
+ "codegen_changed",
+ "${{ steps.filter.outputs.codegen_changed }}",
+ )
.add_step(checkout("Checkout Repo", false))
.add_step(paths_filter())
}
+/// TypeScript checks against the committed generated code. Whether that code
+/// is fresh is [`generated_code`]'s job, so a Rust-only change does not start
+/// this one.
fn typescript() -> Job {
Job::default()
.needs(vec!["path-check".to_string()])
.cond(Expression::new(
- "needs.path-check.outputs.should_run == 'true' || needs.path-check.outputs.api_changed == 'true'",
+ "needs.path-check.outputs.should_run == 'true'",
))
.name("Typecheck")
- .runs_on(typecheck_runner())
+ .runs_on(mid_web_runner())
.add_step(checkout("Checkout Repo", false))
- .add_step(steps::mount_web_cache_volume(true))
+ .add_step(steps::mount_web_cache_volume(false))
.add_step(steps::setup_nix())
.add_step(steps::setup_reqs_web("Setup Prereqs", false))
- .add_step(steps::configure_namespace_sccache_when(
- vars::WEB_SCCACHE_NAME,
- "needs.path-check.outputs.api_changed == 'true'",
- ))
- .add_step(generate_api_types())
- .add_step(check_specta_types())
- .add_step(show_sccache_stats())
.add_step(check_dynamic_ui_schema())
.add_step(check_types())
.add_step(check_collaboration_types())
@@ -89,6 +91,56 @@ fn typescript() -> Job {
.add_step(steps::teardown_nix())
}
+/// Everything generated from Rust binaries: the OpenAPI clients in `apps/web`
+/// (`gen-api`), the specta wasm wire types, the SDK's specs and generated
+/// layer, and the MCP tool reference in `apps/docs`. They used to be checked by
+/// three jobs in three workflows, each compiling the workspace again; one job
+/// on one target dir builds the binaries once. The SDK regeneration reuses the
+/// specs `gen-api` just wrote rather than rerunning it.
+fn generated_code() -> Job {
+ Job::default()
+ .needs(vec!["path-check".to_string()])
+ .cond(Expression::new(
+ "needs.path-check.outputs.codegen_changed == 'true'",
+ ))
+ .name("Generated Code Check")
+ .runs_on(mid_web_runner())
+ // The generators run once and are thrown away, so debug info is pure
+ // cost: it made the 15 binaries 12.7 GB instead of 466 MB. Job-level,
+ // so every cargo call here (the build step, gen-api, the docs and DCS
+ // scripts, the specta recipes) resolves the same profile and reuses
+ // one set of artifacts. No `profile.dev.package` overrides exist, so
+ // this alone covers dependencies too.
+ .add_env(("CARGO_PROFILE_DEV_DEBUG", "0"))
+ // Link with mold, as the Rust check and test jobs do. Job-level for
+ // the same reason: RUSTFLAGS is part of every unit's fingerprint.
+ .add_env(("RUSTFLAGS", "-C link-arg=-fuse-ld=mold"))
+ .add_step(checkout("Checkout Repo", false))
+ .add_step(steps::mount_web_cache_volume(true))
+ .add_step(steps::setup_nix())
+ .add_step(steps::setup_reqs_web("Setup Prereqs", false))
+ .add_step(steps::configure_namespace_sccache(vars::WEB_SCCACHE_NAME))
+ .add_step(build_schema_binaries())
+ .add_step(generate_api_types())
+ .add_step(regenerate_sdk())
+ .add_step(verify_fresh(
+ "Verify SDK generated code is fresh",
+ "packages/sdk",
+ "packages/sdk generated code is stale. Run 'just update-generated' in packages/sdk and commit the result.",
+ ))
+ .add_step(regenerate_tool_pages())
+ .add_step(verify_fresh(
+ "Verify docs tool reference is fresh",
+ "apps/docs",
+ "apps/docs tool reference is stale. Run 'bun run generate:tools' in apps/docs and commit the result.",
+ ))
+ // Last: its `cargo run -p` builds resolve features per package, so
+ // running them earlier would make the steps above recompile.
+ .add_step(check_specta_types())
+ .add_step(steps::show_sccache_stats())
+ .add_step(steps::teardown_nix())
+}
+
fn biome_check() -> Job {
gated_web_job("Biome Check")
.add_step(checkout("Checkout Repo", true))
@@ -102,8 +154,9 @@ fn biome_check() -> Job {
fn test() -> Job {
gated_web_job("Test")
+ .runs_on(mid_web_runner())
.add_step(checkout("Checkout Repo", false))
- .add_step(steps::mount_web_cache_volume(false))
+ .add_step(steps::mount_web_test_cache_volume())
.add_step(steps::setup_nix())
.add_step(steps::setup_reqs_web("Setup", true))
.add_step(build_agent_fold_for_tests())
@@ -128,7 +181,7 @@ fn build() -> Job {
// Match preview/deploy capacity for Vite's chunk-rendering memory peak.
.runs_on(runners::Runner::Mid.with_cache_tag(vars::WEB_CI_CACHE_TAG))
.add_step(checkout("Checkout Repo", false))
- .add_step(steps::mount_web_build_cache_volume())
+ .add_step(steps::mount_web_build_check_cache_volume())
.add_step(steps::setup_nix())
.add_step(steps::setup_reqs_web("Setup", false))
.add_step(steps::configure_namespace_sccache(vars::WEB_SCCACHE_NAME))
@@ -147,6 +200,7 @@ fn status_check() -> Job {
.needs(vec![
"path-check".to_string(),
"typescript".to_string(),
+ "generated-code".to_string(),
"biome-check".to_string(),
"test".to_string(),
"cycles".to_string(),
@@ -179,12 +233,14 @@ fn checkout(name: &str, full_history: bool) -> Step<Use> {
fn paths_filter() -> Step<Use> {
let artifact_paths = crate::workflows::web_artifact_paths::yaml_list(" ");
- // `api_changed` is only the inputs to `bun run gen-api`. xtask, flake.nix,
- // the Nix dev-shell action, and this workflow file do not change OpenAPI
- // output, so they must not start Typecheck (which compiles the schema
- // binaries). The Nix dev-shell action is also omitted from `should_run`
- // so Typecheck (which is `should_run || api_changed`) stays off for a
- // shell-only tweak. Workflow YAML drift is `check generated workflows`.
+ // `codegen_changed` is the union of what feeds [`generated_code`]: the
+ // Rust sources and manifests the schema binaries compile from, the
+ // `gen-api` scripts, and the SDK and docs packages that hold generated
+ // output. xtask, flake.nix, the Nix dev-shell action, and this workflow
+ // file do not change generated output, so they must not start that job.
+ // The Nix dev-shell action is also omitted from `should_run` so Typecheck
+ // stays off for a shell-only tweak. Workflow YAML drift is `check
+ // generated workflows`.
Step::new("Filter changed paths")
.uses(
"dorny",
@@ -195,17 +251,28 @@ fn paths_filter() -> Step<Use> {
.add_with((
"filters",
format!(
- "should_run:\n{artifact_paths} - 'services/lexical-service/**'\n - 'crates/ai_tools/src/display_results/schema.generated.json'\n - '.github/actions/setup-reqs-web/**'\napi_changed:\n - 'crates/**/*.rs'\n - 'services/**/*.rs'\n - 'Cargo.toml'\n - 'Cargo.lock'\n - 'apps/web/scripts/generate-api-schema.ts'\n - 'apps/web/scripts/services.ts'\n - '.github/actions/setup-reqs-web/**'\n"
+ "should_run:\n{artifact_paths} - 'services/lexical-service/**'\n - 'crates/ai_tools/src/display_results/schema.generated.json'\n - '.github/actions/setup-reqs-web/**'\ncodegen_changed:\n - 'crates/**/*.rs'\n - 'services/**/*.rs'\n - 'Cargo.toml'\n - 'Cargo.lock'\n - 'apps/web/scripts/generate-api-schema.ts'\n - 'apps/web/scripts/services.ts'\n - 'packages/sdk/**'\n - 'apps/docs/**'\n - '.github/actions/setup-reqs-web/**'\n"
),
))
}
+/// Build every schema binary the later steps run in one cargo invocation:
+/// the OpenAPI bins (`gen-api` owns the list) and `gen_tool_schemas`, which
+/// `gen-api`'s DCS tool types and the docs pages both run. Without
+/// `--package`, cargo resolves features across the whole workspace, so the
+/// scripts' own `cargo build --bin …` calls resolve the same artifacts and
+/// finish without compiling.
+fn build_schema_binaries() -> Step<Run> {
+ Step::new("Build schema binaries").run(indoc::indoc! {r#"
+ set -euo pipefail
+ read -ra openapi_bins <<< "$(cd apps/web && bun scripts/generate-api-schema.ts --print-cargo-bins)"
+ SQLX_OFFLINE=true cargo build "${openapi_bins[@]}" --bin gen_tool_schemas
+ "#})
+}
+
fn generate_api_types() -> Step<Run> {
Step::new("Generate API Types")
.run("bun run gen-api -- --check")
- .if_condition(Expression::new(
- "needs.path-check.outputs.api_changed == 'true'",
- ))
.working_directory(xtask_paths::repo_dir!("apps/web"))
}
@@ -222,23 +289,46 @@ fn check_specta_types() -> Step<Run> {
exit 1
fi
"#})
- .if_condition(Expression::new(
- "needs.path-check.outputs.api_changed == 'true'",
- ))
.working_directory(xtask_paths::repo_dir!("apps/web"))
}
-fn show_sccache_stats() -> Step<Run> {
- Step::new("show sccache stats")
- .run("sccache --show-stats || true")
- .if_condition(Expression::new(
- "always() && needs.path-check.outputs.api_changed == 'true'",
- ))
+/// The tail of `just update-generated` in `packages/sdk`: its first line reruns
+/// `gen-api`, whose specs the step before this one already wrote, so only the
+/// copy into `specs/` and the client generation run here.
+fn regenerate_sdk() -> Step<Run> {
+ Step::new("Regenerate SDK code")
+ .run(indoc::indoc! {r#"
+ bun install --frozen-lockfile
+ bun run sync-specs
+ bun run generate
+ "#})
+ .working_directory(xtask_paths::repo_dir!("packages/sdk"))
+}
+
+/// Renders `apps/docs/AI/mcp/tools/` from the `gen_tool_schemas` binary, so a
+/// tool's schema or description change cannot leave the docs site behind.
+fn regenerate_tool_pages() -> Step<Run> {
+ Step::new("Regenerate MCP tool pages")
+ .run("bun run generate:tools")
+ .working_directory(xtask_paths::repo_dir!("apps/docs"))
}
+fn verify_fresh(name: &str, dir: &str, stale_message: &str) -> Step<Run> {
+ Step::new(name).run(format!(
+ r#"if [ -n "$(git status --porcelain -- {dir})" ]; then
+ echo "{stale_message}"
+ git status --porcelain -- {dir}
+ git diff -- {dir} | head -200
+ exit 1
+fi
+"#
+ ))
+}
+
+/// `type-check` runs the native TypeScript 7 compiler.
fn check_types() -> Step<Run> {
Step::new("Check Types")
- .run("bun run --bun --silent tsc --project ./tsconfig.json")
+ .run("bun run type-check")
.working_directory(xtask_paths::repo_dir!("apps/web"))
}
@@ -286,9 +376,15 @@ fn build_agent_fold_for_tests() -> Step<Run> {
.timeout_minutes(15u32)
}
+/// `bun run test` enables Vitest's filesystem module cache; the cache volume
+/// keeps it warm across jobs.
fn run_tests() -> Step<Run> {
Step::new("Test")
- .run("bunx vitest")
+ .run("bun run test")
+ .add_env((
+ "VITEST_MODULE_CACHE_DIR",
+ vars::VITEST_MODULE_CACHE_VOLUME_DIR,
+ ))
.working_directory(xtask_paths::repo_dir!("apps/web"))
}
@@ -315,6 +411,7 @@ fn collaboration_cycles_import_check() -> Step<Run> {
fn run_build() -> Step<Run> {
Step::new("Build")
.run("just build-dev")
+ .add_env(("WASM_PKG_CACHE_DIR", vars::WASM_PKG_CACHE_DIR))
.working_directory(xtask_paths::repo_dir!("apps/web"))
}
@@ -322,6 +419,7 @@ fn check_job_results() -> Step<Run> {
Step::new("Check job results").run(indoc::indoc! {r#"
echo "path-check: ${{ needs.path-check.result }}"
echo "typescript: ${{ needs.typescript.result }}"
+ echo "generated-code: ${{ needs.generated-code.result }}"
echo "biome-check: ${{ needs.biome-check.result }}"
echo "test: ${{ needs.test.result }}"
echo "cycles: ${{ needs.cycles.result }}"
@@ -330,6 +428,7 @@ fn check_job_results() -> Step<Run> {
# Fail if any job failed (skipped and success are both OK)
if [[ "${{ needs.path-check.result }}" == "failure" ]] || \
[[ "${{ needs.typescript.result }}" == "failure" ]] || \
+ [[ "${{ needs.generated-code.result }}" == "failure" ]] || \
[[ "${{ needs.biome-check.result }}" == "failure" ]] || \
[[ "${{ needs.test.result }}" == "failure" ]] || \
[[ "${{ needs.cycles.result }}" == "failure" ]] || \
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rs
index 51f201764f..4b16e69dff 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rs
@@ -1,20 +1,32 @@
use super::*;
-#[test]
-fn nix_dev_shell_does_not_start_typecheck() {
- let yaml = web_app_check_main().to_string().expect("workflow yaml");
+fn filters(yaml: &str) -> (&str, &str) {
let filters = yaml
.split("filters: |")
.nth(1)
.and_then(|rest| rest.split("id: filter").next())
.expect("path-check filters");
- let should_run = filters
- .split("api_changed:")
- .next()
- .expect("should_run block");
+ filters
+ .split_once("codegen_changed:")
+ .expect("should_run block, then codegen_changed block")
+}
+
+/// A job's rendered YAML, from its id up to the next job's id. Slicing by id
+/// keeps the job's `needs:` and `if:`, which render before its `name:`.
+fn job<'a>(yaml: &'a str, id: &str, next_id: &str) -> &'a str {
+ yaml.split(&format!("\n {id}:\n"))
+ .nth(1)
+ .and_then(|rest| rest.split(&format!("\n {next_id}:\n")).next())
+ .unwrap_or_else(|| panic!("{id} job"))
+}
+
+#[test]
+fn nix_dev_shell_does_not_start_typecheck() {
+ let yaml = web_app_check_main().to_string().expect("workflow yaml");
+ let (should_run, _) = filters(&yaml);
assert!(
!should_run.contains("setup-nix-dev-shell"),
- "Typecheck is should_run || api_changed; nix-dev-shell must not be in should_run: {should_run}"
+ "Typecheck runs on should_run; nix-dev-shell must not be in it: {should_run}"
);
assert!(
should_run.contains("setup-reqs-web"),
@@ -25,11 +37,7 @@ fn nix_dev_shell_does_not_start_typecheck() {
#[test]
fn typecheck_runs_lexical_service_check_and_tests() {
let yaml = web_app_check_main().to_string().expect("workflow yaml");
- let typescript = yaml
- .split("name: Typecheck")
- .nth(1)
- .and_then(|rest| rest.split("name: Biome Check").next())
- .expect("Typecheck job");
+ let typescript = job(&yaml, "typescript", "generated-code");
assert!(
typescript.contains("bun run check"),
"Typecheck must type-check lexical-service: {typescript}"
@@ -45,20 +53,125 @@ fn typecheck_runs_lexical_service_check_and_tests() {
}
#[test]
-fn typecheck_checks_both_specta_exports_when_rust_changes() {
+fn typecheck_compiles_no_rust() {
+ let yaml = web_app_check_main().to_string().expect("workflow yaml");
+ let typescript = job(&yaml, "typescript", "generated-code");
+ for rust in ["gen-api", "cargo", "sccache", "just gen-"] {
+ assert!(
+ !typescript.contains(rust),
+ "generated code is checked in its own job, so Typecheck must not run `{rust}`: {typescript}"
+ );
+ }
+ assert!(
+ typescript.contains("should_run == 'true'") && !typescript.contains("codegen_changed"),
+ "a Rust-only change does not start Typecheck: {typescript}"
+ );
+}
+
+/// The OpenAPI clients, specta types, SDK and docs tool pages all come from
+/// workspace binaries, so one job builds them once and checks all four.
+#[test]
+fn generated_code_checks_every_rust_generated_artifact_in_one_job() {
let yaml = web_app_check_main().to_string().expect("workflow yaml");
- let step = yaml
- .split("name: Check Specta Types")
+ let generated = job(&yaml, "generated-code", "biome-check");
+ // Specta runs last: its `cargo run -p` builds resolve features per
+ // package and would otherwise make the docs build recompile.
+ let order = [
+ "Build schema binaries",
+ "bun run gen-api -- --check",
+ "bun run sync-specs",
+ "git status --porcelain -- packages/sdk",
+ "bun run generate:tools",
+ "git status --porcelain -- apps/docs",
+ "just gen-agent-fold-types",
+ "just gen-database-sql-types",
+ ];
+ let positions: Vec<usize> = order
+ .iter()
+ .map(|needle| {
+ generated
+ .find(needle)
+ .unwrap_or_else(|| panic!("missing `{needle}`: {generated}"))
+ })
+ .collect();
+ assert!(
+ positions.windows(2).all(|pair| pair[0] < pair[1]),
+ "the binaries build first, then each artifact is checked: {generated}"
+ );
+ let sdk = generated
+ .split("name: Regenerate SDK code")
.nth(1)
.and_then(|rest| rest.split("- name:").next())
- .expect("specta types step");
+ .expect("SDK regeneration step");
+ assert!(
+ !sdk.contains("gen-api") && !sdk.contains("update-generated"),
+ "the SDK reuses the specs gen-api wrote instead of rerunning it: {sdk}"
+ );
+ assert!(
+ generated.contains("nsc cache sccache setup --cache_name web-ci"),
+ "the build uses the shared web-ci remote sccache: {generated}"
+ );
+}
+
+/// Debug info only slows the throwaway generator build. Set at job level so
+/// every cargo call in the job shares the profile and reuses one build.
+#[test]
+fn generated_code_builds_without_debug_info_for_every_step() {
+ let yaml = web_app_check_main().to_string().expect("workflow yaml");
+ let generated = job(&yaml, "generated-code", "biome-check");
+ let (header, steps) = generated
+ .split_once("\n steps:\n")
+ .expect("job header, then steps");
assert!(
- step.contains("just gen-agent-fold-types") && step.contains("just gen-database-sql-types"),
- "both wasm crates' TypeScript must be regenerated: {step}"
+ header.contains("CARGO_PROFILE_DEV_DEBUG: '0'"),
+ "the job env turns debug info off: {header}"
);
assert!(
- step.contains("api_changed == 'true'"),
- "the exporters compile Rust, so only a Rust change runs them: {step}"
+ header.contains("RUSTFLAGS: '-C link-arg=-fuse-ld=mold'"),
+ "the job links with mold like the Rust jobs: {header}"
+ );
+ for flag in ["CARGO_PROFILE_DEV_DEBUG", "RUSTFLAGS", "--config"] {
+ assert!(
+ !steps.contains(flag),
+ "no step may override `{flag}`, or its build stops reusing the others: {steps}"
+ );
+ }
+}
+
+#[test]
+fn codegen_filter_covers_every_input_of_the_generated_code_job() {
+ let yaml = web_app_check_main().to_string().expect("workflow yaml");
+ let (_, codegen) = filters(&yaml);
+ for path in [
+ "'crates/**/*.rs'",
+ "'services/**/*.rs'",
+ "'Cargo.toml'",
+ "'Cargo.lock'",
+ "'apps/web/scripts/generate-api-schema.ts'",
+ "'apps/web/scripts/services.ts'",
+ "'packages/sdk/**'",
+ "'apps/docs/**'",
+ ] {
+ assert!(
+ codegen.contains(path),
+ "codegen_changed must include {path}: {codegen}"
+ );
+ }
+ let generated = job(&yaml, "generated-code", "biome-check");
+ assert!(generated.contains("needs.path-check.outputs.codegen_changed == 'true'"));
+}
+
+#[test]
+fn status_check_requires_generated_code() {
+ let yaml = web_app_check_main().to_string().expect("workflow yaml");
+ let status = yaml
+ .split("\n status-check:\n")
+ .nth(1)
+ .expect("status check job");
+ assert!(status.contains("- generated-code"), "{status}");
+ assert!(
+ status.contains("needs.generated-code.result }}\" == \"failure\""),
+ "{status}"
);
}
@@ -134,7 +247,7 @@ fn test_job_runs_signup_browser_tests_after_vitest() {
let wasm = test_job
.find("just build-agent-fold-wasm")
.expect("build WASM before the test setup hook");
- let vitest = test_job.find("bunx vitest").expect("vitest step");
+ let vitest = test_job.find("bun run test").expect("vitest step");
assert!(wasm < vitest, "cold WASM compilation precedes Vitest");
let browser = test_job
.find("just test-signup-browser")
@@ -148,3 +261,25 @@ fn test_job_runs_signup_browser_tests_after_vitest() {
"the Test job installs Playwright's Chromium: {test_job}"
);
}
+
+#[test]
+fn test_job_keeps_vitest_module_cache_on_the_cache_volume() {
+ let yaml = web_app_check_main().to_string().expect("workflow yaml");
+ let start = yaml.find("name: Test\n").expect("Test job");
+ let end = yaml[start..]
+ .find("name: Cycles Import Check")
+ .map_or(yaml.len(), |offset| start + offset);
+ let test_job = &yaml[start..end];
+ let mounted = format!(" {}\n", vars::VITEST_MODULE_CACHE_VOLUME_DIR);
+ assert!(
+ test_job.contains(&mounted),
+ "the cache volume mounts the Vitest module cache: {test_job}"
+ );
+ assert!(
+ test_job.contains(&format!(
+ "VITEST_MODULE_CACHE_DIR: {}",
+ vars::VITEST_MODULE_CACHE_VOLUME_DIR
+ )),
+ "vitest writes its module cache to the mounted path: {test_job}"
+ );
+}