Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
ci: speed up PR checks across Rust and web (#7588)
teo6df5d6bb5bd6
diff --git a/.github/workflows/code_check_cloud_storage.yml b/.github/workflows/code_check_cloud_storage.ymlindex 4834e6b750..31dd4c1741 100644--- a/.github/workflows/code_check_cloud_storage.yml+++ b/.github/workflows/code_check_cloud_storage.yml@@ -19,12 +19,12 @@ env:- mainjobs:path-check:- runs-on: namespace-profile-linux-small+ runs-on: namespace-profile-linux-small;overrides.cache-tag=nextest-filteroutputs:should_run: ${{ steps.filter.outputs.should_run }}+ doppler_candidates: ${{ steps.filter.outputs.doppler_candidates }}rust_packages: ${{ steps.nextest-filter.outputs.rust_packages }}skip_tests: ${{ steps.nextest-filter.outputs.skip_tests }}- doppler_config_bins: ${{ steps.doppler-bins.outputs.doppler_config_bins }}steps:- name: Checkoutuses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd@@ -65,6 +65,13 @@ jobs:- '.github/actions/setup-sccache/**'- '.github/services-config.json'- .github/workflows/code_check_cloud_storage.yml+ doppler_candidates:+ - '.github/services-config.json'+ - '**/src/config.rs'+ - '**/src/doppler_config.rs'+ - '**/Cargo.toml'+ - 'tooling/xtask/crates/xtask_doppler_bins/**'+ - .github/workflows/code_check_cloud_storage.yml- name: compute changed filesif: steps.filter.outputs.should_run == 'true'run: |@@ -77,8 +84,26 @@ jobs:if [ -z "${GITHUB_BASE_REF:-}" ]; thencompare_rev="$(git rev-parse HEAD~1)"else- git fetch origin "$GITHUB_BASE_REF:refs/remotes/origin/$GITHUB_BASE_REF"- if ! compare_rev="$(git merge-base "origin/${GITHUB_BASE_REF}" HEAD)"; then+ git fetch --no-tags origin "$GITHUB_BASE_REF:refs/remotes/origin/$GITHUB_BASE_REF"++ # The checkout holds only the PR merge commit and its two parents. Normally+ # the first parent is on the base branch, so the merge-base is immediate.+ # Stacked PRs can arrive with a merge commit built on another branch's tip;+ # deepen the PR side until its history reaches the base branch instead of+ # falling back to the full suite.+ compare_rev=""+ for deepen in 0 200 2000; do+ if [ "$deepen" -gt 0 ]; then+ echo "No merge-base with origin/${GITHUB_BASE_REF} yet; deepening PR history by ${deepen}"+ git fetch --no-tags --deepen="$deepen" origin "$GITHUB_SHA" || break+ fi+ if compare_rev="$(git merge-base "origin/${GITHUB_BASE_REF}" HEAD)"; then+ break+ fi+ compare_rev=""+ done++ if [ -z "$compare_rev" ]; thenecho "Unable to find merge-base for origin/${GITHUB_BASE_REF}; falling back to full test suite" >&2: > /tmp/changed-filesexit 0@@ -88,52 +113,87 @@ jobs:printf '%s\n' "$compare_rev" > /tmp/base-revisiongit diff --name-only --no-renames "$compare_rev" "$GITHUB_SHA" > /tmp/changed-filesshell: bash- - id: doppler-bins- name: compute affected Doppler config bins- if: steps.filter.outputs.should_run == 'true'- run: |- set -euo pipefail-- doppler_config_bins="$(cargo run --manifest-path tooling/xtask/Cargo.toml -- doppler-bins /tmp/changed-files)"- {- echo 'doppler_config_bins<<__DOPPLER_CONFIG_BINS__'- if [ -n "$doppler_config_bins" ]; then- printf '%s\n' "$doppler_config_bins"- fi- echo '__DOPPLER_CONFIG_BINS__'- } >> "$GITHUB_OUTPUT"- shell: bash+ - name: Mount nextest-filter binary cache+ if: steps.filter.outputs.should_run == 'true' && ((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository)+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0+ with:+ path: /home/runner/.cache/macro-nextest-filter+ continue-on-error: true- id: nextest-filtername: compute nextest package filterif: steps.filter.outputs.should_run == 'true'run: |set -euo pipefail- # Writes `rust_packages` for the check/test jobs:- # all — compile and run the whole workspace- # none — no Rust packages to lint/test (Nix-only, top-level JSON, docs, …)- # <names> — space-separated workspace packages (changed + reverse deps)- #- # `skip_tests=true` keeps clippy on a package set while the live-Postgres- # test job stays skipped (used for `.sqlx`-only diffs).+ # Writes for the check/test jobs:+ # rust_packages — all | none | space-separated packages (changed + reverse deps)+ # skip_tests — true keeps clippy on rust_packages while the live-Postgres+ # test job stays skipped (`.sqlx`-only or clippy.toml-only diffs)#- # Root Cargo/toolchain/config edits force `all`. Cargo.lock is handled by- # determinator's old/new graph comparison. flake.nix, the Nix shell action,- # and this workflow file used to live on the full-suite list and turned every- # dev-shell tweak into a full-suite rebuild.+ # Toolchain and cargo/nextest config edits force `all`. Root Cargo.toml and+ # Cargo.lock go to the xtask: determinator's old/new graph comparison scopes+ # lockfile, member, and workspace-dependency changes, and the xtask returns+ # `all` for any other root-manifest edit (profiles, lints, patches, …).+ # Cross.toml and deny.toml configure other tools and select nothing here.+ # clippy.toml changes lint results but not test outcomes (handled below).+ # flake.nix, the Nix shell action, and this workflow file used to live on the+ # full-suite list and turned every dev-shell tweak into a full-suite rebuild.++ # The filter runs as a release binary cached per source tree: a debug build+ # spends ~20 s in guppy on this runner, and compiling it took ~40 s more. The+ # key covers every input of the build, so a hit is exactly what `cargo build`+ # would produce. FILTER_CACHE_DIR is a Namespace cache volume for same-repo+ # PRs and a plain scratch directory for forks.+ filter_bin=""+ ensure_filter_bin() {+ [ -n "$filter_bin" ] && return 0+ local key bin_dir staging+ key="$(git rev-parse \+ HEAD:tooling/xtask/crates/xtask_nextest_filter \+ HEAD:tooling/xtask/crates/xtask_graph \+ HEAD:tooling/xtask/crates/xtask_paths \+ HEAD:Cargo.lock \+ HEAD:rust-toolchain.toml | sha256sum | cut -c1-16)"+ bin_dir="$FILTER_CACHE_DIR/bin/$key"+ filter_bin="$bin_dir/xtask_nextest_filter"+ if [ -x "$filter_bin" ]; then+ echo "Using cached nextest-filter build $key"+ touch "$bin_dir"+ return 0+ fi- echo "skip_tests=false" >> "$GITHUB_OUTPUT"+ echo "Building nextest-filter $key"+ CARGO_TARGET_DIR="$FILTER_CACHE_DIR/target" \+ cargo build --release --locked --quiet -p xtask_nextest_filter+ staging="$(mktemp -d "$FILTER_CACHE_DIR/bin/.staging.XXXXXX")"+ cp "$FILTER_CACHE_DIR/target/release/xtask_nextest_filter" "$staging/"+ # Another job may have published the same key meanwhile; either copy works.+ mv -T "$staging" "$bin_dir" 2>/dev/null || rm -rf "$staging"+ # Drop builds no PR has used for a week (every Cargo.lock change adds one).+ find "$FILTER_CACHE_DIR/bin" -mindepth 1 -maxdepth 1 -type d -mtime +7 \+ -exec rm -rf {} + || true+ }+ mkdir -p "$FILTER_CACHE_DIR/bin"++ emit() {+ echo "rust_packages=$1" >> "$GITHUB_OUTPUT"+ echo "skip_tests=$2" >> "$GITHUB_OUTPUT"+ exit 0+ }if [ ! -s /tmp/changed-files ]; thenecho "Unknown or empty change set; running all tests"- echo "rust_packages=all" >> "$GITHUB_OUTPUT"- exit 0+ emit all falsefi- if grep -qE '^(Cargo\.toml|rust-toolchain\.toml|Cross\.toml|clippy\.toml|deny\.toml|\.cargo/.*|\.config/.*)$' /tmp/changed-files; then- echo "Workspace-level Cargo/toolchain change detected; running all tests"- echo "rust_packages=all" >> "$GITHUB_OUTPUT"- exit 0+ if grep -qE '^(rust-toolchain\.toml|\.cargo/.*|\.config/.*)$' /tmp/changed-files; then+ echo "Toolchain or cargo config change detected; running all tests"+ emit all false+ fi++ clippy_config_changed=false+ if grep -qx 'clippy\.toml' /tmp/changed-files; then+ clippy_config_changed=truefi# The test job compiles queries against live Postgres (SQLX_OFFLINE is unset@@ -142,22 +202,33 @@ jobs:# still compile against the cache rather than skip the check job.if ! grep -qvE '^\.sqlx/' /tmp/changed-files; thenecho "Only .sqlx changes detected; clippy all, skip live-Postgres tests"- echo "rust_packages=all" >> "$GITHUB_OUTPUT"- echo "skip_tests=true" >> "$GITHUB_OUTPUT"- exit 0+ emit all truefi- packages="$(cargo run --manifest-path tooling/xtask/Cargo.toml -- nextest-filter /tmp/changed-files "$(< /tmp/base-revision)")"+ ensure_filter_bin+ packages="$("$filter_bin" /tmp/changed-files "$(< /tmp/base-revision)")"++ # One package list drives both clippy and tests, so a clippy.toml edit can+ # only skip tests when nothing else selected a package.+ if [ "$clippy_config_changed" = true ]; then+ if [ -z "$packages" ] || [ "$packages" = "none" ]; then+ echo "Only clippy.toml affects Rust; clippy all, skip live-Postgres tests"+ emit all true+ fi+ echo "clippy.toml changed alongside package changes; running all tests"+ emit all false+ fiif [ -z "$packages" ] || [ "$packages" = "none" ]; thenecho "No package-specific Rust changes detected; running no tests"- echo "rust_packages=none" >> "$GITHUB_OUTPUT"- exit 0+ emit none falsefiecho "rust packages: $packages"- echo "rust_packages=$packages" >> "$GITHUB_OUTPUT"+ emit "$packages" falseshell: bash+ env:+ FILTER_CACHE_DIR: /home/runner/.cache/macro-nextest-filtercheck:needs:- path-check@@ -211,47 +282,6 @@ jobs:elseecho "::warning::Namespace remote sccache setup failed; using local cache fallback"fi- - name: validate Doppler configs- if: needs.path-check.outputs.doppler_config_bins != ''- run: |- set -euo pipefail-- if [ -z "${DOPPLER_TOKEN:-}" ]; then- echo "DOPPLER_TOKEN secret is required to validate Doppler configs" >&2- exit 1- fi-- if [ -z "${RUSTC_WRAPPER:-}" ]; then- echo "RUSTC_WRAPPER is required so Doppler config binaries build through sccache" >&2- exit 1- fi-- bins=()- cargo_args=(build --locked --all-features)- while IFS= read -r bin; do- if [ -z "$bin" ]; then- continue- fi-- bins+=("$bin")- cargo_args+=(--bin "$bin")- done <<< "$DOPPLER_CONFIG_BINS"-- if [ "${#bins[@]}" -eq 0 ]; then- echo "No Doppler config binaries to validate"- exit 0- fi-- echo "Building affected Doppler config binaries with RUSTC_WRAPPER=$RUSTC_WRAPPER"- printf ' %s\n' "${bins[@]}"-- cargo "${cargo_args[@]}"- for bin in "${bins[@]}"; do- "./target/debug/$bin"- done- env:- DOPPLER_CONFIG_BINS: ${{ needs.path-check.outputs.doppler_config_bins }}- DOPPLER_TOKEN: ${{ secrets.DOPPLER_TOKEN }}- name: fmtrun: cargo fmt --check- name: clippy@@ -306,6 +336,161 @@ jobs:- name: Teardown Nixif: always()uses: './.github/actions/teardown-nix'+ doppler-config:+ needs:+ - path-check+ if: needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false && needs.path-check.outputs.doppler_candidates == 'true'+ runs-on: namespace-profile-linux-rust-ci;overrides.cache-tag=sccache-ci+ env:+ RUSTFLAGS: '-Dwarnings -Dclippy::disallowed_methods -C link-arg=-fuse-ld=mold'+ RUSTDOCFLAGS: '-Dwarnings'+ SQLX_OFFLINE: 'true'+ steps:+ - name: Checkout+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd+ with:+ clean: 'false'+ persist-credentials: 'false'+ fetch-depth: '2'+ - name: compute changed files+ run: |+ set -euo pipefail++ # Compare against the merge-base so a deleted file still appears. `--no-renames`+ # turns a rename into a delete+add pair so the old path is attributed to its+ # package (otherwise git reports only the new name).++ if [ -z "${GITHUB_BASE_REF:-}" ]; then+ compare_rev="$(git rev-parse HEAD~1)"+ else+ git fetch --no-tags origin "$GITHUB_BASE_REF:refs/remotes/origin/$GITHUB_BASE_REF"++ # The checkout holds only the PR merge commit and its two parents. Normally+ # the first parent is on the base branch, so the merge-base is immediate.+ # Stacked PRs can arrive with a merge commit built on another branch's tip;+ # deepen the PR side until its history reaches the base branch instead of+ # falling back to the full suite.+ compare_rev=""+ for deepen in 0 200 2000; do+ if [ "$deepen" -gt 0 ]; then+ echo "No merge-base with origin/${GITHUB_BASE_REF} yet; deepening PR history by ${deepen}"+ git fetch --no-tags --deepen="$deepen" origin "$GITHUB_SHA" || break+ fi+ if compare_rev="$(git merge-base "origin/${GITHUB_BASE_REF}" HEAD)"; then+ break+ fi+ compare_rev=""+ done++ if [ -z "$compare_rev" ]; then+ echo "Unable to find merge-base for origin/${GITHUB_BASE_REF}; falling back to full test suite" >&2+ : > /tmp/changed-files+ exit 0+ fi+ fi++ printf '%s\n' "$compare_rev" > /tmp/base-revision+ git diff --name-only --no-renames "$compare_rev" "$GITHUB_SHA" > /tmp/changed-files+ shell: bash+ - name: Mount Namespace cache volume+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0+ with:+ cache: rust+ path: /nix+ continue-on-error: true+ - name: Setup Nix+ uses: './.github/actions/setup-nix'+ - name: Setup Nix dev shell+ uses: './.github/actions/setup-nix-dev-shell'+ - name: Configure Namespace remote sccache+ if: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository+ run: |-+ set -euo pipefail+ env_file="$(mktemp "$RUNNER_TEMP/namespace-sccache.XXXXXX")"+ trap 'rm -f "$env_file"' EXIT+ if nsc cache sccache setup --cache_name sccache-ci > "$env_file"; then+ # Register credential values as masked BEFORE exporting: GitHub only+ # masks `secrets.*`, so without this SCCACHE_WEBDAV_TOKEN — a broad,+ # ~24h Namespace workspace token (registry + cache write) — printed+ # verbatim in every subsequent step's env dump. Masking is selective by+ # key name: masking non-secrets like the endpoint URL or key prefix+ # would redact those strings everywhere in the logs.+ while IFS= read -r line; do+ k="${line%%=*}"+ v="${line#*=}"+ [ -n "$v" ] && [ "$k" != "$line" ] || continue+ case "$k" in+ *TOKEN*|*SECRET*|*PASSWORD*) echo "::add-mask::$v" ;;+ esac+ done < "$env_file"+ cat "$env_file" >> "$GITHUB_ENV"+ # Force the next compiler invocation to start a server with the new remote+ # backend even if a setup hook happened to launch one already.+ sccache --stop-server >/dev/null 2>&1 || true+ else+ echo "::warning::Namespace remote sccache setup failed; using local cache fallback"+ fi+ - id: doppler-bins+ name: compute affected Doppler config bins+ run: |+ set -euo pipefail++ doppler_config_bins="$(cargo run --manifest-path tooling/xtask/Cargo.toml -- doppler-bins /tmp/changed-files)"+ {+ echo 'doppler_config_bins<<__DOPPLER_CONFIG_BINS__'+ if [ -n "$doppler_config_bins" ]; then+ printf '%s\n' "$doppler_config_bins"+ fi+ echo '__DOPPLER_CONFIG_BINS__'+ } >> "$GITHUB_OUTPUT"+ shell: bash+ - name: validate Doppler configs+ if: steps.doppler-bins.outputs.doppler_config_bins != ''+ run: |+ set -euo pipefail++ if [ -z "${DOPPLER_TOKEN:-}" ]; then+ echo "DOPPLER_TOKEN secret is required to validate Doppler configs" >&2+ exit 1+ fi++ if [ -z "${RUSTC_WRAPPER:-}" ]; then+ echo "RUSTC_WRAPPER is required so Doppler config binaries build through sccache" >&2+ exit 1+ fi++ bins=()+ cargo_args=(build --locked --all-features)+ while IFS= read -r bin; do+ if [ -z "$bin" ]; then+ continue+ fi++ bins+=("$bin")+ cargo_args+=(--bin "$bin")+ done <<< "$DOPPLER_CONFIG_BINS"++ if [ "${#bins[@]}" -eq 0 ]; then+ echo "No Doppler config binaries to validate"+ exit 0+ fi++ echo "Building affected Doppler config binaries with RUSTC_WRAPPER=$RUSTC_WRAPPER"+ printf ' %s\n' "${bins[@]}"++ cargo "${cargo_args[@]}"+ for bin in "${bins[@]}"; do+ "./target/debug/$bin"+ done+ env:+ DOPPLER_CONFIG_BINS: ${{ steps.doppler-bins.outputs.doppler_config_bins }}+ DOPPLER_TOKEN: ${{ secrets.DOPPLER_TOKEN }}+ - name: show sccache stats+ if: always()+ run: sccache --show-stats || true+ - name: Teardown Nix+ if: always()+ uses: './.github/actions/teardown-nix'test:needs:- path-check@@ -400,7 +585,7 @@ jobs:donedocker exec "$postgres_container" psql -U user -d macrodb -c "SHOW max_locks_per_transaction;"- name: prepare tests- run: just setup_test_envs && just initialize_dbs+ run: just setup_test_envs && just initialize_dbs && just setup_test_template- name: run testsrun: |set -euo pipefail@@ -454,6 +639,7 @@ jobs:needs:- path-check- check+ - doppler-config- testif: always()name: Cloud Storage Status Check@@ -463,11 +649,13 @@ jobs:run: |echo "path-check: ${{ needs.path-check.result }}"echo "check: ${{ needs.check.result }}"+ echo "doppler-config: ${{ needs.doppler-config.result }}"echo "test: ${{ needs.test.result }}"# Fail if any job failed (skipped and success are both OK)if [[ "${{ needs.path-check.result }}" == "failure" ]] || \[[ "${{ needs.check.result }}" == "failure" ]] || \+ [[ "${{ needs.doppler-config.result }}" == "failure" ]] || \[[ "${{ needs.test.result }}" == "failure" ]]; thenecho "❌ One or more jobs failed"exit 1diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.ymldeleted file mode 100644index 6d8a75c72a..0000000000--- a/.github/workflows/docs-check.yml+++ /dev/null@@ -1,84 +0,0 @@-# DO NOT EDIT — regenerate with `cargo x workflows` (from the repository root).-# Source: tooling/xtask/crates/xtask_workflows/src/workflows/docs_check.rs-name: Docs Check-'on':- pull_request:- branches:- - main- paths:- - apps/docs/**- - crates/**/*.rs- - Cargo.toml- - Cargo.lock- - '.github/workflows/docs-check.yml'-jobs:- check-docs:- name: Docs Generated Code Check- runs-on: namespace-profile-linux-mid;overrides.cache-tag=web-ci- steps:- - name: Checkout- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd- with:- clean: 'false'- persist-credentials: 'false'- - name: Mount Namespace cache volume- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0- with:- cache: nix- path: |-- /home/runner/.bun/install/cache- /home/runner/.cargo/registry- /home/runner/.cargo/git- continue-on-error: true- - name: Setup Rust- uses: './.github/actions/setup-rust'- with:- sccache: 'true'- rust-cache: 'false'- - name: Setup Bun- uses: oven-sh/setup-bun@v2- - name: Configure Namespace remote sccache- if: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository- run: |-- set -euo pipefail- env_file="$(mktemp "$RUNNER_TEMP/namespace-sccache.XXXXXX")"- trap 'rm -f "$env_file"' EXIT- if nsc cache sccache setup --cache_name web-ci > "$env_file"; then- # Register credential values as masked BEFORE exporting: GitHub only- # masks `secrets.*`, so without this SCCACHE_WEBDAV_TOKEN — a broad,- # ~24h Namespace workspace token (registry + cache write) — printed- # verbatim in every subsequent step's env dump. Masking is selective by- # key name: masking non-secrets like the endpoint URL or key prefix- # would redact those strings everywhere in the logs.- while IFS= read -r line; do- k="${line%%=*}"- v="${line#*=}"- [ -n "$v" ] && [ "$k" != "$line" ] || continue- case "$k" in- *TOKEN*|*SECRET*|*PASSWORD*) echo "::add-mask::$v" ;;- esac- done < "$env_file"- cat "$env_file" >> "$GITHUB_ENV"- # Force the next compiler invocation to start a server with the new remote- # backend even if a setup hook happened to launch one already.- sccache --stop-server >/dev/null 2>&1 || true- else- echo "::warning::Namespace remote sccache setup failed; using local cache fallback"- fi- - name: Regenerate MCP tool pages- run: bun run generate:tools- working-directory: apps/docs- - name: show sccache stats- if: always()- run: sccache --show-stats || true- - name: Verify generated docs are fresh- run: |- if [ -n "$(git status --porcelain -- apps/docs)" ]; then- echo "apps/docs tool reference is stale. Run 'bun run generate:tools' in apps/docs and commit the result."- git status --porcelain -- apps/docs- git diff -- apps/docs | head -200- exit 1- fi-concurrency:- group: ${{ github.workflow }}-${{ github.ref }}-check- cancel-in-progress: truediff --git a/.github/workflows/sdk-check.yml b/.github/workflows/sdk-check.ymlindex 24d40616dc..cb857e8c57 100644--- a/.github/workflows/sdk-check.yml+++ b/.github/workflows/sdk-check.yml@@ -7,11 +7,6 @@ name: SDK Check- mainpaths:- packages/sdk/**- - crates/**/*.rs- - Cargo.toml- - Cargo.lock- - apps/web/scripts/generate-api-schema.ts- - apps/web/scripts/services.ts- '.github/workflows/sdk-check.yml'push:branches:@@ -45,74 +40,6 @@ jobs:- name: Check endpoint coveragerun: bun run coverageworking-directory: packages/sdk- check-sdk:- if: ${{ github.event_name == 'pull_request' }}- name: SDK Generated Code Check- runs-on: namespace-profile-linux-mid;overrides.cache-tag=web-ci- steps:- - name: Checkout- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd- with:- clean: 'false'- persist-credentials: 'false'- - name: Mount Namespace cache volume- uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0- with:- cache: nix- path: |-- /home/runner/.bun/install/cache- /home/runner/.cargo/registry- /home/runner/.cargo/git- continue-on-error: true- - name: Setup Nix- uses: './.github/actions/setup-nix'- - name: Setup Prereqs- uses: './.github/actions/setup-reqs-web'- - name: Configure Namespace remote sccache- if: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository- run: |-- set -euo pipefail- env_file="$(mktemp "$RUNNER_TEMP/namespace-sccache.XXXXXX")"- trap 'rm -f "$env_file"' EXIT- if nsc cache sccache setup --cache_name web-ci > "$env_file"; then- # Register credential values as masked BEFORE exporting: GitHub only- # masks `secrets.*`, so without this SCCACHE_WEBDAV_TOKEN — a broad,- # ~24h Namespace workspace token (registry + cache write) — printed- # verbatim in every subsequent step's env dump. Masking is selective by- # key name: masking non-secrets like the endpoint URL or key prefix- # would redact those strings everywhere in the logs.- while IFS= read -r line; do- k="${line%%=*}"- v="${line#*=}"- [ -n "$v" ] && [ "$k" != "$line" ] || continue- case "$k" in- *TOKEN*|*SECRET*|*PASSWORD*) echo "::add-mask::$v" ;;- esac- done < "$env_file"- cat "$env_file" >> "$GITHUB_ENV"- # Force the next compiler invocation to start a server with the new remote- # backend even if a setup hook happened to launch one already.- sccache --stop-server >/dev/null 2>&1 || true- else- echo "::warning::Namespace remote sccache setup failed; using local cache fallback"- fi- - name: Regenerate SDK code- run: just update-generated- working-directory: packages/sdk- - name: show sccache stats- if: always()- run: sccache --show-stats || true- - name: Verify generated code is fresh- run: |- if [ -n "$(git status --porcelain -- packages/sdk)" ]; then- echo "packages/sdk generated code is stale. Run 'just update-generated' in packages/sdk and commit the result."- git status --porcelain -- packages/sdk- git diff -- packages/sdk | head -200- exit 1- fi- - name: Teardown Nix- if: always()- uses: './.github/actions/teardown-nix'concurrency:group: ${{ github.workflow }}-${{ github.ref }}-checkcancel-in-progress: truediff --git a/.github/workflows/web-app-check-main.yml b/.github/workflows/web-app-check-main.ymlindex 02af6c9f61..142cf8e024 100644--- a/.github/workflows/web-app-check-main.yml+++ b/.github/workflows/web-app-check-main.yml@@ -15,7 +15,7 @@ jobs:runs-on: namespace-profile-linux-tiny-no-cacheoutputs:should_run: ${{ steps.filter.outputs.should_run }}- api_changed: ${{ steps.filter.outputs.api_changed }}+ codegen_changed: ${{ steps.filter.outputs.codegen_changed }}steps:- name: Checkout Repouses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10@@ -45,21 +45,63 @@ jobs:- 'services/lexical-service/**'- 'crates/ai_tools/src/display_results/schema.generated.json'- '.github/actions/setup-reqs-web/**'- api_changed:+ codegen_changed:- 'crates/**/*.rs'- 'services/**/*.rs'- 'Cargo.toml'- 'Cargo.lock'- 'apps/web/scripts/generate-api-schema.ts'- 'apps/web/scripts/services.ts'+ - 'packages/sdk/**'+ - 'apps/docs/**'- '.github/actions/setup-reqs-web/**'typescript:needs:- path-check- if: needs.path-check.outputs.should_run == 'true' || needs.path-check.outputs.api_changed == 'true'+ if: needs.path-check.outputs.should_run == 'true'name: Typecheckruns-on: namespace-profile-linux-mid;overrides.cache-tag=web-cisteps:+ - name: Checkout Repo+ uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10+ - name: Mount Namespace cache volume+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0+ with:+ cache: nix+ path: /home/runner/.bun/install/cache+ continue-on-error: true+ - name: Setup Nix+ uses: './.github/actions/setup-nix'+ - name: Setup Prereqs+ uses: './.github/actions/setup-reqs-web'+ - name: Check Dynamic UI Schema+ run: bun run check-dynamic-ui-schema+ working-directory: apps/web+ - name: Check Types+ run: bun run type-check+ working-directory: apps/web+ - name: Check Collaboration Package Types+ run: bun run type-check+ working-directory: packages/collaboration+ - name: Check Lexical Service Types+ run: bun run check+ working-directory: services/lexical-service+ - name: Test Lexical Service Endpoints+ run: bun test src+ working-directory: services/lexical-service+ - name: Teardown Nix+ if: always()+ uses: './.github/actions/teardown-nix'+ generated-code:+ needs:+ - path-check+ if: needs.path-check.outputs.codegen_changed == 'true'+ name: Generated Code Check+ runs-on: namespace-profile-linux-mid;overrides.cache-tag=web-ci+ env:+ CARGO_PROFILE_DEV_DEBUG: '0'+ RUSTFLAGS: '-C link-arg=-fuse-ld=mold'+ steps:- name: Checkout Repouses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10- name: Mount Namespace cache volume@@ -76,7 +118,7 @@ jobs:- name: Setup Prereqsuses: './.github/actions/setup-reqs-web'- name: Configure Namespace remote sccache- if: ((github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repository) && (needs.path-check.outputs.api_changed == 'true')+ if: (github.event_name != 'pull_request' && github.event_name != 'pull_request_target') || github.event.pull_request.head.repo.full_name == github.repositoryrun: |-set -euo pipefailenv_file="$(mktemp "$RUNNER_TEMP/namespace-sccache.XXXXXX")"@@ -103,12 +145,40 @@ jobs:elseecho "::warning::Namespace remote sccache setup failed; using local cache fallback"fi+ - name: Build schema binaries+ run: |+ set -euo pipefail+ read -ra openapi_bins <<< "$(cd apps/web && bun scripts/generate-api-schema.ts --print-cargo-bins)"+ SQLX_OFFLINE=true cargo build "${openapi_bins[@]}" --bin gen_tool_schemas- name: Generate API Types- if: needs.path-check.outputs.api_changed == 'true'run: bun run gen-api -- --checkworking-directory: apps/web+ - name: Regenerate SDK code+ run: |+ bun install --frozen-lockfile+ bun run sync-specs+ bun run generate+ working-directory: packages/sdk+ - name: Verify SDK generated code is fresh+ run: |+ if [ -n "$(git status --porcelain -- packages/sdk)" ]; then+ echo "packages/sdk generated code is stale. Run 'just update-generated' in packages/sdk and commit the result."+ git status --porcelain -- packages/sdk+ git diff -- packages/sdk | head -200+ exit 1+ fi+ - name: Regenerate MCP tool pages+ run: bun run generate:tools+ working-directory: apps/docs+ - name: Verify docs tool reference is fresh+ run: |+ if [ -n "$(git status --porcelain -- apps/docs)" ]; then+ echo "apps/docs tool reference is stale. Run 'bun run generate:tools' in apps/docs and commit the result."+ git status --porcelain -- apps/docs+ git diff -- apps/docs | head -200+ exit 1+ fi- name: Check Specta Types- if: needs.path-check.outputs.api_changed == 'true'run: |just gen-agent-fold-typesjust gen-database-sql-types@@ -118,23 +188,8 @@ jobs:fiworking-directory: apps/web- name: show sccache stats- if: always() && needs.path-check.outputs.api_changed == 'true'+ if: always()run: sccache --show-stats || true- - name: Check Dynamic UI Schema- run: bun run check-dynamic-ui-schema- working-directory: apps/web- - name: Check Types- run: bun run --bun --silent tsc --project ./tsconfig.json- working-directory: apps/web- - name: Check Collaboration Package Types- run: bun run type-check- working-directory: packages/collaboration- - name: Check Lexical Service Types- run: bun run check- working-directory: services/lexical-service- - name: Test Lexical Service Endpoints- run: bun test src- working-directory: services/lexical-service- name: Teardown Nixif: always()uses: './.github/actions/teardown-nix'@@ -172,7 +227,7 @@ jobs:- path-checkif: needs.path-check.outputs.should_run == 'true'name: Test- runs-on: namespace-profile-linux-small;overrides.cache-tag=web-ci+ runs-on: namespace-profile-linux-mid;overrides.cache-tag=web-cisteps:- name: Checkout Repouses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10@@ -180,7 +235,9 @@ jobs:uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0with:cache: nix- path: /home/runner/.bun/install/cache+ path: |-+ /home/runner/.bun/install/cache+ /home/runner/.cache/vitest-modulescontinue-on-error: true- name: Setup Nixuses: './.github/actions/setup-nix'@@ -193,7 +250,9 @@ jobs:timeout-minutes: 15working-directory: apps/web- name: Test- run: bunx vitest+ run: bun run test+ env:+ VITEST_MODULE_CACHE_DIR: /home/runner/.cache/vitest-modulesworking-directory: apps/web- name: Sign-up Browser Testsrun: just test-signup-browser@@ -246,6 +305,7 @@ jobs:/home/runner/.cargo/registry/home/runner/.cargo/git/home/runner/.cache/.wasm-pack+ /home/runner/.cache/wasm-pkgscontinue-on-error: true- name: Setup Nixuses: './.github/actions/setup-nix'@@ -283,6 +343,8 @@ jobs:run: sccache --start-server- name: Buildrun: just build-dev+ env:+ WASM_PKG_CACHE_DIR: /home/runner/.cache/wasm-pkgsworking-directory: apps/web- name: show sccache statsif: always()@@ -294,6 +356,7 @@ jobs:needs:- path-check- typescript+ - generated-code- biome-check- test- cycles@@ -306,6 +369,7 @@ jobs:run: |echo "path-check: ${{ needs.path-check.result }}"echo "typescript: ${{ needs.typescript.result }}"+ echo "generated-code: ${{ needs.generated-code.result }}"echo "biome-check: ${{ needs.biome-check.result }}"echo "test: ${{ needs.test.result }}"echo "cycles: ${{ needs.cycles.result }}"@@ -314,6 +378,7 @@ jobs:# Fail if any job failed (skipped and success are both OK)if [[ "${{ needs.path-check.result }}" == "failure" ]] || \[[ "${{ needs.typescript.result }}" == "failure" ]] || \+ [[ "${{ needs.generated-code.result }}" == "failure" ]] || \[[ "${{ needs.biome-check.result }}" == "failure" ]] || \[[ "${{ needs.test.result }}" == "failure" ]] || \[[ "${{ needs.cycles.result }}" == "failure" ]] || \diff --git a/Cargo.lock b/Cargo.lockindex c3c2ae9fc5..79839c422b 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -21812,6 +21812,7 @@ dependencies = ["determinator","guppy","tempfile",+ "toml 0.8.23","xtask_graph","xtask_paths",]diff --git a/Cargo.toml b/Cargo.tomlindex d5db97c841..725a5d921d 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -437,6 +437,19 @@ ammonia = "4.1.4"html-escape = "0.2"subtle = "2.6.1"+# cache-turso's storage tests load thousands of rows into the embedded SQL engine+# to assert VM step budgets. Unoptimized, the engine made them the slowest tests+# in the Rust suite (34 s for the crate); at opt-level 2 the crate's tests take+# 2.6 s. The assertions count VM steps, which optimization does not change.+[profile.dev.package.turso_core]+opt-level = 2++[profile.dev.package.turso_parser]+opt-level = 2++[profile.dev.package.turso_ext]+opt-level = 2+# Retain Rust DWARF for local cache profiling while keeping release-level LLVM# optimization. wasm-pack's custom profile metadata prevents the final# wasm-opt pass from discarding or rewriting these locations.diff --git a/apps/web/justfile b/apps/web/justfileindex 7005674673..af00c8ad3d 100644--- a/apps/web/justfile+++ b/apps/web/justfile@@ -244,7 +244,7 @@ cycles-ci:# (gitignored). Unconditional — CI app builds must never skip it (schema or# engine changes alter the wasm without a cache-wasm version bump).build-cache-wasm:- CARGO_TARGET_DIR="{{cache_wasm_target}}" wasm-pack build {{justfile_directory()}}/../../crates/client/cache-wasm --target web --release --out-dir {{justfile_directory()}}/src/lib/graphql-cache/wasm+ scripts/cached-wasm-build.sh cache-wasm {{justfile_directory()}}/src/lib/graphql-cache/wasm -- env CARGO_TARGET_DIR="{{cache_wasm_target}}" wasm-pack build {{justfile_directory()}}/../../crates/client/cache-wasm --target web --release --out-dir {{justfile_directory()}}/src/lib/graphql-cache/wasm# Build the production browser harness served by Vite preview.build-cache-wasm-browser-production: build-cache-wasm build-cache-wasm-browser-test-hooks@@ -278,7 +278,7 @@ build-cache-wasm-profile:# (gitignored). `--no-default-features` drops the crate's `cli` feature, whose# tokio/clap do not build for wasm32; the fold itself needs neither.build-agent-fold-wasm:- CARGO_TARGET_DIR="{{agent_fold_wasm_target}}" wasm-pack build {{justfile_directory()}}/../../crates/folds/agent_fold --target web --release --out-dir {{justfile_directory()}}/src/lib/core/agent-fold/wasm -- --no-default-features+ scripts/cached-wasm-build.sh agent_fold {{justfile_directory()}}/src/lib/core/agent-fold/wasm -- env CARGO_TARGET_DIR="{{agent_fold_wasm_target}}" wasm-pack build {{justfile_directory()}}/../../crates/folds/agent_fold --target web --release --out-dir {{justfile_directory()}}/src/lib/core/agent-fold/wasm -- --no-default-features# Dev fast path: build the agent-fold wasm pkg only when it is missing or any# fold source is newer than the built artifact. A stale artifact folds with the@@ -295,7 +295,7 @@ ensure-agent-fold-wasm:# Build the database-sql wasm module (the SQL engine's staged `Query`) into# src/lib/core/database-sql/wasm (gitignored).build-database-sql-wasm:- CARGO_TARGET_DIR="{{database_sql_wasm_target}}" wasm-pack build {{justfile_directory()}}/../../crates/database_sql --target web --release --out-dir {{justfile_directory()}}/src/lib/core/database-sql/wasm+ scripts/cached-wasm-build.sh database_sql {{justfile_directory()}}/src/lib/core/database-sql/wasm -- env CARGO_TARGET_DIR="{{database_sql_wasm_target}}" wasm-pack build {{justfile_directory()}}/../../crates/database_sql --target web --release --out-dir {{justfile_directory()}}/src/lib/core/database-sql/wasm# Dev fast path: build the database-sql wasm pkg only when it is missing or# any source it is compiled from is newer than the built artifact: the@@ -324,7 +324,7 @@ ensure-database-sql-wasm:# worker by the pptx block) into src/lib/core/pptx-engine/wasm (gitignored).# SIMD speeds up rasterization; every browser the app supports has it.build-pptx-engine-wasm:- CARGO_TARGET_DIR="{{pptx_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/pptx_engine --target web --release --out-dir {{justfile_directory()}}/src/lib/core/pptx-engine/wasm+ scripts/cached-wasm-build.sh pptx_engine {{justfile_directory()}}/src/lib/core/pptx-engine/wasm -- env CARGO_TARGET_DIR="{{pptx_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/pptx_engine --target web --release --out-dir {{justfile_directory()}}/src/lib/core/pptx-engine/wasm# Dev fast path: build the pptx-engine wasm pkg only when it is missing or any# engine source is newer than the built artifact.@@ -340,7 +340,7 @@ ensure-pptx-engine-wasm:# Build the docx-engine wasm module (DOCX layout, rendering and editing, hosted# in a worker by the write block) into src/lib/core/docx-engine/wasm (gitignored).build-docx-engine-wasm:- CARGO_TARGET_DIR="{{docx_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/docx_engine --target web --release --out-dir {{justfile_directory()}}/src/lib/core/docx-engine/wasm+ scripts/cached-wasm-build.sh docx_engine {{justfile_directory()}}/src/lib/core/docx-engine/wasm -- env CARGO_TARGET_DIR="{{docx_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/docx_engine --target web --release --out-dir {{justfile_directory()}}/src/lib/core/docx-engine/wasm# Dev fast path: build the docx-engine wasm pkg only when it is missing or any# engine source (including the pptx engine it builds on) is newer than it.@@ -357,7 +357,7 @@ ensure-docx-engine-wasm:# Build the fig-engine wasm module (.fig parsing and tile rasterization, hosted# in workers by the fig block) into src/lib/core/fig-engine/wasm (gitignored).build-fig-engine-wasm:- CARGO_TARGET_DIR="{{fig_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/fig_engine --target web --profile fig-engine-wasm --out-dir {{justfile_directory()}}/src/lib/core/fig-engine/wasm+ scripts/cached-wasm-build.sh fig_engine {{justfile_directory()}}/src/lib/core/fig-engine/wasm -- env CARGO_TARGET_DIR="{{fig_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/fig_engine --target web --profile fig-engine-wasm --out-dir {{justfile_directory()}}/src/lib/core/fig-engine/wasm# Dev fast path: build the fig-engine wasm pkg only when it is missing or any# engine source is newer than the built artifact.@@ -374,7 +374,7 @@ ensure-fig-engine-wasm:# compositing, hosted in workers by the psd block) into# src/lib/core/psd-engine/wasm (gitignored).build-psd-engine-wasm:- CARGO_TARGET_DIR="{{psd_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/psd_engine --target web --profile psd-engine-wasm --out-dir {{justfile_directory()}}/src/lib/core/psd-engine/wasm+ scripts/cached-wasm-build.sh psd_engine {{justfile_directory()}}/src/lib/core/psd-engine/wasm -- env CARGO_TARGET_DIR="{{psd_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/psd_engine --target web --profile psd-engine-wasm --out-dir {{justfile_directory()}}/src/lib/core/psd-engine/wasm# Dev fast path: build the psd-engine wasm pkg only when it is missing or any# engine source (or the fig engine it draws text with) is newer than it.@@ -392,7 +392,7 @@ ensure-psd-engine-wasm:# rendering, hosted in workers by the ai block) into# src/lib/core/ai-engine/wasm (gitignored).build-ai-engine-wasm:- CARGO_TARGET_DIR="{{ai_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/ai_engine --target web --profile ai-engine-wasm --out-dir {{justfile_directory()}}/src/lib/core/ai-engine/wasm+ scripts/cached-wasm-build.sh ai_engine {{justfile_directory()}}/src/lib/core/ai-engine/wasm -- env CARGO_TARGET_DIR="{{ai_engine_wasm_target}}" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C target-feature=+simd128" wasm-pack build {{justfile_directory()}}/../../crates/ai_engine --target web --profile ai-engine-wasm --out-dir {{justfile_directory()}}/src/lib/core/ai-engine/wasm# Dev fast path: build the ai-engine wasm pkg only when it is missing or any# engine source (or the fig engine it draws text with) is newer than it.diff --git a/apps/web/knip.ts b/apps/web/knip.tsindex 74880e5eb8..61ee34d549 100644--- a/apps/web/knip.ts+++ b/apps/web/knip.ts@@ -60,6 +60,8 @@ const config: KnipConfig = {'libheif-js','@tailwindcss/vite','bebop',+ // TypeScript 7's compiler, run by path from `type-check`.+ 'typescript-native',],ignoreWorkspaces: ['../../packages/loro-mirror', '../../packages/sdk'],diff --git a/apps/web/package.json b/apps/web/package.jsonindex b14c6ee532..53641f592c 100644--- a/apps/web/package.json+++ b/apps/web/package.json@@ -10,7 +10,7 @@"build": "vite build -c vite.config.ts && bun run postbuild:semver && bun run postbuild:bundle-manifest","postbuild:semver": "echo $npm_package_version+$(git rev-parse --short HEAD) > ./dist/semver.txt","postbuild:bundle-manifest": "bun ./scripts/write-bundle-manifest.mjs",- "test": "bunx vitest run",+ "test": "bunx vitest run --fsModuleCache","native:e2e:build": "bash tests/native/build.sh","native:e2e": "bash tests/native/run.sh","native:e2e:smoke": "bash tests/native/run.sh --smoke",@@ -31,7 +31,7 @@"ui-audit": "bun scripts/ui-audit.ts","gen-agent-fold-types": "just gen-agent-fold-types","gen-database-sql-types": "just gen-database-sql-types",- "type-check": "tsc --noEmit --skipLibCheck --project tsconfig.json",+ "type-check": "bun ../../node_modules/typescript-native/bin/tsc --noEmit --skipLibCheck --project tsconfig.json","check": "bun check-graphql-cache-schema && bun check-dynamic-ui-schema && bun type-check && bun biome check","preview": "vite preview -c vite.config.ts --outDir dist --base /app","preview:dev": "vite preview -c vite.config.ts --outDir dist --base /app",@@ -219,6 +219,7 @@"orval": "^7.21.0","typedoc": "^0.22.18","typescript": "^5.9.3",+ "typescript-native": "npm:[email protected]","vite": "^6.4.1","vite-plugin-solid": "^2.11.8","vite-plugin-solid-svg": "^0.8.1",diff --git a/apps/web/scripts/cached-wasm-build.sh b/apps/web/scripts/cached-wasm-build.shnew file mode 100755index 0000000000..0228c519f9--- /dev/null+++ b/apps/web/scripts/cached-wasm-build.sh@@ -0,0 +1,58 @@+#!/usr/bin/env bash+# Runs a wasm package build, or restores its output from WASM_PKG_CACHE_DIR when+# nothing the package is built from has changed. Without WASM_PKG_CACHE_DIR it+# only runs the build, so local `just build-*` behaves as before.+#+# The cache key hashes the contents of the crate's workspace dependency closure+# (.github/workspace-dep-closures.json), the root manifests and lockfile, the+# toolchain pins, the shared assets build scripts read, and the build command.+#+# usage: cached-wasm-build.sh <closure package> <out dir> -- <build command...>+set -euo pipefail++package="$1"+out_dir="$2"+shift 2+if [ "${1:-}" = "--" ]; then+ shift+fi++if [ -z "${WASM_PKG_CACHE_DIR:-}" ]; then+ exec "$@"+fi++root="$(git rev-parse --show-toplevel)"+mapfile -t dirs < <(jq -r --arg p "$package" '.closures[$p] // empty | .[]' \+ "$root/.github/workspace-dep-closures.json")+if [ "${#dirs[@]}" -eq 0 ]; then+ echo "no dependency closure for $package; building without the cache" >&2+ exec "$@"+fi++key="$(+ {+ (cd "$root" && git ls-files -z -- "${dirs[@]}" Cargo.toml Cargo.lock \+ rust-toolchain.toml flake.lock .cargo static_assets | xargs -0 sha256sum)+ printf '%s\n' "$@"+ } | sha256sum | cut -c1-32+)"+entry="$WASM_PKG_CACHE_DIR/$package-$key"++if [ -d "$entry" ]; then+ echo "$package: unchanged inputs, reusing the cached wasm package ($key)"+ rm -rf "$out_dir"+ mkdir -p "$(dirname "$out_dir")"+ cp -a "$entry" "$out_dir"+ touch "$entry"+ exit 0+fi++"$@"++mkdir -p "$WASM_PKG_CACHE_DIR"+# Entries are small and keyed by content; drop ones no build has used for a week.+find "$WASM_PKG_CACHE_DIR" -mindepth 1 -maxdepth 1 -mtime +7 -exec rm -rf {} + || true+tmp="$(mktemp -d "$WASM_PKG_CACHE_DIR/.tmp-XXXXXX")"+cp -a "$out_dir/." "$tmp/"+# A concurrent job may have stored the same key first; keep that one.+mv -T "$tmp" "$entry" 2>/dev/null || rm -rf "$tmp"diff --git a/apps/web/scripts/generate-api-schema.ts b/apps/web/scripts/generate-api-schema.tsindex 6bd9f881f3..3b7308ac30 100644--- a/apps/web/scripts/generate-api-schema.ts+++ b/apps/web/scripts/generate-api-schema.ts@@ -2,6 +2,7 @@// bun run scripts/generate-api-schema.ts <service-name> // generate for specific service// bun run scripts/generate-api-schema.ts // generate for all services// bun run scripts/generate-api-schema.ts --check // verify types are up to date (for CI)+// bun run scripts/generate-api-schema.ts --print-cargo-bins // print the `--bin` args phase 1 builds (for CI)//// This script generates OpenAPI schemas by running the local Rust binaries// instead of fetching from deployed services.@@ -37,7 +38,12 @@ const getServiceClientsDir = () =>path.resolve(import.meta.dirname, "../src/lib/service-clients");// Parse argumentsconst getTargetServices = () =>- process.argv.slice(2).filter((arg) => arg !== "--check");+ process.argv+ .slice(2)+ .filter((arg) => arg !== "--check" && arg !== "--print-cargo-bins");++const openApiBinArgs = (crateNames: string[]) =>+ crateNames.flatMap((crate) => ["--bin", `${crate}_openapi`]);// Build all OpenAPI binaries in a single cargo invocation (cargo parallelizes internally).async function buildOpenApiBinaries(@@ -46,7 +52,7 @@ async function buildOpenApiBinaries(): Promise<void> {if (crateNames.length === 0) return;- const binArgs = crateNames.flatMap((crate) => ["--bin", `${crate}_openapi`]);+ const binArgs = openApiBinArgs(crateNames);console.log(`Building ${crateNames.length} OpenAPI binaries...`);console.log(`cargo build args: ${binArgs.join(" ")}`);@@ -216,6 +222,13 @@ async function main() {.map((s) => serviceToCrate[s.name]).filter((crate): crate is string => !!crate);+ // CI builds these together with other schema binaries in one cargo+ // invocation, so the phase 1 build below finds them fresh.+ if (process.argv.includes("--print-cargo-bins")) {+ console.log(openApiBinArgs(crateNames).join(" "));+ return;+ }+console.log(`\nProcessing ${servicesToProcess.length} service(s)...\n`);// Phase 1: Build all binaries (skipped when OPENAPI_BINS_DIR is set)diff --git a/apps/web/scripts/generate-dcs-tools.ts b/apps/web/scripts/generate-dcs-tools.tsindex 23a0dd98bc..6d8d2150dd 100644--- a/apps/web/scripts/generate-dcs-tools.ts+++ b/apps/web/scripts/generate-dcs-tools.ts@@ -59,7 +59,10 @@ type FrontendSchemas = z.infer<typeof FrontendSchemasValidator>;async function buildAndRunSchemaGenerator(): Promise<void> {console.log('Building gen_tool_schemas binary...');- await $`cd ${rustWorkspaceDir} && SQLX_OFFLINE=true cargo build --package ai_tools --bin gen_tool_schemas`;+ // No `--package`: like the OpenAPI bins and apps/docs' generator, resolve+ // features across the workspace, so one build serves every caller instead+ // of `--package ai_tools` compiling a second feature set of its deps.+ await $`cd ${rustWorkspaceDir} && SQLX_OFFLINE=true cargo build --bin gen_tool_schemas`;console.log('Build complete.');await $`rm -rf ${path.join(aiToolsDir, 'schemas')}`.quiet();diff --git a/apps/web/src/features/block-spreadsheet/README.md b/apps/web/src/features/block-spreadsheet/README.mdindex f764c73799..74cb39a72c 100644--- a/apps/web/src/features/block-spreadsheet/README.md+++ b/apps/web/src/features/block-spreadsheet/README.md@@ -463,13 +463,15 @@ group, SmartArt and an EMF, as Excel writes them) and corpus workbooks. Exportswhich draws the same charts (surface charts aside, which it cannot draw) andrebuilds pivot tables from edited data.-`core/xlsx-corpus.test.ts` imports, recalculates, exports and reimports the-real-world workbooks in `core/xlsx-fixtures/real-world/` (finance models,-government statistics and open-source test suites; provenance and licenses are in-its `manifest.json`). Its snapshot records counts, warnings, how many formulas-recalculate to Excel's cached results, and the first round-trip difference, so a-change in fidelity shows up in review. Set `XLSX_CORPUS_VERBOSE=1` to print-mismatch examples.+`core/xlsx-corpus-1.test.ts` through `core/xlsx-corpus-4.test.ts` import,+recalculate, export and reimport the real-world workbooks in+`core/xlsx-fixtures/real-world/` (finance models, government statistics and+open-source test suites; provenance and licenses are in its `manifest.json`).+Each file runs one slice of the corpus through `core/xlsx-corpus-suite.ts`, so+the workbooks run on several Vitest workers. Their snapshots record counts,+warnings, how many formulas recalculate to Excel's cached results, and the first+round-trip difference, so a change in fidelity shows up in review. Set+`XLSX_CORPUS_VERBOSE=1` to print mismatch examples.## Verificationdiff --git a/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-1.test.ts.snap b/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-1.test.ts.snapnew file mode 100644index 0000000000..d4a8a7b73e--- /dev/null+++ b/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-1.test.ts.snap@@ -0,0 +1,326 @@+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html++exports[`real-world XLSX corpus > bls-consumer-expenditure-2024-by-income… 1`] = `+{+ "calculation": {+ "compared": 0,+ "matched": 0,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 5323,+ "formulas": 0,+ "roundTrip": "identical",+ "sheets": 1,+ "styledCells": 5353,+ "warnings": [+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ ],+}+`;++exports[`real-world XLSX corpus > fred-unrate-cpiaucsl-payems-export.xlsx 1`] = `+{+ "calculation": {+ "compared": 0,+ "matched": 0,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 4024,+ "formulas": 0,+ "roundTrip": "identical",+ "sheets": 2,+ "styledCells": 2965,+ "warnings": [],+}+`;++exports[`real-world XLSX corpus > github-alibhatti23-dcf-comps-lbo-merger… 1`] = `+{+ "calculation": {+ "compared": 432,+ "matched": 432,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 843,+ "formulas": 432,+ "roundTrip": "identical",+ "sheets": 8,+ "styledCells": 831,+ "warnings": [+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ ],+}+`;++exports[`real-world XLSX corpus > ironcalc-financial-pmt.xlsx 1`] = `+{+ "calculation": {+ "compared": 309,+ "matched": 309,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 503,+ "formulas": 309,+ "roundTrip": "identical",+ "sheets": 1,+ "styledCells": 353,+ "warnings": [],+}+`;++exports[`real-world XLSX corpus > ironcalc-template-gantt-project-tracker… 1`] = `+{+ "calculation": {+ "compared": 441,+ "matched": 441,+ "mismatchedFunctions": {},+ "volatile": 3,+ },+ "cells": 629,+ "formulas": 444,+ "roundTrip": "identical",+ "sheets": 4,+ "styledCells": 5541,+ "warnings": [+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",+ ],+}+`;++exports[`real-world XLSX corpus > libreoffice-forum-mso-en4-134670-sales-… 1`] = `+{+ "calculation": {+ "compared": 516,+ "matched": 398,+ "mismatchedFunctions": {+ "(no function)": 50,+ "FREQUENCY": 19,+ "IF": 19,+ "IFERROR": 19,+ "MATCH": 19,+ "ROUND": 49,+ "ROW": 19,+ "SIGN": 19,+ "SUM": 19,+ },+ "volatile": 0,+ },+ "cells": 10300,+ "formulas": 516,+ "roundTrip": "identical",+ "sheets": 5,+ "styledCells": 10319,+ "warnings": [+ "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Text rotation, shrink-to-fit and right-to-left layout are not imported.",+ ],+}+`;++exports[`real-world XLSX corpus > obr-efo-ready-reckoner-spring-2025.xlsx 1`] = `+{+ "calculation": {+ "compared": 4601,+ "matched": 4601,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 8608,+ "formulas": 4601,+ "roundTrip": "identical",+ "sheets": 27,+ "styledCells": 12262,+ "warnings": [+ "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",+ "Hyperlinks are imported as display text; link targets are not retained.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Rich text is imported as plain text with the cell’s formatting.",+ ],+}+`;++exports[`real-world XLSX corpus > openxmlsdk-business-startup-costs-pl-te… 1`] = `+{+ "calculation": {+ "compared": 214,+ "matched": 214,+ "mismatchedFunctions": {},+ "volatile": 4,+ },+ "cells": 566,+ "formulas": 218,+ "roundTrip": "identical",+ "sheets": 6,+ "styledCells": 3731,+ "warnings": [+ "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",+ "Font sizes are rounded and limited to 8–36 pt.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Rich text is imported as plain text with the cell’s formatting.",+ ],+}+`;++exports[`real-world XLSX corpus > packt-wazobia-3-statement-dcf-valuation… 1`] = `+{+ "calculation": {+ "compared": 1118,+ "matched": 1118,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 1843,+ "formulas": 1118,+ "roundTrip": "identical",+ "sheets": 6,+ "styledCells": 3821,+ "warnings": [+ "Excel sheet protection is not imported; Macro sharing controls determine edit access.",+ "External workbook links are not supported; formulas that use them keep their last calculated values.",+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Names that refer to other workbooks are not imported.",+ "What-if data tables are imported as their last calculated values.",+ ],+}+`;++exports[`real-world XLSX corpus > poi-formula-eval-test-data.xlsx 1`] = `+{+ "calculation": {+ "compared": 1281,+ "matched": 1226,+ "mismatchedFunctions": {+ "ADDRESS": 1,+ "AND": 1,+ "AVEDEV": 3,+ "DATE": 5,+ "DAY": 2,+ "DEVSQ": 2,+ "INDEX": 1,+ "LOOKUP": 1,+ "MAXA": 1,+ "MINA": 1,+ "MODE": 4,+ "MONTH": 2,+ "STDEV": 2,+ "SUMIF": 1,+ "SUMPRODUCT": 1,+ "SUMSQ": 2,+ "SUMX2MY2": 6,+ "SUMX2PY2": 6,+ "SUMXMY2": 5,+ "T": 2,+ "VAR": 2,+ "VARP": 2,+ "YEAR": 2,+ },+ "volatile": 14,+ },+ "cells": 4160,+ "formulas": 1295,+ "roundTrip": "identical",+ "sheets": 4,+ "styledCells": 591,+ "warnings": [+ "Hyperlinks are imported as display text; link targets are not retained.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",+ "Text rotation, shrink-to-fit and right-to-left layout are not imported.",+ ],+}+`;++exports[`real-world XLSX corpus > sec-jpmorgan-10k-fy2024-financial-repor… 1`] = `+{+ "calculation": {+ "compared": 0,+ "matched": 0,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 30568,+ "formulas": 0,+ "roundTrip": "identical",+ "sheets": 234,+ "styledCells": 30568,+ "warnings": [+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Text longer than 10,000 characters was shortened to fit Macro cells.",+ ],+}+`;++exports[`real-world XLSX corpus > sheetjs-formula-stress-test.xlsx 1`] = `+{+ "calculation": {+ "compared": 439,+ "matched": 415,+ "mismatchedFunctions": {+ "AREAS": 1,+ "BESSELY": 1,+ "CHAR": 2,+ "CODE": 1,+ "CONVERT": 1,+ "DATEVALUE": 1,+ "ECMA.CEILING": 1,+ "FREQUENCY": 1,+ "GROWTH": 1,+ "IMDIV": 1,+ "IMSUM": 1,+ "LINEST": 2,+ "LOGEST": 2,+ "MINVERSE": 1,+ "MMULT": 1,+ "PHONETIC": 1,+ "TEXT": 1,+ "TREND": 2,+ "XIRR": 1,+ "XNPV": 1,+ },+ "volatile": 8,+ },+ "cells": 1845,+ "formulas": 447,+ "roundTrip": "identical",+ "sheets": 10,+ "styledCells": 164,+ "warnings": [+ "Macro cannot calculate these Excel functions yet, so their formulas show errors: ECMA.CEILING, JIS, PHONETIC, YEN.",+ "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",+ ],+}+`;++exports[`real-world XLSX corpus > zenodo-surewave-floating-pv-cost-and-fi… 1`] = `+{+ "calculation": {+ "compared": 1888,+ "matched": 1888,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 3671,+ "formulas": 1888,+ "roundTrip": "identical",+ "sheets": 12,+ "styledCells": 42476,+ "warnings": [+ "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",+ "Hyperlinks are imported as display text; link targets are not retained.",+ "Macro cannot calculate these Excel functions yet, so their formulas show errors: __XLUDF.UNSUPPORTED.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ ],+}+`;diff --git a/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-2.test.ts.snap b/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-2.test.ts.snapnew file mode 100644index 0000000000..2ed90a6c17--- /dev/null+++ b/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-2.test.ts.snap@@ -0,0 +1,291 @@+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html++exports[`real-world XLSX corpus > bea-gdp-key-source-data-2q2026-third.xl… 1`] = `+{+ "calculation": {+ "compared": 0,+ "matched": 0,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 7816,+ "formulas": 0,+ "roundTrip": "identical",+ "sheets": 4,+ "styledCells": 9255,+ "warnings": [+ "Hyperlinks are imported as display text; link targets are not retained.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Rich text is imported as plain text with the cell’s formatting.",+ ],+}+`;++exports[`real-world XLSX corpus > closedxml-template-table-source-pivot-t… 1`] = `+{+ "calculation": {+ "compared": 1450,+ "matched": 1449,+ "mismatchedFunctions": {+ "XLL.DSSDATA": 1,+ },+ "volatile": 0,+ },+ "cells": 1789,+ "formulas": 1450,+ "roundTrip": "identical",+ "sheets": 28,+ "styledCells": 2126,+ "warnings": [+ "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",+ "Macro cannot calculate these Excel functions yet, so their formulas show errors: _XLL.DSSDATA.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",+ ],+}+`;++exports[`real-world XLSX corpus > ironcalc-template-travel-expenses-track… 1`] = `+{+ "calculation": {+ "compared": 40,+ "matched": 40,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 382,+ "formulas": 40,+ "roundTrip": "identical",+ "sheets": 2,+ "styledCells": 1475,+ "warnings": [+ "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",+ ],+}+`;++exports[`real-world XLSX corpus > libreoffice-missing-path-external-shift… 1`] = `+{+ "calculation": {+ "compared": 4027,+ "matched": 4027,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 8334,+ "formulas": 4027,+ "roundTrip": "identical",+ "sheets": 3,+ "styledCells": 8397,+ "warnings": [+ "External workbook links are not supported; formulas that use them keep their last calculated values.",+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Names that refer to other workbooks are not imported.",+ ],+}+`;++exports[`real-world XLSX corpus > obr-public-finances-databank-2026-09.xl… 1`] = `+{+ "calculation": {+ "compared": 5938,+ "matched": 5938,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 14387,+ "formulas": 5938,+ "roundTrip": "identical",+ "sheets": 7,+ "styledCells": 20760,+ "warnings": [+ "Chart sheets and other non-grid sheets are not imported.",+ "External workbook links are not supported; formulas that use them keep their last calculated values.",+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Names that refer to other workbooks are not imported.",+ "Rich text is imported as plain text with the cell’s formatting.",+ "Special font effects are simplified to the supported text formatting.",+ ],+}+`;++exports[`real-world XLSX corpus > ons-business-demography-2024-reference-… 1`] = `+{+ "calculation": {+ "compared": 1808,+ "matched": 1808,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 94578,+ "formulas": 1808,+ "roundTrip": "identical",+ "sheets": 45,+ "styledCells": 110323,+ "warnings": [+ "Hyperlinks are imported as display text; link targets are not retained.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Rich text is imported as plain text with the cell’s formatting.",+ "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",+ ],+}+`;++exports[`real-world XLSX corpus > packt-project-finance-model.xlsx 1`] = `+{+ "calculation": {+ "compared": 2973,+ "matched": 2973,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 4088,+ "formulas": 2973,+ "roundTrip": "identical",+ "sheets": 10,+ "styledCells": 5437,+ "warnings": [+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",+ ],+}+`;++exports[`real-world XLSX corpus > poi-number-format-tests-date1904.xlsx 1`] = `+{+ "calculation": {+ "compared": 325,+ "matched": 88,+ "mismatchedFunctions": {+ "TEXT": 237,+ },+ "volatile": 0,+ },+ "cells": 1314,+ "formulas": 325,+ "roundTrip": "identical",+ "sheets": 2,+ "styledCells": 1322,+ "warnings": [+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",+ "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",+ ],+}+`;++exports[`real-world XLSX corpus > poi-structured-refs-lots-with-lookups-b… 1`] = `+{+ "calculation": {+ "compared": 3476,+ "matched": 3476,+ "mismatchedFunctions": {},+ "volatile": 3798,+ },+ "cells": 37696,+ "formulas": 7274,+ "roundTrip": "identical",+ "sheets": 5,+ "styledCells": 4022,+ "warnings": [+ "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",+ "External workbook links are not supported; formulas that use them keep their last calculated values.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Names that refer to other workbooks are not imported.",+ ],+}+`;++exports[`real-world XLSX corpus > sba-table-of-size-standards-2023.xlsx 1`] = `+{+ "calculation": {+ "compared": 0,+ "matched": 0,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 5873,+ "formulas": 0,+ "roundTrip": "identical",+ "sheets": 6,+ "styledCells": 5739,+ "warnings": [+ "Excel filters are retained for export; Macro displays all rows.",+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",+ "Hyperlinks are imported as display text; link targets are not retained.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Rich text is imported as plain text with the cell’s formatting.",+ ],+}+`;++exports[`real-world XLSX corpus > sec-educational-development-corp-10k-fy… 1`] = `+{+ "calculation": {+ "compared": 0,+ "matched": 0,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 3835,+ "formulas": 0,+ "roundTrip": "identical",+ "sheets": 81,+ "styledCells": 3835,+ "warnings": [+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Text longer than 10,000 characters was shortened to fit Macro cells.",+ ],+}+`;++exports[`real-world XLSX corpus > sheetjs-metadata-2019-dynamic-arrays-ri… 1`] = `+{+ "calculation": {+ "compared": 8,+ "matched": 1,+ "mismatchedFunctions": {+ "FV": 6,+ "UNIQUE": 1,+ },+ "volatile": 0,+ },+ "cells": 30,+ "formulas": 8,+ "roundTrip": "identical",+ "sheets": 1,+ "styledCells": 4,+ "warnings": [+ "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",+ "Linked data types and pictures in cells are imported as their displayed values.",+ "Macro cannot calculate these Excel functions yet, so their formulas show errors: _FV.",+ ],+}+`;++exports[`real-world XLSX corpus > zenodo-debt-capacity-calculator.xlsx 1`] = `+{+ "calculation": {+ "compared": 110,+ "matched": 110,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 225,+ "formulas": 110,+ "roundTrip": "identical",+ "sheets": 2,+ "styledCells": 328,+ "warnings": [+ "External workbook links are not supported; formulas that use them keep their last calculated values.",+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Names that refer to other workbooks are not imported.",+ "Rich text is imported as plain text with the cell’s formatting.",+ ],+}+`;diff --git a/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-3.test.ts.snap b/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-3.test.ts.snapnew file mode 100644index 0000000000..4f0206ef47--- /dev/null+++ b/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-3.test.ts.snap@@ -0,0 +1,286 @@+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html++exports[`real-world XLSX corpus > census-monthly-retail-trade-sales-1992-… 1`] = `+{+ "calculation": {+ "compared": 0,+ "matched": 0,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 52106,+ "formulas": 0,+ "roundTrip": "identical",+ "sheets": 35,+ "styledCells": 60252,+ "warnings": [+ "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",+ ],+}+`;++exports[`real-world XLSX corpus > closedxml-data-table-formula-what-if.xl… 1`] = `+{+ "calculation": {+ "compared": 9,+ "matched": 9,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 111,+ "formulas": 9,+ "roundTrip": "identical",+ "sheets": 9,+ "styledCells": 4,+ "warnings": [+ "What-if data tables are imported as their last calculated values.",+ ],+}+`;++exports[`real-world XLSX corpus > ironcalc-example-pivots-charts-comments… 1`] = `+{+ "calculation": {+ "compared": 36,+ "matched": 36,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 154,+ "formulas": 36,+ "roundTrip": "identical",+ "sheets": 10,+ "styledCells": 128,+ "warnings": [+ "Chart sheets and other non-grid sheets are not imported.",+ "Diagonal borders are not imported.",+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Patterned fills are shown as a blended solid color.",+ "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",+ "Rich text is imported as plain text with the cell’s formatting.",+ ],+}+`;++exports[`real-world XLSX corpus > irs-soi-2023-partnerships-table3-balanc… 1`] = `+{+ "calculation": {+ "compared": 0,+ "matched": 0,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 722,+ "formulas": 0,+ "roundTrip": "identical",+ "sheets": 1,+ "styledCells": 8677,+ "warnings": [+ "Font sizes are rounded and limited to 8–36 pt.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ ],+}+`;++exports[`real-world XLSX corpus > libreoffice-functions-excel-2010.xlsx 1`] = `+{+ "calculation": {+ "compared": 517,+ "matched": 513,+ "mismatchedFunctions": {+ "AND": 1,+ "RANK.EQ": 1,+ "ROUND": 2,+ },+ "volatile": 0,+ },+ "cells": 973,+ "formulas": 517,+ "roundTrip": "identical",+ "sheets": 2,+ "styledCells": 281,+ "warnings": [+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ ],+}+`;++exports[`real-world XLSX corpus > nrel-sam-ppa-single-owner-project-finan… 1`] = `+{+ "calculation": {+ "compared": 16248,+ "matched": 16158,+ "mismatchedFunctions": {+ "(no function)": 64,+ "SUM": 26,+ },+ "volatile": 0,+ },+ "cells": 18067,+ "formulas": 16248,+ "roundTrip": "identical",+ "sheets": 3,+ "styledCells": 28010,+ "warnings": [+ "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Rich text is imported as plain text with the cell’s formatting.",+ ],+}+`;++exports[`real-world XLSX corpus > packt-loan-amortization-schedule.xlsx 1`] = `+{+ "calculation": {+ "compared": 96,+ "matched": 96,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 138,+ "formulas": 96,+ "roundTrip": "identical",+ "sheets": 1,+ "styledCells": 138,+ "warnings": [],+}+`;++exports[`real-world XLSX corpus > poi-cvlkra-kyc-file-structure-tika3163.… 1`] = `+{+ "calculation": {+ "compared": 103,+ "matched": 103,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 2987,+ "formulas": 103,+ "roundTrip": "identical",+ "sheets": 30,+ "styledCells": 52242,+ "warnings": [+ "Excel sheet protection is not imported; Macro sharing controls determine edit access.",+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",+ "Hyperlinks are imported as display text; link targets are not retained.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Rich text is imported as plain text with the cell’s formatting.",+ "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",+ ],+}+`;++exports[`real-world XLSX corpus > poi-data-validation-table-range-county-… 1`] = `+{+ "calculation": {+ "compared": 87,+ "matched": 87,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 6615,+ "formulas": 87,+ "roundTrip": "identical",+ "sheets": 2,+ "styledCells": 176,+ "warnings": [+ "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",+ "Font sizes are rounded and limited to 8–36 pt.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ ],+}+`;++exports[`real-world XLSX corpus > sheetjs-large-strings.xlsx 1`] = `+{+ "calculation": {+ "compared": 21492,+ "matched": 21492,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 56913,+ "formulas": 21492,+ "roundTrip": "identical",+ "sheets": 3,+ "styledCells": 35238,+ "warnings": [],+}+`;++exports[`real-world XLSX corpus > worldbank-commodity-prices-pink-sheet-m… 1`] = `+{+ "calculation": {+ "compared": 0,+ "matched": 0,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 72988,+ "formulas": 0,+ "roundTrip": "identical",+ "sheets": 5,+ "styledCells": 85343,+ "warnings": [+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Special font effects are simplified to the supported text formatting.",+ ],+}+`;++exports[`real-world XLSX corpus > zenodo-ccs-cement-post-tax-dcf-model.xl… 1`] = `+{+ "calculation": {+ "compared": 1752,+ "matched": 1752,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 16138,+ "formulas": 1752,+ "roundTrip": "identical",+ "sheets": 12,+ "styledCells": 20467,+ "warnings": [+ "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",+ "Excel filters are retained for export; Macro displays all rows.",+ "Hyperlinks are imported as display text; link targets are not retained.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Patterned fills are shown as a blended solid color.",+ "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",+ "Rich text is imported as plain text with the cell’s formatting.",+ "Text rotation, shrink-to-fit and right-to-left layout are not imported.",+ ],+}+`;++exports[`real-world XLSX corpus > zenodo-circular-economy-business-case-c… 1`] = `+{+ "calculation": {+ "compared": 2825,+ "matched": 2825,+ "mismatchedFunctions": {},+ "volatile": 3,+ },+ "cells": 4665,+ "formulas": 2828,+ "roundTrip": "identical",+ "sheets": 14,+ "styledCells": 20074,+ "warnings": [+ "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",+ "Excel sheet protection is not imported; Macro sharing controls determine edit access.",+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Rich text is imported as plain text with the cell’s formatting.",+ ],+}+`;diff --git a/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-4.test.ts.snap b/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-4.test.ts.snapnew file mode 100644index 0000000000..667b7efbd7--- /dev/null+++ b/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus-4.test.ts.snap@@ -0,0 +1,288 @@+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html++exports[`real-world XLSX corpus > census-susb-2022-us-sector-large-employ… 1`] = `+{+ "calculation": {+ "compared": 0,+ "matched": 0,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 2464,+ "formulas": 0,+ "roundTrip": "identical",+ "sheets": 2,+ "styledCells": 2483,+ "warnings": [+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Rich text is imported as plain text with the cell’s formatting.",+ ],+}+`;++exports[`real-world XLSX corpus > exceljs-test-issue-877-spare-parts-inve… 1`] = `+{+ "calculation": {+ "compared": 89,+ "matched": 89,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 977,+ "formulas": 89,+ "roundTrip": "identical",+ "sheets": 5,+ "styledCells": 3786,+ "warnings": [+ "Excel sheet protection is not imported; Macro sharing controls determine edit access.",+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",+ "Hyperlinks are imported as display text; link targets are not retained.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Rich text is imported as plain text with the cell’s formatting.",+ "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",+ "Text rotation, shrink-to-fit and right-to-left layout are not imported.",+ ],+}+`;++exports[`real-world XLSX corpus > figshare-ucl-social-enterprise-financia… 1`] = `+{+ "calculation": {+ "compared": 1861,+ "matched": 1861,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 65799,+ "formulas": 1861,+ "roundTrip": "identical",+ "sheets": 17,+ "styledCells": 101113,+ "warnings": [+ "Excel filters are retained for export; Macro displays all rows.",+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Pivot tables over other workbooks or data connections show their last values in Macro; the download keeps them with the data Excel saved, to refresh in Excel.",+ ],+}+`;++exports[`real-world XLSX corpus > gsa-per-diem-rates-fy2026-master.xlsx 1`] = `+{+ "calculation": {+ "compared": 0,+ "matched": 0,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 4927,+ "formulas": 0,+ "roundTrip": "identical",+ "sheets": 1,+ "styledCells": 5209,+ "warnings": [],+}+`;++exports[`real-world XLSX corpus > ironcalc-tables-structured-references.x… 1`] = `+{+ "calculation": {+ "compared": 201,+ "matched": 192,+ "mismatchedFunctions": {+ "SUBTOTAL": 9,+ },+ "volatile": 4,+ },+ "cells": 831,+ "formulas": 205,+ "roundTrip": "identical",+ "sheets": 9,+ "styledCells": 327,+ "warnings": [+ "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",+ "Some table references could not be converted; those formulas show their last calculated values.",+ ],+}+`;++exports[`real-world XLSX corpus > ironcalc-template-invoice.xlsx 1`] = `+{+ "calculation": {+ "compared": 23,+ "matched": 23,+ "mismatchedFunctions": {},+ "volatile": 2,+ },+ "cells": 96,+ "formulas": 25,+ "roundTrip": "identical",+ "sheets": 3,+ "styledCells": 230,+ "warnings": [+ "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",+ "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",+ ],+}+`;++exports[`real-world XLSX corpus > libreoffice-column-style-autofilter-lin… 1`] = `+{+ "calculation": {+ "compared": 27,+ "matched": 27,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 62015,+ "formulas": 27,+ "roundTrip": "identical",+ "sheets": 24,+ "styledCells": 3694,+ "warnings": [+ "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",+ "Pivot tables over other workbooks or data connections show their last values in Macro; the download keeps them with the data Excel saved, to refresh in Excel.",+ "Some pivot tables over other workbooks or data connections are downloaded without the data Excel saved with them; Excel fills them again when they refresh.",+ ],+}+`;++exports[`real-world XLSX corpus > libreoffice-forum-mso-de-104083-pivot-w… 1`] = `+{+ "calculation": {+ "compared": 2303,+ "matched": 2303,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 9524,+ "formulas": 2303,+ "roundTrip": "identical",+ "sheets": 2,+ "styledCells": 12208,+ "warnings": [+ "Excel filters are retained for export; Macro displays all rows.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",+ ],+}+`;++exports[`real-world XLSX corpus > nrel-sam-residential-solar-cash-flow.xl… 1`] = `+{+ "calculation": {+ "compared": 22431,+ "matched": 22431,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 23003,+ "formulas": 22431,+ "roundTrip": "identical",+ "sheets": 2,+ "styledCells": 32545,+ "warnings": [+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Patterned fills are shown as a blended solid color.",+ "Rich text is imported as plain text with the cell’s formatting.",+ ],+}+`;++exports[`real-world XLSX corpus > packt365-dynamic-array-functions-workbo… 1`] = `+{+ "calculation": {+ "compared": 302,+ "matched": 301,+ "mismatchedFunctions": {+ "FILTER": 1,+ },+ "volatile": 0,+ },+ "cells": 1239,+ "formulas": 302,+ "roundTrip": "identical",+ "sheets": 3,+ "styledCells": 1411,+ "warnings": [+ "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",+ "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",+ "External workbook links are not supported; formulas that use them keep their last calculated values.",+ "Linked data types and pictures in cells are imported as their displayed values.",+ "Names that refer to other workbooks are not imported.",+ ],+}+`;++exports[`real-world XLSX corpus > poi-average-tax-rates-ireland.xlsx 1`] = `+{+ "calculation": {+ "compared": 0,+ "matched": 0,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 1110,+ "formulas": 0,+ "roundTrip": "identical",+ "sheets": 3,+ "styledCells": 1360,+ "warnings": [+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ ],+}+`;++exports[`real-world XLSX corpus > poi-conditional-formatting-samples.xlsx 1`] = `+{+ "calculation": {+ "compared": 352,+ "matched": 351,+ "mismatchedFunctions": {+ "SUBTOTAL": 1,+ },+ "volatile": 0,+ },+ "cells": 1723,+ "formulas": 352,+ "roundTrip": "identical",+ "sheets": 18,+ "styledCells": 1838,+ "warnings": [+ "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Rich text is imported as plain text with the cell’s formatting.",+ "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",+ ],+}+`;++exports[`real-world XLSX corpus > zenodo-offshore-wind-lcoe-dcf-model.xlsx 1`] = `+{+ "calculation": {+ "compared": 4000,+ "matched": 4000,+ "mismatchedFunctions": {},+ "volatile": 0,+ },+ "cells": 5012,+ "formulas": 4000,+ "roundTrip": "identical",+ "sheets": 9,+ "styledCells": 6665,+ "warnings": [+ "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",+ "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",+ "Hyperlinks are imported as display text; link targets are not retained.",+ "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",+ "Rich text is imported as plain text with the cell’s formatting.",+ "Some table references could not be converted; those formulas show their last calculated values.",+ ],+}+`;diff --git a/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus.test.ts.snap b/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus.test.ts.snapdeleted file mode 100644index 56f1140dc3..0000000000--- a/apps/web/src/features/block-spreadsheet/core/__snapshots__/xlsx-corpus.test.ts.snap+++ /dev/null@@ -1,1188 +0,0 @@-// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html--exports[`real-world XLSX corpus > bea-gdp-key-source-data-2q2026-third.xl… 1`] = `-{- "calculation": {- "compared": 0,- "matched": 0,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 7816,- "formulas": 0,- "roundTrip": "identical",- "sheets": 4,- "styledCells": 9255,- "warnings": [- "Hyperlinks are imported as display text; link targets are not retained.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Rich text is imported as plain text with the cell’s formatting.",- ],-}-`;--exports[`real-world XLSX corpus > bls-consumer-expenditure-2024-by-income… 1`] = `-{- "calculation": {- "compared": 0,- "matched": 0,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 5323,- "formulas": 0,- "roundTrip": "identical",- "sheets": 1,- "styledCells": 5353,- "warnings": [- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- ],-}-`;--exports[`real-world XLSX corpus > census-monthly-retail-trade-sales-1992-… 1`] = `-{- "calculation": {- "compared": 0,- "matched": 0,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 52106,- "formulas": 0,- "roundTrip": "identical",- "sheets": 35,- "styledCells": 60252,- "warnings": [- "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",- ],-}-`;--exports[`real-world XLSX corpus > census-susb-2022-us-sector-large-employ… 1`] = `-{- "calculation": {- "compared": 0,- "matched": 0,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 2464,- "formulas": 0,- "roundTrip": "identical",- "sheets": 2,- "styledCells": 2483,- "warnings": [- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Rich text is imported as plain text with the cell’s formatting.",- ],-}-`;--exports[`real-world XLSX corpus > closedxml-data-table-formula-what-if.xl… 1`] = `-{- "calculation": {- "compared": 9,- "matched": 9,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 111,- "formulas": 9,- "roundTrip": "identical",- "sheets": 9,- "styledCells": 4,- "warnings": [- "What-if data tables are imported as their last calculated values.",- ],-}-`;--exports[`real-world XLSX corpus > closedxml-template-table-source-pivot-t… 1`] = `-{- "calculation": {- "compared": 1450,- "matched": 1449,- "mismatchedFunctions": {- "XLL.DSSDATA": 1,- },- "volatile": 0,- },- "cells": 1789,- "formulas": 1450,- "roundTrip": "identical",- "sheets": 28,- "styledCells": 2126,- "warnings": [- "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",- "Macro cannot calculate these Excel functions yet, so their formulas show errors: _XLL.DSSDATA.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",- ],-}-`;--exports[`real-world XLSX corpus > exceljs-test-issue-877-spare-parts-inve… 1`] = `-{- "calculation": {- "compared": 89,- "matched": 89,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 977,- "formulas": 89,- "roundTrip": "identical",- "sheets": 5,- "styledCells": 3786,- "warnings": [- "Excel sheet protection is not imported; Macro sharing controls determine edit access.",- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",- "Hyperlinks are imported as display text; link targets are not retained.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Rich text is imported as plain text with the cell’s formatting.",- "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",- "Text rotation, shrink-to-fit and right-to-left layout are not imported.",- ],-}-`;--exports[`real-world XLSX corpus > figshare-ucl-social-enterprise-financia… 1`] = `-{- "calculation": {- "compared": 1861,- "matched": 1861,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 65799,- "formulas": 1861,- "roundTrip": "identical",- "sheets": 17,- "styledCells": 101113,- "warnings": [- "Excel filters are retained for export; Macro displays all rows.",- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Pivot tables over other workbooks or data connections show their last values in Macro; the download keeps them with the data Excel saved, to refresh in Excel.",- ],-}-`;--exports[`real-world XLSX corpus > fred-unrate-cpiaucsl-payems-export.xlsx 1`] = `-{- "calculation": {- "compared": 0,- "matched": 0,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 4024,- "formulas": 0,- "roundTrip": "identical",- "sheets": 2,- "styledCells": 2965,- "warnings": [],-}-`;--exports[`real-world XLSX corpus > github-alibhatti23-dcf-comps-lbo-merger… 1`] = `-{- "calculation": {- "compared": 432,- "matched": 432,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 843,- "formulas": 432,- "roundTrip": "identical",- "sheets": 8,- "styledCells": 831,- "warnings": [- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- ],-}-`;--exports[`real-world XLSX corpus > gsa-per-diem-rates-fy2026-master.xlsx 1`] = `-{- "calculation": {- "compared": 0,- "matched": 0,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 4927,- "formulas": 0,- "roundTrip": "identical",- "sheets": 1,- "styledCells": 5209,- "warnings": [],-}-`;--exports[`real-world XLSX corpus > ironcalc-example-pivots-charts-comments… 1`] = `-{- "calculation": {- "compared": 36,- "matched": 36,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 154,- "formulas": 36,- "roundTrip": "identical",- "sheets": 10,- "styledCells": 128,- "warnings": [- "Chart sheets and other non-grid sheets are not imported.",- "Diagonal borders are not imported.",- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Patterned fills are shown as a blended solid color.",- "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",- "Rich text is imported as plain text with the cell’s formatting.",- ],-}-`;--exports[`real-world XLSX corpus > ironcalc-financial-pmt.xlsx 1`] = `-{- "calculation": {- "compared": 309,- "matched": 309,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 503,- "formulas": 309,- "roundTrip": "identical",- "sheets": 1,- "styledCells": 353,- "warnings": [],-}-`;--exports[`real-world XLSX corpus > ironcalc-tables-structured-references.x… 1`] = `-{- "calculation": {- "compared": 201,- "matched": 192,- "mismatchedFunctions": {- "SUBTOTAL": 9,- },- "volatile": 4,- },- "cells": 831,- "formulas": 205,- "roundTrip": "identical",- "sheets": 9,- "styledCells": 327,- "warnings": [- "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",- "Some table references could not be converted; those formulas show their last calculated values.",- ],-}-`;--exports[`real-world XLSX corpus > ironcalc-template-gantt-project-tracker… 1`] = `-{- "calculation": {- "compared": 441,- "matched": 441,- "mismatchedFunctions": {},- "volatile": 3,- },- "cells": 629,- "formulas": 444,- "roundTrip": "identical",- "sheets": 4,- "styledCells": 5541,- "warnings": [- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",- ],-}-`;--exports[`real-world XLSX corpus > ironcalc-template-invoice.xlsx 1`] = `-{- "calculation": {- "compared": 23,- "matched": 23,- "mismatchedFunctions": {},- "volatile": 2,- },- "cells": 96,- "formulas": 25,- "roundTrip": "identical",- "sheets": 3,- "styledCells": 230,- "warnings": [- "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",- ],-}-`;--exports[`real-world XLSX corpus > ironcalc-template-travel-expenses-track… 1`] = `-{- "calculation": {- "compared": 40,- "matched": 40,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 382,- "formulas": 40,- "roundTrip": "identical",- "sheets": 2,- "styledCells": 1475,- "warnings": [- "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",- ],-}-`;--exports[`real-world XLSX corpus > irs-soi-2023-partnerships-table3-balanc… 1`] = `-{- "calculation": {- "compared": 0,- "matched": 0,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 722,- "formulas": 0,- "roundTrip": "identical",- "sheets": 1,- "styledCells": 8677,- "warnings": [- "Font sizes are rounded and limited to 8–36 pt.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- ],-}-`;--exports[`real-world XLSX corpus > libreoffice-column-style-autofilter-lin… 1`] = `-{- "calculation": {- "compared": 27,- "matched": 27,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 62015,- "formulas": 27,- "roundTrip": "identical",- "sheets": 24,- "styledCells": 3694,- "warnings": [- "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",- "Pivot tables over other workbooks or data connections show their last values in Macro; the download keeps them with the data Excel saved, to refresh in Excel.",- "Some pivot tables over other workbooks or data connections are downloaded without the data Excel saved with them; Excel fills them again when they refresh.",- ],-}-`;--exports[`real-world XLSX corpus > libreoffice-forum-mso-de-104083-pivot-w… 1`] = `-{- "calculation": {- "compared": 2303,- "matched": 2303,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 9524,- "formulas": 2303,- "roundTrip": "identical",- "sheets": 2,- "styledCells": 12208,- "warnings": [- "Excel filters are retained for export; Macro displays all rows.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",- ],-}-`;--exports[`real-world XLSX corpus > libreoffice-forum-mso-en4-134670-sales-… 1`] = `-{- "calculation": {- "compared": 516,- "matched": 398,- "mismatchedFunctions": {- "(no function)": 50,- "FREQUENCY": 19,- "IF": 19,- "IFERROR": 19,- "MATCH": 19,- "ROUND": 49,- "ROW": 19,- "SIGN": 19,- "SUM": 19,- },- "volatile": 0,- },- "cells": 10300,- "formulas": 516,- "roundTrip": "identical",- "sheets": 5,- "styledCells": 10319,- "warnings": [- "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Text rotation, shrink-to-fit and right-to-left layout are not imported.",- ],-}-`;--exports[`real-world XLSX corpus > libreoffice-functions-excel-2010.xlsx 1`] = `-{- "calculation": {- "compared": 517,- "matched": 513,- "mismatchedFunctions": {- "AND": 1,- "RANK.EQ": 1,- "ROUND": 2,- },- "volatile": 0,- },- "cells": 973,- "formulas": 517,- "roundTrip": "identical",- "sheets": 2,- "styledCells": 281,- "warnings": [- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- ],-}-`;--exports[`real-world XLSX corpus > libreoffice-missing-path-external-shift… 1`] = `-{- "calculation": {- "compared": 4027,- "matched": 4027,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 8334,- "formulas": 4027,- "roundTrip": "identical",- "sheets": 3,- "styledCells": 8397,- "warnings": [- "External workbook links are not supported; formulas that use them keep their last calculated values.",- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Names that refer to other workbooks are not imported.",- ],-}-`;--exports[`real-world XLSX corpus > nrel-sam-ppa-single-owner-project-finan… 1`] = `-{- "calculation": {- "compared": 16248,- "matched": 16158,- "mismatchedFunctions": {- "(no function)": 64,- "SUM": 26,- },- "volatile": 0,- },- "cells": 18067,- "formulas": 16248,- "roundTrip": "identical",- "sheets": 3,- "styledCells": 28010,- "warnings": [- "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Rich text is imported as plain text with the cell’s formatting.",- ],-}-`;--exports[`real-world XLSX corpus > nrel-sam-residential-solar-cash-flow.xl… 1`] = `-{- "calculation": {- "compared": 22431,- "matched": 22431,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 23003,- "formulas": 22431,- "roundTrip": "identical",- "sheets": 2,- "styledCells": 32545,- "warnings": [- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Patterned fills are shown as a blended solid color.",- "Rich text is imported as plain text with the cell’s formatting.",- ],-}-`;--exports[`real-world XLSX corpus > obr-efo-ready-reckoner-spring-2025.xlsx 1`] = `-{- "calculation": {- "compared": 4601,- "matched": 4601,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 8608,- "formulas": 4601,- "roundTrip": "identical",- "sheets": 27,- "styledCells": 12262,- "warnings": [- "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",- "Hyperlinks are imported as display text; link targets are not retained.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Rich text is imported as plain text with the cell’s formatting.",- ],-}-`;--exports[`real-world XLSX corpus > obr-public-finances-databank-2026-09.xl… 1`] = `-{- "calculation": {- "compared": 5938,- "matched": 5938,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 14387,- "formulas": 5938,- "roundTrip": "identical",- "sheets": 7,- "styledCells": 20760,- "warnings": [- "Chart sheets and other non-grid sheets are not imported.",- "External workbook links are not supported; formulas that use them keep their last calculated values.",- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Names that refer to other workbooks are not imported.",- "Rich text is imported as plain text with the cell’s formatting.",- "Special font effects are simplified to the supported text formatting.",- ],-}-`;--exports[`real-world XLSX corpus > ons-business-demography-2024-reference-… 1`] = `-{- "calculation": {- "compared": 1808,- "matched": 1808,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 94578,- "formulas": 1808,- "roundTrip": "identical",- "sheets": 45,- "styledCells": 110323,- "warnings": [- "Hyperlinks are imported as display text; link targets are not retained.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Rich text is imported as plain text with the cell’s formatting.",- "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",- ],-}-`;--exports[`real-world XLSX corpus > openxmlsdk-business-startup-costs-pl-te… 1`] = `-{- "calculation": {- "compared": 214,- "matched": 214,- "mismatchedFunctions": {},- "volatile": 4,- },- "cells": 566,- "formulas": 218,- "roundTrip": "identical",- "sheets": 6,- "styledCells": 3731,- "warnings": [- "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",- "Font sizes are rounded and limited to 8–36 pt.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Rich text is imported as plain text with the cell’s formatting.",- ],-}-`;--exports[`real-world XLSX corpus > packt-loan-amortization-schedule.xlsx 1`] = `-{- "calculation": {- "compared": 96,- "matched": 96,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 138,- "formulas": 96,- "roundTrip": "identical",- "sheets": 1,- "styledCells": 138,- "warnings": [],-}-`;--exports[`real-world XLSX corpus > packt-project-finance-model.xlsx 1`] = `-{- "calculation": {- "compared": 2973,- "matched": 2973,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 4088,- "formulas": 2973,- "roundTrip": "identical",- "sheets": 10,- "styledCells": 5437,- "warnings": [- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",- ],-}-`;--exports[`real-world XLSX corpus > packt-wazobia-3-statement-dcf-valuation… 1`] = `-{- "calculation": {- "compared": 1118,- "matched": 1118,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 1843,- "formulas": 1118,- "roundTrip": "identical",- "sheets": 6,- "styledCells": 3821,- "warnings": [- "Excel sheet protection is not imported; Macro sharing controls determine edit access.",- "External workbook links are not supported; formulas that use them keep their last calculated values.",- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Names that refer to other workbooks are not imported.",- "What-if data tables are imported as their last calculated values.",- ],-}-`;--exports[`real-world XLSX corpus > packt365-dynamic-array-functions-workbo… 1`] = `-{- "calculation": {- "compared": 302,- "matched": 301,- "mismatchedFunctions": {- "FILTER": 1,- },- "volatile": 0,- },- "cells": 1239,- "formulas": 302,- "roundTrip": "identical",- "sheets": 3,- "styledCells": 1411,- "warnings": [- "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",- "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",- "External workbook links are not supported; formulas that use them keep their last calculated values.",- "Linked data types and pictures in cells are imported as their displayed values.",- "Names that refer to other workbooks are not imported.",- ],-}-`;--exports[`real-world XLSX corpus > poi-average-tax-rates-ireland.xlsx 1`] = `-{- "calculation": {- "compared": 0,- "matched": 0,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 1110,- "formulas": 0,- "roundTrip": "identical",- "sheets": 3,- "styledCells": 1360,- "warnings": [- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- ],-}-`;--exports[`real-world XLSX corpus > poi-conditional-formatting-samples.xlsx 1`] = `-{- "calculation": {- "compared": 352,- "matched": 351,- "mismatchedFunctions": {- "SUBTOTAL": 1,- },- "volatile": 0,- },- "cells": 1723,- "formulas": 352,- "roundTrip": "identical",- "sheets": 18,- "styledCells": 1838,- "warnings": [- "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Rich text is imported as plain text with the cell’s formatting.",- "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",- ],-}-`;--exports[`real-world XLSX corpus > poi-cvlkra-kyc-file-structure-tika3163.… 1`] = `-{- "calculation": {- "compared": 103,- "matched": 103,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 2987,- "formulas": 103,- "roundTrip": "identical",- "sheets": 30,- "styledCells": 52242,- "warnings": [- "Excel sheet protection is not imported; Macro sharing controls determine edit access.",- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",- "Hyperlinks are imported as display text; link targets are not retained.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Rich text is imported as plain text with the cell’s formatting.",- "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",- ],-}-`;--exports[`real-world XLSX corpus > poi-data-validation-table-range-county-… 1`] = `-{- "calculation": {- "compared": 87,- "matched": 87,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 6615,- "formulas": 87,- "roundTrip": "identical",- "sheets": 2,- "styledCells": 176,- "warnings": [- "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",- "Font sizes are rounded and limited to 8–36 pt.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- ],-}-`;--exports[`real-world XLSX corpus > poi-formula-eval-test-data.xlsx 1`] = `-{- "calculation": {- "compared": 1281,- "matched": 1226,- "mismatchedFunctions": {- "ADDRESS": 1,- "AND": 1,- "AVEDEV": 3,- "DATE": 5,- "DAY": 2,- "DEVSQ": 2,- "INDEX": 1,- "LOOKUP": 1,- "MAXA": 1,- "MINA": 1,- "MODE": 4,- "MONTH": 2,- "STDEV": 2,- "SUMIF": 1,- "SUMPRODUCT": 1,- "SUMSQ": 2,- "SUMX2MY2": 6,- "SUMX2PY2": 6,- "SUMXMY2": 5,- "T": 2,- "VAR": 2,- "VARP": 2,- "YEAR": 2,- },- "volatile": 14,- },- "cells": 4160,- "formulas": 1295,- "roundTrip": "identical",- "sheets": 4,- "styledCells": 591,- "warnings": [- "Hyperlinks are imported as display text; link targets are not retained.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",- "Text rotation, shrink-to-fit and right-to-left layout are not imported.",- ],-}-`;--exports[`real-world XLSX corpus > poi-number-format-tests-date1904.xlsx 1`] = `-{- "calculation": {- "compared": 325,- "matched": 88,- "mismatchedFunctions": {- "TEXT": 237,- },- "volatile": 0,- },- "cells": 1314,- "formulas": 325,- "roundTrip": "identical",- "sheets": 2,- "styledCells": 1322,- "warnings": [- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",- "Text number formats on numeric, boolean, date or formula cells are reset to preserve the original value types.",- ],-}-`;--exports[`real-world XLSX corpus > poi-structured-refs-lots-with-lookups-b… 1`] = `-{- "calculation": {- "compared": 3476,- "matched": 3476,- "mismatchedFunctions": {},- "volatile": 3798,- },- "cells": 37696,- "formulas": 7274,- "roundTrip": "identical",- "sheets": 5,- "styledCells": 4022,- "warnings": [- "Excel table references are converted to cell ranges; tables are imported as ordinary cells.",- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",- "External workbook links are not supported; formulas that use them keep their last calculated values.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Names that refer to other workbooks are not imported.",- ],-}-`;--exports[`real-world XLSX corpus > sba-table-of-size-standards-2023.xlsx 1`] = `-{- "calculation": {- "compared": 0,- "matched": 0,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 5873,- "formulas": 0,- "roundTrip": "identical",- "sheets": 6,- "styledCells": 5739,- "warnings": [- "Excel filters are retained for export; Macro displays all rows.",- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",- "Hyperlinks are imported as display text; link targets are not retained.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Rich text is imported as plain text with the cell’s formatting.",- ],-}-`;--exports[`real-world XLSX corpus > sec-educational-development-corp-10k-fy… 1`] = `-{- "calculation": {- "compared": 0,- "matched": 0,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 3835,- "formulas": 0,- "roundTrip": "identical",- "sheets": 81,- "styledCells": 3835,- "warnings": [- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Text longer than 10,000 characters was shortened to fit Macro cells.",- ],-}-`;--exports[`real-world XLSX corpus > sec-jpmorgan-10k-fy2024-financial-repor… 1`] = `-{- "calculation": {- "compared": 0,- "matched": 0,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 30568,- "formulas": 0,- "roundTrip": "identical",- "sheets": 234,- "styledCells": 30568,- "warnings": [- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Text longer than 10,000 characters was shortened to fit Macro cells.",- ],-}-`;--exports[`real-world XLSX corpus > sheetjs-formula-stress-test.xlsx 1`] = `-{- "calculation": {- "compared": 439,- "matched": 415,- "mismatchedFunctions": {- "AREAS": 1,- "BESSELY": 1,- "CHAR": 2,- "CODE": 1,- "CONVERT": 1,- "DATEVALUE": 1,- "ECMA.CEILING": 1,- "FREQUENCY": 1,- "GROWTH": 1,- "IMDIV": 1,- "IMSUM": 1,- "LINEST": 2,- "LOGEST": 2,- "MINVERSE": 1,- "MMULT": 1,- "PHONETIC": 1,- "TEXT": 1,- "TREND": 2,- "XIRR": 1,- "XNPV": 1,- },- "volatile": 8,- },- "cells": 1845,- "formulas": 447,- "roundTrip": "identical",- "sheets": 10,- "styledCells": 164,- "warnings": [- "Macro cannot calculate these Excel functions yet, so their formulas show errors: ECMA.CEILING, JIS, PHONETIC, YEN.",- "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",- ],-}-`;--exports[`real-world XLSX corpus > sheetjs-large-strings.xlsx 1`] = `-{- "calculation": {- "compared": 21492,- "matched": 21492,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 56913,- "formulas": 21492,- "roundTrip": "identical",- "sheets": 3,- "styledCells": 35238,- "warnings": [],-}-`;--exports[`real-world XLSX corpus > sheetjs-metadata-2019-dynamic-arrays-ri… 1`] = `-{- "calculation": {- "compared": 8,- "matched": 1,- "mismatchedFunctions": {- "FV": 6,- "UNIQUE": 1,- },- "volatile": 0,- },- "cells": 30,- "formulas": 8,- "roundTrip": "identical",- "sheets": 1,- "styledCells": 4,- "warnings": [- "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",- "Linked data types and pictures in cells are imported as their displayed values.",- "Macro cannot calculate these Excel functions yet, so their formulas show errors: _FV.",- ],-}-`;--exports[`real-world XLSX corpus > worldbank-commodity-prices-pink-sheet-m… 1`] = `-{- "calculation": {- "compared": 0,- "matched": 0,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 72988,- "formulas": 0,- "roundTrip": "identical",- "sheets": 5,- "styledCells": 85343,- "warnings": [- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Special font effects are simplified to the supported text formatting.",- ],-}-`;--exports[`real-world XLSX corpus > zenodo-ccs-cement-post-tax-dcf-model.xl… 1`] = `-{- "calculation": {- "compared": 1752,- "matched": 1752,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 16138,- "formulas": 1752,- "roundTrip": "identical",- "sheets": 12,- "styledCells": 20467,- "warnings": [- "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",- "Excel filters are retained for export; Macro displays all rows.",- "Hyperlinks are imported as display text; link targets are not retained.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Patterned fills are shown as a blended solid color.",- "Pivot tables show their last values in Macro; Excel rebuilds them from their data when the downloaded file opens.",- "Rich text is imported as plain text with the cell’s formatting.",- "Text rotation, shrink-to-fit and right-to-left layout are not imported.",- ],-}-`;--exports[`real-world XLSX corpus > zenodo-circular-economy-business-case-c… 1`] = `-{- "calculation": {- "compared": 2825,- "matched": 2825,- "mismatchedFunctions": {},- "volatile": 3,- },- "cells": 4665,- "formulas": 2828,- "roundTrip": "identical",- "sheets": 14,- "styledCells": 20074,- "warnings": [- "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",- "Excel sheet protection is not imported; Macro sharing controls determine edit access.",- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Rich text is imported as plain text with the cell’s formatting.",- ],-}-`;--exports[`real-world XLSX corpus > zenodo-debt-capacity-calculator.xlsx 1`] = `-{- "calculation": {- "compared": 110,- "matched": 110,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 225,- "formulas": 110,- "roundTrip": "identical",- "sheets": 2,- "styledCells": 328,- "warnings": [- "External workbook links are not supported; formulas that use them keep their last calculated values.",- "Hidden sheets are shown in Macro; their visibility is retained in Excel downloads.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Names that refer to other workbooks are not imported.",- "Rich text is imported as plain text with the cell’s formatting.",- ],-}-`;--exports[`real-world XLSX corpus > zenodo-offshore-wind-lcoe-dcf-model.xlsx 1`] = `-{- "calculation": {- "compared": 4000,- "matched": 4000,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 5012,- "formulas": 4000,- "roundTrip": "identical",- "sheets": 9,- "styledCells": 6665,- "warnings": [- "Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.",- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",- "Hyperlinks are imported as display text; link targets are not retained.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- "Rich text is imported as plain text with the cell’s formatting.",- "Some table references could not be converted; those formulas show their last calculated values.",- ],-}-`;--exports[`real-world XLSX corpus > zenodo-surewave-floating-pv-cost-and-fi… 1`] = `-{- "calculation": {- "compared": 1888,- "matched": 1888,- "mismatchedFunctions": {},- "volatile": 0,- },- "cells": 3671,- "formulas": 1888,- "roundTrip": "identical",- "sheets": 12,- "styledCells": 42476,- "warnings": [- "Dynamic array formulas are recalculated by Macro; cached spill values are discarded and their cell formatting is retained.",- "Frozen panes are retained for Excel export; Macro uses its own scrolling view.",- "Hyperlinks are imported as display text; link targets are not retained.",- "Macro cannot calculate these Excel functions yet, so their formulas show errors: __XLUDF.UNSUPPORTED.",- "Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.",- ],-}-`;diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-1.test.ts b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-1.test.tsnew file mode 100644index 0000000000..d6b5c42360--- /dev/null+++ b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-1.test.ts@@ -0,0 +1,3 @@+import { describeCorpusSlice } from './xlsx-corpus-suite';++describeCorpusSlice(1);diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-2.test.ts b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-2.test.tsnew file mode 100644index 0000000000..53f595762c--- /dev/null+++ b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-2.test.ts@@ -0,0 +1,3 @@+import { describeCorpusSlice } from './xlsx-corpus-suite';++describeCorpusSlice(2);diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-3.test.ts b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-3.test.tsnew file mode 100644index 0000000000..86fc4f9585--- /dev/null+++ b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-3.test.ts@@ -0,0 +1,3 @@+import { describeCorpusSlice } from './xlsx-corpus-suite';++describeCorpusSlice(3);diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-4.test.ts b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-4.test.tsnew file mode 100644index 0000000000..cbf2362b07--- /dev/null+++ b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-4.test.ts@@ -0,0 +1,3 @@+import { describeCorpusSlice } from './xlsx-corpus-suite';++describeCorpusSlice(4);diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-suite.ts b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-suite.tsnew file mode 100644index 0000000000..dba820ac31--- /dev/null+++ b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus-suite.ts@@ -0,0 +1,152 @@+import { readFileSync } from 'node:fs';+import { createRequire } from 'node:module';+import { initSync } from '@ironcalc/wasm';+import { createInitializedSpreadsheetCalculator } from '@macro-inc/spreadsheet/calculation';+import { afterAll, beforeAll, describe, expect, it } from 'vitest';+import type { SpreadsheetCalculator, WorkbookCalculation } from './calculation';+import type { WorkbookFileData } from './workbook-file-types';+import { decodeXlsx, encodeXlsx } from './xlsx-codec';+import {+ type CalculationFidelity,+ calculateImported,+ calculationFidelity,+ corpusBytes,+ corpusEntries,+} from './xlsx-fixtures/corpus';++/**+ * The real-world corpus runs from `xlsx-corpus-1.test.ts` through+ * `xlsx-corpus-4.test.ts`, one slice each, so Vitest spreads the workbooks+ * across workers instead of running them serially in one file for a minute.+ * A slice takes every `CORPUS_SLICES`th manifest entry, so new workbooks+ * spread across the files.+ */+export const CORPUS_SLICES = 4;++type CorpusSummary = {+ error?: string;+ sheets?: number;+ cells?: number;+ formulas?: number;+ styledCells?: number;+ calculation?: Omit<CalculationFidelity, 'examples'> | string;+ roundTrip?: 'identical' | string;+ warnings?: string[];+};++function counts(workbook: WorkbookFileData) {+ const cells = workbook.sheets.flatMap((sheet) => Object.values(sheet.cells));+ return {+ sheets: workbook.sheets.length,+ cells: cells.filter((cell) => cell.value !== '').length,+ formulas: cells.filter(+ (cell) => cell.value.startsWith('=') && cell.format !== 'text'+ ).length,+ styledCells: cells.filter((cell) => Object.keys(cell).length > 1).length,+ };+}++/** The first difference between two imports, or `identical`. */+function roundTripDifference(+ first: WorkbookFileData,+ second: WorkbookFileData+): string {+ if (first.sheets.length !== second.sheets.length)+ return `sheet count ${first.sheets.length} → ${second.sheets.length}`;+ for (const [index, sheet] of first.sheets.entries()) {+ const other = second.sheets[index];+ if (sheet.name !== other.name) return `sheet name ${sheet.name}`;+ const addresses = new Set([+ ...Object.keys(sheet.cells),+ ...Object.keys(other.cells),+ ]);+ for (const address of addresses) {+ const before = JSON.stringify(sheet.cells[address] ?? null);+ const after = JSON.stringify(other.cells[address] ?? null);+ if (before !== after)+ return `${sheet.name}!${address} ${before} → ${after}`;+ }+ if (JSON.stringify(sheet.metadata) !== JSON.stringify(other.metadata))+ return `${sheet.name} metadata`;+ if (+ JSON.stringify(sheet.columnWidths) !== JSON.stringify(other.columnWidths)+ )+ return `${sheet.name} column widths`;+ }+ return 'identical';+}++/**+ * Imports, recalculates and round-trips slice `slice` (1-based) of the corpus.+ * `corpusEntries` fails when a workbook lacks provenance in `manifest.json`.+ */+export function describeCorpusSlice(slice: number) {+ if (!Number.isInteger(slice) || slice < 1 || slice > CORPUS_SLICES)+ throw new Error(`Corpus slice must be 1–${CORPUS_SLICES}, got ${slice}`);+ const entries = corpusEntries().filter(+ (_, index) => index % CORPUS_SLICES === slice - 1+ );++ let calculator: SpreadsheetCalculator;+ beforeAll(() => {+ initSync({+ module: readFileSync(+ createRequire(import.meta.url).resolve('@ironcalc/wasm/wasm_bg.wasm')+ ),+ });+ calculator = createInitializedSpreadsheetCalculator();+ });+ afterAll(() => calculator.dispose());++ // Snapshot names stay `real-world XLSX corpus > <file>` in every slice.+ describe('real-world XLSX corpus', () => {+ it.each(entries)(+ '$file',+ async (entry) => {+ const bytes = corpusBytes(entry);+ const summary: CorpusSummary = {};+ let imported: WorkbookFileData;+ try {+ imported = await decodeXlsx(bytes);+ } catch (error) {+ summary.error =+ error instanceof Error ? error.message : String(error);+ expect(summary).toMatchSnapshot();+ return;+ }+ Object.assign(summary, counts(imported));+ summary.warnings = [...imported.warnings].sort();+ let values: WorkbookCalculation = {};+ if (entry.skipCalculation) summary.calculation = entry.skipCalculation;+ else {+ values = calculateImported(calculator, imported, bytes);+ const { examples, ...calculation } = calculationFidelity(+ bytes,+ imported,+ values+ );+ summary.calculation = calculation;+ if (process.env.XLSX_CORPUS_VERBOSE)+ console.info(entry.file, JSON.stringify(calculation), examples);+ }+ try {+ const exported = await encodeXlsx({+ sheets: imported.sheets.map((sheet, index) => ({+ ...sheet,+ values: values[String(index)],+ })),+ images: imported.images,+ });+ summary.roundTrip = roundTripDifference(+ imported,+ await decodeXlsx(exported.bytes)+ );+ } catch (error) {+ summary.roundTrip = `export failed: ${error instanceof Error ? error.message : String(error)}`;+ }+ expect(summary).toMatchSnapshot();+ },+ 120_000+ );+ });+}diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-corpus.test.ts b/apps/web/src/features/block-spreadsheet/core/xlsx-corpus.test.tsdeleted file mode 100644index 37951e0fa3..0000000000--- a/apps/web/src/features/block-spreadsheet/core/xlsx-corpus.test.ts+++ /dev/null@@ -1,129 +0,0 @@-import { readFileSync } from 'node:fs';-import { createRequire } from 'node:module';-import { initSync } from '@ironcalc/wasm';-import { createInitializedSpreadsheetCalculator } from '@macro-inc/spreadsheet/calculation';-import { afterAll, beforeAll, describe, expect, it } from 'vitest';-import type { SpreadsheetCalculator, WorkbookCalculation } from './calculation';-import type { WorkbookFileData } from './workbook-file-types';-import { decodeXlsx, encodeXlsx } from './xlsx-codec';-import {- type CalculationFidelity,- calculateImported,- calculationFidelity,- corpusBytes,- corpusEntries,-} from './xlsx-fixtures/corpus';--let calculator: SpreadsheetCalculator;-beforeAll(() => {- initSync({- module: readFileSync(- createRequire(import.meta.url).resolve('@ironcalc/wasm/wasm_bg.wasm')- ),- });- calculator = createInitializedSpreadsheetCalculator();-});-afterAll(() => calculator.dispose());--type CorpusSummary = {- error?: string;- sheets?: number;- cells?: number;- formulas?: number;- styledCells?: number;- calculation?: Omit<CalculationFidelity, 'examples'> | string;- roundTrip?: 'identical' | string;- warnings?: string[];-};--function counts(workbook: WorkbookFileData) {- const cells = workbook.sheets.flatMap((sheet) => Object.values(sheet.cells));- return {- sheets: workbook.sheets.length,- cells: cells.filter((cell) => cell.value !== '').length,- formulas: cells.filter(- (cell) => cell.value.startsWith('=') && cell.format !== 'text'- ).length,- styledCells: cells.filter((cell) => Object.keys(cell).length > 1).length,- };-}--/** The first difference between two imports, or `identical`. */-function roundTripDifference(- first: WorkbookFileData,- second: WorkbookFileData-): string {- if (first.sheets.length !== second.sheets.length)- return `sheet count ${first.sheets.length} → ${second.sheets.length}`;- for (const [index, sheet] of first.sheets.entries()) {- const other = second.sheets[index];- if (sheet.name !== other.name) return `sheet name ${sheet.name}`;- const addresses = new Set([- ...Object.keys(sheet.cells),- ...Object.keys(other.cells),- ]);- for (const address of addresses) {- const before = JSON.stringify(sheet.cells[address] ?? null);- const after = JSON.stringify(other.cells[address] ?? null);- if (before !== after)- return `${sheet.name}!${address} ${before} → ${after}`;- }- if (JSON.stringify(sheet.metadata) !== JSON.stringify(other.metadata))- return `${sheet.name} metadata`;- if (- JSON.stringify(sheet.columnWidths) !== JSON.stringify(other.columnWidths)- )- return `${sheet.name} column widths`;- }- return 'identical';-}--describe('real-world XLSX corpus', () => {- it.each(corpusEntries())(- '$file',- async (entry) => {- const bytes = corpusBytes(entry);- const summary: CorpusSummary = {};- let imported: WorkbookFileData;- try {- imported = await decodeXlsx(bytes);- } catch (error) {- summary.error = error instanceof Error ? error.message : String(error);- expect(summary).toMatchSnapshot();- return;- }- Object.assign(summary, counts(imported));- summary.warnings = [...imported.warnings].sort();- let values: WorkbookCalculation = {};- if (entry.skipCalculation) summary.calculation = entry.skipCalculation;- else {- values = calculateImported(calculator, imported, bytes);- const { examples, ...calculation } = calculationFidelity(- bytes,- imported,- values- );- summary.calculation = calculation;- if (process.env.XLSX_CORPUS_VERBOSE)- console.info(entry.file, JSON.stringify(calculation), examples);- }- try {- const exported = await encodeXlsx({- sheets: imported.sheets.map((sheet, index) => ({- ...sheet,- values: values[String(index)],- })),- images: imported.images,- });- summary.roundTrip = roundTripDifference(- imported,- await decodeXlsx(exported.bytes)- );- } catch (error) {- summary.roundTrip = `export failed: ${error instanceof Error ? error.message : String(error)}`;- }- expect(summary).toMatchSnapshot();- },- 120_000- );-});diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.md b/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.mdindex e763c1f2a9..9a09ad93b4 100644--- a/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.md+++ b/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.md@@ -16,7 +16,7 @@ Additional generated-in-test workbooks exercise the 1904 date system, stored Exc## Real-world corpus-`real-world/` holds published workbooks that Excel, LibreOffice and other producers wrote: financial models (project finance, DCF, LBO, loan schedules, three-statement models), small-business and government statistics workbooks, and files from open-source spreadsheet test suites. `corpus.ts` lists them from `real-world/manifest.json` and fails if a workbook has no recorded provenance. `../xlsx-corpus.test.ts` imports, recalculates and round-trips each one and snapshots the results.+`real-world/` holds published workbooks that Excel, LibreOffice and other producers wrote: financial models (project finance, DCF, LBO, loan schedules, three-statement models), small-business and government statistics workbooks, and files from open-source spreadsheet test suites. `corpus.ts` lists them from `real-world/manifest.json` and fails if a workbook has no recorded provenance. `../xlsx-corpus-suite.ts` imports, recalculates and round-trips each one and snapshots the results, split across `../xlsx-corpus-1.test.ts` through `../xlsx-corpus-4.test.ts` so the workbooks run in parallel.Every file is redistributable. `manifest.json` records each file's source page and license:diff --git a/apps/web/src/lib/queries/favorites/graphql.ts b/apps/web/src/lib/queries/favorites/graphql.tsindex 5d6bf96ba3..f791dd84e2 100644--- a/apps/web/src/lib/queries/favorites/graphql.ts+++ b/apps/web/src/lib/queries/favorites/graphql.ts@@ -1,6 +1,7 @@import {createUrqlMutation,createUrqlQuery,+ type UrqlMutationExecutorArgs,type UrqlQueryResult,} from '@app/lib/urql-solid';import { optimisticMutationDispositionOf } from '@graphql-cache/exchange/optimistic';@@ -171,7 +172,10 @@ function createFavoriteMutation<>(() => ({mutation: options.mutation,client: getGraphqlSoupClient(),- execute: async ({ client, input }) => {+ execute: async ({+ client,+ input,+ }: UrqlMutationExecutorArgs<Data, Variables, Input>) => {const result = await options.execute(client, input);// Validate before onSuccess, so malformed responses take the error// lifecycle too. Server-side failures are already transport errors,diff --git a/apps/web/src/lib/service-clients/service-storage/util/uploadDocx.ts b/apps/web/src/lib/service-clients/service-storage/util/uploadDocx.tsindex 849af4d9f7..2b722851cc 100644--- a/apps/web/src/lib/service-clients/service-storage/util/uploadDocx.ts+++ b/apps/web/src/lib/service-clients/service-storage/util/uploadDocx.ts@@ -1,4 +1,5 @@import { ENABLE_DOCX_TO_PDF } from '@core/constant/featureFlags';+import type { MinimalWebSocket } from '@macro-inc/collaboration/websocket/platform/minimal-websocket';import {createWebsocketPromiseChain,JobTypeEnum,@@ -7,7 +8,7 @@ import {const PROMISE_TIMEOUT_MS = 15000; // 15 seconds-export function uploadDocx(ws: WebSocket) {+export function uploadDocx(ws: MinimalWebSocket) {const websocketChain = createWebsocketPromiseChain({jobType: JobTypeEnum.DocxUpload,diff --git a/apps/web/src/lib/service-clients/service-storage/util/websocketPromiseChain.ts b/apps/web/src/lib/service-clients/service-storage/util/websocketPromiseChain.tsindex 3517ab7949..e8670ddc97 100644--- a/apps/web/src/lib/service-clients/service-storage/util/websocketPromiseChain.ts+++ b/apps/web/src/lib/service-clients/service-storage/util/websocketPromiseChain.ts@@ -1,5 +1,5 @@import type { ResultError } from '@core/util/result';-+import type { MinimalWebSocket } from '@macro-inc/collaboration/websocket/platform/minimal-websocket';import { err, ok, type Result } from 'neverthrow';import { v7 as uuid7 } from 'uuid';@@ -30,7 +30,7 @@ export function createWebsocketPromiseChain<jobType: JobTypeEnum;getInputData: (...args: FunctionInputArgType) => any | Promise<any>;timeoutMs: number;- ws: WebSocket;+ ws: MinimalWebSocket;},listenersChain: L) {diff --git a/apps/web/vite.base.ts b/apps/web/vite.base.tsindex 5ea4476bdd..601b538077 100644--- a/apps/web/vite.base.ts+++ b/apps/web/vite.base.ts@@ -187,6 +187,9 @@ export const createAppViteConfig = (): UserConfigFn => {outDir: 'dist',emptyOutDir: true,minify: !NO_MINIFY,+ // The gzip-size console report re-compresses every chunk, adding ~20 s+ // to each build; nothing reads it.+ reportCompressedSize: false,rollupOptions: {input: {app: resolve(__dirname, 'index.html'),diff --git a/apps/web/vitest.config.ts b/apps/web/vitest.config.tsindex c9a93464c7..1743d7416a 100644--- a/apps/web/vitest.config.ts+++ b/apps/web/vitest.config.ts@@ -1,9 +1,55 @@+import { createHash } from 'node:crypto';+import {+ mkdirSync,+ readdirSync,+ readFileSync,+ rmSync,+ statSync,+ writeFileSync,+} from 'node:fs';+import { join } from 'node:path';import { fileURLToPath } from 'node:url';import solidPlugin from 'vite-plugin-solid';import solidSvg from 'vite-plugin-solid-svg';import tsconfigPaths from 'vite-tsconfig-paths';import { configDefaults, defineConfig } from 'vitest/config';+const MODULE_CACHE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;++/**+ * Directory for Vitest's filesystem module cache, which `--fsModuleCache`+ * (the `test` script) turns on for every project. Cached transforms embed+ * resolved import paths, so entries live under a key of the lockfile and+ * tsconfig; key directories unused for a week are deleted. CI points+ * `VITEST_MODULE_CACHE_DIR` at a cache volume that persists across jobs.+ */+function moduleCachePath(): string {+ const root =+ process.env.VITEST_MODULE_CACHE_DIR ??+ fileURLToPath(new URL('./.cache/vitest-modules', import.meta.url));+ const key = createHash('sha256')+ .update(readFileSync(new URL('../../bun.lock', import.meta.url)))+ .update(readFileSync(new URL('./tsconfig.json', import.meta.url)))+ .digest('hex')+ .slice(0, 16);+ const dir = join(root, key);+ mkdirSync(dir, { recursive: true });+ writeFileSync(join(dir, '.last-used'), '');+ for (const entry of readdirSync(root)) {+ if (entry === key) continue;+ const stale = join(root, entry);+ try {+ const lastUsed = statSync(join(stale, '.last-used')).mtimeMs;+ if (Date.now() - lastUsed > MODULE_CACHE_MAX_AGE_MS) {+ rmSync(stale, { recursive: true, force: true });+ }+ } catch {+ // Not a key directory, or another run removed it first.+ }+ }+ return dir;+}+export default defineConfig({plugins: [tsconfigPaths(),@@ -29,6 +75,8 @@ export default defineConfig({},test: {exclude: [...configDefaults.exclude],+ // Projects without their own path fall back to the root's.+ fsModuleCachePath: moduleCachePath(),projects: [{extends: './src/features/scheduling/vitest.config.ts',diff --git a/bun.lock b/bun.lockindex dd790338a1..8b85f3cc6f 100644--- a/bun.lock+++ b/bun.lock@@ -217,6 +217,7 @@"orval": "^7.21.0","typedoc": "^0.22.18","typescript": "^5.9.3",+ "typescript-native": "npm:[email protected]","vite": "^6.4.1","vite-plugin-solid": "^2.11.8","vite-plugin-solid-svg": "^0.8.1",@@ -249,6 +250,7 @@"bebop-tools": "3.1.3","jsdom": "^26.1.0","typescript": "^5.9.3",+ "typescript-native": "npm:[email protected]","vite-plugin-solid": "^2.11.8","vite-plugin-wasm": "^3.5.0","vitest": "5.0.1",@@ -1842,6 +1844,46 @@"@types/yauzl": ["@types/[email protected]", "", { "dependencies": { "@types/node": "*" } }, "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q=="],+ "@typescript/typescript-aix-ppc64": ["@typescript/[email protected]", "", { "os": "aix", "cpu": "ppc64" }, "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ=="],++ "@typescript/typescript-darwin-arm64": ["@typescript/[email protected]", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA=="],++ "@typescript/typescript-darwin-x64": ["@typescript/[email protected]", "", { "os": "darwin", "cpu": "x64" }, "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA=="],++ "@typescript/typescript-freebsd-arm64": ["@typescript/[email protected]", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ=="],++ "@typescript/typescript-freebsd-x64": ["@typescript/[email protected]", "", { "os": "freebsd", "cpu": "x64" }, "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw=="],++ "@typescript/typescript-linux-arm": ["@typescript/[email protected]", "", { "os": "linux", "cpu": "arm" }, "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ=="],++ "@typescript/typescript-linux-arm64": ["@typescript/[email protected]", "", { "os": "linux", "cpu": "arm64" }, "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ=="],++ "@typescript/typescript-linux-loong64": ["@typescript/[email protected]", "", { "os": "linux", "cpu": "none" }, "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ=="],++ "@typescript/typescript-linux-mips64el": ["@typescript/[email protected]", "", { "os": "linux", "cpu": "none" }, "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA=="],++ "@typescript/typescript-linux-ppc64": ["@typescript/[email protected]", "", { "os": "linux", "cpu": "ppc64" }, "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA=="],++ "@typescript/typescript-linux-riscv64": ["@typescript/[email protected]", "", { "os": "linux", "cpu": "none" }, "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ=="],++ "@typescript/typescript-linux-s390x": ["@typescript/[email protected]", "", { "os": "linux", "cpu": "s390x" }, "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw=="],++ "@typescript/typescript-linux-x64": ["@typescript/[email protected]", "", { "os": "linux", "cpu": "x64" }, "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A=="],++ "@typescript/typescript-netbsd-arm64": ["@typescript/[email protected]", "", { "os": "none", "cpu": "arm64" }, "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA=="],++ "@typescript/typescript-netbsd-x64": ["@typescript/[email protected]", "", { "os": "none", "cpu": "x64" }, "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA=="],++ "@typescript/typescript-openbsd-arm64": ["@typescript/[email protected]", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ=="],++ "@typescript/typescript-openbsd-x64": ["@typescript/[email protected]", "", { "os": "openbsd", "cpu": "x64" }, "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg=="],++ "@typescript/typescript-sunos-x64": ["@typescript/[email protected]", "", { "os": "sunos", "cpu": "x64" }, "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g=="],++ "@typescript/typescript-win32-arm64": ["@typescript/[email protected]", "", { "os": "win32", "cpu": "arm64" }, "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ=="],++ "@typescript/typescript-win32-x64": ["@typescript/[email protected]", "", { "os": "win32", "cpu": "x64" }, "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g=="],+"@typespec/ts-http-runtime": ["@typespec/[email protected]", "", { "dependencies": { "http-proxy-agent": "^7.0.0", "https-proxy-agent": "^7.0.0", "tslib": "^2.6.2" } }, "sha512-jIXhD0eWQ1JA6ln/5Dltyx22UxWNrw0hZmhy2rlv6m6KgF7kplHx3g0fzi09lNmTJQRR91OlemYp3xFnvDK9og=="],"@ungap/structured-clone": ["@ungap/[email protected]", "", {}, "sha512-60YRaenCQcVjYEKOcG824+DRGGIQ3VKErcBoAEDJZz5bKIs2ZG+X/H9Nk+Q6EVkwJk5QNApxbrc5QtBSwtrXAg=="],@@ -3638,6 +3680,8 @@"typescript": ["[email protected]", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],+ "typescript-native": ["[email protected]", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="],+"uc.micro": ["[email protected]", "", {}, "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A=="],"uhyphen": ["[email protected]", "", {}, "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="],diff --git a/crates/ai_tools/src/test/booking_links.rs b/crates/ai_tools/src/test/booking_links.rsindex 72fecff9f2..ddd43d1e3c 100644--- a/crates/ai_tools/src/test/booking_links.rs+++ b/crates/ai_tools/src/test/booking_links.rs@@ -25,6 +25,11 @@ use model_user::UserContext;use rootcause::Report;use uuid::Uuid;+/// CI clones every test database from a `template1` that already holds the macrodb+/// schema (`just setup_test_template`). This test builds its own schema, so it+/// starts from an empty `public` schema.+const EMPTY_PUBLIC_SCHEMA: &str = "DROP SCHEMA public CASCADE; CREATE SCHEMA public;";+#[derive(Clone)]struct NoAuth;impl MacroAuthorizationService for NoAuth {@@ -90,6 +95,10 @@ async fn execute(context: Context, tool: &str, args: Value) -> Result<Value, Str#[sqlx::test(migrations = false)]async fn confirmed_booking_is_saved_and_bookable_without_touching_other_links(pool: sqlx::PgPool) {+ sqlx::raw_sql(EMPTY_PUBLIC_SCHEMA)+ .execute(&pool)+ .await+ .unwrap();sqlx::raw_sql("CREATE TABLE \"User\" (id text PRIMARY KEY); CREATE TABLE team (id uuid PRIMARY KEY); INSERT INTO \"User\" VALUES ('macro|[email protected]');").execute(&pool).await.unwrap();sqlx::raw_sql(include_str!("../../../macro_db_client/migrations/20260918164303_calendar_scheduling.sql"diff --git a/crates/calendar_scheduling/src/inbound/router/test.rs b/crates/calendar_scheduling/src/inbound/router/test.rsindex c161ce3e6b..cb74a298e5 100644--- a/crates/calendar_scheduling/src/inbound/router/test.rs+++ b/crates/calendar_scheduling/src/inbound/router/test.rs@@ -7,6 +7,11 @@ use model_user::UserContext;use rootcause::Report;use std::sync::atomic::{AtomicBool, Ordering};+/// CI clones every test database from a `template1` that already holds the macrodb+/// schema (`just setup_test_template`). These tests build their own schema, so they+/// start from an empty `public` schema.+const EMPTY_PUBLIC_SCHEMA: &str = "DROP SCHEMA public CASCADE; CREATE SCHEMA public;";+#[derive(Clone)]struct Identity;impl MacroAuthorizationService for Identity {@@ -76,6 +81,10 @@ impl Calendars for Calendar {#[sqlx::test(migrations = false)]async fn http_booking_receipt_recovery_reschedule_cancel_and_team_auth(pool: sqlx::PgPool) {+ sqlx::raw_sql(EMPTY_PUBLIC_SCHEMA)+ .execute(&pool)+ .await+ .unwrap();sqlx::raw_sql("CREATE TABLE \"User\" (id text PRIMARY KEY); CREATE TABLE team (id uuid PRIMARY KEY); INSERT INTO \"User\" VALUES ('macro|[email protected]');").execute(&pool).await.unwrap();sqlx::raw_sql(include_str!("../../../../macro_db_client/migrations/20260918164303_calendar_scheduling.sql"diff --git a/crates/calendar_scheduling/src/outbound/postgres/test.rs b/crates/calendar_scheduling/src/outbound/postgres/test.rsindex 11993863c3..6ab0419775 100644--- a/crates/calendar_scheduling/src/outbound/postgres/test.rs+++ b/crates/calendar_scheduling/src/outbound/postgres/test.rs@@ -1,7 +1,16 @@use super::*;use chrono::Duration;+/// CI clones every test database from a `template1` that already holds the macrodb+/// schema (`just setup_test_template`). These tests build their own schema, so they+/// start from an empty `public` schema.+const EMPTY_PUBLIC_SCHEMA: &str = "DROP SCHEMA public CASCADE; CREATE SCHEMA public;";+async fn setup(pool: &PgPool) {+ sqlx::raw_sql(EMPTY_PUBLIC_SCHEMA)+ .execute(pool)+ .await+ .unwrap();sqlx::raw_sql("CREATE TABLE \"User\" (id text PRIMARY KEY); CREATE TABLE team (id uuid PRIMARY KEY);",)diff --git a/crates/codex_cloud_agents/src/outbound/postgres_journal/test.rs b/crates/codex_cloud_agents/src/outbound/postgres_journal/test.rsindex f21891c7ec..4bbb40d619 100644--- a/crates/codex_cloud_agents/src/outbound/postgres_journal/test.rs+++ b/crates/codex_cloud_agents/src/outbound/postgres_journal/test.rs@@ -1,6 +1,11 @@use super::*;use uuid::Uuid;+/// CI clones every test database from a `template1` that already holds the macrodb+/// schema (`just setup_test_template`). These tests build their own schema, so they+/// start from an empty `public` schema.+const EMPTY_PUBLIC_SCHEMA: &str = "DROP SCHEMA public CASCADE; CREATE SCHEMA public;";+#[sqlx::test(migrations = false)]async fn journal_survives_takeover_and_rejects_stale_or_foreign_attachments(pool: PgPool) {tables(&pool).await;@@ -114,6 +119,10 @@ async fn journal_survives_takeover_and_rejects_stale_or_foreign_attachments(pool}async fn tables(pool: &PgPool) {+ sqlx::raw_sql(EMPTY_PUBLIC_SCHEMA)+ .execute(pool)+ .await+ .unwrap();sqlx::raw_sql("CREATE TABLE agent_session(id uuid PRIMARY KEY, manager_replica_id uuid, manager_fence bigint NOT NULL)").execute(pool).await.unwrap();sqlx::raw_sql(include_str!(diff --git a/crates/cursor_cloud_agents/src/outbound/postgres_journal/test.rs b/crates/cursor_cloud_agents/src/outbound/postgres_journal/test.rsindex 04a0dada3b..7f242b5fae 100644--- a/crates/cursor_cloud_agents/src/outbound/postgres_journal/test.rs+++ b/crates/cursor_cloud_agents/src/outbound/postgres_journal/test.rs@@ -1,8 +1,17 @@use super::*;use uuid::Uuid;+/// CI clones every test database from a `template1` that already holds the macrodb+/// schema (`just setup_test_template`). These tests build their own schema, so they+/// start from an empty `public` schema.+const EMPTY_PUBLIC_SCHEMA: &str = "DROP SCHEMA public CASCADE; CREATE SCHEMA public;";+#[sqlx::test(migrations = false)]async fn append_is_ordered_fenced_and_session_scoped(pool: PgPool) {+ sqlx::raw_sql(EMPTY_PUBLIC_SCHEMA)+ .execute(&pool)+ .await+ .unwrap();sqlx::raw_sql("CREATE TABLE agent_session(id uuid PRIMARY KEY, manager_replica_id uuid, manager_fence bigint NOT NULL);CREATE TABLE agent_session_log(id uuid PRIMARY KEY, agent_session_id uuid NOT NULL REFERENCES agent_session(id));diff --git a/crates/macro_db_client/tests/normalize_macro_bot_mention_ids.rs b/crates/macro_db_client/tests/normalize_macro_bot_mention_ids.rsindex 64827d34bf..c8ea53ad0d 100644--- a/crates/macro_db_client/tests/normalize_macro_bot_mention_ids.rs+++ b/crates/macro_db_client/tests/normalize_macro_bot_mention_ids.rs@@ -1,5 +1,10 @@use sqlx::{Pool, Postgres};+/// CI clones every test database from a `template1` that already holds the macrodb+/// schema (`just setup_test_template`). These tests build their own schema, so they+/// start from an empty `public` schema.+const EMPTY_PUBLIC_SCHEMA: &str = "DROP SCHEMA public CASCADE; CREATE SCHEMA public;";+const NORMALIZED_MACRO_BOT_ID: &str = "bot|00000000-0000-0000-0000-00000000a1a1";const OTHER_UUID: &str = "11111111-1111-1111-1111-111111111111";const MIGRATION: &str =@@ -180,6 +185,7 @@ async fn migration_normalizes_bare_uuid_message_senders(}async fn create_minimal_tables(pool: &Pool<Postgres>) -> anyhow::Result<()> {+ sqlx::raw_sql(EMPTY_PUBLIC_SCHEMA).execute(pool).await?;sqlx::raw_sql(r#"CREATE TABLE comms_messages (diff --git a/crates/macro_db_migrator/src/test.rs b/crates/macro_db_migrator/src/test.rsindex e2031a95c4..aab977d63b 100644--- a/crates/macro_db_migrator/src/test.rs+++ b/crates/macro_db_migrator/src/test.rs@@ -2,9 +2,18 @@ use super::MACRO_DB_MIGRATIONS;use sqlx::{PgPool, migrate::Migrator};use std::borrow::Cow;+/// CI clones every test database from a `template1` that already holds the macrodb+/// schema (`just setup_test_template`). These tests build their own schema, so they+/// start from an empty `public` schema.+const EMPTY_PUBLIC_SCHEMA: &str = "DROP SCHEMA public CASCADE; CREATE SCHEMA public;";+const SCHEMA_DROP_VERSION: i64 = 20261002150422;async fn before_drop(pool: &PgPool) {+ sqlx::raw_sql(EMPTY_PUBLIC_SCHEMA)+ .execute(pool)+ .await+ .unwrap();let migrator = Migrator {migrations: Cow::Owned(MACRO_DB_MIGRATIONSdiff --git a/crates/notification_state/src/postgres/test/migration.rs b/crates/notification_state/src/postgres/test/migration.rsindex 06efafe0bd..fd049f048b 100644--- a/crates/notification_state/src/postgres/test/migration.rs+++ b/crates/notification_state/src/postgres/test/migration.rs@@ -1,6 +1,11 @@use crate::NotificationState;use sqlx::{Connection, PgPool};+/// CI clones every test database from a `template1` that already holds the macrodb+/// schema (`just setup_test_template`). These tests build their own schema, so they+/// start from an empty `public` schema.+const EMPTY_PUBLIC_SCHEMA: &str = "DROP SCHEMA public CASCADE; CREATE SCHEMA public;";+const LEGACY_SCHEMA: &str = include_str!("../../../../macro_db_client/migrations/20260126170641_create_notification_tables.sql");@@ -20,6 +25,7 @@ async fn migration_normalizes_legacy_states_without_changing_timestamps() -> Result<(), sqlx::Error> {// DDL and the pre-cutover fixture cannot be checked against the current// development schema. Execute the real migration scripts in an isolated DB.+ sqlx::raw_sql(EMPTY_PUBLIC_SCHEMA).execute(&pool).await?;sqlx::raw_sql(LEGACY_SCHEMA).execute(&pool).await?;sqlx::raw_sql(LEGACY_INDEXES).execute(&pool).await?;sqlx::raw_sql(@@ -77,6 +83,7 @@ async fn migration_normalizes_legacy_states_without_changing_timestamps(#[sqlx::test(migrations = false)]async fn state_defaults_and_constraints_are_enforced(pool: PgPool) -> Result<(), sqlx::Error> {+ sqlx::raw_sql(EMPTY_PUBLIC_SCHEMA).execute(&pool).await?;sqlx::raw_sql(LEGACY_SCHEMA).execute(&pool).await?;sqlx::raw_sql(LEGACY_INDEXES).execute(&pool).await?;sqlx::raw_sql(UP).execute(&pool).await?;diff --git a/docs/DATABASE_DEVELOPMENT.md b/docs/DATABASE_DEVELOPMENT.mdindex 1482120b5c..b683871be6 100644--- a/docs/DATABASE_DEVELOPMENT.md+++ b/docs/DATABASE_DEVELOPMENT.md@@ -35,6 +35,20 @@ On **Cursor Cloud**, installation already prepares the database and test envs.Start the infrastructure with `bash .cursor/infra.sh`; see [Cursor Cloud](CURSOR_CLOUD.md).Apply any new migrations before testing.+### Pre-migrated test template++CI runs `just setup_test_template` after `initialize_dbs`. It migrates `template1`,+which Postgres clones for every new database, so each `#[sqlx::test]` database+starts with the MacroDB schema instead of replaying every migration. That+replay used to be nearly all of the Rust test suite's runtime. Run the recipe+locally only against a Postgres dedicated to one checkout; every database created+on that server afterwards gets the schema too.++Tests that build their own schema (`#[sqlx::test(migrations = false)]`, or a+crate's test-only schema fixture) must first run+`DROP SCHEMA public CASCADE; CREATE SCHEMA public;`, since their database may+arrive pre-migrated.+## Safe database schema changes### Entity tables and reusable storagediff --git a/nix-support/node_modules-hashes.json b/nix-support/node_modules-hashes.jsonindex 6839c2ba30..7cf66fb599 100644--- a/nix-support/node_modules-hashes.json+++ b/nix-support/node_modules-hashes.json@@ -1,6 +1,6 @@{"nodeModules": {- "aarch64-darwin": "sha256-tjUS6Z66ykQJ+FmBB+TYmrxbur3wu7f5iJbnaZNVieA=",- "x86_64-linux": "sha256-egI40KcIhudsvS0EkjSMOi0gnAy6cKwG/vfxi7W8Wow="+ "aarch64-darwin": "sha256-9bFwe5o3iBLBrlBlRQ3hmeq0A7VjmDbsxGTjOpFnSqY=",+ "x86_64-linux": "sha256-9WeocJXG6/po7vLNPjue7JUErg1K7nJ5Q7X5yZ6XrpE="}}diff --git a/nix/cloud-storage.nix b/nix/cloud-storage.nixindex 1f732c893c..f128d44d27 100644--- a/nix/cloud-storage.nix+++ b/nix/cloud-storage.nix@@ -55,8 +55,10 @@# `.sh` is required so `include_str!` of# `crates/agent_harness/container/ensure_ready.sh` survives the prune;# `.ttf` and `.xml` for the fonts and preset shape definitions that- # `crates/pptx_engine` embeds with `include_bytes!`.- assetFilter = path: _type: builtins.match ".*\\.(md|html|txt|json|canvas|sql|sh|ttf|xml)$" path != null;+ # `crates/pptx_engine` embeds with `include_bytes!`; `.graphql` for the+ # `static_assets/schema.graphql` that `cache-core`'s build script reads;+ # `.jsonl` for the recorded sessions `agent_fold` tests embed.+ assetFilter = path: _type: builtins.match ".*\\.(md|html|txt|json|jsonl|canvas|sql|sh|ttf|xml|graphql)$" path != null;# Rust local HTTP clients embed only the public proxy CA, never its keys.localCaFilter = path: _type: pkgs.lib.hasSuffix "/infra/local/certs/ca.pem" (toString path);binFilter = path: _type: builtins.match ".*\\.bin$" path != null;@@ -226,7 +228,9 @@doCheck = false;doInstallCargoArtifacts = true;cargoExtraArgs = "--locked --all-features --workspace --lib --exclude sync_service";- RUSTFLAGS = "-Dwarnings" + pkgs.lib.optionalString isLinux " -C link-arg=-fuse-ld=mold";+ # Keep RUSTFLAGS identical to cargoArtifacts: any difference changes+ # cargo's fingerprint and recompiles every dependency in this layer.+ # The clippy checks enforce `-D warnings`.RUSTDOCFLAGS = "-Dwarnings";});diff --git a/packages/collaboration/package.json b/packages/collaboration/package.jsonindex fe534d90c3..15c79c4834 100644--- a/packages/collaboration/package.json+++ b/packages/collaboration/package.json@@ -15,7 +15,7 @@"generate-sync-schema": "bun scripts/generate-schema.ts","lint": "biome check .","test": "vitest run --config vitest.config.ts",- "type-check": "tsc --noEmit --project tsconfig.json"+ "type-check": "bun ../../node_modules/typescript-native/bin/tsc --noEmit --project tsconfig.json"},"dependencies": {"@loro-mirror/core": "workspace:*",@@ -38,6 +38,7 @@"bebop-tools": "3.1.3","jsdom": "^26.1.0","typescript": "^5.9.3",+ "typescript-native": "npm:[email protected]","vite-plugin-solid": "^2.11.8","vite-plugin-wasm": "^3.5.0","vitest": "5.0.1",diff --git a/packages/sdk/RELEASING.md b/packages/sdk/RELEASING.mdindex 48badbcae4..29bee1e62b 100644--- a/packages/sdk/RELEASING.md+++ b/packages/sdk/RELEASING.md@@ -34,8 +34,9 @@ npm pack --dry-run --ignore-scriptsThe compiled JavaScript and declaration files referenced by `exports` mustappear in the package listing. `coverage` checks endpoint wrappers, not testcoverage. `SDK Check` runs the same package suite on the PR and again on the-merge to `main`, and additionally checks generated-code freshness on the PR; if-it reports staleness, run `just update-generated` and review the diff. Run+merge to `main`. Generated-code freshness is checked on the PR by the web app+workflow's `Generated Code Check`; if it reports staleness, run+`just update-generated` and review the diff. Run`just check` from the repository root before committing.Forgetting to bump publishes nothing, which is the intended failure mode: open adiff --git a/services/authentication_service/src/api/context.rs b/services/authentication_service/src/api/context.rsindex eb092f47bd..02e8852462 100644--- a/services/authentication_service/src/api/context.rs+++ b/services/authentication_service/src/api/context.rs@@ -58,7 +58,7 @@ use sqlx::PgPool;use tokio_util::task::TaskTracker;use crate::microsoft_token_cipher::MicrosoftTokenCipher;-use authentication_service::service::signup_policy::SignupPolicy;+use crate::service::signup_policy::SignupPolicy;use cursor_api_key::cipher::CursorApiKeyCipher;pub(crate) type NotificationIngressType = SqsNotificationIngress<SqsQueue>;@@ -148,16 +148,14 @@ pub(crate) struct ApiContext {pub macro_cache_client: Arc<MacroCache>,pub stripe_client: Arc<stripe::Client>,pub subscription_checkout: Arc<- authentication_service::service::subscription_checkout::CheckoutService<- authentication_service::outbound::subscription_checkout::StripeCheckoutGateway<- GtmInviteServiceType,- >,+ crate::service::subscription_checkout::CheckoutService<+ crate::outbound::subscription_checkout::StripeCheckoutGateway<GtmInviteServiceType>,>,>,pub document_storage_service_client:Arc<document_storage_service_client::DocumentStorageServiceClient>,pub user_deletion: Arc<- authentication_service::outbound::user_deletion::UserDeletionAdapter<+ crate::outbound::user_deletion::UserDeletionAdapter<TeamsServiceType,onboarding::outbound::pg_onboarding_repo::PgOnboardingRepo,>,diff --git a/services/authentication_service/src/api/email/verify_email_link.rs b/services/authentication_service/src/api/email/verify_email_link.rsindex 4c839a9818..8de7813495 100644--- a/services/authentication_service/src/api/email/verify_email_link.rs+++ b/services/authentication_service/src/api/email/verify_email_link.rs@@ -7,7 +7,7 @@ use axum::{use macro_middleware::tracking::ClientIp;use crate::api::{context::ApiContext, utils::default_redirect_url};-use authentication_service::service::user::create_user::create_user_profile;+use crate::service::user::create_user::create_user_profile;use model::response::{EmptyResponse, ErrorResponse};diff --git a/services/authentication_service/src/api/user/stripe/create_checkout_session_v2.rs b/services/authentication_service/src/api/user/stripe/create_checkout_session_v2.rsindex 825dc13666..3dd6cfdf7e 100644--- a/services/authentication_service/src/api/user/stripe/create_checkout_session_v2.rs+++ b/services/authentication_service/src/api/user/stripe/create_checkout_session_v2.rs@@ -1,4 +1,4 @@-use authentication_service::service::subscription_checkout::CheckoutRequest;+use crate::service::subscription_checkout::CheckoutRequest;use axum::{Json, extract::State};use entity_access::domain::models::OwnerTeamRole;use entity_access::domain::ports::EntityAccessService;diff --git a/services/authentication_service/src/api/user/stripe/shared.rs b/services/authentication_service/src/api/user/stripe/shared.rsindex 0022044c4f..3b79750356 100644--- a/services/authentication_service/src/api/user/stripe/shared.rs+++ b/services/authentication_service/src/api/user/stripe/shared.rs@@ -1,4 +1,4 @@-use authentication_service::{+use crate::{outbound::subscription_checkout::StripeCheckoutError,service::subscription_checkout::CheckoutError,};diff --git a/services/authentication_service/src/api/webhooks/user/create_user_webhook.rs b/services/authentication_service/src/api/webhooks/user/create_user_webhook.rsindex b104dae883..2e7cee8919 100644--- a/services/authentication_service/src/api/webhooks/user/create_user_webhook.rs+++ b/services/authentication_service/src/api/webhooks/user/create_user_webhook.rs@@ -4,8 +4,8 @@ mod test;use analytics_client::{MetaActionSource, MetaUserData};use std::{collections::HashSet, future::Future};+use crate::service::signup_policy::SignupPolicy;use anyhow::Context;-use authentication_service::service::signup_policy::SignupPolicy;use axum::{extract::{self, State},http::StatusCode,@@ -14,6 +14,10 @@ use axum::{use macro_authorization::{InternalOnly, MacroAuthorizationExtractor};use rand::Rng;+use crate::service::user::create_user::create_user;+use crate::service::user::support_channel_welcome::{+ AuthorizedSupportChannelMessages, post_support_channel_welcome,+};use crate::{api::{context::{ApiContext, AuthorizationService},@@ -21,10 +25,6 @@ use crate::{},rate_limit_config::RATE_LIMIT_CONFIG,};-use authentication_service::service::user::create_user::create_user;-use authentication_service::service::user::support_channel_welcome::{- AuthorizedSupportChannelMessages, post_support_channel_welcome,-};use channels::domain::{models::{ChannelType, CreateChannelRequest},ports::ChannelService,diff --git a/services/authentication_service/src/api/webhooks/user/create_user_webhook/test.rs b/services/authentication_service/src/api/webhooks/user/create_user_webhook/test.rsindex f5bde3eafc..8d0b83859d 100644--- a/services/authentication_service/src/api/webhooks/user/create_user_webhook/test.rs+++ b/services/authentication_service/src/api/webhooks/user/create_user_webhook/test.rs@@ -3,7 +3,7 @@ use std::sync::{atomic::{AtomicUsize, Ordering},};-use authentication_service::service::signup_policy::SignupPolicy;+use crate::service::signup_policy::SignupPolicy;use axum::http::StatusCode;use macro_user_id::email::Email;use model::authentication::webhooks::{diff --git a/services/authentication_service/src/api/webhooks/user/delete_user_webhook.rs b/services/authentication_service/src/api/webhooks/user/delete_user_webhook.rsindex 6d1c1bb7e6..a5b786a903 100644--- a/services/authentication_service/src/api/webhooks/user/delete_user_webhook.rs+++ b/services/authentication_service/src/api/webhooks/user/delete_user_webhook.rs@@ -232,7 +232,7 @@ async fn delete_user(.iter().map(|id| MacroUserIdStr::try_from(id.clone())).collect::<Result<Vec<_>, _>>()?;- authentication_service::service::user::delete_user::delete_user_data(+ crate::service::user::delete_user::delete_user_data(ctx.user_deletion.as_ref(),¯o_user.id,&users,diff --git a/services/authentication_service/src/config.rs b/services/authentication_service/src/config.rsindex 5afdaa6beb..75a644a3a8 100644--- a/services/authentication_service/src/config.rs+++ b/services/authentication_service/src/config.rs@@ -1,7 +1,7 @@use std::sync::LazyLock;+use crate::service::signup_policy::SignupPolicy;use anyhow::Context;-use authentication_service::service::signup_policy::SignupPolicy;use database_env_vars::{DatabaseUrl, RedisUri};use gtm_invite::domain::models::{GtmInviteConfig, PromoCode};use macro_auth::InternalApiKey;@@ -287,8 +287,9 @@ impl Config {self.signup_policy_for_environment(self.environment)}- /// Resolves the signup policy for an explicit environment.- pub(crate) fn signup_policy_for_environment(+ /// Resolves the signup policy for an explicit environment. Public for the+ /// Doppler config check binary, which validates both environments.+ pub fn signup_policy_for_environment(&self,environment: Environment,) -> anyhow::Result<SignupPolicy> {diff --git a/services/authentication_service/src/config/test.rs b/services/authentication_service/src/config/test.rsindex 7430e009a6..28490f8fb3 100644--- a/services/authentication_service/src/config/test.rs+++ b/services/authentication_service/src/config/test.rs@@ -1,4 +1,4 @@-use authentication_service::service::signup_policy::SignupPolicyDenial;+use crate::service::signup_policy::SignupPolicyDenial;use super::*;diff --git a/services/authentication_service/src/doppler_config.rs b/services/authentication_service/src/doppler_config.rsindex 1593ecc815..e053ffe5d3 100644--- a/services/authentication_service/src/doppler_config.rs+++ b/services/authentication_service/src/doppler_config.rs@@ -1,10 +1,6 @@-#![allow(unused)]-#![recursion_limit = "256"]-+use authentication_service::Config;use macro_env::Environment;-mod config;-const DOPPLER_PROJECT: &str = "authentication-service";#[tokio::main]@@ -16,7 +12,7 @@ async fn main() -> anyhow::Result<()> {.build().expect("able to grab doppler project");- let dev_config = dev.load::<config::Config>().await?;+ let dev_config = dev.load::<Config>().await?;dev_config.signup_policy_for_environment(Environment::Develop)?;let prd = doppler_config::DopplerConfig::builder()@@ -26,7 +22,7 @@ async fn main() -> anyhow::Result<()> {.build().expect("able to grab doppler project");- let prd_config = prd.load::<config::Config>().await?;+ let prd_config = prd.load::<Config>().await?;prd_config.signup_policy_for_environment(Environment::Production)?;Ok(())diff --git a/services/authentication_service/src/lib.rs b/services/authentication_service/src/lib.rsindex d83bf12274..601b1458ea 100644--- a/services/authentication_service/src/lib.rs+++ b/services/authentication_service/src/lib.rs@@ -1,2 +1,19 @@+#![recursion_limit = "256"]+//! Authentication service. The service binary, the OpenAPI generator, and the+//! Doppler config check all link this library, so the service compiles once+//! and caches as one rlib.++mod api;+mod config;+mod generate_password;+mod microsoft_token_cipher;pub mod outbound;+mod rate_limit_config;+mod server;pub mod service;++/// The service's OpenAPI document, rendered by the `*_openapi` binary.+pub use api::swagger::ApiDoc;+/// The service configuration, loaded from Doppler by the config check binary.+pub use config::Config;+pub use server::run;diff --git a/services/authentication_service/src/main.rs b/services/authentication_service/src/main.rsindex 6e140fdcf6..de7eff7e0b 100644--- a/services/authentication_service/src/main.rs+++ b/services/authentication_service/src/main.rs@@ -1,655 +1,11 @@+// `run()`'s future type is deep enough to need the library's recursion limit+// wherever it is awaited.#![recursion_limit = "256"]-use analytics_client::{- AnalyticsClient, AnalyticsClientConfig, GoogleAnalyticsConfig, MetaConfig, PostHogConfig,-};-use anyhow::{Context, anyhow};-use channels::{- domain::{- service::ChannelServiceImpl,- side_effects::{ChannelSideEffectService, SpawnedChannelEventDispatcher},- },- outbound::{- connection_gateway_realtime::ConnectionGatewayChannelRealtimePublisher,- contacts_dispatcher::ContactsChannelDispatcher,- notification_sender::NotificationChannelSender,- pg_channel_reference_share_permissions::PgChannelReferenceSharePermissions,- pg_channels_repo::PgChannelsRepo, pg_side_effect_context::PgChannelSideEffectContext,- },-};-use config::{Config, Environment};-use connection_gateway_client::ConnectionGatewayClient;-use contacts::{domain::service::SqsContactsIngress, outbound::ingress::SqsContactsQueue};-use document_storage_service_client::DocumentStorageServiceClient;-use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};-use foreign_entity::{- domain::service::ForeignEntityServiceImpl,- outbound::pg_foreign_entity_repo::PgForeignEntityRepo,-};-use github::{- domain::service::{GithubLinkConfig, GithubLinkServiceImpl},- outbound::{- github_auth_client::GithubAuthImpl, github_oauth_client::GithubOauthImpl,- pg_github_repo::PgGithubRepo,- },-};-use github_pull_requests::{- domain::service::GithubPullRequestServiceImpl,- outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo,-};-use loops_client::LoopsClient;-use macro_auth::middleware::decode_jwt::JwtValidationArgs;-use macro_authorization::{- InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationState,- PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,-};-use macro_entrypoint::MacroEntrypoint;-use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};-use macro_service_urls::{- AppServiceUrl, ConnectionGatewayUrl, DocumentStorageServiceUrl, EmailServiceUrl,-};-use native_app_service::{- domain::{models::PlatformData, service::NativeAppServiceImpl},- outbound::DefaultBundleFetcher,-};-use notification::outbound::queue::SqsQueue;-use notification::{- domain::service::SqsNotificationIngress, outbound::rate_limit::RedisRateLimitAdapter,-};-use rate_limit::domain::service::RateLimitServiceImpl;-use roles_and_permissions::{- domain::service::UserRolesAndPermissionsServiceImpl, outbound::pgpool::MacroDB,-};-use secretsmanager_client::SecretManager;-use sqlx::postgres::PgPoolOptions;-use teams::{- domain::team_service::TeamServiceImpl,- outbound::{- contacts_enqueuer::ContactsIngressEnqueuer, customer_repo::CustomerRepositoryImpl,- team_analytics::AnalyticsClientTeamAnalytics, team_repo::TeamRepositoryImpl,- },-};--use gtm_invite::{- domain::service::GtmInviteServiceImpl, outbound::pg_gtm_invite_repo::PgGtmInviteRepo,-};-use referral::{- domain::service::ReferralServiceImpl,- outbound::{pg_referral_repo::PgReferralRepo, stripe_discount_client::StripeDiscountClient},-};--use crate::{- api::context::{- ApiContext, AuthorizationService, MacroApiTokenContext, MacroApiTokenExpirySeconds,- MacroApiTokenIssuer, MacroApiTokenPrivateSecretKey, StripeWebhookSecretKey,- },- microsoft_token_cipher::{- EnvelopeMicrosoftTokenCipher, KmsDataKeyProvider, MicrosoftTokenCipher,- },-};-use std::{sync::Arc, time::Duration};-use tokio_util::task::TaskTracker;-mod api;-mod config;-mod generate_password;-mod microsoft_token_cipher;-mod rate_limit_config;+use macro_entrypoint::MacroEntrypoint;#[tokio::main]async fn main() -> anyhow::Result<()> {MacroEntrypoint::default().init();- let env = Environment::new_or_prod();-- // One SDK config is sufficient for every AWS client in this process.- let aws_config = macro_aws_config::get_macro_aws_config().await;- let secretsmanager_client = secretsmanager_client::SecretsManager::new(- aws_sdk_secretsmanager::Client::new(&aws_config),- );-- // Parse our configuration from the environment.- let config = Config::from_env().context("expected to be able to generate config")?;- let signup_policy = Arc::new(- config- .signup_policy()- .context("invalid signup policy configuration")?,- );- let gtm_invite_config = config- .gtm_invite_config()- .context("invalid GTM invite link configuration")?;- let microsoft_credentials = config- .microsoft_credentials()- .context("invalid Microsoft OAuth configuration")?;- let microsoft_token_cipher = microsoft_credentials.as_ref().map(|credentials| {- Arc::new(EnvelopeMicrosoftTokenCipher::new(KmsDataKeyProvider::new(- aws_sdk_kms::Client::new(&aws_config),- credentials.token_kms_key_id.clone(),- ))) as Arc<dyn MicrosoftTokenCipher>- });-- // A separate KMS key from the Microsoft one by design: sharing it would- // grant whatever decrypts Cursor keys access to the key protecting- // everyone's mailbox credentials.- let cursor_api_key_cipher = Arc::new(cursor_api_key::cipher::KmsCursorApiKeyCipher::new(- cursor_api_key::cipher::AwsKmsCiphertexts::new(- aws_sdk_kms::Client::new(&aws_config),- config.cursor_api_key_kms_key_id()?,- ),- )) as Arc<dyn cursor_api_key::cipher::CursorApiKeyCipher>;-- let internal_api_key = config.internal_api_key.clone();-- let stripe_webhook_secret = secretsmanager_client- .get_maybe_secret_value(env, StripeWebhookSecretKey::new()?)- .await?;-- tracing::trace!("initialized config");-- let (min_connections, max_connections): (u32, u32) = match config.environment {- Environment::Production => (5, 25),- Environment::Develop => (1, 25),- Environment::Local => (1, 10),- };-- let db = PgPoolOptions::new()- .min_connections(min_connections)- .max_connections(max_connections)- .connect(&config.database_url)- .await- .context("could not connect to db")?;-- tracing::trace!(- min_connections,- max_connections,- "initialized db connection"- );-- // Macro API token- let macro_api_token_private_key = secretsmanager_client- .get_maybe_secret_value(config.environment, MacroApiTokenPrivateSecretKey::new()?)- .await?;-- let fusionauth_api_key = match config.environment {- Environment::Local => config.fusionauth_api_key_secret_key.to_string().clone(),- _ => secretsmanager_client- .get_secret_value(&config.fusionauth_api_key_secret_key)- .await- .context("unable to get secret")?- .to_string(),- };-- let fusionauth_client_secret = match config.environment {- Environment::Local => config.fusionauth_client_secret_key.to_string().clone(),- _ => secretsmanager_client- .get_secret_value(&config.fusionauth_client_secret_key)- .await- .context("unable to get secret")?- .to_string(),- };-- let stripe_client_secret = match config.environment {- Environment::Local => config.stripe_secret_key.to_string().clone(),- _ => secretsmanager_client- .get_secret_value(&config.stripe_secret_key)- .await- .context("unable to get secret")?- .to_string(),- };-- let google_client_secret = match config.environment {- Environment::Local => config.google_client_secret_key.to_string().clone(),- _ => secretsmanager_client- .get_secret_value(&config.google_client_secret_key)- .await- .context("unable to get google client secret")?- .to_string(),- };-- let fusionauth_public_url = config- .fusionauth_public_url- .value()- .unwrap_or(config.fusionauth_base_url.as_ref())- .to_owned();- let auth_client = fusionauth::FusionAuthClient::new(- fusionauth_api_key,- config.fusionauth_client_id.to_string().clone(),- fusionauth_client_secret,- config.fusionauth_base_url.to_string().clone(),- config.fusionauth_oauth_redirect_uri.to_string().clone(),- config.google_client_id.to_string().clone(),- google_client_secret,- )- .with_public_url(fusionauth_public_url);- let auth_client = match microsoft_credentials {- Some(credentials) => auth_client.with_microsoft_credentials(- credentials.client_id,- credentials.client_secret,- credentials.tenant_id,- ),- None => auth_client,- };- tracing::trace!("initialized auth client");-- let document_storage_service_client = DocumentStorageServiceClient::new(- config.service_internal_auth_key.to_string().clone(),- DocumentStorageServiceUrl::new()?.to_string(),- );- tracing::trace!("initialized document storage service client");-- let email_service_client =- email::outbound::EmailServiceHttpClient::new(EmailServiceUrl::new()?.to_string());- tracing::trace!("initialized email service client");-- let macro_cache_client =- macro_cache_client::MacroCache::new(config.redis_uri.to_string().as_str());-- tracing::trace!("initialized redis client");-- let stripe_client = stripe::Client::new(stripe_client_secret);- tracing::trace!("initialized stripe client");-- // `from_env` routes to local SMTP (Mailpit) when SMTP_HOST is set, else SES.- let ses_client = ses_client::Ses::from_env(- aws_sdk_sesv2::Client::new(&aws_config),- &config.environment.to_string(),- );-- let jwt_args =- JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)- .await?;- let authorization_state = MacroAuthorizationState::new(Arc::new(AuthorizationService::new(- MacroAuthJwtValidator::new(jwt_args.clone()),- InternalAuthConfig {- api_key: internal_api_key.to_string(),- default_user_id: None,- },- macro_authorization::NoBotAuthorizer,- PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),- )));-- let redis_client = redis::Client::open(config.redis_uri.to_string().as_str())- .context("failed to create redis client")?;- let redis_multiplexed_conn = redis_client- .get_multiplexed_async_connection()- .await- .context("failed to get multiplexed redis connection")?;-- let notification_queue = macro_queues::NotificationIngressQueue::new();- let search_event_queue = macro_queues::SearchEventQueue::new();- let link_manager_queue = macro_queues::LinkManagerQueue::new();- let email_backfill_queue = macro_queues::EmailBackfillQueue::new();- let ingress_queue = SqsQueue::new(- aws_sdk_sqs::Client::new(¯o_aws_config::get_macro_aws_config().await),- notification_queue.to_string(),- );- let notification_ingress_service = SqsNotificationIngress {- queue: ingress_queue,- };- tracing::trace!("initialized notification ingress service");-- let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(&aws_config))- .search_event_queue(&search_event_queue)- .email_link_manager_queue(&link_manager_queue)- .email_backfill_queue(&email_backfill_queue);- tracing::trace!("initialized sqs client");-- // Initialize analytics client with configured providers- let analytics_client = Arc::new(AnalyticsClient::new(AnalyticsClientConfig {- google_analytics: config- .ga_measurement_id- .value()- .zip(config.ga_api_secret.value())- .map(|(measurement_id, api_secret)| {- tracing::info!("configuring Google Analytics");- GoogleAnalyticsConfig {- measurement_id: measurement_id.to_string(),- api_secret: api_secret.to_string(),- }- }),- meta: config- .meta_pixel_id- .value()- .zip(config.meta_access_token.value())- .map(|(pixel_id, access_token)| {- tracing::info!("configuring Meta Conversions API");- MetaConfig {- pixel_id: pixel_id.to_string(),- access_token: access_token.to_string(),- test_event_code: config.meta_test_event_code.value().map(str::to_string),- }- }),- posthog: config.posthog_api_key.value().map(|api_key| {- tracing::info!("configuring PostHog");- PostHogConfig {- api_key: api_key.to_string(),- host: config- .posthog_host- .value()- .map(str::to_string)- .unwrap_or_else(|| "https://us.i.posthog.com".to_string()),- }- }),- }));- tracing::trace!("initialized analytics client");-- // Initialize Loops client. Production and develop sign-ups are added to- // Loops; on localhost, or wherever no API key is configured, this is a- // no-op. Note there is one Loops audience — a develop sign-up creates a- // real contact and can trigger live workflows, so leave the key unset in- // any environment that should not send.- let loops_client = match (config.environment, config.loops_api_key.value()) {- (Environment::Production | Environment::Develop, Some(api_key)) => {- tracing::info!("configuring Loops");- LoopsClient::new(api_key.to_string())- }- _ => LoopsClient::noop(),- };- tracing::trace!("initialized loops client");-- let user_roles_and_permissions_macro_db = MacroDB::new(db.clone());-- let user_roles_and_permissions_service = UserRolesAndPermissionsServiceImpl::new(- user_roles_and_permissions_macro_db.clone(),- user_roles_and_permissions_macro_db,- );-- let stripe_prices = api::StripePrices {- premium: config.stripe_price_id.to_string(),- max: config- .stripe_max_price_id- .value()- .filter(|id| !id.trim().is_empty())- .map(str::to_owned),- };- if stripe_prices.max.is_none() {- tracing::warn!("STRIPE_MAX_PRICE_ID is not set; the Max plan cannot be sold");- }-- let teams_repo_impl = TeamRepositoryImpl::new(db.clone());- let customer_repo_impl =- CustomerRepositoryImpl::new(stripe_client.clone(), stripe_prices.seat_prices());- let favorites_service = favorites::domain::service::FavoritesServiceImpl::new(- favorites::outbound::pg_favorites_repo::PgFavoritesRepo::new(db.clone()),- );- let team_crm_settings_repo_impl =- teams::outbound::team_crm_settings_repo::TeamCrmSettingsRepositoryImpl::new(db.clone());-- let notification_ingress_service = Arc::new(notification_ingress_service);-- let crm_enqueuer = teams::outbound::crm_enqueuer::SqsCrmEnqueuer::new(sqs_client.clone());- let sqs_client = Arc::new(sqs_client);- let contacts_ingress = Arc::new(SqsContactsIngress {- queue: SqsContactsQueue::new(- aws_sdk_sqs::Client::new(&aws_config),- macro_queues::ContactsQueue::new().to_string(),- ),- });- let contacts_enqueuer = ContactsIngressEnqueuer::new(contacts_ingress.clone());- let team_analytics = AnalyticsClientTeamAnalytics::new(analytics_client.clone());- let event_broker_tracker = TaskTracker::new();- let macro_event_broker = MacroEventBrokerService::new(- KafkaEventPublisher::new(config.kafka_brokers.as_ref())- .context("failed to create kafka event publisher")?,- event_broker_tracker.clone(),- );- let entity_access_service_impl = Arc::new(EntityAccessServiceImpl::new(- PgAccessRepository::new(db.clone()),- ));- let connection_gateway_client = Arc::new(ConnectionGatewayClient::new(- config.service_internal_auth_key.to_string(),- ConnectionGatewayUrl::new()?.to_string(),- ));- // Authentication creates channels and posts support welcome messages in-process, so its- // channel service must dispatch the same realtime, notification, contact, and broker side- // effects as the document-storage channel API. Channel broker events drive live search- // indexing.- let channel_side_effects = ChannelSideEffectService::new(- PgChannelSideEffectContext::new(db.clone()),- ConnectionGatewayChannelRealtimePublisher::new(connection_gateway_client.clone()),- NotificationChannelSender::new(notification_ingress_service.clone()),- ContactsChannelDispatcher::new(contacts_ingress),- )- .with_macro_event_broker(macro_event_broker.clone());- let channel_event_dispatcher = SpawnedChannelEventDispatcher::new(channel_side_effects);- let channel_service = ChannelServiceImpl::with_dependencies(- PgChannelsRepo::new(db.clone()),- channel_event_dispatcher.clone(),- PgChannelReferenceSharePermissions::new(db.clone(), entity_access_service_impl.clone()),- );- // The welcome message goes through the shared message service so it gets the- // same persistence and delivery as every other channel message.- let channel_messages: Arc<dyn messages::domain::api::MessageCommands> = Arc::new(- messages::domain::service::MessageService::new(- messages::outbound::pg_message_repo::PgMessageRepository::new(db.clone())- .with_initiatives(initiative::domain::lookup::InitiativeLookup::new(- initiative::outbound::PgInitiativeRepo::new(db.clone()),- ))- .with_crm(crm::outbound::lookup::PgCrmParentReader::new(db.clone())),- messages::domain::effects::MessageEffects::new(- messages::outbound::broker::BrokerMessagePublisher::new(macro_event_broker.clone()),- messages::domain::ports::NoMessageEventPublisher,- channels::domain::message_delivery::ChannelMessageDelivery::new(- PgChannelsRepo::new(db.clone()),- channel_event_dispatcher,- PgChannelReferenceSharePermissions::new(- db.clone(),- entity_access_service_impl.clone(),- ),- messages::outbound::connection_gateway::ConnectionGatewayMessages(- connection_gateway_client,- ),- ),- ),- )- .with_group_recipients(channels::domain::group_mentions::ChannelGroupRecipients(- PgChannelsRepo::new(db.clone()),- ))- .with_references(- messages::outbound::entity_access_audience::EntityAccessMessageReferences(- (*entity_access_service_impl).clone(),- ),- ),- );-- let foreign_entity_service =- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone()));-- let github_link_service_impl = GithubLinkServiceImpl::new(- PgGithubRepo::new(db.clone()),- GithubOauthImpl::default(),- GithubAuthImpl::new(auth_client.clone(), redis_multiplexed_conn),- GithubPullRequestServiceImpl::new(- foreign_entity_service,- PgGithubPullRequestRepo::new(db.clone()),- ),- GithubLinkConfig {- client_id: config.github_client_id.to_string(),- client_secret: config.github_client_secret.to_string(),- idp_id: config.github_idp_id.to_string(),- },- );-- let rate_limit = RateLimitServiceImpl {- repo: RedisRateLimitAdapter {- redis: redis_client,- },- };- let referral_service = ReferralServiceImpl {- repo: PgReferralRepo::new(db.clone()),- discount_client: StripeDiscountClient::new(- stripe_client.clone(),- 10000, /*100$ credit, in cents*/- ),- notification_ingress: notification_ingress_service.clone(),- };- let gtm_invite_service = GtmInviteServiceImpl {- repo: PgGtmInviteRepo::new(db.clone()),- config: gtm_invite_config,- };-- let codex_connection = if let Some(key_id) = config.codex_oauth_kms_key_id() {- let cipher = Arc::new(codex_connection::outbound::cipher::EnvelopeCipher::new(- aws_sdk_kms::Client::new(&aws_config),- key_id,- )?);- let repository = Arc::new(- codex_connection::outbound::postgres::PostgresRepository::new(db.clone(), cipher),- );- let provider = codex_cloud_agents::outbound::openai::OpenAi::new()- .map_err(|_| anyhow::anyhow!("failed to initialize Codex OAuth client"))?;- Some(- Arc::new(codex_connection::domain::ConnectionServiceImpl::new(- repository, provider,- )) as Arc<dyn codex_connection::domain::ConnectionService>,- )- } else {- None- };-- let stripe_client = Arc::new(stripe_client);- let gtm_invite_service = Arc::new(gtm_invite_service);- let subscription_checkout = Arc::new(- authentication_service::service::subscription_checkout::CheckoutService::new(- authentication_service::outbound::subscription_checkout::StripeCheckoutGateway::new(- db.clone(),- stripe_client.clone(),- gtm_invite_service.clone(),- stripe_prices.seat_prices(),- ),- ),- );- let ai_payment_gateway = ai_billing::outbound::StripePaymentGateway::new(stripe_client.clone());- let ai_pricing = config.ai_pricing();- let ai_billing_service = Arc::new(- ai_billing::domain::BillingServiceImpl::new(- ai_billing::outbound::RolesTeamsEntitlementSource::new(- user_roles_and_permissions_service.clone(),- teams_repo_impl.clone(),- ),- ai_billing::outbound::PgUsageReader::new(db.clone()),- ai_billing::outbound::PgBillingRepo::new(db.clone(), ai_pricing),- ai_payment_gateway.clone(),- ai_pricing,- )- .with_enforcement(config.enable_ai_usage_enforcement)- .with_billing(config.enable_ai_usage_billing),- );- let teams_service_impl = TeamServiceImpl::new_with_analytics(- teams_repo_impl.clone(),- customer_repo_impl,- channel_service.clone(),- user_roles_and_permissions_service.clone(),- notification_ingress_service.clone(),- crm_enqueuer,- team_crm_settings_repo_impl,- team_analytics,- )- .with_contacts_enqueuer(contacts_enqueuer)- .with_event_broker(macro_event_broker)- .with_open_seat_release((*ai_billing_service).clone());- let teams_service = Arc::new(teams_service_impl);- let document_storage_service_client = Arc::new(document_storage_service_client);- // The harness and scheduled-action services validate the fleet-wide- // internal key, not this service's own inbound key.- let user_deletion = Arc::new(- authentication_service::outbound::user_deletion::UserDeletionAdapter::new(- db.clone(),- document_storage_service_client.clone(),- teams_service.clone(),- onboarding::outbound::pg_onboarding_repo::PgOnboardingRepo::new(db.clone()),- stripe_client.clone(),- config.service_internal_auth_key.to_string(),- macro_service_urls::AgentHarnessServiceUrl::new()?.to_string(),- macro_service_urls::ScheduledActionServiceUrl::new()?.to_string(),- )- .map_err(|error| anyhow::anyhow!("{error:?}"))?,- );-- let server_result = api::setup_and_serve(- ApiContext {- db,- github_link_service: Arc::new(github_link_service_impl),- auth_client: Arc::new(auth_client),- microsoft_token_cipher,- cursor_api_key_cipher,- codex_connection,- macro_cache_client: Arc::new(macro_cache_client),- stripe_client,- subscription_checkout,- document_storage_service_client,- user_deletion,- email_service_client: Arc::new(email_service_client),- ses_client: Arc::new(ses_client),- notification_ingress_service,- sqs_client,- environment: config.environment,- signup_policy,- rate_limit_service: rate_limit,- calendar_scope_enabled: config.calendar_scope_enabled,- jwt_args,- authorization_state,- token_context: MacroApiTokenContext {- issuer: MacroApiTokenIssuer::new()?,- macro_api_token_private_key,- expiry_seconds: MacroApiTokenExpirySeconds::new()?- .as_ref()- .parse()- .context("failed to parse MACRO_API_TOKEN_EXPIRY_SECONDS as usize")?,- },- internal_api_key,- stripe_webhook_secret,- user_roles_and_permissions_service: Arc::new(user_roles_and_permissions_service),- teams_service,- channel_service: Arc::new(channel_service),- channel_messages,- favorites_service: Arc::new(favorites_service),- entity_access_service: entity_access_service_impl,- referral_service: Arc::new(referral_service),- gtm_invite_service,- native_app_service: Arc::new(NativeAppServiceImpl {- bundle_fetcher: DefaultBundleFetcher::new(- AppServiceUrl::new_for_environment(config.environment)- .context("failed to resolve app service URL")?- .parse_url()- .context("failed to parse app service URL")?,- ),- bundle_policy: native_app_service::domain::models::BundleUpdatePolicy::from_env()- .map_err(|err| {- anyhow!("failed to load bundle update policy: {err}")- })?,- platform_data: PlatformData {- ios_development_team_id: IOS_DEVELOPMENT_TEAM_ID.to_string(),- ios_app_bundle_id: IOS_APP_BUNDLE_ID.to_string(),- },- }),- loops_client: Arc::new(loops_client),- analytics_client,- stripe_prices,- ai_billing_service,- ai_payment_gateway: Arc::new(ai_payment_gateway),- },- config.port,- )- .await;-- tracing::info!("waiting for event broker publishes to drain");- event_broker_tracker.close();- match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {- Ok(()) => tracing::info!("event broker publishes drained"),- Err(error) => {- tracing::warn!(- error=?error,- timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),- "timed out waiting for event broker publishes to drain"- );- }- }-- server_result+ authentication_service::run().await}--const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);--// SAFETY: this is not a secret value-const IOS_DEVELOPMENT_TEAM_ID: &str = "TY74Q77JBD";-// SAFETY: this is not a secret value-const IOS_APP_BUNDLE_ID: &str = "com.macro.app.prod";diff --git a/services/authentication_service/src/openapi.rs b/services/authentication_service/src/openapi.rsindex 196a3f0025..60a543a982 100644--- a/services/authentication_service/src/openapi.rs+++ b/services/authentication_service/src/openapi.rs@@ -1,17 +1,6 @@-#![allow(unused)]--mod api;-mod config;-mod generate_password;-mod microsoft_token_cipher;-mod rate_limit_config;-mod service;-+use authentication_service::ApiDoc;use utoipa::OpenApi;fn main() {- println!(- "{}",- api::swagger::ApiDoc::openapi().to_pretty_json().unwrap()- );+ println!("{}", ApiDoc::openapi().to_pretty_json().unwrap());}diff --git a/services/authentication_service/src/server.rs b/services/authentication_service/src/server.rsnew file mode 100644index 0000000000..9c068785d8--- /dev/null+++ b/services/authentication_service/src/server.rs@@ -0,0 +1,651 @@+//! The service's composition root: reads config and wires every adapter and+//! domain service into the HTTP server.++use analytics_client::{+ AnalyticsClient, AnalyticsClientConfig, GoogleAnalyticsConfig, MetaConfig, PostHogConfig,+};+use anyhow::{Context, anyhow};+use channels::{+ domain::{+ service::ChannelServiceImpl,+ side_effects::{ChannelSideEffectService, SpawnedChannelEventDispatcher},+ },+ outbound::{+ connection_gateway_realtime::ConnectionGatewayChannelRealtimePublisher,+ contacts_dispatcher::ContactsChannelDispatcher,+ notification_sender::NotificationChannelSender,+ pg_channel_reference_share_permissions::PgChannelReferenceSharePermissions,+ pg_channels_repo::PgChannelsRepo, pg_side_effect_context::PgChannelSideEffectContext,+ },+};+use connection_gateway_client::ConnectionGatewayClient;+use contacts::{domain::service::SqsContactsIngress, outbound::ingress::SqsContactsQueue};+use document_storage_service_client::DocumentStorageServiceClient;+use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};+use foreign_entity::{+ domain::service::ForeignEntityServiceImpl,+ outbound::pg_foreign_entity_repo::PgForeignEntityRepo,+};+use github::{+ domain::service::{GithubLinkConfig, GithubLinkServiceImpl},+ outbound::{+ github_auth_client::GithubAuthImpl, github_oauth_client::GithubOauthImpl,+ pg_github_repo::PgGithubRepo,+ },+};+use github_pull_requests::{+ domain::service::GithubPullRequestServiceImpl,+ outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo,+};+use loops_client::LoopsClient;+use macro_auth::middleware::decode_jwt::JwtValidationArgs;+use macro_authorization::{+ InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationState,+ PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,+};+use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};+use macro_service_urls::{+ AppServiceUrl, ConnectionGatewayUrl, DocumentStorageServiceUrl, EmailServiceUrl,+};+use native_app_service::{+ domain::{models::PlatformData, service::NativeAppServiceImpl},+ outbound::DefaultBundleFetcher,+};+use notification::outbound::queue::SqsQueue;+use notification::{+ domain::service::SqsNotificationIngress, outbound::rate_limit::RedisRateLimitAdapter,+};+use rate_limit::domain::service::RateLimitServiceImpl;+use roles_and_permissions::{+ domain::service::UserRolesAndPermissionsServiceImpl, outbound::pgpool::MacroDB,+};+use secretsmanager_client::SecretManager;+use sqlx::postgres::PgPoolOptions;+use teams::{+ domain::team_service::TeamServiceImpl,+ outbound::{+ contacts_enqueuer::ContactsIngressEnqueuer, customer_repo::CustomerRepositoryImpl,+ team_analytics::AnalyticsClientTeamAnalytics, team_repo::TeamRepositoryImpl,+ },+};++use gtm_invite::{+ domain::service::GtmInviteServiceImpl, outbound::pg_gtm_invite_repo::PgGtmInviteRepo,+};+use referral::{+ domain::service::ReferralServiceImpl,+ outbound::{pg_referral_repo::PgReferralRepo, stripe_discount_client::StripeDiscountClient},+};++use crate::{+ api::{+ self,+ context::{+ ApiContext, AuthorizationService, MacroApiTokenContext, MacroApiTokenExpirySeconds,+ MacroApiTokenIssuer, MacroApiTokenPrivateSecretKey, StripeWebhookSecretKey,+ },+ },+ config::{Config, Environment},+ microsoft_token_cipher::{+ EnvelopeMicrosoftTokenCipher, KmsDataKeyProvider, MicrosoftTokenCipher,+ },+};+use std::{sync::Arc, time::Duration};+use tokio_util::task::TaskTracker;++/// Build the service from the environment and serve until shutdown.+pub async fn run() -> anyhow::Result<()> {+ let env = Environment::new_or_prod();++ // One SDK config is sufficient for every AWS client in this process.+ let aws_config = macro_aws_config::get_macro_aws_config().await;+ let secretsmanager_client = secretsmanager_client::SecretsManager::new(+ aws_sdk_secretsmanager::Client::new(&aws_config),+ );++ // Parse our configuration from the environment.+ let config = Config::from_env().context("expected to be able to generate config")?;+ let signup_policy = Arc::new(+ config+ .signup_policy()+ .context("invalid signup policy configuration")?,+ );+ let gtm_invite_config = config+ .gtm_invite_config()+ .context("invalid GTM invite link configuration")?;+ let microsoft_credentials = config+ .microsoft_credentials()+ .context("invalid Microsoft OAuth configuration")?;+ let microsoft_token_cipher = microsoft_credentials.as_ref().map(|credentials| {+ Arc::new(EnvelopeMicrosoftTokenCipher::new(KmsDataKeyProvider::new(+ aws_sdk_kms::Client::new(&aws_config),+ credentials.token_kms_key_id.clone(),+ ))) as Arc<dyn MicrosoftTokenCipher>+ });++ // A separate KMS key from the Microsoft one by design: sharing it would+ // grant whatever decrypts Cursor keys access to the key protecting+ // everyone's mailbox credentials.+ let cursor_api_key_cipher = Arc::new(cursor_api_key::cipher::KmsCursorApiKeyCipher::new(+ cursor_api_key::cipher::AwsKmsCiphertexts::new(+ aws_sdk_kms::Client::new(&aws_config),+ config.cursor_api_key_kms_key_id()?,+ ),+ )) as Arc<dyn cursor_api_key::cipher::CursorApiKeyCipher>;++ let internal_api_key = config.internal_api_key.clone();++ let stripe_webhook_secret = secretsmanager_client+ .get_maybe_secret_value(env, StripeWebhookSecretKey::new()?)+ .await?;++ tracing::trace!("initialized config");++ let (min_connections, max_connections): (u32, u32) = match config.environment {+ Environment::Production => (5, 25),+ Environment::Develop => (1, 25),+ Environment::Local => (1, 10),+ };++ let db = PgPoolOptions::new()+ .min_connections(min_connections)+ .max_connections(max_connections)+ .connect(&config.database_url)+ .await+ .context("could not connect to db")?;++ tracing::trace!(+ min_connections,+ max_connections,+ "initialized db connection"+ );++ // Macro API token+ let macro_api_token_private_key = secretsmanager_client+ .get_maybe_secret_value(config.environment, MacroApiTokenPrivateSecretKey::new()?)+ .await?;++ let fusionauth_api_key = match config.environment {+ Environment::Local => config.fusionauth_api_key_secret_key.to_string().clone(),+ _ => secretsmanager_client+ .get_secret_value(&config.fusionauth_api_key_secret_key)+ .await+ .context("unable to get secret")?+ .to_string(),+ };++ let fusionauth_client_secret = match config.environment {+ Environment::Local => config.fusionauth_client_secret_key.to_string().clone(),+ _ => secretsmanager_client+ .get_secret_value(&config.fusionauth_client_secret_key)+ .await+ .context("unable to get secret")?+ .to_string(),+ };++ let stripe_client_secret = match config.environment {+ Environment::Local => config.stripe_secret_key.to_string().clone(),+ _ => secretsmanager_client+ .get_secret_value(&config.stripe_secret_key)+ .await+ .context("unable to get secret")?+ .to_string(),+ };++ let google_client_secret = match config.environment {+ Environment::Local => config.google_client_secret_key.to_string().clone(),+ _ => secretsmanager_client+ .get_secret_value(&config.google_client_secret_key)+ .await+ .context("unable to get google client secret")?+ .to_string(),+ };++ let fusionauth_public_url = config+ .fusionauth_public_url+ .value()+ .unwrap_or(config.fusionauth_base_url.as_ref())+ .to_owned();+ let auth_client = fusionauth::FusionAuthClient::new(+ fusionauth_api_key,+ config.fusionauth_client_id.to_string().clone(),+ fusionauth_client_secret,+ config.fusionauth_base_url.to_string().clone(),+ config.fusionauth_oauth_redirect_uri.to_string().clone(),+ config.google_client_id.to_string().clone(),+ google_client_secret,+ )+ .with_public_url(fusionauth_public_url);+ let auth_client = match microsoft_credentials {+ Some(credentials) => auth_client.with_microsoft_credentials(+ credentials.client_id,+ credentials.client_secret,+ credentials.tenant_id,+ ),+ None => auth_client,+ };+ tracing::trace!("initialized auth client");++ let document_storage_service_client = DocumentStorageServiceClient::new(+ config.service_internal_auth_key.to_string().clone(),+ DocumentStorageServiceUrl::new()?.to_string(),+ );+ tracing::trace!("initialized document storage service client");++ let email_service_client =+ email::outbound::EmailServiceHttpClient::new(EmailServiceUrl::new()?.to_string());+ tracing::trace!("initialized email service client");++ let macro_cache_client =+ macro_cache_client::MacroCache::new(config.redis_uri.to_string().as_str());++ tracing::trace!("initialized redis client");++ let stripe_client = stripe::Client::new(stripe_client_secret);+ tracing::trace!("initialized stripe client");++ // `from_env` routes to local SMTP (Mailpit) when SMTP_HOST is set, else SES.+ let ses_client = ses_client::Ses::from_env(+ aws_sdk_sesv2::Client::new(&aws_config),+ &config.environment.to_string(),+ );++ let jwt_args =+ JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)+ .await?;+ let authorization_state = MacroAuthorizationState::new(Arc::new(AuthorizationService::new(+ MacroAuthJwtValidator::new(jwt_args.clone()),+ InternalAuthConfig {+ api_key: internal_api_key.to_string(),+ default_user_id: None,+ },+ macro_authorization::NoBotAuthorizer,+ PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),+ )));++ let redis_client = redis::Client::open(config.redis_uri.to_string().as_str())+ .context("failed to create redis client")?;+ let redis_multiplexed_conn = redis_client+ .get_multiplexed_async_connection()+ .await+ .context("failed to get multiplexed redis connection")?;++ let notification_queue = macro_queues::NotificationIngressQueue::new();+ let search_event_queue = macro_queues::SearchEventQueue::new();+ let link_manager_queue = macro_queues::LinkManagerQueue::new();+ let email_backfill_queue = macro_queues::EmailBackfillQueue::new();+ let ingress_queue = SqsQueue::new(+ aws_sdk_sqs::Client::new(¯o_aws_config::get_macro_aws_config().await),+ notification_queue.to_string(),+ );+ let notification_ingress_service = SqsNotificationIngress {+ queue: ingress_queue,+ };+ tracing::trace!("initialized notification ingress service");++ let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(&aws_config))+ .search_event_queue(&search_event_queue)+ .email_link_manager_queue(&link_manager_queue)+ .email_backfill_queue(&email_backfill_queue);+ tracing::trace!("initialized sqs client");++ // Initialize analytics client with configured providers+ let analytics_client = Arc::new(AnalyticsClient::new(AnalyticsClientConfig {+ google_analytics: config+ .ga_measurement_id+ .value()+ .zip(config.ga_api_secret.value())+ .map(|(measurement_id, api_secret)| {+ tracing::info!("configuring Google Analytics");+ GoogleAnalyticsConfig {+ measurement_id: measurement_id.to_string(),+ api_secret: api_secret.to_string(),+ }+ }),+ meta: config+ .meta_pixel_id+ .value()+ .zip(config.meta_access_token.value())+ .map(|(pixel_id, access_token)| {+ tracing::info!("configuring Meta Conversions API");+ MetaConfig {+ pixel_id: pixel_id.to_string(),+ access_token: access_token.to_string(),+ test_event_code: config.meta_test_event_code.value().map(str::to_string),+ }+ }),+ posthog: config.posthog_api_key.value().map(|api_key| {+ tracing::info!("configuring PostHog");+ PostHogConfig {+ api_key: api_key.to_string(),+ host: config+ .posthog_host+ .value()+ .map(str::to_string)+ .unwrap_or_else(|| "https://us.i.posthog.com".to_string()),+ }+ }),+ }));+ tracing::trace!("initialized analytics client");++ // Initialize Loops client. Production and develop sign-ups are added to+ // Loops; on localhost, or wherever no API key is configured, this is a+ // no-op. Note there is one Loops audience — a develop sign-up creates a+ // real contact and can trigger live workflows, so leave the key unset in+ // any environment that should not send.+ let loops_client = match (config.environment, config.loops_api_key.value()) {+ (Environment::Production | Environment::Develop, Some(api_key)) => {+ tracing::info!("configuring Loops");+ LoopsClient::new(api_key.to_string())+ }+ _ => LoopsClient::noop(),+ };+ tracing::trace!("initialized loops client");++ let user_roles_and_permissions_macro_db = MacroDB::new(db.clone());++ let user_roles_and_permissions_service = UserRolesAndPermissionsServiceImpl::new(+ user_roles_and_permissions_macro_db.clone(),+ user_roles_and_permissions_macro_db,+ );++ let stripe_prices = api::StripePrices {+ premium: config.stripe_price_id.to_string(),+ max: config+ .stripe_max_price_id+ .value()+ .filter(|id| !id.trim().is_empty())+ .map(str::to_owned),+ };+ if stripe_prices.max.is_none() {+ tracing::warn!("STRIPE_MAX_PRICE_ID is not set; the Max plan cannot be sold");+ }++ let teams_repo_impl = TeamRepositoryImpl::new(db.clone());+ let customer_repo_impl =+ CustomerRepositoryImpl::new(stripe_client.clone(), stripe_prices.seat_prices());+ let favorites_service = favorites::domain::service::FavoritesServiceImpl::new(+ favorites::outbound::pg_favorites_repo::PgFavoritesRepo::new(db.clone()),+ );+ let team_crm_settings_repo_impl =+ teams::outbound::team_crm_settings_repo::TeamCrmSettingsRepositoryImpl::new(db.clone());++ let notification_ingress_service = Arc::new(notification_ingress_service);++ let crm_enqueuer = teams::outbound::crm_enqueuer::SqsCrmEnqueuer::new(sqs_client.clone());+ let sqs_client = Arc::new(sqs_client);+ let contacts_ingress = Arc::new(SqsContactsIngress {+ queue: SqsContactsQueue::new(+ aws_sdk_sqs::Client::new(&aws_config),+ macro_queues::ContactsQueue::new().to_string(),+ ),+ });+ let contacts_enqueuer = ContactsIngressEnqueuer::new(contacts_ingress.clone());+ let team_analytics = AnalyticsClientTeamAnalytics::new(analytics_client.clone());+ let event_broker_tracker = TaskTracker::new();+ let macro_event_broker = MacroEventBrokerService::new(+ KafkaEventPublisher::new(config.kafka_brokers.as_ref())+ .context("failed to create kafka event publisher")?,+ event_broker_tracker.clone(),+ );+ let entity_access_service_impl = Arc::new(EntityAccessServiceImpl::new(+ PgAccessRepository::new(db.clone()),+ ));+ let connection_gateway_client = Arc::new(ConnectionGatewayClient::new(+ config.service_internal_auth_key.to_string(),+ ConnectionGatewayUrl::new()?.to_string(),+ ));+ // Authentication creates channels and posts support welcome messages in-process, so its+ // channel service must dispatch the same realtime, notification, contact, and broker side+ // effects as the document-storage channel API. Channel broker events drive live search+ // indexing.+ let channel_side_effects = ChannelSideEffectService::new(+ PgChannelSideEffectContext::new(db.clone()),+ ConnectionGatewayChannelRealtimePublisher::new(connection_gateway_client.clone()),+ NotificationChannelSender::new(notification_ingress_service.clone()),+ ContactsChannelDispatcher::new(contacts_ingress),+ )+ .with_macro_event_broker(macro_event_broker.clone());+ let channel_event_dispatcher = SpawnedChannelEventDispatcher::new(channel_side_effects);+ let channel_service = ChannelServiceImpl::with_dependencies(+ PgChannelsRepo::new(db.clone()),+ channel_event_dispatcher.clone(),+ PgChannelReferenceSharePermissions::new(db.clone(), entity_access_service_impl.clone()),+ );+ // The welcome message goes through the shared message service so it gets the+ // same persistence and delivery as every other channel message.+ let channel_messages: Arc<dyn messages::domain::api::MessageCommands> = Arc::new(+ messages::domain::service::MessageService::new(+ messages::outbound::pg_message_repo::PgMessageRepository::new(db.clone())+ .with_initiatives(initiative::domain::lookup::InitiativeLookup::new(+ initiative::outbound::PgInitiativeRepo::new(db.clone()),+ ))+ .with_crm(crm::outbound::lookup::PgCrmParentReader::new(db.clone())),+ messages::domain::effects::MessageEffects::new(+ messages::outbound::broker::BrokerMessagePublisher::new(macro_event_broker.clone()),+ messages::domain::ports::NoMessageEventPublisher,+ channels::domain::message_delivery::ChannelMessageDelivery::new(+ PgChannelsRepo::new(db.clone()),+ channel_event_dispatcher,+ PgChannelReferenceSharePermissions::new(+ db.clone(),+ entity_access_service_impl.clone(),+ ),+ messages::outbound::connection_gateway::ConnectionGatewayMessages(+ connection_gateway_client,+ ),+ ),+ ),+ )+ .with_group_recipients(channels::domain::group_mentions::ChannelGroupRecipients(+ PgChannelsRepo::new(db.clone()),+ ))+ .with_references(+ messages::outbound::entity_access_audience::EntityAccessMessageReferences(+ (*entity_access_service_impl).clone(),+ ),+ ),+ );++ let foreign_entity_service =+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone()));++ let github_link_service_impl = GithubLinkServiceImpl::new(+ PgGithubRepo::new(db.clone()),+ GithubOauthImpl::default(),+ GithubAuthImpl::new(auth_client.clone(), redis_multiplexed_conn),+ GithubPullRequestServiceImpl::new(+ foreign_entity_service,+ PgGithubPullRequestRepo::new(db.clone()),+ ),+ GithubLinkConfig {+ client_id: config.github_client_id.to_string(),+ client_secret: config.github_client_secret.to_string(),+ idp_id: config.github_idp_id.to_string(),+ },+ );++ let rate_limit = RateLimitServiceImpl {+ repo: RedisRateLimitAdapter {+ redis: redis_client,+ },+ };+ let referral_service = ReferralServiceImpl {+ repo: PgReferralRepo::new(db.clone()),+ discount_client: StripeDiscountClient::new(+ stripe_client.clone(),+ 10000, /*100$ credit, in cents*/+ ),+ notification_ingress: notification_ingress_service.clone(),+ };+ let gtm_invite_service = GtmInviteServiceImpl {+ repo: PgGtmInviteRepo::new(db.clone()),+ config: gtm_invite_config,+ };++ let codex_connection = if let Some(key_id) = config.codex_oauth_kms_key_id() {+ let cipher = Arc::new(codex_connection::outbound::cipher::EnvelopeCipher::new(+ aws_sdk_kms::Client::new(&aws_config),+ key_id,+ )?);+ let repository = Arc::new(+ codex_connection::outbound::postgres::PostgresRepository::new(db.clone(), cipher),+ );+ let provider = codex_cloud_agents::outbound::openai::OpenAi::new()+ .map_err(|_| anyhow::anyhow!("failed to initialize Codex OAuth client"))?;+ Some(+ Arc::new(codex_connection::domain::ConnectionServiceImpl::new(+ repository, provider,+ )) as Arc<dyn codex_connection::domain::ConnectionService>,+ )+ } else {+ None+ };++ let stripe_client = Arc::new(stripe_client);+ let gtm_invite_service = Arc::new(gtm_invite_service);+ let subscription_checkout =+ Arc::new(crate::service::subscription_checkout::CheckoutService::new(+ crate::outbound::subscription_checkout::StripeCheckoutGateway::new(+ db.clone(),+ stripe_client.clone(),+ gtm_invite_service.clone(),+ stripe_prices.seat_prices(),+ ),+ ));+ let ai_payment_gateway = ai_billing::outbound::StripePaymentGateway::new(stripe_client.clone());+ let ai_pricing = config.ai_pricing();+ let ai_billing_service = Arc::new(+ ai_billing::domain::BillingServiceImpl::new(+ ai_billing::outbound::RolesTeamsEntitlementSource::new(+ user_roles_and_permissions_service.clone(),+ teams_repo_impl.clone(),+ ),+ ai_billing::outbound::PgUsageReader::new(db.clone()),+ ai_billing::outbound::PgBillingRepo::new(db.clone(), ai_pricing),+ ai_payment_gateway.clone(),+ ai_pricing,+ )+ .with_enforcement(config.enable_ai_usage_enforcement)+ .with_billing(config.enable_ai_usage_billing),+ );+ let teams_service_impl = TeamServiceImpl::new_with_analytics(+ teams_repo_impl.clone(),+ customer_repo_impl,+ channel_service.clone(),+ user_roles_and_permissions_service.clone(),+ notification_ingress_service.clone(),+ crm_enqueuer,+ team_crm_settings_repo_impl,+ team_analytics,+ )+ .with_contacts_enqueuer(contacts_enqueuer)+ .with_event_broker(macro_event_broker)+ .with_open_seat_release((*ai_billing_service).clone());+ let teams_service = Arc::new(teams_service_impl);+ let document_storage_service_client = Arc::new(document_storage_service_client);+ // The harness and scheduled-action services validate the fleet-wide+ // internal key, not this service's own inbound key.+ let user_deletion = Arc::new(+ crate::outbound::user_deletion::UserDeletionAdapter::new(+ db.clone(),+ document_storage_service_client.clone(),+ teams_service.clone(),+ onboarding::outbound::pg_onboarding_repo::PgOnboardingRepo::new(db.clone()),+ stripe_client.clone(),+ config.service_internal_auth_key.to_string(),+ macro_service_urls::AgentHarnessServiceUrl::new()?.to_string(),+ macro_service_urls::ScheduledActionServiceUrl::new()?.to_string(),+ )+ .map_err(|error| anyhow::anyhow!("{error:?}"))?,+ );++ let server_result = api::setup_and_serve(+ ApiContext {+ db,+ github_link_service: Arc::new(github_link_service_impl),+ auth_client: Arc::new(auth_client),+ microsoft_token_cipher,+ cursor_api_key_cipher,+ codex_connection,+ macro_cache_client: Arc::new(macro_cache_client),+ stripe_client,+ subscription_checkout,+ document_storage_service_client,+ user_deletion,+ email_service_client: Arc::new(email_service_client),+ ses_client: Arc::new(ses_client),+ notification_ingress_service,+ sqs_client,+ environment: config.environment,+ signup_policy,+ rate_limit_service: rate_limit,+ calendar_scope_enabled: config.calendar_scope_enabled,+ jwt_args,+ authorization_state,+ token_context: MacroApiTokenContext {+ issuer: MacroApiTokenIssuer::new()?,+ macro_api_token_private_key,+ expiry_seconds: MacroApiTokenExpirySeconds::new()?+ .as_ref()+ .parse()+ .context("failed to parse MACRO_API_TOKEN_EXPIRY_SECONDS as usize")?,+ },+ internal_api_key,+ stripe_webhook_secret,+ user_roles_and_permissions_service: Arc::new(user_roles_and_permissions_service),+ teams_service,+ channel_service: Arc::new(channel_service),+ channel_messages,+ favorites_service: Arc::new(favorites_service),+ entity_access_service: entity_access_service_impl,+ referral_service: Arc::new(referral_service),+ gtm_invite_service,+ native_app_service: Arc::new(NativeAppServiceImpl {+ bundle_fetcher: DefaultBundleFetcher::new(+ AppServiceUrl::new_for_environment(config.environment)+ .context("failed to resolve app service URL")?+ .parse_url()+ .context("failed to parse app service URL")?,+ ),+ bundle_policy: native_app_service::domain::models::BundleUpdatePolicy::from_env()+ .map_err(|err| {+ anyhow!("failed to load bundle update policy: {err}")+ })?,+ platform_data: PlatformData {+ ios_development_team_id: IOS_DEVELOPMENT_TEAM_ID.to_string(),+ ios_app_bundle_id: IOS_APP_BUNDLE_ID.to_string(),+ },+ }),+ loops_client: Arc::new(loops_client),+ analytics_client,+ stripe_prices,+ ai_billing_service,+ ai_payment_gateway: Arc::new(ai_payment_gateway),+ },+ config.port,+ )+ .await;++ tracing::info!("waiting for event broker publishes to drain");+ event_broker_tracker.close();+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {+ Ok(()) => tracing::info!("event broker publishes drained"),+ Err(error) => {+ tracing::warn!(+ error=?error,+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),+ "timed out waiting for event broker publishes to drain"+ );+ }+ }++ server_result+}++const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);++// SAFETY: this is not a secret value+const IOS_DEVELOPMENT_TEAM_ID: &str = "TY74Q77JBD";+// SAFETY: this is not a secret value+const IOS_APP_BUNDLE_ID: &str = "com.macro.app.prod";diff --git a/services/document_cognition_service/src/api/context/test.rs b/services/document_cognition_service/src/api/context/test.rsindex 1b27f37319..01256ec3b2 100644--- a/services/document_cognition_service/src/api/context/test.rs+++ b/services/document_cognition_service/src/api/context/test.rs@@ -68,6 +68,10 @@ pub async fn test_model_access(ctx: &ApiContext) -> DcsChatModelAccess {pub struct MockConnectionRepo;impl MockConnectionRepo {+ #[expect(+ clippy::new_ret_no_self,+ reason = "tests take the mock as the trait object ApiContext stores"+ )]pub fn new() -> Arc<dyn ConnectionRepo> {Arc::new(Self)}@@ -134,6 +138,10 @@ pub struct MockStreamRepo {}impl MockStreamRepo {+ #[expect(+ clippy::new_ret_no_self,+ reason = "tests take the mock as the trait object ApiContext stores"+ )]pub fn new() -> Arc<dyn StreamRepo> {let (tx, _) = broadcast::channel(16);Arc::new(Self { tx })diff --git a/services/document_cognition_service/src/lib.rs b/services/document_cognition_service/src/lib.rsnew file mode 100644index 0000000000..63957096eb--- /dev/null+++ b/services/document_cognition_service/src/lib.rs@@ -0,0 +1,14 @@+#![recursion_limit = "256"]+//! Document cognition service. The service binary and the OpenAPI generator+//! both link this library, so the service compiles once and caches as one rlib.++mod api;+mod config;+mod core;+mod model;+mod server;+mod service;++/// The service's OpenAPI document, rendered by the `*_openapi` binary.+pub use api::swagger::ApiDoc;+pub use server::run;diff --git a/services/document_cognition_service/src/main.rs b/services/document_cognition_service/src/main.rsindex f970aa05db..a41e6c8023 100644--- a/services/document_cognition_service/src/main.rs+++ b/services/document_cognition_service/src/main.rs@@ -1,984 +1,12 @@+// `run()`'s future type is deep enough to need the library's recursion limit+// wherever it is awaited.#![recursion_limit = "256"]-use crate::api::context::ApiContext;-use ai_tools::{NoOpCallRtcClient, NoOpConnectionService, NoOpNotificationIngress};-use anyhow::Context;-use bots::outbound::pg_bots_repo::PgBotsRepo;-use call::domain::service::{CallRecordQueryServiceImpl, CallServiceImpl};-use call::inbound::toolset::CallToolContext;-use call::outbound::pg_call_repo::PgCallRepo;-use call::outbound::s3_recording_storage::S3RecordingStorage;-use channels::{- domain::list_service::ChannelListServiceImpl, outbound::pg_channels_repo::PgChannelsRepo,-};-use config::{Config, Environment};-use document_storage_service_client::DocumentStorageServiceClient;-use documents::{- domain::{models::CloudFrontConfig, service::DocumentServiceImpl},- inbound::toolset::DocumentToolContext,- outbound::{pg_document_repo::PgDocumentRepo, s3_upload_url::S3UploadUrlAdapter},-};-use email::domain::ports::ReadonlyEmailPreviewAdapter;-use email::domain::service::EmailServiceImpl;-use email::outbound::EmailPgRepo;-use email_service_client::{EmailServiceClient, EmailServiceClientExternal};-use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};-use entity_registry::OwnerGrantPolicy;-use entity_registry_db_utils::OwnedEntityRegistrar;-use foreign_entity::{- domain::service::ForeignEntityServiceImpl,- outbound::pg_foreign_entity_repo::PgForeignEntityRepo,-};-use frecency::domain::services::FrecencyQueryServiceImpl;-use frecency::outbound::postgres::FrecencyPgStorage;-use github_pull_requests::{- domain::service::GithubPullRequestServiceImpl,- outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo,-};-use macro_auth::middleware::decode_jwt::JwtValidationArgs;-use macro_authorization::{- InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationServiceImpl,- MacroAuthorizationState, PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,-};-use macro_entrypoint::MacroEntrypoint;-use macro_service_urls::{- AuthServiceUrl, CalendarServiceUrl, ConnectionGatewayUrl, DocumentStorageServiceUrl,- EmailServiceUrl, LexicalServiceUrl, StaticFileServiceUrl, SyncServiceUrl,-};-use notification::domain::service::{- NotificationReaderService, PlatformArnConfig, SqsNotificationIngress,-};-use notification::outbound::queue::SqsQueue;-use notification::outbound::repository::DbNotificationRepository;-use notification::outbound::websocket::ConnectionGatewayClient;-use readonly_pool::ReadOnlyPool;-use search_service_client::SearchServiceClient;-use sqlx::postgres::PgPoolOptions;-use std::{sync::Arc, time::Duration};-use stream::outbound::redis_pg::RedisPostgresStreamRepo;-use sync_service_client::SyncServiceClient;-use tokio_util::task::TaskTracker;-mod api;-mod config;-mod core;-mod model;-mod service;+use macro_entrypoint::MacroEntrypoint;#[tokio::main]#[tracing::instrument(err)]async fn main() -> anyhow::Result<()> {MacroEntrypoint::default().init();-- let aws_config = macro_aws_config::get_macro_aws_config().await;-- let secretsmanager_client = secretsmanager_client::SecretsManager::new(- aws_sdk_secretsmanager::Client::new(&aws_config),- );-- // Parse our configuration from the environment.- let config = Config::from_env()- .context("expected to be able to generate config")?- .resolve_remote_secrets(Environment::new_or_prod(), &secretsmanager_client)- .await- .context("expected to be able to resolve config secrets")?;- let non_user_owners = config.non_user_owners()?;-- tracing::info!("initialized config");-- let (min_connections, max_connections): (u32, u32) = match config.environment {- Environment::Production => (5, 30),- Environment::Develop => (3, 20),- Environment::Local => (3, 10),- };-- let db = PgPoolOptions::new()- .min_connections(min_connections)- .max_connections(max_connections)- .connect(&config.database_url)- .await- .context("failed to connect to macrodb")?;-- tracing::info!(- min_connections,- max_connections,- "initialized db connection"- );-- let dynamodb_client = aws_sdk_dynamodb::Client::new(&aws_config);- let queue_aws_client = aws_sdk_sqs::Client::new(&aws_config);-- let document_text_extractor_queue = macro_queues::DocumentTextExtractorQueue::new();- let chat_delete_queue = macro_queues::ChatDeleteQueue::new();- let email_scheduled_queue = macro_queues::EmailScheduledQueue::new();- let gmail_ops_queue = macro_queues::GmailOpsQueue::new();- let ai_projection_queue = macro_queues::AiProjectionQueue::new();- let notification_queue = macro_queues::NotificationIngressQueue::new();- let sqs_client = sqs_client::SQS::new(queue_aws_client)- .document_text_extractor_queue(&document_text_extractor_queue)- .chat_delete_queue(&chat_delete_queue)- .email_scheduled_queue(&email_scheduled_queue)- .gmail_ops_queue(&gmail_ops_queue)- .ai_projection_queue(&ai_projection_queue);-- let internal_api_key = config.internal_api_key.to_string();-- let document_storage_client = DocumentStorageServiceClient::new(- config- .document_storage_service_auth_key- .as_ref()- .to_string(),- DocumentStorageServiceUrl::new()?.to_string(),- );-- tracing::info!("initialized dss client");-- let sync_service_client = SyncServiceClient::new(- config.sync_service_auth_key.as_ref().to_string(),- SyncServiceUrl::new()?.to_string(),- );- tracing::info!("initialized sync service client");-- let search_service_client = SearchServiceClient::new(- config- .document_storage_service_auth_key- .as_ref()- .to_string(),- DocumentStorageServiceUrl::new()?.to_string(),- );- tracing::info!("initialized search service client");-- let jwt_args =- JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)- .await- .context("failed to create jwt validation args")?;-- let authorization_state =- MacroAuthorizationState::new(Arc::new(MacroAuthorizationServiceImpl::new(- MacroAuthJwtValidator::new(jwt_args),- InternalAuthConfig {- api_key: internal_api_key.clone(),- default_user_id: None,- },- macro_authorization::NoBotAuthorizer,- PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),- )));-- let lexical_client = Arc::new(lexical_client::LexicalClient::new(- internal_api_key.clone(),- LexicalServiceUrl::new()?.to_string(),- ));-- let email_service_client = Arc::new(EmailServiceClient::new(- internal_api_key.clone(),- EmailServiceUrl::new()?.to_string(),- ));-- tracing::info!("initialized static file service client");-- // Initialize Redis client for stream service- let redis_client = Arc::new(- redis::Client::open(config.redis_host.as_ref())- .inspect(|client| {- client- .get_connection()- .map(|_| tracing::trace!("initialized redis connection"))- .inspect_err(|e| {- tracing::error!(error=?e, "failed to connect to redis");- })- .expect("redis connetion required");- })- .context("failed to connect to redis")?,- );- let stream_repo = RedisPostgresStreamRepo::new((*redis_client).clone(), db.clone()).obj();-- tracing::info!("initialized stream repo");-- let connection_manager =- connection_gateway::service::dynamodb::create_dynamo_db_connection_manager(dynamodb_client)- .await- .context("failed to create connection manager")?;-- tracing::info!("initialized connection repo");- let connection_gateway_client = Arc::new(ConnectionGatewayClient::new(- internal_api_key.clone(),- ConnectionGatewayUrl::new()?.to_string(),- ));-- let ingress_queue = SqsQueue::new(- aws_sdk_sqs::Client::new(&aws_config),- notification_queue.to_string(),- );- let notification_ingress_service = Arc::new(SqsNotificationIngress {- queue: ingress_queue,- });-- let notification_reader_queue = SqsQueue::new(- aws_sdk_sqs::Client::new(&aws_config),- notification_queue.to_string(),- );- let notification_reader_service = NotificationReaderService {- repository: DbNotificationRepository::new(db.clone()),- queue: ai_tools::ToolNotificationQueue::Sqs(notification_reader_queue),- sns_endpoint: ai_tools::NoOpSnsEndpointManager,- platform_config: PlatformArnConfig {- apns_platform_arn: String::new(),- fcm_platform_arn: String::new(),- apns_voip_platform_arn: String::new(),- },- realtime: notification::domain::ports::NoopNotificationRealtimePublisher,- };- let notification_tool_context =- notification::inbound::ai_tool::NotificationToolContext::new(notification_reader_service);-- tracing::info!("initialized notification ingress service");- let entity_access_service = Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(- db.clone(),- )));-- let frecency_storage = FrecencyPgStorage::new(db.clone());- let frecency_service = FrecencyQueryServiceImpl::new(frecency_storage.clone());- let crm_service = crm::domain::service::CrmServiceImpl::new(- crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),- crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,- );- let email_service = EmailServiceImpl::new(- EmailPgRepo::new(db.clone()),- frecency_service.clone(),- email::domain::ports::NoOpEnqueuer,- crm_service.clone(),- entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(- entity_access_management::outbound::PgRepository::new(db.clone()),- ),- 0,- );- let channels_service = ChannelListServiceImpl::new(- PgChannelsRepo::new(db.clone()),- PgChannelsRepo::new(db.clone()),- frecency_storage,- );- let email_service_for_tools: Arc<ai_tools::ToolEmailService> = Arc::new(email_service.clone());- let github_pull_request_service = GithubPullRequestServiceImpl::new(- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),- PgGithubPullRequestRepo::new(db.clone()),- );- let soup_service = Arc::new(soup::domain::service::SoupImpl::new(- soup::outbound::pg_soup_repo::PgSoupRepo::new(ReadOnlyPool(db.clone())),- frecency_service,- ReadonlyEmailPreviewAdapter(email_service),- channels_service,- CallRecordQueryServiceImpl::new(PgCallRepo::new(db.clone())),- crm::domain::service::NoOpCrmService,- github_pull_request_service,- ));-- tracing::info!("initialized soup service");-- let s3_client = macro_aws_config::s3_client().await;- let document_files = documents::outbound::s3_document_files::S3DocumentFiles::new(- db.clone(),- s3_client.clone(),- config.document_storage_bucket.to_string(),- );- let s3_upload_adapter = S3UploadUrlAdapter::new(- s3_client,- config.document_storage_bucket.to_string(),- config.docx_document_upload_bucket.to_string(),- );- let owned_entity_registrar =- OwnedEntityRegistrar::new(OwnerGrantPolicy::new(PgBotsRepo::new(db.clone())));- let document_repo = PgDocumentRepo::new(db.clone(), owned_entity_registrar.clone());-- let cloudfront_config = CloudFrontConfig {- distribution_url: config- .document_storage_service_cloudfront_distribution_url- .to_string(),- signer_public_key_id: config- .document_storage_service_cloudfront_signer_public_key_id- .to_string(),- signer_private_key: config- .document_storage_service_cloudfront_signer_private_key- .as_ref()- .to_string(),- presigned_url_expiry_seconds: 3600,- browser_cache_expiry_seconds: 86400,- };- let event_broker_tracker = TaskTracker::new();- let macro_event_broker = macro_event_broker::MacroEventBrokerService::new(- macro_event_broker::KafkaEventPublisher::new(config.kafka_brokers.as_ref())- .context("failed to create kafka event publisher")?,- event_broker_tracker.clone(),- );- let properties_service = ai_tools::build_properties_service_with_broker(- db.clone(),- entity_access_service.clone(),- macro_event_broker.clone(),- );- let task_properties_service = ai_tools::build_task_properties_adapter(- db.clone(),- properties_service.clone(),- entity_access_service.clone(),- );- // The import pipeline sets the same task system properties on imported- // Linear issues (status, priority, due date, assignee).- let task_properties_for_import = task_properties_service.clone();- let document_properties_for_import =- import::outbound::document_properties::DocumentPropertiesApplicator::new(- properties_service.clone(),- );- let document_service = DocumentServiceImpl::new(- document_repo,- cloudfront_config,- sync_service_client.clone(),- s3_upload_adapter,- task_properties_service,- NoOpConnectionService,- entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(- entity_access_management::outbound::PgRepository::new(db.clone()),- ),- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),- macro_event_broker.clone(),- );- // Messages sent by AI tools (chat, agents) dispatch the same side effects- // as the document-storage message API, so mentions, replies and document- // comments notify recipients and stream to connected clients.- let channels_connection_gateway =- Arc::new(connection_gateway_client::ConnectionGatewayClient::new(- internal_api_key.clone(),- ConnectionGatewayUrl::new()?.to_string(),- ));- let side_effect_clients = ai_tools::ChannelSideEffectClients {- connection_gateway: channels_connection_gateway.clone(),- sqs: aws_sdk_sqs::Client::new(&aws_config),- macro_event_broker: macro_event_broker.clone(),- };- let lexical_client_for_tools = (*lexical_client).clone();- let document_tool_context = DocumentToolContext::new(- document_service,- (*entity_access_service).clone(),- lexical_client_for_tools,- sync_service_client.clone(),- documents::outbound::editing_worker_client::ReqwestEditingWorkerClient::new(- config.ai_editing_worker_url.clone(),- std::sync::Arc::new(reqwest::Client::new()),- ),- config.document_permission_jwt.as_ref().to_string(),- ai_tools::build_message_service_with_side_effects(- db.clone(),- lexical_client.clone(),- &side_effect_clients,- ),- )- .with_presentation_files(Arc::new(document_files.clone()))- .with_design_files(Arc::new(document_files.clone()))- .with_photoshop_files(Arc::new(document_files.clone()))- .with_illustrator_files(Arc::new(document_files));-- tracing::info!("initialized document tool context");-- let attachment_provider = attachment::provider::AttachmentProvider {- document: documents::inbound::attachment::DocumentAttachmentService::new(- document_tool_context.service.clone(),- document_tool_context.entity_access_service.clone(),- document_tool_context.lexical_client.clone(),- ),- email_thread: email::inbound::attachment::EmailAttachmentService::new(- email_service_for_tools.clone(),- entity_access_service.clone(),- ),- chat: chat::inbound::attachment::ChatAttachmentService::new(- Arc::new(chat::outbound::postgres::PgChatRepo::new(- db.clone(),- owned_entity_registrar.clone(),- )),- entity_access_service.clone(),- ),- channel: channels::inbound::attachment::ChannelAttachmentService::new(- Arc::new(PgChannelsRepo::new(db.clone())),- entity_access_service.clone(),- ),- static_file: static_file::inbound::attachment::StaticFileAttachmentService::new(Arc::new(- static_file::outbound::CdnStaticFileRepo::new(StaticFileServiceUrl::new()?.to_string()),- )),- };- let message_service = Arc::new(- chat::domain::service::MessageServiceImpl::new(- chat::outbound::postgres::PgChatRepo::new(db.clone(), owned_entity_registrar.clone()),- attachment_provider,- )- .with_event_broker(macro_event_broker.clone()),- );-- tracing::info!("initialized attachment provider");-- let email_service_client_external = Arc::new(EmailServiceClientExternal::new(- email_service_client.url().to_owned(),- ));-- let search_service_client = Arc::new(search_service_client);-- let properties_tool_context = ai_tools::build_properties_tool_context(- properties_service.clone(),- entity_access_service.clone(),- );-- tracing::info!("initialized properties tool context");-- let user_email_service = Arc::new(- EmailServiceImpl::new(- EmailPgRepo::new(db.clone()),- FrecencyQueryServiceImpl::new(FrecencyPgStorage::new(db.clone())),- sqs_client.clone(),- crm_service.clone(),- entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(- entity_access_management::outbound::PgRepository::new(db.clone()),- ),- 0,- )- .with_macro_event_broker(macro_event_broker.clone()),- );- let email_tool_context = email::inbound::toolset::EmailToolContext::new(- user_email_service.clone(),- Arc::new(email::domain::ports::NoOpGmailTokenProvider),- Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(- db.clone(),- ))),- lexical_client.clone(),- );-- tracing::info!("initialized email tool context");-- let call_service = CallServiceImpl::new(- PgCallRepo::new(db.clone()),- NoOpCallRtcClient,- NoOpConnectionService,- (*entity_access_service).clone(),- NoOpNotificationIngress,- None::<S3RecordingStorage>,- String::new(),- );- let call_tool_context = CallToolContext::new(call_service, (*entity_access_service).clone());-- tracing::info!("initialized call tool context");-- let chat_tool_context = chat::inbound::toolset::ChatToolContext::new(- chat::domain::service::ChatServiceImpl::new(- chat::outbound::postgres::PgChatRepo::new(db.clone(), owned_entity_registrar.clone()),- Arc::new(ai_toolset::AsyncToolCollection::new()),- (),- entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(- entity_access_management::outbound::PgRepository::new(db.clone()),- ),- ),- (*entity_access_service).clone(),- );-- tracing::info!("initialized chat tool context");-- let channel_tool_context = ai_tools::build_channel_tool_context_with_side_effects(- db.clone(),- lexical_client.clone(),- &side_effect_clients,- );- let user_permissions_service = Arc::new(- roles_and_permissions::domain::service::UserRolesAndPermissionsServiceImpl::new(- roles_and_permissions::outbound::pgpool::MacroDB::new(db.clone()),- roles_and_permissions::outbound::pgpool::MacroDB::new(db.clone()),- ),- );-- // The AI billing gate reads allowances, credits, and overage state here;- // collection (Stripe) lives in the authentication service, which the- // recorder below asks to settle once a payer runs past their allowance- // and ENABLE_AI_USAGE_BILLING is enabled. This instance never settles.- let auth_service_client = Arc::new(authentication_service_client::AuthServiceClient::new(- config- .authentication_service_secret_key- .as_ref()- .to_string(),- AuthServiceUrl::new()?.to_string(),- ));- let ai_pricing = config.ai_pricing();- let ai_billing = Arc::new(- ai_billing::domain::BillingServiceImpl::new(- ai_billing::outbound::RolesTeamsEntitlementSource::new(- (*user_permissions_service).clone(),- teams::outbound::team_repo::TeamRepositoryImpl::new(db.clone()),- ),- ai_billing::outbound::PgUsageReader::new(db.clone()),- ai_billing::outbound::PgBillingRepo::new(db.clone(), ai_pricing),- ai_billing::outbound::HttpPaymentGateway::new(auth_service_client.clone()),- ai_pricing,- )- .with_enforcement(config.enable_ai_usage_enforcement),- );- let admission: Arc<dyn ai_billing::AiAdmissionService> =- Arc::new(ai_billing::BillingAdmissionService::new(- ai_billing.clone(),- config.enable_ai_usage_enforcement,- ));- let recorder: Arc<dyn ai_usage::UsageRecorder> =- Arc::new(ai_billing::outbound::SettlingUsageRecorder::new(- Arc::new(- ai_usage::domain::service::UsageServiceImpl::new(- ai_usage::outbound::PgUsageRepo::new(db.clone()),- )- .with_enforcement(config.enable_ai_usage_enforcement),- ),- ai_billing.clone(),- ai_billing::outbound::HttpSettlementTrigger::new(auth_service_client),- config.enable_ai_usage_billing,- ));-- // Per-attempt observations use a separate journal and never feed the- // legacy settlement trigger, which remains counted-usage-only and gated by- // ENABLE_AI_USAGE_BILLING.- let recorder = ai_usage::with_tracking(recorder, ai_usage::pg_tracking(db.clone()));-- // The import pipeline: staged/imported external items, gather jobs over- // the user's connectors, and the Haiku import job. Built before the tool- // service context so the chat toolset gets a wired import context.- let mcp_encryption_key = mcp_client::domain::models::AesKey::try_from(- config.mcp_credentials_key_secret_name.as_ref(),- )- .context("invalid MCP credentials encryption key")?;- let mcp_server_repo =- mcp_client::outbound::pg_server_repo::PgServerRepo::new(db.clone(), mcp_encryption_key);-- // The Pipedream MCP stack, fully separate from the native one above- // (own endpoints, own table, own toolset). Without credentials its- // endpoints answer 501 and its toolsets come up empty. Connect-flow- // webhooks need both URI and secret; a half-set pair is ignored so we- // never mint a callback we cannot serve.- let pipedream_client: ai_tools::ToolPipedreamConnection = match (- config.pipedream_client_id.value(),- config.pipedream_client_secret.value(),- config.pipedream_project_id.value(),- ) {- (Some(client_id), Some(client_secret), Some(project_id)) => {- let webhook_uri = match (- config.pipedream_webhook_uri.value(),- config.pipedream_webhook_secret.value(),- ) {- (Some(uri), Some(_)) => Some(uri.to_owned()),- (None, None) => None,- _ => {- tracing::warn!(- "ignoring incomplete Pipedream webhook config; set both PIPEDREAM_WEBHOOK_URI and PIPEDREAM_WEBHOOK_SECRET"- );- None- }- };- let client = pipedream_mcp::outbound::api::PipedreamClient::new(- pipedream_mcp::outbound::api::PipedreamConfig {- client_id: client_id.to_owned(),- client_secret: client_secret.to_owned(),- project_id: project_id.to_owned(),- environment: config- .pipedream_environment- .value()- .unwrap_or(match config.environment {- Environment::Production => "production",- _ => "development",- })- .to_owned(),- api_url: config- .pipedream_api_url- .value()- .unwrap_or(pipedream_mcp::outbound::api::DEFAULT_API_URL)- .to_owned(),- mcp_url: config- .pipedream_mcp_url- .value()- .unwrap_or(pipedream_mcp::outbound::api::DEFAULT_MCP_URL)- .to_owned(),- allowed_origins: match config.pipedream_allowed_origins.value() {- Some(origins) => origins- .split(',')- .map(|origin| origin.trim().to_owned())- .filter(|origin| !origin.is_empty())- .collect(),- None => match config.environment {- Environment::Production => vec!["https://macro.com".to_owned()],- Environment::Develop => vec![- "https://dev.macro.com".to_owned(),- "http://localhost:3000".to_owned(),- ],- Environment::Local => vec!["http://localhost:3000".to_owned()],- },- },- },- )- .context("failed to build Pipedream client")?;- Some(Arc::new(match webhook_uri {- Some(uri) => client.with_webhook_uri(uri),- None => client,- }))- }- _ => {- tracing::info!("Pipedream credentials not set; Pipedream MCP connectors disabled");- None- }- };- let pipedream_repo =- pipedream_mcp::outbound::pg_connection_repo::PgConnectionRepo::new(db.clone());-- // The one sanctioned meeting point of the two MCP stacks: agents load- // tools through this selector, which prefers a user's Pipedream- // connectors and falls back to the native ones (see `mcp_select`).- // The import service's live Slack source shares this selector.- let mcp_selector: Arc<ai_tools::ToolMcpSelector> = Arc::new(mcp_select::McpToolSelector::new(- Arc::new(mcp_server_repo.clone()),- Arc::new(pipedream_repo.clone()),- Arc::new(pipedream_client.clone()),- ));-- // Nudges the user's connected clients when import rows flip, so setup- // sections and chat surfaces update immediately instead of on the next- // poll (see import::outbound::gateway_notifier).- let import_notify = import::outbound::gateway_notifier::gateway_import_notify(- channels_connection_gateway.clone(),- );-- let entity_creator = ai_tools::ToolEntityCreator {- document_creator: document_tool_context.creator.clone(),- entity_access_service: entity_access_service.clone(),- channel_service: channel_tool_context.service.clone(),- task_properties: task_properties_for_import,- document_properties: document_properties_for_import,- team_repository: ai_tools::build_team_repository(db.clone()),- };- let import_service = Arc::new(- import::domain::service::ImportServiceImpl::new(- import::outbound::pg_import_repo::PgImportRepo::new(db.clone()),- mcp_selector.clone(),- Arc::new(entity_creator),- recorder.clone(),- )- .with_slack_source(Arc::new(- import::outbound::mcp_slack_source::McpSlackSource::new(mcp_selector.clone()),- ))- .with_admission(admission.clone())- .with_notifier(import_notify),- );-- // No boot-time recovery: running batches heartbeat their rows, and the- // read path reaps rows whose heartbeat stopped — safe with multiple- // replicas, where a boot-time sweep would clobber other instances' jobs.- tracing::info!("initialized import service");-- let project_tool_context = ai_tools::build_project_tool_context(- db.clone(),- macro_event_broker.clone(),- entity_access_service.clone(),- document_tool_context.service.clone(),- chat_tool_context.service.clone(),- user_email_service.clone(),- );-- let initiative_tool_context = ai_tools::build_initiative_tool_context(- db.clone(),- &document_tool_context,- properties_service.clone(),- entity_access_service.clone(),- macro_event_broker.clone(),- );-- let databases_tool_context = ai_tools::build_databases_tool_context(- db.clone(),- entity_access_service.clone(),- ai_tools::ToolTableEventPublisher::Gateway(- databases::outbound::gateway_event_publisher::GatewayTableEventPublisher::new(- channels_connection_gateway.as_ref().clone(),- ),- ),- ai_tools::MaybeToolEventBroker::Real(macro_event_broker.clone()),- );- let databases_sql_tool_context = ai_tools::build_databases_sql_tool_context(- &databases_tool_context,- soup_service.clone(),- db.clone(),- );-- let tool_service_context = ai_tools::ToolServiceContext {- search_service_client: search_service_client.clone(),- email_service_client: email_service_client_external.clone(),- soup_service: soup_service.clone(),- email_service: email_service_for_tools.clone(),- activity_tool_context: ai_tools::build_activity_tool_context(- db.clone(),- properties_service,- entity_access_service.clone(),- ),- image_generation_tool_context: ai_tools::build_image_generation_tool_context(- &document_tool_context,- ai_tools::build_image_generator_from_env(),- recorder.clone(),- )?,- document_tool_context: document_tool_context.clone(),- properties_tool_context: properties_tool_context.clone(),- email_tool_context: email_tool_context.clone(),- call_tool_context: call_tool_context.clone(),- booking_link_tool_context: ai_tools::build_booking_link_tool_context(- db.clone(),- config.environment,- ),- calendar_tool_context: ai_tools::build_calendar_tool_context(- db.clone(),- CalendarServiceUrl::new()?,- internal_api_key.clone(),- ),- notification_tool_context: notification_tool_context.clone(),- databases_tool_context,- databases_sql_tool_context,- import_tool_context: import::inbound::toolset::ImportToolContext::wired(- import_service.clone(),- ),- chat_tool_context,- channel_tool_context,- bot_tool_context: ai_tools::build_bot_tool_context(- db.clone(),- ai_tools::MaybeToolEventBroker::Real(macro_event_broker.clone()),- entity_access_service.clone(),- DocumentStorageServiceUrl::new()?.to_string(),- pipedream_client.clone(),- ),- coding_agent_tool_context: ai_tools::build_coding_agent_tool_context(- macro_service_urls::AgentHarnessServiceUrl::new()?,- internal_api_key.clone(),- )?,- project_tool_context,- initiative_tool_context,- team_tool_context: ai_tools::build_team_tool_context(db.clone()),- crm_tool_context: ai_tools::build_crm_tool_context(db.clone()),- skill_tool_context: ai_tools::build_skill_tool_context(- search_service_client.clone(),- soup_service.clone(),- &document_tool_context,- ),- schedule_tool_context: ai_tools::build_routine_tool_context()?,- anthropic_tool_context: ai_tools::build_anthropic_tool_context(),- admission,- recorder,- usage_context: ai_usage::UsageContext::system(ai_usage::AiFeature::Chat),- };- let all_tools = ai_tools::tools_for(ai_tools::AiHost::Chat);- let all_tools_toolset = all_tools.toolset.clone();- let all_tools_prompt: Arc<dyn std::fmt::Display + Send + Sync> =- Arc::new(all_tools.prompt.to_string());-- // Build memory service- let memory_repo = memory::outbound::pg_memory_repo::PgMemoryRepo::new(db.clone());- let memory_service = Arc::new(memory::domain::service::MemoryServiceImpl::new(- memory_repo,- tool_service_context.clone(),- all_tools,- ));-- tracing::info!("initialized memory service");-- // Build the AI cost service. It backs both the admin query/pricing router- // and the usage recorder threaded through the tool service context.- let usage_service = Arc::new(- ai_usage::domain::service::UsageServiceImpl::new(ai_usage::outbound::PgUsageRepo::new(- db.clone(),- ))- .with_enforcement(config.enable_ai_usage_enforcement),- );-- tracing::info!("initialized ai cost service");-- // Generator that materializes projections by running their prompt through- // the shared AI toolset, attributed to the requesting user.- let projection_generator =- ai_projections::outbound::agent_generator::AgentProjectionGenerator::new(- tool_service_context.clone(),- ai_tools::tools_for(ai_tools::AiHost::Chat),- );- // Notifier that pushes finished materializations to the target's connected- // clients through the connection gateway.- let projection_notifier =- ai_projections::outbound::gateway_notifier::GatewayProjectionNotifier::new(Arc::new(- connection_gateway_client::ConnectionGatewayClient::new(- internal_api_key.clone(),- ConnectionGatewayUrl::new()?.to_string(),- ),- ));- let ai_projections_service_impl =- ai_projections::domain::ai_projection_service::AiProjectionServiceImpl::new(- ai_projections::outbound::ai_projection_repo::AiProjectionRepositoryImpl::new(- db.clone(),- ),- sqs_client.clone(),- projection_generator,- projection_notifier,- );-- // Spawn the inbound worker that polls ai_projection_queue and materializes- // projection instances. It owns its own clone of the service.- let ai_projection_worker = ai_projections::worker::AiProjectionWorker::new(- sqs_worker::SQSWorker::new(- aws_sdk_sqs::Client::new(&aws_config),- ai_projection_queue.to_string(),- 10,- 10,- ),- ai_projections_service_impl.clone(),- );- let ai_projection_worker_task = tokio::spawn(async move {- ai_projection_worker.poll().await;- });-- let ai_projections_service = Arc::new(ai_projections_service_impl);-- tracing::info!("initialized ai projections service");-- // The onboarding flow drives the import pipeline: reads/hooks start- // auto-importing gather runs for authenticated connectors, and- // completion deletes unreserved onboarding-staged candidates.- let onboarding_service = Arc::new(onboarding::domain::service::OnboardingServiceImpl::new(- onboarding::outbound::pg_onboarding_repo::PgOnboardingRepo::new(db.clone()),- Arc::new(mcp_server_repo.clone()),- import_service.clone(),- mcp_selector.clone(),- ));-- tracing::info!("initialized onboarding service");-- let mcp_public_url = &config.mcp_public_url;- let mcp_client_metadata = mcp_client::domain::models::OAuthClientMetadata::new(- format!("{mcp_public_url}/mcp/servers/auth/client-metadata"),- format!("{mcp_public_url}/mcp/servers/auth/callback"),- );- let mcp_oauth_state_store =- mcp_client::outbound::redis_state_store::RedisOAuthStateStore::new(redis_client.clone());- let mcp_pre_registered =- mcp_client::domain::provider_registry::PreRegisteredProviders::from_env()?;- let mcp_oauth = mcp_client::outbound::oauth::OAuthService::new(- mcp_server_repo.clone(),- mcp_oauth_state_store,- mcp_client_metadata.clone(),- mcp_pre_registered,- );- // The moment a connector finishes OAuth, reconcile onboarding for that- // user — gather jobs start before the user even returns to their- // original tab. Spawned so the callback response never waits on them.- let onboarding_for_auth_hook = onboarding_service.clone();- let mcp_auth_hook: mcp_client::inbound::axum_router::McpAuthCompletedHook =- Arc::new(move |record: mcp_client::domain::models::McpServerRecord| {- let service = onboarding_for_auth_hook.clone();- Box::pin(async move {- tokio::spawn(async move {- use onboarding::domain::service::OnboardingService;- if let Err(e) = service.reconcile(record.user_id).await {- tracing::warn!(error = ?e, "post-auth onboarding reconcile failed");- }- });- })- });- let mcp_state = mcp_client::inbound::McpRouterState::new(- mcp_server_repo,- mcp_oauth,- authorization_state.clone(),- mcp_client_metadata,- )- .with_auth_completed_hook(mcp_auth_hook);-- // The Pipedream stack gets the same post-connect reconcile hook as the- // native one: a finished Connect flow starts gather jobs immediately.- let onboarding_for_pipedream_hook = onboarding_service.clone();- let pipedream_auth_hook: pipedream_mcp::inbound::axum_router::PipedreamAuthCompletedHook =- Arc::new(- move |connection: pipedream_mcp::domain::models::PipedreamConnection| {- let service = onboarding_for_pipedream_hook.clone();- Box::pin(async move {- tokio::spawn(async move {- use onboarding::domain::service::OnboardingService;- if let Err(e) = service.reconcile(connection.user_id).await {- tracing::warn!(error = ?e, "post-connect onboarding reconcile failed");- }- });- })- },- );- let pipedream_state = pipedream_mcp::inbound::PipedreamRouterState::new(- pipedream_repo,- pipedream_client,- authorization_state.clone(),- )- .with_auth_completed_hook(pipedream_auth_hook);-- let api_result = api::setup_and_serve(ApiContext {- db: db.clone(),- email_service_client_external,- sqs_client: Arc::new(sqs_client),- document_storage_client: Arc::new(document_storage_client),- search_service_client,- authorization_state,- user_permissions_service,- non_user_owners,- ai_admission: tool_service_context.admission.clone(),- ai_billing,- internal_api_key: config.internal_api_key.clone(),- config: Arc::new(config),- notification_ingress_service,- connection_repo: connection_manager.persistence,- connection_gateway_client,- soup_service,- email_service: email_service_for_tools,- stream_repo,- document_tool_context,- memory_service,- usage_service,- ai_projections_service,- properties_tool_context,- email_tool_context: email_tool_context.clone(),- call_tool_context,- tool_service_context,- all_tools: all_tools_toolset,- all_tools_prompt,- entity_access_service,- message_service,- ai_stream_registry: service::ai_stream_registry::AiStreamRegistry::new(- redis_client.clone(),- ),- mcp_state,- pipedream_state,- mcp_selector,- import_service,- onboarding_service,- macro_event_broker: macro_event_broker.clone(),- })- .await- .context("failed to setup and serve api");-- ai_projection_worker_task.abort();- match ai_projection_worker_task.await {- Err(error) if error.is_cancelled() => {- tracing::info!("ai projection worker stopped");- }- Err(error) => {- tracing::error!(error = ?error, "ai projection worker exited unexpectedly");- }- Ok(()) => {- tracing::error!(- error = "worker exited naturally",- "ai projection worker exited unexpectedly"- );- }- }-- tracing::info!("waiting for event broker publishes to drain");- event_broker_tracker.close();- match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {- Ok(()) => tracing::info!("event broker publishes drained"),- Err(error) => tracing::warn!(- error=?error,- timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),- "timed out waiting for event broker publishes to drain"- ),- }-- api_result+ document_cognition_service::run().await}--const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);diff --git a/services/document_cognition_service/src/openapi.rs b/services/document_cognition_service/src/openapi.rsindex 09b0b1c676..9ecbe00785 100644--- a/services/document_cognition_service/src/openapi.rs+++ b/services/document_cognition_service/src/openapi.rs@@ -1,19 +1,6 @@-#![recursion_limit = "256"]-#![allow(unused)]--mod api;-mod config;-mod core;-mod model;-mod service;--pub use config::Config;-+use document_cognition_service::ApiDoc;use utoipa::OpenApi;fn main() {- println!(- "{}",- api::swagger::ApiDoc::openapi().to_pretty_json().unwrap()- );+ println!("{}", ApiDoc::openapi().to_pretty_json().unwrap());}diff --git a/services/document_cognition_service/src/server.rs b/services/document_cognition_service/src/server.rsnew file mode 100644index 0000000000..3b9edac569--- /dev/null+++ b/services/document_cognition_service/src/server.rs@@ -0,0 +1,979 @@+//! The service's composition root: reads config and wires every adapter and+//! domain service into the HTTP server.++use crate::{+ api::{self, context::ApiContext},+ config::{Config, Environment},+ service,+};+use ai_tools::{NoOpCallRtcClient, NoOpConnectionService, NoOpNotificationIngress};+use anyhow::Context;+use bots::outbound::pg_bots_repo::PgBotsRepo;+use call::domain::service::{CallRecordQueryServiceImpl, CallServiceImpl};+use call::inbound::toolset::CallToolContext;+use call::outbound::pg_call_repo::PgCallRepo;+use call::outbound::s3_recording_storage::S3RecordingStorage;+use channels::{+ domain::list_service::ChannelListServiceImpl, outbound::pg_channels_repo::PgChannelsRepo,+};+use document_storage_service_client::DocumentStorageServiceClient;+use documents::{+ domain::{models::CloudFrontConfig, service::DocumentServiceImpl},+ inbound::toolset::DocumentToolContext,+ outbound::{pg_document_repo::PgDocumentRepo, s3_upload_url::S3UploadUrlAdapter},+};+use email::domain::ports::ReadonlyEmailPreviewAdapter;+use email::domain::service::EmailServiceImpl;+use email::outbound::EmailPgRepo;+use email_service_client::{EmailServiceClient, EmailServiceClientExternal};+use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};+use entity_registry::OwnerGrantPolicy;+use entity_registry_db_utils::OwnedEntityRegistrar;+use foreign_entity::{+ domain::service::ForeignEntityServiceImpl,+ outbound::pg_foreign_entity_repo::PgForeignEntityRepo,+};+use frecency::domain::services::FrecencyQueryServiceImpl;+use frecency::outbound::postgres::FrecencyPgStorage;+use github_pull_requests::{+ domain::service::GithubPullRequestServiceImpl,+ outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo,+};+use macro_auth::middleware::decode_jwt::JwtValidationArgs;+use macro_authorization::{+ InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationServiceImpl,+ MacroAuthorizationState, PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,+};+use macro_service_urls::{+ AuthServiceUrl, CalendarServiceUrl, ConnectionGatewayUrl, DocumentStorageServiceUrl,+ EmailServiceUrl, LexicalServiceUrl, StaticFileServiceUrl, SyncServiceUrl,+};+use notification::domain::service::{+ NotificationReaderService, PlatformArnConfig, SqsNotificationIngress,+};+use notification::outbound::queue::SqsQueue;+use notification::outbound::repository::DbNotificationRepository;+use notification::outbound::websocket::ConnectionGatewayClient;+use readonly_pool::ReadOnlyPool;+use search_service_client::SearchServiceClient;+use sqlx::postgres::PgPoolOptions;+use std::{sync::Arc, time::Duration};+use stream::outbound::redis_pg::RedisPostgresStreamRepo;+use sync_service_client::SyncServiceClient;+use tokio_util::task::TaskTracker;++/// Build the service from the environment and serve until shutdown.+pub async fn run() -> anyhow::Result<()> {+ let aws_config = macro_aws_config::get_macro_aws_config().await;++ let secretsmanager_client = secretsmanager_client::SecretsManager::new(+ aws_sdk_secretsmanager::Client::new(&aws_config),+ );++ // Parse our configuration from the environment.+ let config = Config::from_env()+ .context("expected to be able to generate config")?+ .resolve_remote_secrets(Environment::new_or_prod(), &secretsmanager_client)+ .await+ .context("expected to be able to resolve config secrets")?;+ let non_user_owners = config.non_user_owners()?;++ tracing::info!("initialized config");++ let (min_connections, max_connections): (u32, u32) = match config.environment {+ Environment::Production => (5, 30),+ Environment::Develop => (3, 20),+ Environment::Local => (3, 10),+ };++ let db = PgPoolOptions::new()+ .min_connections(min_connections)+ .max_connections(max_connections)+ .connect(&config.database_url)+ .await+ .context("failed to connect to macrodb")?;++ tracing::info!(+ min_connections,+ max_connections,+ "initialized db connection"+ );++ let dynamodb_client = aws_sdk_dynamodb::Client::new(&aws_config);+ let queue_aws_client = aws_sdk_sqs::Client::new(&aws_config);++ let document_text_extractor_queue = macro_queues::DocumentTextExtractorQueue::new();+ let chat_delete_queue = macro_queues::ChatDeleteQueue::new();+ let email_scheduled_queue = macro_queues::EmailScheduledQueue::new();+ let gmail_ops_queue = macro_queues::GmailOpsQueue::new();+ let ai_projection_queue = macro_queues::AiProjectionQueue::new();+ let notification_queue = macro_queues::NotificationIngressQueue::new();+ let sqs_client = sqs_client::SQS::new(queue_aws_client)+ .document_text_extractor_queue(&document_text_extractor_queue)+ .chat_delete_queue(&chat_delete_queue)+ .email_scheduled_queue(&email_scheduled_queue)+ .gmail_ops_queue(&gmail_ops_queue)+ .ai_projection_queue(&ai_projection_queue);++ let internal_api_key = config.internal_api_key.to_string();++ let document_storage_client = DocumentStorageServiceClient::new(+ config+ .document_storage_service_auth_key+ .as_ref()+ .to_string(),+ DocumentStorageServiceUrl::new()?.to_string(),+ );++ tracing::info!("initialized dss client");++ let sync_service_client = SyncServiceClient::new(+ config.sync_service_auth_key.as_ref().to_string(),+ SyncServiceUrl::new()?.to_string(),+ );+ tracing::info!("initialized sync service client");++ let search_service_client = SearchServiceClient::new(+ config+ .document_storage_service_auth_key+ .as_ref()+ .to_string(),+ DocumentStorageServiceUrl::new()?.to_string(),+ );+ tracing::info!("initialized search service client");++ let jwt_args =+ JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)+ .await+ .context("failed to create jwt validation args")?;++ let authorization_state =+ MacroAuthorizationState::new(Arc::new(MacroAuthorizationServiceImpl::new(+ MacroAuthJwtValidator::new(jwt_args),+ InternalAuthConfig {+ api_key: internal_api_key.clone(),+ default_user_id: None,+ },+ macro_authorization::NoBotAuthorizer,+ PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),+ )));++ let lexical_client = Arc::new(lexical_client::LexicalClient::new(+ internal_api_key.clone(),+ LexicalServiceUrl::new()?.to_string(),+ ));++ let email_service_client = Arc::new(EmailServiceClient::new(+ internal_api_key.clone(),+ EmailServiceUrl::new()?.to_string(),+ ));++ tracing::info!("initialized static file service client");++ // Initialize Redis client for stream service+ let redis_client = Arc::new(+ redis::Client::open(config.redis_host.as_ref())+ .inspect(|client| {+ client+ .get_connection()+ .map(|_| tracing::trace!("initialized redis connection"))+ .inspect_err(|e| {+ tracing::error!(error=?e, "failed to connect to redis");+ })+ .expect("redis connetion required");+ })+ .context("failed to connect to redis")?,+ );+ let stream_repo = RedisPostgresStreamRepo::new((*redis_client).clone(), db.clone()).obj();++ tracing::info!("initialized stream repo");++ let connection_manager =+ connection_gateway::service::dynamodb::create_dynamo_db_connection_manager(dynamodb_client)+ .await+ .context("failed to create connection manager")?;++ tracing::info!("initialized connection repo");+ let connection_gateway_client = Arc::new(ConnectionGatewayClient::new(+ internal_api_key.clone(),+ ConnectionGatewayUrl::new()?.to_string(),+ ));++ let ingress_queue = SqsQueue::new(+ aws_sdk_sqs::Client::new(&aws_config),+ notification_queue.to_string(),+ );+ let notification_ingress_service = Arc::new(SqsNotificationIngress {+ queue: ingress_queue,+ });++ let notification_reader_queue = SqsQueue::new(+ aws_sdk_sqs::Client::new(&aws_config),+ notification_queue.to_string(),+ );+ let notification_reader_service = NotificationReaderService {+ repository: DbNotificationRepository::new(db.clone()),+ queue: ai_tools::ToolNotificationQueue::Sqs(notification_reader_queue),+ sns_endpoint: ai_tools::NoOpSnsEndpointManager,+ platform_config: PlatformArnConfig {+ apns_platform_arn: String::new(),+ fcm_platform_arn: String::new(),+ apns_voip_platform_arn: String::new(),+ },+ realtime: notification::domain::ports::NoopNotificationRealtimePublisher,+ };+ let notification_tool_context =+ notification::inbound::ai_tool::NotificationToolContext::new(notification_reader_service);++ tracing::info!("initialized notification ingress service");+ let entity_access_service = Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(+ db.clone(),+ )));++ let frecency_storage = FrecencyPgStorage::new(db.clone());+ let frecency_service = FrecencyQueryServiceImpl::new(frecency_storage.clone());+ let crm_service = crm::domain::service::CrmServiceImpl::new(+ crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),+ crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,+ );+ let email_service = EmailServiceImpl::new(+ EmailPgRepo::new(db.clone()),+ frecency_service.clone(),+ email::domain::ports::NoOpEnqueuer,+ crm_service.clone(),+ entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(+ entity_access_management::outbound::PgRepository::new(db.clone()),+ ),+ 0,+ );+ let channels_service = ChannelListServiceImpl::new(+ PgChannelsRepo::new(db.clone()),+ PgChannelsRepo::new(db.clone()),+ frecency_storage,+ );+ let email_service_for_tools: Arc<ai_tools::ToolEmailService> = Arc::new(email_service.clone());+ let github_pull_request_service = GithubPullRequestServiceImpl::new(+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),+ PgGithubPullRequestRepo::new(db.clone()),+ );+ let soup_service = Arc::new(soup::domain::service::SoupImpl::new(+ soup::outbound::pg_soup_repo::PgSoupRepo::new(ReadOnlyPool(db.clone())),+ frecency_service,+ ReadonlyEmailPreviewAdapter(email_service),+ channels_service,+ CallRecordQueryServiceImpl::new(PgCallRepo::new(db.clone())),+ crm::domain::service::NoOpCrmService,+ github_pull_request_service,+ ));++ tracing::info!("initialized soup service");++ let s3_client = macro_aws_config::s3_client().await;+ let document_files = documents::outbound::s3_document_files::S3DocumentFiles::new(+ db.clone(),+ s3_client.clone(),+ config.document_storage_bucket.to_string(),+ );+ let s3_upload_adapter = S3UploadUrlAdapter::new(+ s3_client,+ config.document_storage_bucket.to_string(),+ config.docx_document_upload_bucket.to_string(),+ );+ let owned_entity_registrar =+ OwnedEntityRegistrar::new(OwnerGrantPolicy::new(PgBotsRepo::new(db.clone())));+ let document_repo = PgDocumentRepo::new(db.clone(), owned_entity_registrar.clone());++ let cloudfront_config = CloudFrontConfig {+ distribution_url: config+ .document_storage_service_cloudfront_distribution_url+ .to_string(),+ signer_public_key_id: config+ .document_storage_service_cloudfront_signer_public_key_id+ .to_string(),+ signer_private_key: config+ .document_storage_service_cloudfront_signer_private_key+ .as_ref()+ .to_string(),+ presigned_url_expiry_seconds: 3600,+ browser_cache_expiry_seconds: 86400,+ };+ let event_broker_tracker = TaskTracker::new();+ let macro_event_broker = macro_event_broker::MacroEventBrokerService::new(+ macro_event_broker::KafkaEventPublisher::new(config.kafka_brokers.as_ref())+ .context("failed to create kafka event publisher")?,+ event_broker_tracker.clone(),+ );+ let properties_service = ai_tools::build_properties_service_with_broker(+ db.clone(),+ entity_access_service.clone(),+ macro_event_broker.clone(),+ );+ let task_properties_service = ai_tools::build_task_properties_adapter(+ db.clone(),+ properties_service.clone(),+ entity_access_service.clone(),+ );+ // The import pipeline sets the same task system properties on imported+ // Linear issues (status, priority, due date, assignee).+ let task_properties_for_import = task_properties_service.clone();+ let document_properties_for_import =+ import::outbound::document_properties::DocumentPropertiesApplicator::new(+ properties_service.clone(),+ );+ let document_service = DocumentServiceImpl::new(+ document_repo,+ cloudfront_config,+ sync_service_client.clone(),+ s3_upload_adapter,+ task_properties_service,+ NoOpConnectionService,+ entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(+ entity_access_management::outbound::PgRepository::new(db.clone()),+ ),+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),+ macro_event_broker.clone(),+ );+ // Messages sent by AI tools (chat, agents) dispatch the same side effects+ // as the document-storage message API, so mentions, replies and document+ // comments notify recipients and stream to connected clients.+ let channels_connection_gateway =+ Arc::new(connection_gateway_client::ConnectionGatewayClient::new(+ internal_api_key.clone(),+ ConnectionGatewayUrl::new()?.to_string(),+ ));+ let side_effect_clients = ai_tools::ChannelSideEffectClients {+ connection_gateway: channels_connection_gateway.clone(),+ sqs: aws_sdk_sqs::Client::new(&aws_config),+ macro_event_broker: macro_event_broker.clone(),+ };+ let lexical_client_for_tools = (*lexical_client).clone();+ let document_tool_context = DocumentToolContext::new(+ document_service,+ (*entity_access_service).clone(),+ lexical_client_for_tools,+ sync_service_client.clone(),+ documents::outbound::editing_worker_client::ReqwestEditingWorkerClient::new(+ config.ai_editing_worker_url.clone(),+ std::sync::Arc::new(reqwest::Client::new()),+ ),+ config.document_permission_jwt.as_ref().to_string(),+ ai_tools::build_message_service_with_side_effects(+ db.clone(),+ lexical_client.clone(),+ &side_effect_clients,+ ),+ )+ .with_presentation_files(Arc::new(document_files.clone()))+ .with_design_files(Arc::new(document_files.clone()))+ .with_photoshop_files(Arc::new(document_files.clone()))+ .with_illustrator_files(Arc::new(document_files));++ tracing::info!("initialized document tool context");++ let attachment_provider = attachment::provider::AttachmentProvider {+ document: documents::inbound::attachment::DocumentAttachmentService::new(+ document_tool_context.service.clone(),+ document_tool_context.entity_access_service.clone(),+ document_tool_context.lexical_client.clone(),+ ),+ email_thread: email::inbound::attachment::EmailAttachmentService::new(+ email_service_for_tools.clone(),+ entity_access_service.clone(),+ ),+ chat: chat::inbound::attachment::ChatAttachmentService::new(+ Arc::new(chat::outbound::postgres::PgChatRepo::new(+ db.clone(),+ owned_entity_registrar.clone(),+ )),+ entity_access_service.clone(),+ ),+ channel: channels::inbound::attachment::ChannelAttachmentService::new(+ Arc::new(PgChannelsRepo::new(db.clone())),+ entity_access_service.clone(),+ ),+ static_file: static_file::inbound::attachment::StaticFileAttachmentService::new(Arc::new(+ static_file::outbound::CdnStaticFileRepo::new(StaticFileServiceUrl::new()?.to_string()),+ )),+ };+ let message_service = Arc::new(+ chat::domain::service::MessageServiceImpl::new(+ chat::outbound::postgres::PgChatRepo::new(db.clone(), owned_entity_registrar.clone()),+ attachment_provider,+ )+ .with_event_broker(macro_event_broker.clone()),+ );++ tracing::info!("initialized attachment provider");++ let email_service_client_external = Arc::new(EmailServiceClientExternal::new(+ email_service_client.url().to_owned(),+ ));++ let search_service_client = Arc::new(search_service_client);++ let properties_tool_context = ai_tools::build_properties_tool_context(+ properties_service.clone(),+ entity_access_service.clone(),+ );++ tracing::info!("initialized properties tool context");++ let user_email_service = Arc::new(+ EmailServiceImpl::new(+ EmailPgRepo::new(db.clone()),+ FrecencyQueryServiceImpl::new(FrecencyPgStorage::new(db.clone())),+ sqs_client.clone(),+ crm_service.clone(),+ entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(+ entity_access_management::outbound::PgRepository::new(db.clone()),+ ),+ 0,+ )+ .with_macro_event_broker(macro_event_broker.clone()),+ );+ let email_tool_context = email::inbound::toolset::EmailToolContext::new(+ user_email_service.clone(),+ Arc::new(email::domain::ports::NoOpGmailTokenProvider),+ Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(+ db.clone(),+ ))),+ lexical_client.clone(),+ );++ tracing::info!("initialized email tool context");++ let call_service = CallServiceImpl::new(+ PgCallRepo::new(db.clone()),+ NoOpCallRtcClient,+ NoOpConnectionService,+ (*entity_access_service).clone(),+ NoOpNotificationIngress,+ None::<S3RecordingStorage>,+ String::new(),+ );+ let call_tool_context = CallToolContext::new(call_service, (*entity_access_service).clone());++ tracing::info!("initialized call tool context");++ let chat_tool_context = chat::inbound::toolset::ChatToolContext::new(+ chat::domain::service::ChatServiceImpl::new(+ chat::outbound::postgres::PgChatRepo::new(db.clone(), owned_entity_registrar.clone()),+ Arc::new(ai_toolset::AsyncToolCollection::new()),+ (),+ entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(+ entity_access_management::outbound::PgRepository::new(db.clone()),+ ),+ ),+ (*entity_access_service).clone(),+ );++ tracing::info!("initialized chat tool context");++ let channel_tool_context = ai_tools::build_channel_tool_context_with_side_effects(+ db.clone(),+ lexical_client.clone(),+ &side_effect_clients,+ );+ let user_permissions_service = Arc::new(+ roles_and_permissions::domain::service::UserRolesAndPermissionsServiceImpl::new(+ roles_and_permissions::outbound::pgpool::MacroDB::new(db.clone()),+ roles_and_permissions::outbound::pgpool::MacroDB::new(db.clone()),+ ),+ );++ // The AI billing gate reads allowances, credits, and overage state here;+ // collection (Stripe) lives in the authentication service, which the+ // recorder below asks to settle once a payer runs past their allowance+ // and ENABLE_AI_USAGE_BILLING is enabled. This instance never settles.+ let auth_service_client = Arc::new(authentication_service_client::AuthServiceClient::new(+ config+ .authentication_service_secret_key+ .as_ref()+ .to_string(),+ AuthServiceUrl::new()?.to_string(),+ ));+ let ai_pricing = config.ai_pricing();+ let ai_billing = Arc::new(+ ai_billing::domain::BillingServiceImpl::new(+ ai_billing::outbound::RolesTeamsEntitlementSource::new(+ (*user_permissions_service).clone(),+ teams::outbound::team_repo::TeamRepositoryImpl::new(db.clone()),+ ),+ ai_billing::outbound::PgUsageReader::new(db.clone()),+ ai_billing::outbound::PgBillingRepo::new(db.clone(), ai_pricing),+ ai_billing::outbound::HttpPaymentGateway::new(auth_service_client.clone()),+ ai_pricing,+ )+ .with_enforcement(config.enable_ai_usage_enforcement),+ );+ let admission: Arc<dyn ai_billing::AiAdmissionService> =+ Arc::new(ai_billing::BillingAdmissionService::new(+ ai_billing.clone(),+ config.enable_ai_usage_enforcement,+ ));+ let recorder: Arc<dyn ai_usage::UsageRecorder> =+ Arc::new(ai_billing::outbound::SettlingUsageRecorder::new(+ Arc::new(+ ai_usage::domain::service::UsageServiceImpl::new(+ ai_usage::outbound::PgUsageRepo::new(db.clone()),+ )+ .with_enforcement(config.enable_ai_usage_enforcement),+ ),+ ai_billing.clone(),+ ai_billing::outbound::HttpSettlementTrigger::new(auth_service_client),+ config.enable_ai_usage_billing,+ ));++ // Per-attempt observations use a separate journal and never feed the+ // legacy settlement trigger, which remains counted-usage-only and gated by+ // ENABLE_AI_USAGE_BILLING.+ let recorder = ai_usage::with_tracking(recorder, ai_usage::pg_tracking(db.clone()));++ // The import pipeline: staged/imported external items, gather jobs over+ // the user's connectors, and the Haiku import job. Built before the tool+ // service context so the chat toolset gets a wired import context.+ let mcp_encryption_key = mcp_client::domain::models::AesKey::try_from(+ config.mcp_credentials_key_secret_name.as_ref(),+ )+ .context("invalid MCP credentials encryption key")?;+ let mcp_server_repo =+ mcp_client::outbound::pg_server_repo::PgServerRepo::new(db.clone(), mcp_encryption_key);++ // The Pipedream MCP stack, fully separate from the native one above+ // (own endpoints, own table, own toolset). Without credentials its+ // endpoints answer 501 and its toolsets come up empty. Connect-flow+ // webhooks need both URI and secret; a half-set pair is ignored so we+ // never mint a callback we cannot serve.+ let pipedream_client: ai_tools::ToolPipedreamConnection = match (+ config.pipedream_client_id.value(),+ config.pipedream_client_secret.value(),+ config.pipedream_project_id.value(),+ ) {+ (Some(client_id), Some(client_secret), Some(project_id)) => {+ let webhook_uri = match (+ config.pipedream_webhook_uri.value(),+ config.pipedream_webhook_secret.value(),+ ) {+ (Some(uri), Some(_)) => Some(uri.to_owned()),+ (None, None) => None,+ _ => {+ tracing::warn!(+ "ignoring incomplete Pipedream webhook config; set both PIPEDREAM_WEBHOOK_URI and PIPEDREAM_WEBHOOK_SECRET"+ );+ None+ }+ };+ let client = pipedream_mcp::outbound::api::PipedreamClient::new(+ pipedream_mcp::outbound::api::PipedreamConfig {+ client_id: client_id.to_owned(),+ client_secret: client_secret.to_owned(),+ project_id: project_id.to_owned(),+ environment: config+ .pipedream_environment+ .value()+ .unwrap_or(match config.environment {+ Environment::Production => "production",+ _ => "development",+ })+ .to_owned(),+ api_url: config+ .pipedream_api_url+ .value()+ .unwrap_or(pipedream_mcp::outbound::api::DEFAULT_API_URL)+ .to_owned(),+ mcp_url: config+ .pipedream_mcp_url+ .value()+ .unwrap_or(pipedream_mcp::outbound::api::DEFAULT_MCP_URL)+ .to_owned(),+ allowed_origins: match config.pipedream_allowed_origins.value() {+ Some(origins) => origins+ .split(',')+ .map(|origin| origin.trim().to_owned())+ .filter(|origin| !origin.is_empty())+ .collect(),+ None => match config.environment {+ Environment::Production => vec!["https://macro.com".to_owned()],+ Environment::Develop => vec![+ "https://dev.macro.com".to_owned(),+ "http://localhost:3000".to_owned(),+ ],+ Environment::Local => vec!["http://localhost:3000".to_owned()],+ },+ },+ },+ )+ .context("failed to build Pipedream client")?;+ Some(Arc::new(match webhook_uri {+ Some(uri) => client.with_webhook_uri(uri),+ None => client,+ }))+ }+ _ => {+ tracing::info!("Pipedream credentials not set; Pipedream MCP connectors disabled");+ None+ }+ };+ let pipedream_repo =+ pipedream_mcp::outbound::pg_connection_repo::PgConnectionRepo::new(db.clone());++ // The one sanctioned meeting point of the two MCP stacks: agents load+ // tools through this selector, which prefers a user's Pipedream+ // connectors and falls back to the native ones (see `mcp_select`).+ // The import service's live Slack source shares this selector.+ let mcp_selector: Arc<ai_tools::ToolMcpSelector> = Arc::new(mcp_select::McpToolSelector::new(+ Arc::new(mcp_server_repo.clone()),+ Arc::new(pipedream_repo.clone()),+ Arc::new(pipedream_client.clone()),+ ));++ // Nudges the user's connected clients when import rows flip, so setup+ // sections and chat surfaces update immediately instead of on the next+ // poll (see import::outbound::gateway_notifier).+ let import_notify = import::outbound::gateway_notifier::gateway_import_notify(+ channels_connection_gateway.clone(),+ );++ let entity_creator = ai_tools::ToolEntityCreator {+ document_creator: document_tool_context.creator.clone(),+ entity_access_service: entity_access_service.clone(),+ channel_service: channel_tool_context.service.clone(),+ task_properties: task_properties_for_import,+ document_properties: document_properties_for_import,+ team_repository: ai_tools::build_team_repository(db.clone()),+ };+ let import_service = Arc::new(+ import::domain::service::ImportServiceImpl::new(+ import::outbound::pg_import_repo::PgImportRepo::new(db.clone()),+ mcp_selector.clone(),+ Arc::new(entity_creator),+ recorder.clone(),+ )+ .with_slack_source(Arc::new(+ import::outbound::mcp_slack_source::McpSlackSource::new(mcp_selector.clone()),+ ))+ .with_admission(admission.clone())+ .with_notifier(import_notify),+ );++ // No boot-time recovery: running batches heartbeat their rows, and the+ // read path reaps rows whose heartbeat stopped — safe with multiple+ // replicas, where a boot-time sweep would clobber other instances' jobs.+ tracing::info!("initialized import service");++ let project_tool_context = ai_tools::build_project_tool_context(+ db.clone(),+ macro_event_broker.clone(),+ entity_access_service.clone(),+ document_tool_context.service.clone(),+ chat_tool_context.service.clone(),+ user_email_service.clone(),+ );++ let initiative_tool_context = ai_tools::build_initiative_tool_context(+ db.clone(),+ &document_tool_context,+ properties_service.clone(),+ entity_access_service.clone(),+ macro_event_broker.clone(),+ );++ let databases_tool_context = ai_tools::build_databases_tool_context(+ db.clone(),+ entity_access_service.clone(),+ ai_tools::ToolTableEventPublisher::Gateway(+ databases::outbound::gateway_event_publisher::GatewayTableEventPublisher::new(+ channels_connection_gateway.as_ref().clone(),+ ),+ ),+ ai_tools::MaybeToolEventBroker::Real(macro_event_broker.clone()),+ );+ let databases_sql_tool_context = ai_tools::build_databases_sql_tool_context(+ &databases_tool_context,+ soup_service.clone(),+ db.clone(),+ );++ let tool_service_context = ai_tools::ToolServiceContext {+ search_service_client: search_service_client.clone(),+ email_service_client: email_service_client_external.clone(),+ soup_service: soup_service.clone(),+ email_service: email_service_for_tools.clone(),+ activity_tool_context: ai_tools::build_activity_tool_context(+ db.clone(),+ properties_service,+ entity_access_service.clone(),+ ),+ image_generation_tool_context: ai_tools::build_image_generation_tool_context(+ &document_tool_context,+ ai_tools::build_image_generator_from_env(),+ recorder.clone(),+ )?,+ document_tool_context: document_tool_context.clone(),+ properties_tool_context: properties_tool_context.clone(),+ email_tool_context: email_tool_context.clone(),+ call_tool_context: call_tool_context.clone(),+ booking_link_tool_context: ai_tools::build_booking_link_tool_context(+ db.clone(),+ config.environment,+ ),+ calendar_tool_context: ai_tools::build_calendar_tool_context(+ db.clone(),+ CalendarServiceUrl::new()?,+ internal_api_key.clone(),+ ),+ notification_tool_context: notification_tool_context.clone(),+ databases_tool_context,+ databases_sql_tool_context,+ import_tool_context: import::inbound::toolset::ImportToolContext::wired(+ import_service.clone(),+ ),+ chat_tool_context,+ channel_tool_context,+ bot_tool_context: ai_tools::build_bot_tool_context(+ db.clone(),+ ai_tools::MaybeToolEventBroker::Real(macro_event_broker.clone()),+ entity_access_service.clone(),+ DocumentStorageServiceUrl::new()?.to_string(),+ pipedream_client.clone(),+ ),+ coding_agent_tool_context: ai_tools::build_coding_agent_tool_context(+ macro_service_urls::AgentHarnessServiceUrl::new()?,+ internal_api_key.clone(),+ )?,+ project_tool_context,+ initiative_tool_context,+ team_tool_context: ai_tools::build_team_tool_context(db.clone()),+ crm_tool_context: ai_tools::build_crm_tool_context(db.clone()),+ skill_tool_context: ai_tools::build_skill_tool_context(+ search_service_client.clone(),+ soup_service.clone(),+ &document_tool_context,+ ),+ schedule_tool_context: ai_tools::build_routine_tool_context()?,+ anthropic_tool_context: ai_tools::build_anthropic_tool_context(),+ admission,+ recorder,+ usage_context: ai_usage::UsageContext::system(ai_usage::AiFeature::Chat),+ };+ let all_tools = ai_tools::tools_for(ai_tools::AiHost::Chat);+ let all_tools_toolset = all_tools.toolset.clone();+ let all_tools_prompt: Arc<dyn std::fmt::Display + Send + Sync> =+ Arc::new(all_tools.prompt.to_string());++ // Build memory service+ let memory_repo = memory::outbound::pg_memory_repo::PgMemoryRepo::new(db.clone());+ let memory_service = Arc::new(memory::domain::service::MemoryServiceImpl::new(+ memory_repo,+ tool_service_context.clone(),+ all_tools,+ ));++ tracing::info!("initialized memory service");++ // Build the AI cost service. It backs both the admin query/pricing router+ // and the usage recorder threaded through the tool service context.+ let usage_service = Arc::new(+ ai_usage::domain::service::UsageServiceImpl::new(ai_usage::outbound::PgUsageRepo::new(+ db.clone(),+ ))+ .with_enforcement(config.enable_ai_usage_enforcement),+ );++ tracing::info!("initialized ai cost service");++ // Generator that materializes projections by running their prompt through+ // the shared AI toolset, attributed to the requesting user.+ let projection_generator =+ ai_projections::outbound::agent_generator::AgentProjectionGenerator::new(+ tool_service_context.clone(),+ ai_tools::tools_for(ai_tools::AiHost::Chat),+ );+ // Notifier that pushes finished materializations to the target's connected+ // clients through the connection gateway.+ let projection_notifier =+ ai_projections::outbound::gateway_notifier::GatewayProjectionNotifier::new(Arc::new(+ connection_gateway_client::ConnectionGatewayClient::new(+ internal_api_key.clone(),+ ConnectionGatewayUrl::new()?.to_string(),+ ),+ ));+ let ai_projections_service_impl =+ ai_projections::domain::ai_projection_service::AiProjectionServiceImpl::new(+ ai_projections::outbound::ai_projection_repo::AiProjectionRepositoryImpl::new(+ db.clone(),+ ),+ sqs_client.clone(),+ projection_generator,+ projection_notifier,+ );++ // Spawn the inbound worker that polls ai_projection_queue and materializes+ // projection instances. It owns its own clone of the service.+ let ai_projection_worker = ai_projections::worker::AiProjectionWorker::new(+ sqs_worker::SQSWorker::new(+ aws_sdk_sqs::Client::new(&aws_config),+ ai_projection_queue.to_string(),+ 10,+ 10,+ ),+ ai_projections_service_impl.clone(),+ );+ let ai_projection_worker_task = tokio::spawn(async move {+ ai_projection_worker.poll().await;+ });++ let ai_projections_service = Arc::new(ai_projections_service_impl);++ tracing::info!("initialized ai projections service");++ // The onboarding flow drives the import pipeline: reads/hooks start+ // auto-importing gather runs for authenticated connectors, and+ // completion deletes unreserved onboarding-staged candidates.+ let onboarding_service = Arc::new(onboarding::domain::service::OnboardingServiceImpl::new(+ onboarding::outbound::pg_onboarding_repo::PgOnboardingRepo::new(db.clone()),+ Arc::new(mcp_server_repo.clone()),+ import_service.clone(),+ mcp_selector.clone(),+ ));++ tracing::info!("initialized onboarding service");++ let mcp_public_url = &config.mcp_public_url;+ let mcp_client_metadata = mcp_client::domain::models::OAuthClientMetadata::new(+ format!("{mcp_public_url}/mcp/servers/auth/client-metadata"),+ format!("{mcp_public_url}/mcp/servers/auth/callback"),+ );+ let mcp_oauth_state_store =+ mcp_client::outbound::redis_state_store::RedisOAuthStateStore::new(redis_client.clone());+ let mcp_pre_registered =+ mcp_client::domain::provider_registry::PreRegisteredProviders::from_env()?;+ let mcp_oauth = mcp_client::outbound::oauth::OAuthService::new(+ mcp_server_repo.clone(),+ mcp_oauth_state_store,+ mcp_client_metadata.clone(),+ mcp_pre_registered,+ );+ // The moment a connector finishes OAuth, reconcile onboarding for that+ // user — gather jobs start before the user even returns to their+ // original tab. Spawned so the callback response never waits on them.+ let onboarding_for_auth_hook = onboarding_service.clone();+ let mcp_auth_hook: mcp_client::inbound::axum_router::McpAuthCompletedHook =+ Arc::new(move |record: mcp_client::domain::models::McpServerRecord| {+ let service = onboarding_for_auth_hook.clone();+ Box::pin(async move {+ tokio::spawn(async move {+ use onboarding::domain::service::OnboardingService;+ if let Err(e) = service.reconcile(record.user_id).await {+ tracing::warn!(error = ?e, "post-auth onboarding reconcile failed");+ }+ });+ })+ });+ let mcp_state = mcp_client::inbound::McpRouterState::new(+ mcp_server_repo,+ mcp_oauth,+ authorization_state.clone(),+ mcp_client_metadata,+ )+ .with_auth_completed_hook(mcp_auth_hook);++ // The Pipedream stack gets the same post-connect reconcile hook as the+ // native one: a finished Connect flow starts gather jobs immediately.+ let onboarding_for_pipedream_hook = onboarding_service.clone();+ let pipedream_auth_hook: pipedream_mcp::inbound::axum_router::PipedreamAuthCompletedHook =+ Arc::new(+ move |connection: pipedream_mcp::domain::models::PipedreamConnection| {+ let service = onboarding_for_pipedream_hook.clone();+ Box::pin(async move {+ tokio::spawn(async move {+ use onboarding::domain::service::OnboardingService;+ if let Err(e) = service.reconcile(connection.user_id).await {+ tracing::warn!(error = ?e, "post-connect onboarding reconcile failed");+ }+ });+ })+ },+ );+ let pipedream_state = pipedream_mcp::inbound::PipedreamRouterState::new(+ pipedream_repo,+ pipedream_client,+ authorization_state.clone(),+ )+ .with_auth_completed_hook(pipedream_auth_hook);++ let api_result = api::setup_and_serve(ApiContext {+ db: db.clone(),+ email_service_client_external,+ sqs_client: Arc::new(sqs_client),+ document_storage_client: Arc::new(document_storage_client),+ search_service_client,+ authorization_state,+ user_permissions_service,+ non_user_owners,+ ai_admission: tool_service_context.admission.clone(),+ ai_billing,+ internal_api_key: config.internal_api_key.clone(),+ config: Arc::new(config),+ notification_ingress_service,+ connection_repo: connection_manager.persistence,+ connection_gateway_client,+ soup_service,+ email_service: email_service_for_tools,+ stream_repo,+ document_tool_context,+ memory_service,+ usage_service,+ ai_projections_service,+ properties_tool_context,+ email_tool_context: email_tool_context.clone(),+ call_tool_context,+ tool_service_context,+ all_tools: all_tools_toolset,+ all_tools_prompt,+ entity_access_service,+ message_service,+ ai_stream_registry: service::ai_stream_registry::AiStreamRegistry::new(+ redis_client.clone(),+ ),+ mcp_state,+ pipedream_state,+ mcp_selector,+ import_service,+ onboarding_service,+ macro_event_broker: macro_event_broker.clone(),+ })+ .await+ .context("failed to setup and serve api");++ ai_projection_worker_task.abort();+ match ai_projection_worker_task.await {+ Err(error) if error.is_cancelled() => {+ tracing::info!("ai projection worker stopped");+ }+ Err(error) => {+ tracing::error!(error = ?error, "ai projection worker exited unexpectedly");+ }+ Ok(()) => {+ tracing::error!(+ error = "worker exited naturally",+ "ai projection worker exited unexpectedly"+ );+ }+ }++ tracing::info!("waiting for event broker publishes to drain");+ event_broker_tracker.close();+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {+ Ok(()) => tracing::info!("event broker publishes drained"),+ Err(error) => tracing::warn!(+ error=?error,+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),+ "timed out waiting for event broker publishes to drain"+ ),+ }++ api_result+}++const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);diff --git a/services/document_storage_service/src/lib.rs b/services/document_storage_service/src/lib.rsindex 65880be0e3..24899104fd 100644--- a/services/document_storage_service/src/lib.rs+++ b/services/document_storage_service/src/lib.rs@@ -1 +1,14 @@+#![recursion_limit = "256"]+//! Document storage service. The service binary and the OpenAPI generator both+//! link this library, so the service compiles once and caches as one rlib.++mod api;+mod config;pub mod model;+mod outbound;+mod server;+mod service;++/// The service's OpenAPI document, rendered by the `*_openapi` binary.+pub use api::swagger::ApiDoc;+pub use server::run;diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rsindex bcd09e2c0d..f6ac13d731 100644--- a/services/document_storage_service/src/main.rs+++ b/services/document_storage_service/src/main.rs@@ -1,2077 +1,13 @@+// `run()`'s future type is deep enough to need the library's recursion limit+// wherever it is awaited.#![recursion_limit = "256"]-use crate::{- api::{- MACRO_INTERNAL_USER_ID,- context::{- ApiContext, AuthorizationService, DocumentStorageServiceAuthKey, TaskPropertiesAdapter,- },- },- config::{- CalEventTypeContentNamesKey, CalWebhookSecretKey, MetaAccessToken, MetaPixelId,- MetaTestEventCode,- },- service::s3::S3,-};-use agent_session_realtime::{- domain::service::AgentSessionLogConsumerService, outbound::log_topic_consumer::LogTopicConsumer,-};-use analytics_client::{AnalyticsClient, AnalyticsClientConfig, MetaConfig};-use anyhow::Context;-use bots::{domain::service::BotServiceImpl, outbound::pg_bots_repo::PgBotsRepo};-use cal::{- domain::service::{CalConfig, CalEventMeta, CalWebhookServiceImpl},- inbound::cal_webhook_router::CalWebhookRouterState,- outbound::analytics_client::AnalyticsClientSink,-};-use calendar_events::domain::reminder_dispatch::CalendarReminderDispatchService;-use calendar_events::inbound::axum_router::CalendarRouterState;-use calendar_events::inbound::dispatch_worker::CalendarReminderDispatchWorker;-use calendar_events::outbound::notification_notifier::NotificationCalendarReminderNotifier;-use calendar_events::outbound::sqs_dispatch_queue::SqsCalendarDispatchQueue;-use call::{- domain::service::CallServiceImpl,- inbound::axum_router::{CallRouterState, InternalCallRouterState, WebhookRouterState},- outbound::{- ai_call_summarizer::AiCallSummarizer,- livekit_rtc_client::LivekitRtcClient,- pg_call_repo::PgCallRepo,- pg_voice_repo::PgVoiceRepo,- s3_recording_storage::{RecordingCloudFrontConfig, S3RecordingStorage},- },-};-use channel_labels::{- domain::service::ChannelLabelsServiceImpl, inbound::axum_router::ChannelLabelsRouterState,- outbound::pg_channel_labels_repo::PgChannelLabelsRepo,-};-use channels::{- domain::{- list_service::ChannelListServiceImpl,- service::ChannelServiceImpl,- side_effects::{ChannelSideEffectService, SpawnedChannelEventDispatcher},- },- inbound::{axum_router::ChannelsRouterState, list_router::ChannelListRouterState},- outbound::{- connection_gateway_realtime::ConnectionGatewayChannelRealtimePublisher,- contacts_dispatcher::ContactsChannelDispatcher,- notification_sender::NotificationChannelSender,- pg_channel_reference_share_permissions::PgChannelReferenceSharePermissions,- pg_channels_repo::PgChannelsRepo, pg_side_effect_context::PgChannelSideEffectContext,- },-};-use collab_surface::inbound::axum_router::CollabSurfaceRouterState;-use config::{Config, Environment};-use connection::{- domain::service::ConnectionServiceImpl,- outbound::connection_gateway_client::ConnectionGatewayImpl,-};-use connection_gateway_client::client::ConnectionGatewayClient;-use documents_hex::domain::models::CloudFrontConfig;-use documents_hex::domain::service::DocumentServiceImpl;-use documents_hex::inbound::axum_router::DocumentRouterState;-use documents_hex::outbound::pg_document_repo::PgDocumentRepo;-use documents_hex::outbound::s3_upload_url::S3UploadUrlAdapter;-use dynamodb_client::DynamodbClient;-use email::{- domain::{ports::ReadonlyEmailPreviewAdapter, service::EmailServiceImpl},- outbound::EmailPgRepo,-};-use embedding::embedding_provider::openai::TextEmbedding3Small;-use entity_registry::OwnerGrantPolicy;-use entity_registry_db_utils::OwnedEntityRegistrar;-use favorites::{- domain::{mutation_service::FavoritesMutationServiceImpl, service::FavoritesServiceImpl},- inbound::axum_router::FavoritesRouterState,- outbound::pg_favorites_repo::PgFavoritesRepo,-};-use foreign_entity::{- domain::service::ForeignEntityServiceImpl, inbound::axum_router::ForeignEntityRouterState,- outbound::pg_foreign_entity_repo::PgForeignEntityRepo,-};-use frecency::{domain::services::FrecencyQueryServiceImpl, outbound::postgres::FrecencyPgStorage};-use github::domain::service::{- GithubSyncConfig, GithubSyncServiceImpl, InstallationTokenConfig, InstallationTokenService,- PullRequestIndexService,-};-use github::inbound::pull_request_index_router::PullRequestIndexRouterState;-use github::outbound::connection_gateway_realtime::ConnectionGatewayGithubRealtime;-use github::outbound::github_sync_client::GithubSyncClientImpl;-use github::outbound::pg_github_sync_repo::PgGithubSyncRepo;-use github::outbound::pull_request_diff::GithubPullRequestDiffClient;-use github_pull_requests::{- domain::service::{- GithubPullRequestChangesServiceImpl, GithubPullRequestChangesetStore,- GithubPullRequestServiceImpl,- },- inbound::{- axum_router::GithubPullRequestRouterState,- changes_router::GithubPullRequestChangesRouterState,- },- outbound::{- pg_github_pull_request_repo::PgGithubPullRequestRepo,- s3_patch_store::S3GithubPullRequestPatchStore,- },-};-use graphql_scheduled_action::ScheduledActionGraphqlContext;-use harnesses::outbound::pg_harness_repo::PgHarnessRepo;-use initiative::{- domain::service::InitiativeServiceImpl, inbound::axum_router::InitiativeRouterState,- outbound::PgInitiativeRepo,-};-use lexical_client::LexicalClient;-use macro_auth::middleware::decode_jwt::JwtValidationArgs;-use macro_authorization::{- InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationServiceImpl,- MacroAuthorizationState, PgBotAuthorizationRepo, PgBotAuthorizer, PgHarnessAuthorizationRepo,- PgHarnessAuthorizer, PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,-};-use macro_entrypoint::MacroEntrypoint;-use macro_env_var::maybe_env_vars;-use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};-#[cfg(feature = "delete_document_worker")]-use macro_service_urls::AiEditingWorkerUrl;-use macro_service_urls::{- ConnectionGatewayUrl, LexicalServiceUrl, StaticFileServiceUrl, SyncServiceUrl,-};-use macro_sha_count_client::Redis;-use notification::domain::service::{- NotificationReaderService, PlatformArnConfig, SqsNotificationIngress,- WebSocketNotificationConsumerService,-};-use notification::outbound::{notification_consumer::NotificationTopicConsumer, queue::SqsQueue};-use opensearch_client::OpensearchClient;-use projects_hex::{- domain::service::ProjectServiceImpl,- inbound::axum_router::ProjectRouterState,- outbound::{- DynamoBulkUploadAdapter, PgProjectRepo, S3ProjectUploadAdapter, ShaCountAdapter,- SqsProjectSearchIndexer,- },-};-use properties::{- NotificationServiceImpl, PermissionServiceImpl, PropertiesPgRepo, PropertiesServiceImpl,-};-use rate_limit::{RateLimitServiceImpl, RedisRateLimitAdapter};-use reminders::{- domain::service::dispatch::ReminderDispatchService,- inbound::{axum_router::RemindersRouterState, dispatch_worker::DispatchWorker},- outbound::{- notification_notifier::NotificationReminderNotifier, pg_reminders_repo::PgRemindersRepo,- sqs_dispatch_queue::SqsDispatchQueue,- },-};-use scheduled_action::domain::read_service::ScheduledActionReadServiceImpl;-use scheduled_action::outbound::pg_scheduled_action_repo::PgScheduledActionRepo;-use secretsmanager_client::SecretManager;-use soup::{- domain::service::SoupImpl, inbound::axum_router::SoupRouterState,- outbound::pg_soup_repo::PgSoupRepo,-};-use soup_realtime::{- domain::service::{SoupRealtimeConsumerService, SoupRealtimeServiceImpl},- outbound::{- entity_access::EntityAccessExpander, kafka_publisher::KafkaSoupRealtimePublisher,- soup_consumer::SoupTopicConsumer,- },-};-use sqlx::postgres::PgPoolOptions;-use std::{sync::Arc, time::Duration};-use sync_service_client::SyncServiceClient;-use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};-use task_dedup::{- TaskDedupService,- outbound::{- cohere::CohereReranker,- connection_gateway::ConnectionGatewayTaskDedupNotifier,- judge::AgentDuplicateJudge,- postgres::{PgTaskMatchRepo, PgTaskVectorDb},- },-};-use tokio_util::{sync::CancellationToken, task::TaskTracker};-use user_api_key::{- domain::service::UserApiKeyServiceImpl, inbound::axum_router::UserApiKeyRouterState,- outbound::pg_user_api_keys_repo::PgUserApiKeysRepo,-};--mod api;-mod config;-mod model;-mod outbound;-mod service;--const SOUP_CONSUMER_RESTART_MAX_DELAY_SECS: u64 = 60;-const SOUP_CONSUMER_RESTART_ALERT_THRESHOLD: u32 = 5;--fn soup_consumer_restart_delay(consecutive_failures: u32) -> Duration {- let exponent = consecutive_failures.saturating_sub(1).min(6);- Duration::from_secs((1_u64 << exponent).min(SOUP_CONSUMER_RESTART_MAX_DELAY_SECS))-}-maybe_env_vars! {- struct AppleBundleId;- struct SnsApnsVoipPlatformArn;-}+use macro_entrypoint::MacroEntrypoint;#[tokio::main]async fn main() -> anyhow::Result<()> {let entrypoint = MacroEntrypoint::default().init();- let result = run().await;+ let result = document_storage_service::run().await;entrypoint.shutdown();result}--async fn run() -> anyhow::Result<()> {- let env = Environment::new_or_prod();-- let aws_config = macro_aws_config::get_macro_aws_config().await;-- let secretsmanager_client = secretsmanager_client::SecretsManager::new(- aws_sdk_secretsmanager::Client::new(&aws_config),- );-- // Parse our configuration from the environment, then resolve any secret-manager backed values.- let config = Config::from_env()- .context("expected to be able to generate config")?- .resolve_remote_secrets(env, &secretsmanager_client)- .await- .context("expected to be able to resolve config secrets")?;- let non_user_owners = config.non_user_owners()?;-- tracing::trace!("initialized config");-- let (min_connections, max_connections): (u32, u32) = match config.environment {- Environment::Production => (50, 150),- Environment::Develop => (25, 100),- Environment::Local => (15, 50),- };-- let db = PgPoolOptions::new()- .min_connections(min_connections)- .max_connections(max_connections)- .connect(&config.database_url)- .await- .context("could not connect to db")?;-- tracing::trace!(- min_connections,- max_connections,- "initialized db connection"- );-- let readonly_db = match PgPoolOptions::new()- .min_connections(min_connections)- .max_connections(max_connections)- .connect(&config.database_url_readonly)- .await- {- Ok(pool) => {- tracing::trace!("initialized readonly db connection");- pool- }- Err(e) => {- tracing::warn!(error=?e, "failed to connect to readonly db, falling back to primary");- db.clone()- }- };-- let dynamo_db = aws_sdk_dynamodb::Client::new(&aws_config);-- let dynamodb_client = DynamodbClient::new_from_client(- dynamo_db.clone(),- Some(config.bulk_upload_requests_table.as_ref().to_string()),- );- tracing::trace!("initialized dynamodb client");-- let s3_client = macro_aws_config::s3_client().await;-- tracing::trace!("initialized s3 client");-- let search_event_queue = macro_queues::SearchEventQueue::new();- let document_delete_queue = macro_queues::DocumentDeleteQueue::new();- let contacts_queue = macro_queues::ContactsQueue::new();- let notification_queue = macro_queues::NotificationIngressQueue::new();- let gmail_ops_queue = macro_queues::GmailOpsQueue::new();- let reminder_dispatch_queue = macro_queues::ReminderDispatchQueue::new();- let calendar_reminder_dispatch_queue = macro_queues::CalendarReminderDispatchQueue::new();- let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(&aws_config))- .search_event_queue(&search_event_queue)- .document_delete_queue(&document_delete_queue)- .gmail_ops_queue(&gmail_ops_queue);- let webhook_event_queue = webhook::outbound::SqsWebhookQueue::new(- Arc::new(sqs_client.clone()),- macro_queues::WebhookEventQueue::new().to_string(),- config.webhook_queue_max_messages,- config.webhook_queue_wait_time_seconds,- );-- let contacts_ingress = Arc::new(contacts::domain::service::SqsContactsIngress {- queue: contacts::outbound::ingress::SqsContactsQueue::new(- aws_sdk_sqs::Client::new(&aws_config),- contacts_queue.to_string(),- ),- });-- tracing::trace!("initialized sqs client");-- // Redis handles it own connection pool internally. Each time we use redis- // we should be using redis_client.get_connection() to grab a specific connection- let redis_client = redis::Client::open(config.redis_uri.to_string().as_ref())- .expect("could not connect to redis client");-- match redis_client.get_connection().is_err() {- true => {- tracing::error!("unable to connect to redis");- }- false => {- tracing::trace!("initialized redis connection");- }- }-- let dss_auth_key = DocumentStorageServiceAuthKey::new()?;-- let conn_gateway_client = ConnectionGatewayClient::new(- config.internal_api_key.to_string(),- ConnectionGatewayUrl::new()?.to_string(),- );-- let sync_service_client = Arc::new(SyncServiceClient::new(- config.sync_service_auth_key.as_ref().to_string(),- SyncServiceUrl::new()?.to_string(),- ));-- let lexical_client = Arc::new(LexicalClient::new(- config.internal_api_key.to_string(),- LexicalServiceUrl::new()?.to_string(),- ));-- let jwt_validation_args =- JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)- .await?;-- let consumer_cancellation_token = CancellationToken::new();- let consumer_tracker = TaskTracker::new();- let event_broker_tracker = TaskTracker::new();- let macro_event_broker = MacroEventBrokerService::new(- KafkaEventPublisher::new(config.kafka_brokers.as_ref())- .context("failed to create kafka event publisher")?,- event_broker_tracker.clone(),- );- let bots_repo = PgBotsRepo::new(db.clone());- // The agent API checks a newly selected MCP app against Pipedream's- // directory, the same check the ConfigureAgent tool makes.- let bots_service = BotServiceImpl::new(bots_repo, macro_event_broker.clone()).with_mcp_apps(- ai_tools::PipedreamMcpAppCatalog::new(ai_tools::pipedream_client_from_env()?),- );-- let authorization_service: AuthorizationService = MacroAuthorizationServiceImpl::new(- MacroAuthJwtValidator::new(jwt_validation_args.clone()),- InternalAuthConfig {- api_key: dss_auth_key.as_ref().to_string(),- default_user_id: Some(MACRO_INTERNAL_USER_ID.to_string()),- },- PgBotAuthorizer::new(PgBotAuthorizationRepo::new(db.clone())),- PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),- )- .with_harness_authorizer(PgHarnessAuthorizer::new(PgHarnessAuthorizationRepo::new(- db.clone(),- )));- let authorization_state = MacroAuthorizationState::new(Arc::new(authorization_service));- let harnesses_service =- harnesses::domain::service::HarnessServiceImpl::new(PgHarnessRepo::new(db.clone()));-- // Initialize OpenSearch client- let opensearch_client = OpensearchClient::new(- config.opensearch_url.to_string(),- config.opensearch_username.to_string(),- config.opensearch_password.to_string(),- )- .context("unable to create opensearch client")?;-- if let Err(e) = opensearch_client.health().await {- tracing::error!(error=?e, "error connecting to opensearch");- return Err(e);- }- tracing::trace!("initialized opensearch client");-- let frecency_storage = FrecencyPgStorage::new(db.clone());- let frecency_service = FrecencyQueryServiceImpl::new(frecency_storage.clone());- // DSS only reads CRM rows — no-op resolver keeps reqwest/scraper- // out of this binary.- let crm_service = crm::domain::service::CrmServiceImpl::new(- crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),- crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,- );- let email_service = EmailServiceImpl::new(- EmailPgRepo::new(db.clone()),- frecency_service.clone(),- sqs_client.clone(),- crm_service.clone(),- entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(- entity_access_management::outbound::PgRepository::new(db.clone()),- ),- 0,- )- .with_macro_event_broker(macro_event_broker.clone());- let readonly_email_service = ReadonlyEmailPreviewAdapter(- EmailServiceImpl::new(- EmailPgRepo::new(readonly_db.clone()),- frecency_service.clone(),- sqs_client.clone(),- crm_service.clone(),- entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(- entity_access_management::outbound::PgRepository::new(readonly_db.clone()),- ),- 0,- )- .with_macro_event_broker(macro_event_broker.clone()),- );- let system_properties_service =- SystemPropertiesServiceImpl::new(PgSystemPropertiesRepository::new(db.clone()));- let ingress_queue = SqsQueue::new(- aws_sdk_sqs::Client::new(&aws_config),- notification_queue.to_string(),- );- let notification_ingress_service = Arc::new(SqsNotificationIngress {- queue: ingress_queue.clone(),- });- let graphql_notification_reader: Arc<ai_tools::ToolNotificationService> =- Arc::new(NotificationReaderService {- repository: notification::outbound::repository::DbNotificationRepository::new(- db.clone(),- ),- queue: ai_tools::ToolNotificationQueue::Sqs(ingress_queue.clone()),- sns_endpoint: ai_tools::NoOpSnsEndpointManager,- platform_config: PlatformArnConfig {- apns_platform_arn: String::new(),- fcm_platform_arn: String::new(),- apns_voip_platform_arn: String::new(),- },- realtime: notification::domain::ports::NoopNotificationRealtimePublisher,- });- tracing::trace!("initialized notification ingress service");-- let entity_access_service = Arc::new(- entity_access::domain::service::EntityAccessServiceImpl::new(- entity_access::outbound::PgAccessRepository::new(db.clone()),- ),- );-- let permission_checker = PermissionServiceImpl::new(db.clone(), entity_access_service.clone());- let notification_service = NotificationServiceImpl::new(- SqsNotificationIngress {- queue: ingress_queue,- },- db.clone(),- );- let properties_service = Arc::new(- PropertiesServiceImpl::new(- PropertiesPgRepo::new(db.clone()),- Some(permission_checker),- Some(notification_service),- )- .with_event_broker(macro_event_broker.clone())- .with_initiative_assignees(Arc::new(- initiative::domain::assignees::InitiativeAssignees::new(PgInitiativeRepo::new(- db.clone(),- )),- )),- );-- // Create the channel list service used by soup.- // Create the legacy channel list router state for routes mounted under /comms.- let channel_list_state = ChannelListRouterState::new(- ChannelListServiceImpl::new(- PgChannelsRepo::new(db.clone()),- PgChannelsRepo::new(db.clone()),- frecency_storage.clone(),- ),- authorization_state.clone(),- );-- let s3 = Arc::new(S3::new(- s3_client,- config.document_storage_bucket.as_ref(),- config.docx_document_upload_bucket.as_ref(),- config.upload_staging_bucket.as_ref(),- ));- let system_properties_service = Arc::new(system_properties_service);-- let owned_entity_registrar =- OwnedEntityRegistrar::new(OwnerGrantPolicy::new(PgBotsRepo::new(db.clone())));- let document_repo = PgDocumentRepo::new(db.clone(), owned_entity_registrar.clone());- let cloudfront_config = CloudFrontConfig {- distribution_url: config- .document_storage_service_cloudfront_distribution_url- .as_ref()- .to_string(),- signer_public_key_id: config- .document_storage_service_cloudfront_signer_public_key_id- .as_ref()- .to_string(),- signer_private_key: config- .document_storage_service_cloudfront_signer_private_key- .as_ref()- .to_string(),- presigned_url_expiry_seconds: config.document_storage_service_presigned_url_expiry_seconds,- browser_cache_expiry_seconds: config- .document_storage_service_presigned_url_browser_cache_expiry_seconds,- };- let s3_upload_adapter = S3UploadUrlAdapter::new(- macro_aws_config::s3_client().await,- config.document_storage_bucket.as_ref(),- config.docx_document_upload_bucket.as_ref(),- );- let markdown_initializer =- documents_hex::outbound::markdown_init::LexicalSyncMarkdownInitializer::new(- lexical_client.as_ref().clone(),- sync_service_client.as_ref().clone(),- );-- let connection_gateway = Arc::new(ConnectionGatewayImpl::new(conn_gateway_client.clone()));-- let connection_service =- ConnectionServiceImpl::new(entity_access_service.clone(), connection_gateway.clone());-- let entity_access_management_service =- entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(- entity_access_management::outbound::PgRepository::new(db.clone()),- );-- let chat_mutation_service =- Arc::new(chat::domain::service::ChatServiceImpl::new_without_tools(- chat::outbound::postgres::PgChatRepo::new(db.clone(), owned_entity_registrar.clone()),- entity_access_management_service.clone(),- ));-- let project_service = Arc::new(ProjectServiceImpl::new(- PgProjectRepo::new(db.clone(), owned_entity_registrar.clone()),- S3ProjectUploadAdapter::new(- macro_aws_config::s3_client().await,- config.document_storage_bucket.as_ref(),- config.docx_document_upload_bucket.as_ref(),- config.upload_staging_bucket.as_ref(),- ),- DynamoBulkUploadAdapter::new(dynamodb_client.clone()),- ShaCountAdapter::new(Redis::new(redis_client.clone())),- entity_access_management_service.clone(),- SqsProjectSearchIndexer::new(Arc::new(sqs_client.clone()), macro_event_broker.clone()),- if cfg!(feature = "local") {- Some(uuid::uuid!("d50676e2-0a12-4c62-bc07-4b1cb6d8e9bc"))- } else {- None- },- macro_event_broker.clone(),- ));-- let document_service = Arc::new(DocumentServiceImpl::new(- document_repo,- cloudfront_config,- sync_service_client.as_ref().clone(),- s3_upload_adapter,- TaskPropertiesAdapter {- system_properties: system_properties_service.clone(),- properties: properties_service.clone(),- entity_access_service: entity_access_service.clone(),- },- connection_service,- entity_access_management_service.clone(),- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),- macro_event_broker.clone(),- ));-- let foreign_entity_service = Arc::new(ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(- db.clone(),- )));-- let github_sync_service_impl = GithubSyncServiceImpl::new(- GithubSyncConfig {- webhook_secret: config.github_webhook_secret_key.as_ref().to_string(),- github_sync_app_url: config.github_sync_app_url.to_string(),- sync_app_pem: config.github_sync_app_pem_secret_key.as_ref().to_string(),- sync_app_client_id: config.github_sync_app_client_id.to_string(),- sync_app_client_secret: config.github_sync_app_client_secret.to_string(),- installation_state_secret: config.github_installation_state_secret.to_string(),- },- document_service.clone(),- Arc::new(GithubPullRequestServiceImpl::new(- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),- PgGithubPullRequestRepo::new(db.clone()),- )),- (*notification_ingress_service).clone(),- PgGithubSyncRepo::new(db.clone()),- GithubSyncClientImpl::default(),- ConnectionGatewayGithubRealtime::new(conn_gateway_client.clone()),- );-- let github_pull_request_index_state = PullRequestIndexRouterState {- service: Arc::new(PullRequestIndexService::new(- InstallationTokenConfig {- client_id: config.github_sync_app_client_id.to_string(),- private_key_pem: config.github_sync_app_pem_secret_key.as_ref().to_string(),- },- PgGithubSyncRepo::new(db.clone()),- GithubSyncClientImpl::default(),- GithubPullRequestServiceImpl::new(- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),- PgGithubPullRequestRepo::new(db.clone()),- ),- )),- authorization_state: authorization_state.clone(),- };-- let github_pull_request_state = GithubPullRequestRouterState::new(- Arc::new(GithubPullRequestServiceImpl::new(- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),- PgGithubPullRequestRepo::new(db.clone()),- )),- entity_access_service.clone(),- authorization_state.clone(),- );-- let github_pull_request_changes_state = GithubPullRequestChangesRouterState::new(- Arc::new(GithubPullRequestChangesServiceImpl::new(- GithubPullRequestServiceImpl::new(- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),- PgGithubPullRequestRepo::new(db.clone()),- ),- GithubPullRequestChangesetStore::new(- GithubPullRequestDiffClient::new(InstallationTokenService::new(- InstallationTokenConfig {- client_id: config.github_sync_app_client_id.to_string(),- private_key_pem: config.github_sync_app_pem_secret_key.as_ref().to_string(),- },- PgGithubSyncRepo::new(db.clone()),- GithubSyncClientImpl::default(),- )),- PgGithubPullRequestRepo::new(db.clone()),- S3GithubPullRequestPatchStore::new(- macro_aws_config::s3_client().await,- config.github_pull_request_patch_bucket.to_string(),- ),- ),- )),- entity_access_service.clone(),- authorization_state.clone(),- );-- let foreign_entity_state = ForeignEntityRouterState::new(- foreign_entity_service.clone(),- entity_access_service.clone(),- authorization_state.clone(),- );-- // Cal.com webhooks → Meta Lead events. Both secrets are loaded here- // (rather than on Config) to keep cal/Meta wiring colocated.- let cal_webhook_secret = secretsmanager_client- .get_maybe_secret_value(env, CalWebhookSecretKey::new()?)- .await?;- let cal_event_type_content_names_secret = secretsmanager_client- .get_maybe_secret_value(env, CalEventTypeContentNamesKey::new()?)- .await?;-- let analytics_client = Arc::new(AnalyticsClient::new(AnalyticsClientConfig {- google_analytics: None,- meta: Some(MetaConfig {- pixel_id: MetaPixelId::new()?.as_ref().to_string(),- access_token: MetaAccessToken::new()?.as_ref().to_string(),- test_event_code: MetaTestEventCode::new().map(|v| v.as_ref().to_string()),- }),- posthog: None,- }));-- let cal_event_type_meta: std::collections::HashMap<u64, CalEventMeta> =- serde_json::from_str(cal_event_type_content_names_secret.as_ref())- .context("CalEventTypeContentNames secret must be a JSON object mapping eventTypeId (u64) to { content_name: string, value: number (USD) }")?;- let cal_webhook_service = CalWebhookServiceImpl::new(- CalConfig {- webhook_secret: cal_webhook_secret.as_ref().to_string(),- event_type_meta: cal_event_type_meta,- },- AnalyticsClientSink::new(analytics_client.clone()),- );- let cal_webhook_state = CalWebhookRouterState::new(cal_webhook_service);-- // Call service (LiveKit)- let transcription_agent_name = config- .livekit_transcription_agent_name- .value()- .map(|v| v.to_owned());-- let internal_call_secret = config.internal_call_secret.value().map(|v| v.to_owned());-- anyhow::ensure!(- transcription_agent_name.is_none() || internal_call_secret.is_some(),- "LIVEKIT_TRANSCRIPTION_AGENT_NAME is set but INTERNAL_CALL_SECRET is missing — \- the transcription agent will not be able to submit transcripts"- );- let livekit_rtc_client = LivekitRtcClient::new(- config.livekit_server_url.as_ref(),- config.livekit_api_key.as_ref(),- config.livekit_api_secret.as_ref(),- transcription_agent_name,- );- let call_connection_service =- ConnectionServiceImpl::new(entity_access_service.clone(), connection_gateway.clone());- let call_repo = PgCallRepo::new(db.clone());- let egress_config = match (- config::CallRecordingS3Bucket::new(),- config::CallRecordingS3Region::new(),- config::CallRecordingS3AccessKey::new(),- config::CallRecordingS3Secret::new(),- ) {- (Some(bucket), Some(region), Some(access_key), Some(secret)) => {- tracing::info!(bucket = bucket.as_ref(), "call recording enabled");- Some(call::domain::models::EgressS3Config {- bucket: bucket.as_ref().to_string(),- region: region.as_ref().to_string(),- access_key: access_key.as_ref().to_string(),- secret: secret.as_ref().to_string(),- })- }- _ => None,- };- let recording_storage = match &egress_config {- Some(egress_config) => {- let cloudfront_config = RecordingCloudFrontConfig {- distribution_url: config- .document_storage_service_cloudfront_distribution_url- .as_ref()- .to_string(),- signer_public_key_id: config- .document_storage_service_cloudfront_signer_public_key_id- .as_ref()- .to_string(),- signer_private_key: config- .document_storage_service_cloudfront_signer_private_key- .as_ref()- .to_string(),- presigned_url_expiry_seconds: config::CALL_RECORDING_PRESIGNED_URL_EXPIRY_SECONDS,- };- Some(S3RecordingStorage::new(egress_config.bucket.clone(), cloudfront_config).await)- }- None => None,- };- let mut call_service_builder = CallServiceImpl::<_, _, _, _, _, _, AiCallSummarizer>::new(- call_repo,- livekit_rtc_client,- call_connection_service,- (*entity_access_service).clone(),- (*notification_ingress_service).clone(),- recording_storage,- config.livekit_server_url.as_ref(),- )- .with_summarizer(AiCallSummarizer::new(- ai_usage::pg_recorder_with_enforcement(db.clone(), config.enable_ai_usage_enforcement),- ));- if let Some(secret) = internal_call_secret {- call_service_builder = call_service_builder.with_internal_call_secret(secret);- }- if let Some(config) = egress_config {- call_service_builder = call_service_builder.with_egress(config);- }- if let Some(base_url) = config::CallRingStatusBaseUrl::new() {- call_service_builder =- call_service_builder.with_ring_status_base_url(base_url.as_ref().to_owned());- }-- // VoIP push is optional: enabled only when both env vars are present.- // APPLE_BUNDLE_ID: the app bundle ID (e.g. "com.macro.app.prod")- // SNS_APNS_VOIP_PLATFORM_ARN: SNS platform app ARN for APNS_VOIP- //- // Option<VoipPushServiceImpl<...>> is used as the type parameter so the- // type stays stable regardless of whether VoIP push is configured.- let voip_sender = if let (Some(bundle_id), Some(voip_arn)) =- (AppleBundleId::new(), SnsApnsVoipPlatformArn::new())- {- if voip_arn.is_empty() {- tracing::warn!("voip push disabled: SNS_APNS_VOIP_PLATFORM_ARN is set but empty");- None- } else {- let voip_repo =- notification::outbound::repository::DbNotificationRepository::new(db.clone());- let voip_mobile = notification::outbound::mobile::MobilePushAdapter {- push_service: aws_sdk_sns::Client::new(&aws_config),- apns_bundle_id: bundle_id.to_string(),- voip_bundle_id: Some(format!("{}.voip", bundle_id.as_ref())),- };- tracing::info!(- bundle_id = bundle_id.as_ref(),- voip_arn = voip_arn.as_ref(),- "voip push enabled"- );- Some(notification::domain::service::VoipPushServiceImpl::new(- voip_repo,- voip_mobile,- ))- }- } else {- tracing::info!("voip push disabled: APPLE_BUNDLE_ID or SNS_APNS_VOIP_PLATFORM_ARN not set");- None- };- let call_service_builder = call_service_builder.with_voip_push_sender(voip_sender);-- let call_service = Arc::new(- call_service_builder- .with_voice_repo(PgVoiceRepo::new(db.clone()))- .with_event_broker(macro_event_broker.clone()),- );-- consumer_tracker.spawn({- let service = call_service.clone();- let cancellation_token = consumer_cancellation_token.clone();- async move {- cancellation_token- .run_until_cancelled(call::inbound::stale_call_sweeper::run_stale_call_sweeper(- service,- call::inbound::stale_call_sweeper::SWEEP_INTERVAL,- ))- .await;- }- });-- let call_state = CallRouterState::new(- call_service.clone(),- entity_access_service.clone(),- authorization_state.clone(),- );- let call_webhook_state = WebhookRouterState::new(call_service.clone());- let call_public_rate_limiter = RateLimitServiceImpl {- repo: RedisRateLimitAdapter {- redis: redis_client.clone(),- },- };-- let webhook_repository = webhook::outbound::PgRepository::new(db.clone());- let webhook_endpoint_scheme_policy = if matches!(env, Environment::Local) {- webhook::domain::models::WebhookEndpointSchemePolicy::HttpAndHttps- } else {- webhook::domain::models::WebhookEndpointSchemePolicy::HttpsOnly- };- let webhook_http_client =- webhook::outbound::ReqwestWebhookDeliveryClient::new_with_endpoint_scheme_policy(- webhook_endpoint_scheme_policy,- )- .context("failed to create webhook HTTP client")?;- let webhook_service =- webhook::domain::service::WebhookServiceImpl::new_with_endpoint_scheme_policy(- webhook_repository.clone(),- webhook_http_client.clone(),- webhook_endpoint_scheme_policy,- macro_event_broker.clone(),- );- let webhook_rate_limiter = RateLimitServiceImpl {- repo: RedisRateLimitAdapter {- redis: redis_client.clone(),- },- };- let webhook_state = webhook::inbound::axum_router::WebhookRouterState::new(- webhook_service,- webhook_rate_limiter,- authorization_state.clone(),- );-- let (webhook_stream_sender, _) =- tokio::sync::broadcast::channel(webhook::domain::stream::WEBHOOK_STREAM_CHANNEL_CAPACITY);- let sse_stream_service = webhook::domain::stream::WebhookEventStreamServiceImpl::new(- webhook_stream_sender.clone(),- entity_access_service.clone(),- webhook_repository.clone(),- );- let sse_stream_state = webhook::inbound::stream_router::WebhookStreamRouterState::new(- sse_stream_service,- authorization_state.clone(),- );- consumer_tracker.spawn({- let brokers = config.kafka_brokers.as_ref().to_string();- let sender = webhook_stream_sender;- let cancellation_token = consumer_cancellation_token.clone();- async move {- loop {- let result = tokio::select! {- biased;- _ = cancellation_token.cancelled() => break,- result = webhook::inbound::kafka_stream_consumer::run_webhook_stream_consumer(- &brokers,- &sender,- ) => result,- };-- if cancellation_token.is_cancelled() {- break;- }- if let Err(error) = result {- tracing::error!(- error = ?error,- "webhook SSE Kafka consumer stopped"- );- tokio::select! {- biased;- _ = cancellation_token.cancelled() => break,- _ = tokio::time::sleep(Duration::from_secs(1)) => {}- }- }- }- }- });-- let webhook_ingestion_service =- webhook::domain::ingestion::WebhookEventIngestionServiceImpl::new(- entity_access_service.clone(),- webhook_repository,- webhook_event_queue.clone(),- );- let webhook_delivery_repository =- webhook::outbound::PgWebhookDeliveryRepository::new(db.clone());- let webhook_delivery_service = webhook::domain::delivery::WebhookEventDeliveryServiceImpl::new(- webhook_delivery_repository,- webhook_http_client,- );- let webhook_worker = webhook::inbound::worker::WebhookEventWorker::new(- webhook_event_queue,- webhook_delivery_service,- );- let webhook_worker_task = tokio::spawn(async move {- tracing::info!("starting webhook event worker");- webhook_worker.run().await;- });- tokio::spawn(async move {- match webhook_worker_task.await {- Ok(()) => tracing::error!("webhook event worker exited unexpectedly"),- Err(error) => {- tracing::error!(error = ?error, "webhook event worker exited unexpectedly");- }- }- });-- let webhook_consumer_brokers = config.kafka_brokers.as_ref().to_string();- consumer_tracker.spawn({- let cancellation_token = consumer_cancellation_token.clone();- async move {- loop {- if cancellation_token.is_cancelled() {- break;- }-- tracing::info!("starting webhook event consumer");- let result = webhook::inbound::kafka_consumer::run_webhook_event_consumer(- &webhook_consumer_brokers,- webhook_ingestion_service.clone(),- cancellation_token.cancelled(),- )- .await;-- if cancellation_token.is_cancelled() {- break;- }-- match result {- Ok(()) => tracing::error!("webhook event consumer exited unexpectedly"),- Err(error) => {- tracing::error!(error = ?error, "webhook event consumer exited unexpectedly");- }- }-- tokio::select! {- biased;- _ = cancellation_token.cancelled() => break,- _ = tokio::time::sleep(Duration::from_secs(5)) => {}- }- }- }- });-- let activity_consumer_brokers = config.kafka_brokers.as_ref().to_string();- let editing_activity = Arc::new(- documents_hex::outbound::editing_activity::RedisEditingActivityStore::new(- redis_client.clone(),- ),- );- consumer_tracker.spawn({- let cancellation_token = consumer_cancellation_token.clone();- let activity_repo = activity::outbound::pg_activity_repo::PgActivityRepo::new(db.clone());- let activity_realtime = activity::domain::announcements::ActivityAnnouncements::new(- activity::KafkaActivityRealtimePublisher::new(macro_event_broker.clone()),- crate::service::activity::EntityAccessActivityAudience::new(- entity_access_service.clone(),- ),- );- async move {- let consumer = activity::inbound::kafka_consumer::ActivityConsumer::<- _,- crate::service::activity::ActivitySourceEvent,- _,- _,- >::new(- activity_repo,- move |event| {- let editing_activity = editing_activity.clone();- async move {- crate::service::activity::ingest(event, editing_activity.as_ref()).await- }- },- activity_realtime,- );- loop {- if cancellation_token.is_cancelled() {- break;- }-- tracing::info!("starting activity consumer");- let result = consumer- .run(&activity_consumer_brokers, cancellation_token.cancelled())- .await;-- if cancellation_token.is_cancelled() {- break;- }-- match result {- Ok(()) => tracing::error!("activity consumer exited unexpectedly"),- Err(error) => {- tracing::error!(error = ?error, "activity consumer exited unexpectedly");- }- }-- tokio::select! {- biased;- _ = cancellation_token.cancelled() => break,- _ = tokio::time::sleep(Duration::from_secs(5)) => {}- }- }- }- });-- // Links archived calls to the CRM records of the people on them.- let crm_call_link_brokers = config.kafka_brokers.as_ref().to_string();- consumer_tracker.spawn({- let cancellation_token = consumer_cancellation_token.clone();- let consumer = crm::inbound::call_archived::CallArchivedConsumer::new(- crm::domain::call_links::CallRecordLinker::new(- call::domain::service::CallRecordQueryServiceImpl::new(PgCallRepo::new(- db.clone(),- )),- crm::outbound::call_link::PgCallCrmLinker::new(- db.clone(),- (*system_properties_service).clone(),- ),- ),- );- async move {- loop {- if cancellation_token.is_cancelled() {- break;- }-- tracing::info!("starting CRM call-link consumer");- let result = consumer- .run(&crm_call_link_brokers, cancellation_token.cancelled())- .await;-- if cancellation_token.is_cancelled() {- break;- }-- match result {- Ok(()) => tracing::error!("CRM call-link consumer exited unexpectedly"),- Err(error) => {- tracing::error!(error = ?error, "CRM call-link consumer exited unexpectedly");- }- }-- tokio::select! {- biased;- _ = cancellation_token.cancelled() => break,- _ = tokio::time::sleep(Duration::from_secs(5)) => {}- }- }- }- });-- let call_internal_state = InternalCallRouterState::new(call_service.clone());-- // Create the SQS worker for delete document processing before config is moved.- #[cfg(feature = "delete_document_worker")]- let delete_document_worker = sqs_worker::SQSWorker::new(- aws_sdk_sqs::Client::new(&aws_config),- document_delete_queue.to_string(),- config.queue_max_messages,- config.queue_wait_time_seconds,- );-- let sqs_client = Arc::new(sqs_client);- let conn_gateway_client = Arc::new(conn_gateway_client);-- // MacroConfig reads the shared OPENAI_API_KEY from Doppler's APP_SECRETS_JSON- // or the local environment. Validate once before constructing consumers.- let openai_api_key = config.openai_api_key.as_ref().to_owned();- anyhow::ensure!(- !openai_api_key.trim().is_empty(),- "OpenAI API key is required for task dedup embeddings and dictation",- );- let cohere_api_key = config.cohere_api_key.as_ref().to_owned();- anyhow::ensure!(- !cohere_api_key.trim().is_empty(),- "Cohere API key is required for task dedup reranking",- );- let dictation_state = dictation::inbound::axum_router::DictationRouterState::new(- dictation::domain::DictationServiceImpl::new(- dictation::outbound::WhisperTranscriber::new(&config.openai_api_key)?,- dictation::outbound::SymphoniaRecordingInspector,- ai_usage::pg_recorder_with_enforcement(db.clone(), config.enable_ai_usage_enforcement),- ),- RateLimitServiceImpl {- repo: RedisRateLimitAdapter {- redis: redis_client.clone(),- },- },- authorization_state.clone(),- );- let task_dedup_service = Arc::new(TaskDedupService::new(- TextEmbedding3Small::new(openai_api_key),- PgTaskVectorDb::new(db.clone()),- CohereReranker::new(cohere_api_key),- Arc::new(AgentDuplicateJudge::new(- ai_usage::pg_recorder_with_enforcement(db.clone(), config.enable_ai_usage_enforcement),- )),- Arc::new(ConnectionGatewayTaskDedupNotifier::new(- conn_gateway_client.clone(),- )),- Arc::new(PgTaskMatchRepo::new(db.clone())),- ));- let channels_repo = PgChannelsRepo::new(db.clone());- // Built-in agents read the same committed-post stream as hosted and- // external agents, for channel and document posts alike, so the channel- // side effects no longer carry their own trigger queue.- let (bot_trigger_sender, bot_trigger_receiver) = tokio::sync::mpsc::unbounded_channel();-- let channel_side_effects = ChannelSideEffectService::new(- PgChannelSideEffectContext::new(db.clone()),- ConnectionGatewayChannelRealtimePublisher::new(conn_gateway_client.clone()),- NotificationChannelSender::new(notification_ingress_service.clone()),- ContactsChannelDispatcher::new(contacts_ingress.clone()),- )- .with_macro_event_broker(macro_event_broker.clone());-- let channels_service = Arc::new(- ChannelServiceImpl::with_dependencies(- channels_repo,- SpawnedChannelEventDispatcher::new(channel_side_effects.clone()),- PgChannelReferenceSharePermissions::new(db.clone(), entity_access_service.clone()),- )- .with_picture_files(- channels::outbound::static_file_pictures::StaticFileChannelPictures::new(- static_file_service_client::StaticFileServiceClient::new(- config.internal_api_key.to_string(),- StaticFileServiceUrl::new()?.to_string(),- ),- ),- ),- );-- // One message implementation for channels and documents. Committed changes fan- // out to the broker, the local agent queue, and the parent's own delivery: channel- // messages keep their notifications, activity, and sharing side effects and send- // the common `message_update` payload; document discussions get the common- // payload and document comment notifications.- let message_realtime = messages::outbound::connection_gateway::ConnectionGatewayMessages(- conn_gateway_client.clone(),- );- let discussion_delivery = messages::domain::delivery::DiscussionDelivery::new(- messages::outbound::pg_discussion_context::PgDiscussionContext(db.clone())- .with_initiatives(- initiative::domain::lookup::InitiativeLookup::new(- initiative::outbound::PgInitiativeRepo::new(db.clone()),- ),- properties_service.clone(),- )- .with_crm(crm::outbound::lookup::PgCrmParentReader::new(db.clone())),- messages::outbound::entity_access_audience::EntityAccessMessageAudience(- (*entity_access_service).clone(),- ),- message_realtime.clone(),- messages::outbound::notification_sender::MessageNotificationSender(- notification_ingress_service.clone(),- ),- )- .with_sharing(messages::outbound::pg_discussion_context::PgDiscussionContext(db.clone()));- let annotation_service = Arc::new(messages::domain::annotations::AnnotationService::new(- macro_db_client::annotations::repository::PgAnnotationRepository(db.clone()),- (*entity_access_service).clone(),- discussion_delivery.clone(),- ));- let channel_delivery = channels::domain::message_delivery::ChannelMessageDelivery::new(- PgChannelsRepo::new(db.clone()),- SpawnedChannelEventDispatcher::new(channel_side_effects.clone()),- PgChannelReferenceSharePermissions::new(db.clone(), entity_access_service.clone()),- message_realtime,- );- let message_service = Arc::new(- messages::domain::service::MessageService::new(- messages::outbound::pg_message_repo::PgMessageRepository::new(db.clone())- .with_initiatives(initiative::domain::lookup::InitiativeLookup::new(- initiative::outbound::PgInitiativeRepo::new(db.clone()),- ))- .with_crm(crm::outbound::lookup::PgCrmParentReader::new(db.clone())),- messages::domain::effects::MessageEffects::new(- messages::outbound::broker::BrokerMessagePublisher::new(macro_event_broker.clone()),- channel_bots::outbound::conversation::LocalBotPublisher::new(bot_trigger_sender),- messages::domain::delivery::ParentMessagePublisher::new(- channel_delivery,- discussion_delivery,- ),- ),- )- .with_group_recipients(channels::domain::group_mentions::ChannelGroupRecipients(- PgChannelsRepo::new(db.clone()),- ))- .with_mention_extractor(lexical_mention_extractor::LexicalMentionExtractor::new(- lexical_client.clone(),- ))- .with_references(- messages::outbound::entity_access_audience::EntityAccessMessageReferences(- (*entity_access_service).clone(),- ),- ),- );- let message_commands: Arc<dyn messages::domain::api::MessageCommands> = message_service.clone();- let messages_state = messages::inbound::axum_router::MessagesRouterState {- service: message_service.clone(),- access: entity_access_service.clone(),- authorization: authorization_state.clone(),- };-- let teammate_dms_brokers = config.kafka_brokers.as_ref().to_string();- consumer_tracker.spawn({- let cancellation_token = consumer_cancellation_token.clone();- let channels = (*channels_service).clone();- async move {- loop {- if cancellation_token.is_cancelled() {- break;- }-- tracing::info!("starting teammate DM consumer");- let result = channels::inbound::teammate_dms_consumer::run_teammate_dms_consumer(- &teammate_dms_brokers,- channels.clone(),- cancellation_token.cancelled(),- )- .await;-- if cancellation_token.is_cancelled() {- break;- }-- match result {- Ok(()) => tracing::error!("teammate DM consumer exited unexpectedly"),- Err(error) => {- tracing::error!(- error = ?error,- "teammate DM consumer exited unexpectedly"- );- }- }-- tokio::select! {- biased;- _ = cancellation_token.cancelled() => break,- _ = tokio::time::sleep(Duration::from_secs(5)) => {}- }- }- }- });-- // Wire Macro AI to react to mentions with the classic in-channel chat- // reply. The router posts replies through the channel service we just- // built and runs the agent loop in-process with the channel-bot toolset:- // a channel has no composer to finish a chat-deferred user tool in, so- // calendar event creation executes directly and SendEmail is unavailable.- // Agent sessions belong to a different bot entirely- // (`bot_id::MACRO_NEW_BOT_ID`, served by the harness), so the two paths- // can never answer the same mention.- let mut macro_agent_tool_context = ai_tools::build_tool_service_context_from_env(- db.clone(),- event_broker_tracker.clone(),- config.enable_ai_usage_enforcement,- config.ai_pricing(),- )- .await- .context("failed to build Macro agent tool context")?;- // Wire the agent's SendChannelMessage tool to this service's own- // side-effect pipeline so agent-posted messages share the exact instance- // used by the HTTP API, including the in-process bot trigger sender (the- // env builder wires an equivalent pipeline, but without bot triggers).- macro_agent_tool_context.channel_tool_context =- ai_tools::build_channel_tool_context_with_dispatcher(- db.clone(),- std::sync::Arc::new(SpawnedChannelEventDispatcher::new(channel_side_effects)),- message_service.clone(),- );- let macro_agent_tools = ai_tools::tools_for(ai_tools::AiHost::ChannelBot);- let conversation_access = Arc::new(- channel_bots::outbound::conversation::EntityAccessConversation(- entity_access_service.clone(),- ),- );- let admission = macro_agent_tool_context.admission.clone();- let bot_trigger_router = channel_bots::inbound::BotTriggerRouter::new_with_admission(- message_service.clone(),- conversation_access.clone(),- Arc::new(channel_bots::outbound::AgentLoopResponder::new(- macro_agent_tool_context,- macro_agent_tools,- )),- Arc::new(- channel_bots::domain::trigger_detector::MentionOrInferredDetector::new(- message_service.clone(),- conversation_access,- Arc::new(channel_bots::outbound::FastModelTriggerClassifier::new(- ai_usage::pg_recorder_with_enforcement(- db.clone(),- config.enable_ai_usage_enforcement,- ),- )),- )- .with_admission(admission.clone()),- ),- Arc::new(channel_bots::outbound::PrimaryCalendarTimeZones::new(- Arc::new(calendar_events::domain::service::CalendarService::new(- calendar_events::outbound::pg::PgCalendarRepository::new(readonly_db.clone()),- )),- )),- Arc::new(channel_bots::outbound::LexicalCommentMarks::new(- (*lexical_client).clone(),- )),- admission,- );- bot_trigger_router.spawn(bot_trigger_receiver);-- let channel_bot_webhook_state =- bots::inbound::channel_webhook_router::ChannelBotWebhookRouterState::new(- bots_service.clone(),- message_commands.clone(),- (*entity_access_service).clone(),- authorization_state.clone(),- );-- let slack_limits = slack_integration::domain::models::ImportLimits::default();- let slack_enabled = config.slack_import_enabled;- let slack_service = api::context::SlackService::new(- slack_integration::outbound::pg_slack_import_repo::PgSlackImportRepo::new(- db.clone(),- slack_limits,- ),- slack_integration::outbound::s3_storage::S3ImportStorage::new(- macro_aws_config::s3_client().await,- config.upload_staging_bucket.as_ref().to_owned(),- slack_limits,- )- .map_err(|_| anyhow::anyhow!("invalid Slack import storage configuration"))?,- slack_integration::outbound::import_ledger::CanonicalImportLedger::new(- import::outbound::pg_import_repo::PgImportRepo::new(db.clone()),- ),- slack_integration::outbound::gateway_notifier::GatewayImportNotifier::new(- conn_gateway_client.clone(),- ),- api::context::SlackAdminAuthorizer(- entity_access_service.clone(),- entity_access::outbound::PgAccessRepository::new(db.clone()),- ),- Box::new(move |_| slack_enabled),- slack_limits,- )?;- let slack_state = slack_integration::inbound::axum_router::SlackRouterState {- service: Arc::new(slack_service),- entity_access_service: entity_access_service.clone(),- authorization_state: authorization_state.clone(),- };-- // Held by value here and behind an `Arc` in the router state: the impl is a- // pool handle, so cloning is cheap and `SoupImpl` needs an owned service.- // Additional connections for guards spanning email I/O. HTTP requests and- // dispatch share this budget, independently of the main data pool.- let followup_lock_capacity = match config.environment {- Environment::Production => 32,- Environment::Develop => 16,- Environment::Local => 8,- };- let email_followups = reminders::domain::email_followup::service::EmailFollowupService::new(- PgRemindersRepo::with_followup_lock_capacity(db.clone(), followup_lock_capacity),- email_service.clone(),- );- let reminders_service =- reminders::domain::email_followup::reminder_service::EmailRemindersService::new(- email_followups.clone(),- )- .with_entity_access((*entity_access_service).clone());-- let document_creator = documents_hex::domain::create::DocumentCreator::new(- document_service.clone(),- markdown_initializer,- documents_hex::outbound::document_bytes_upload::ReqwestDocumentBytesUploader::default(),- documents_hex::outbound::mention_tracker::LexicalCommsMentionTracker::new(- db.clone(),- lexical_client.clone(),- ),- );- // Forms' ids are their surfaces' ids, so the public surface API asks the- // forms domain which ids are taken.- let collab_surface_service = Arc::new(- collab_surface::outbound::pg_collab_surface_service(- db.clone(),- lexical_client.as_ref().clone(),- sync_service_client.as_ref().clone(),- config.document_permission_jwt.as_ref().to_string(),- )- .with_form_ids(Arc::new(- forms::outbound::collaborative_layout::RepositoryFormIds::new(Arc::new(- forms::outbound::pg_forms_repo::PgFormsRepo::new(db.clone()),- )),- )),- );-- let initiative_service = Arc::new(- InitiativeServiceImpl::new(- PgInitiativeRepo::new(db.clone()),- initiative_description::InitiativeDescriptionSurfacesAdapter::new(- collab_surface_service.clone(),- ),- Arc::new(initiative::outbound::resources::ProjectResources::new(- properties_service.clone(),- system_properties_service.clone(),- entity_access_service.clone(),- )),- )- .with_event_publisher(Arc::new(- initiative::outbound::event_publisher::BrokerInitiativeEventPublisher::new(- macro_event_broker.clone(),- ),- )),- );-- // Shared by the databases router and the unified entity-mutation router.- let databases_service = Arc::new(databases::wiring::build_service(- db.clone(),- entity_access_service.clone(),- databases::outbound::gateway_event_publisher::GatewayTableEventPublisher::new(- conn_gateway_client.as_ref().clone(),- ),- macro_event_broker.clone(),- ));- let forms_service = Arc::new(forms::wiring::build_service(- db.clone(),- databases_service.clone(),- entity_access_service.clone(),- forms::outbound::gateway_event_publisher::GatewayFormEventPublisher::new(- conn_gateway_client.as_ref().clone(),- ),- macro_event_broker.clone(),- collab_surface_service.clone(),- ));-- // Individual initiative reads preserve read-after-write consistency when a- // newly created project opens immediately. Lists retain the replica reader.- // Reuse Soup hydration so detail and mutation metadata include viewer history.- let initiative_entity_soup = Arc::new(- SoupImpl::new(- PgSoupRepo::new(readonly_pool::ReadOnlyPool(db.clone())),- frecency_service.clone(),- ReadonlyEmailPreviewAdapter(email_service.clone()),- ChannelListServiceImpl::new(- PgChannelsRepo::new(db.clone()),- PgChannelsRepo::new(db.clone()),- frecency_storage.clone(),- ),- call::domain::service::CallRecordQueryServiceImpl::new(PgCallRepo::new(db.clone())),- crm_service.clone(),- GithubPullRequestServiceImpl::new(- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),- PgGithubPullRequestRepo::new(db.clone()),- ),- )- .with_agent_branches(agent_changes::outbound::postgres::PgChangesetRepo::new(- db.clone(),- )),- );-- let favorites_service = Arc::new(FavoritesServiceImpl::new(PgFavoritesRepo::new(db.clone())));-- // Keep the replica-backed Soup reader alongside the primary-backed email- // writer in SoupRouterState. REST, GraphQL lists, and realtime hydration- // share this reader; only email mutations and their reply loader use the- // writer service, so normal list traffic never switches to the primary.- let soup_service = Arc::new(- SoupImpl::new(- PgSoupRepo::new(readonly_pool::ReadOnlyPool(readonly_db.clone())),- frecency_service,- readonly_email_service,- ChannelListServiceImpl::new(- PgChannelsRepo::new(readonly_db.clone()),- PgChannelsRepo::new(readonly_db.clone()),- frecency_storage.clone(),- ),- call::domain::service::CallRecordQueryServiceImpl::new(PgCallRepo::new(- readonly_db.clone(),- )),- crm_service.clone(),- GithubPullRequestServiceImpl::new(- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(readonly_db.clone())),- PgGithubPullRequestRepo::new(readonly_db.clone()),- ),- )- .with_favorites(favorites_service.clone())- .with_agent_branches(agent_changes::outbound::postgres::PgChangesetRepo::new(- readonly_db.clone(),- )),- );-- let websocket_notification_consumer_service =- Arc::new(WebSocketNotificationConsumerService::new(- NotificationTopicConsumer::<model_notifications::NotifEvent>::from_env(- config.kafka_brokers.as_ref(),- )- .map_err(|error| {- anyhow::anyhow!("failed to create WebSocket notification topic consumer: {error:?}")- })?,- ));- consumer_tracker.spawn({- let service = Arc::clone(&websocket_notification_consumer_service);- let cancellation_token = consumer_cancellation_token.clone();- async move {- loop {- let result = tokio::select! {- biased;- _ = cancellation_token.cancelled() => break,- result = service.run() => result,- };-- if cancellation_token.is_cancelled() {- break;- }-- let _ = result.inspect_err(|error| {- tracing::error!(- error = ?error,- "WebSocket notification consumer stopped"- );- });-- tokio::select! {- biased;- _ = cancellation_token.cancelled() => break,- _ = tokio::time::sleep(Duration::from_secs(1)) => {}- }- }- }- });-- let activity_realtime_service = Arc::new(activity::ActivityRealtimeConsumerService::new(- activity::ActivityTopicConsumer::from_env(config.kafka_brokers.as_ref()).map_err(- |error| anyhow::anyhow!("failed to create realtime activity topic consumer: {error:?}"),- )?,- ));- consumer_tracker.spawn({- let service = Arc::clone(&activity_realtime_service);- let cancellation_token = consumer_cancellation_token.clone();- async move {- loop {- let result = tokio::select! {- biased;- _ = cancellation_token.cancelled() => break,- result = service.run() => result,- };-- if cancellation_token.is_cancelled() {- break;- }-- let _ = result.inspect_err(|error| {- tracing::error!(- error = ?error,- "realtime activity subscription consumer stopped"- );- });-- tokio::select! {- biased;- _ = cancellation_token.cancelled() => break,- _ = tokio::time::sleep(Duration::from_secs(1)) => {}- }- }- }- });-- let soup_realtime_service = Arc::new(SoupRealtimeConsumerService::new(- SoupTopicConsumer::from_env(config.kafka_brokers.as_ref()).map_err(|error| {- anyhow::anyhow!("failed to create realtime Soup topic consumer: {error:?}")- })?,- ));- consumer_tracker.spawn({- let soup_realtime_service = Arc::clone(&soup_realtime_service);- let cancellation_token = consumer_cancellation_token.clone();- async move {- loop {- let result = tokio::select! {- biased;- _ = cancellation_token.cancelled() => break,- result = soup_realtime_service.run() => result,- };-- if cancellation_token.is_cancelled() {- break;- }-- let _ = result.inspect_err(|error| {- tracing::error!(- error = ?error,- "realtime Soup subscription consumer stopped"- );- });-- tokio::select! {- biased;- _ = cancellation_token.cancelled() => break,- _ = tokio::time::sleep(Duration::from_secs(1)) => {}- }- }- }- });-- let agent_session_log_realtime = Arc::new(AgentSessionLogConsumerService::new(- LogTopicConsumer::from_env(config.kafka_brokers.as_ref()).map_err(|error| {- anyhow::anyhow!("failed to create agent session log topic consumer: {error:?}")- })?,- ));- consumer_tracker.spawn({- let service = Arc::clone(&agent_session_log_realtime);- let cancellation_token = consumer_cancellation_token.clone();- async move {- loop {- let result = tokio::select! {- biased;- _ = cancellation_token.cancelled() => break,- result = service.run() => result,- };-- if cancellation_token.is_cancelled() {- break;- }-- let _ = result.inspect_err(|error| {- tracing::error!(- error = ?error,- "agent session log subscription consumer stopped"- );- });-- tokio::select! {- biased;- _ = cancellation_token.cancelled() => break,- _ = tokio::time::sleep(Duration::from_secs(1)) => {}- }- }- }- });-- consumer_tracker.spawn({- let brokers = config.kafka_brokers.as_ref().to_string();- let entity_access_service = entity_access_service.as_ref().clone();- let macro_event_broker = macro_event_broker.clone();- let cancellation_token = consumer_cancellation_token.clone();- async move {- let mut consecutive_failures = 0_u32;- loop {- if cancellation_token.is_cancelled() {- break;- }-- let fanout_service = SoupRealtimeServiceImpl::new(- EntityAccessExpander::new(entity_access_service.clone()),- KafkaSoupRealtimePublisher::new(macro_event_broker.clone()),- );- tracing::info!("starting realtime Soup entity consumer");- let result = fanout_service- .run_entity_update_consumer(&brokers, cancellation_token.cancelled())- .await;-- if cancellation_token.is_cancelled() {- break;- }-- let restart_delay = match result {- Ok(()) => {- consecutive_failures = 0;- tracing::error!("realtime Soup entity consumer exited unexpectedly");- soup_consumer_restart_delay(1)- }- Err(error) => {- consecutive_failures = consecutive_failures.saturating_add(1);- let restart_delay = soup_consumer_restart_delay(consecutive_failures);- if consecutive_failures >= SOUP_CONSUMER_RESTART_ALERT_THRESHOLD {- tracing::error!(- error = ?error,- consecutive_failures,- restart_delay_secs = restart_delay.as_secs(),- "realtime Soup entity consumer repeatedly failed"- );- } else {- tracing::warn!(- error = ?error,- consecutive_failures,- restart_delay_secs = restart_delay.as_secs(),- "realtime Soup entity consumer failed; scheduling restart"- );- }- restart_delay- }- };-- tokio::select! {- biased;- _ = cancellation_token.cancelled() => break,- _ = tokio::time::sleep(restart_delay) => {}- }- }- }- });-- let favorites_mutation_service = Arc::new(FavoritesMutationServiceImpl::new(- favorites_service.clone(),- entity_access_service.clone(),- ));- let user_api_key_service = Arc::new(UserApiKeyServiceImpl::new(PgUserApiKeysRepo::new(- db.clone(),- )));- let calendar_state = CalendarRouterState::new(- Arc::new(calendar_events::domain::service::CalendarService::new(- calendar_events::outbound::pg::PgCalendarRepository::new(readonly_db.clone()),- )),- authorization_state.clone(),- );-- // Reminder dispatch. An EventBridge rule drops a sweep tick on this queue- // every minute; the sweep fans one message out per due firing, onto the- // same queue, and every task in the pool delivers them in parallel.- let reminder_dispatch_worker = {- let queue = SqsDispatchQueue::new(- aws_sdk_sqs::Client::new(&aws_config),- reminder_dispatch_queue.to_string(),- );- let dispatch_service = ReminderDispatchService::new(- PgRemindersRepo::new(db.clone()),- NotificationReminderNotifier::new((*notification_ingress_service).clone()),- queue.clone(),- );- DispatchWorker::new(- reminders::domain::email_followup::dispatch::EmailReminderDispatch::new(- dispatch_service,- email_followups.clone(),- ),- queue,- )- };-- consumer_tracker.spawn({- let cancellation_token = consumer_cancellation_token.clone();- async move {- tracing::info!("starting reminder dispatch worker");- reminder_dispatch_worker.run(cancellation_token).await;- tracing::info!("reminder dispatch worker stopped");- }- });-- // Calendar event reminder dispatch: same sweep/deliver shape as reminders,- // on its own queue. Behind a master switch — when off, the worker drains- // the minutely tick so the queue neither backs up nor dead-letters.- let calendar_reminder_dispatch_worker = {- let queue = SqsCalendarDispatchQueue::new(- aws_sdk_sqs::Client::new(&aws_config),- calendar_reminder_dispatch_queue.to_string(),- );- let dispatch_service = CalendarReminderDispatchService::new(- calendar_events::outbound::pg::PgCalendarRepository::new(db.clone()),- NotificationCalendarReminderNotifier::new((*notification_ingress_service).clone()),- queue.clone(),- );- CalendarReminderDispatchWorker::new(dispatch_service, queue)- };-- consumer_tracker.spawn({- let cancellation_token = consumer_cancellation_token.clone();- let enabled = config.calendar_reminder_dispatch_enabled;- async move {- if enabled {- tracing::info!("starting calendar reminder dispatch worker");- calendar_reminder_dispatch_worker- .run(cancellation_token)- .await;- tracing::info!("calendar reminder dispatch worker stopped");- } else {- tracing::info!("calendar reminder dispatch disabled; draining its queue");- calendar_reminder_dispatch_worker- .drain(cancellation_token)- .await;- }- }- });-- let redis_sha_client = Arc::new(Redis::new(redis_client));-- let graphql_entity_mutation_service =- Arc::new(service::entity_mutation::DssEntityMutationService::new(- document_service.clone(),- chat_mutation_service,- channels_service.clone(),- call_service.clone(),- Arc::new(email_service.clone()),- project_service.clone(),- databases_service.clone(),- forms_service.clone(),- entity_access_service.clone(),- Arc::new(outbound::entity_mutation::DssEntityLifecycleAdapter::new(- db.clone(),- redis_sha_client.clone(),- sqs_client.clone(),- macro_event_broker.clone(),- )),- ));-- // Routine writes still use the scheduled-action service. Read from the- // primary pool here so the GraphQL list cannot restore stale state after- // a REST write.- let scheduled_action_read_service = Arc::new(ScheduledActionReadServiceImpl::new(- Arc::new(PgScheduledActionRepo::new(- db.clone(),- owned_entity_registrar.clone(),- )),- entity_access_service.clone(),- ));-- let api_context = ApiContext {- dictation_state,- contacts_ingress: contacts_ingress.clone(),- soup_router_state: SoupRouterState::from_arc(- soup_service.clone(),- email_service.clone(),- entity_access_service.clone(),- authorization_state.clone(),- )- .with_favorites_reader(Arc::new(- service::soup_favorites_reader::DssSoupFavoritesReader(favorites_service.clone()),- )),- favorites_state: FavoritesRouterState::new(- favorites_service.clone(),- entity_access_service.clone(),- authorization_state.clone(),- ),- favorites_service,- favorites_mutation_service,- channel_labels_state: ChannelLabelsRouterState::new(- Arc::new(ChannelLabelsServiceImpl::new(PgChannelLabelsRepo::new(- db.clone(),- ))),- entity_access_service.clone(),- authorization_state.clone(),- ),- user_api_key_state: UserApiKeyRouterState::new(- user_api_key_service,- authorization_state.clone(),- ),- slack_state,- reminders_state: RemindersRouterState::new(- Arc::new(reminders_service),- entity_access_service.clone(),- authorization_state.clone(),- ),- graphql_initiative_entity_loader: graphql_initiative::InitiativeEntityLoader(- graphql_soup::soup_item_loader(initiative_entity_soup, Arc::new(email_service.clone())),- ),- graphql_initiative_context: graphql_initiative::InitiativeGraphqlContext::new(- initiative_service.clone(),- entity_access_service.clone(),- ),- graphql_scheduled_action_context: ScheduledActionGraphqlContext::new(- scheduled_action_read_service,- ),- initiative_state: InitiativeRouterState::new(- initiative_service,- entity_access_service.clone(),- authorization_state.clone(),- ),- databases_state: databases::inbound::axum_router::DatabasesRouterState::new(- databases_service,- entity_access_service.clone(),- authorization_state.clone(),- ),- forms_state: forms::inbound::axum_router::FormsRouterState::new(- forms_service,- entity_access_service.clone(),- authorization_state.clone(),- ),- database_starter_state: databases::inbound::starter_router::DatabaseStarterRouterState::new(- Arc::new(databases::domain::starter::DatabaseStarterServiceImpl::new(- databases::outbound::pg_starter::PgDatabaseStarterRepo::new(- db.clone(),- properties::outbound::properties_pg_repo::PropertiesPgRepo::new(db.clone()),- ),- macro_event_broker.clone(),- )),- authorization_state.clone(),- ),- collab_surface_state: CollabSurfaceRouterState::new(- collab_surface_service,- entity_access_service.clone(),- authorization_state.clone(),- ),- graphql_soup_schema: complete_graph::build_schema_from_arcs(- soup_service,- soup_realtime_service,- websocket_notification_consumer_service,- activity_realtime_service,- ),- agent_session_log_subscriptions: complete_graph::agent_session_log_subscriptions(- agent_session_log_realtime,- ),- graphql_notification_reader,- // GraphQL reads the activity log through the readonly pool; the- // Kafka consumer's writer-pool repo above is separate on purpose.- activity_reader: complete_graph::ActivityPortReader::new(Arc::new(- initiative::domain::personal_activity::ProjectVisibleActivityReads::new(- activity::outbound::pg_activity_repo::PgActivityRepo::new(readonly_db.clone()),- entity_access_service.clone(),- ),- )),- graphql_entity_mutation_service,- github_sync_service: Arc::new(github_sync_service_impl),- github_pull_request_index_state,- github_pull_request_state,- github_pull_request_changes_state,- foreign_entity_state,- db: db.clone(),- readonly_db: readonly_pool::ReadOnlyPool(readonly_db.clone()),- redis_client: redis_sha_client,- s3_client: s3,- dynamodb_client: Arc::new(dynamodb_client),- dynamo_db,- macro_event_broker: macro_event_broker.clone(),- sqs_client: sqs_client.clone(),- notification_ingress_service: notification_ingress_service.clone(),- conn_gateway_client: conn_gateway_client.clone(),- sync_service_client: sync_service_client.clone(),- system_properties_service: system_properties_service.clone(),- properties_service: properties_service.clone(),- opensearch_client: Arc::new(opensearch_client),- authorization_state: authorization_state.clone(),- jwt_validation_args,- dss_auth_key,- // Shared frecency storage and legacy channel list routes.- frecency_storage,- channel_list_state,- entity_access_service: entity_access_service.clone(),- calendar_state,- projects_state: ProjectRouterState {- service: project_service,- access_service: entity_access_service.clone(),- authorization_state: authorization_state.clone(),- non_user_owners,- },- documents_state: DocumentRouterState {- service: document_service,- access_service: entity_access_service.clone(),- authorization_state: authorization_state.clone(),- pool: db.clone(),- task_dedup_service,- lexical_client: lexical_client.clone(),- creator: document_creator,- document_permission_jwt_secret: config.document_permission_jwt.as_ref().to_string(),- non_user_owners,- },- config: Arc::new(config),- channel_service: channels_service.clone(),- channels_state: ChannelsRouterState::from_arc(- channels_service,- (*entity_access_service).clone(),- authorization_state.clone(),- ),- messages_state,- annotation_service,- bots_state: bots::inbound::axum_router::BotsRouterState::new(- bots_service.clone(),- (*entity_access_service).clone(),- authorization_state.clone(),- ),- harnesses_state: harnesses::inbound::axum_router::HarnessesRouterState::new(- harnesses_service,- authorization_state.clone(),- ),- channel_bot_webhook_state,- call_state,- call_webhook_state,- call_public_rate_limiter,- webhook_state,- sse_stream_state,- call_internal_state,- cal_webhook_state,- entity_access_management_service,- crm_state: crm::inbound::axum_router::CrmRouterState {- service: Arc::new(crm_service),- stage_service: Arc::new(crm::domain::stages::CrmStageServiceImpl::new(- crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),- crm::outbound::stage_definitions::PropertiesStageDefinitionStore::new(- properties_service.clone(),- ),- )),- entity_access_service: entity_access_service.clone(),- authorization_state: authorization_state.clone(),- },- };-- #[cfg(feature = "delete_document_worker")]- {- let editing_worker_client = Arc::new(- documents_hex::outbound::editing_worker_client::ReqwestEditingWorkerClient::from_url(- AiEditingWorkerUrl::new()?.to_string(),- )- .with_internal_auth_key(Some(api_context.config.internal_api_key.to_string())),- );-- // Spawn the delete document worker.- let delete_worker_ctx = service::delete_document_worker::DeleteDocumentWorkerContext {- worker: Arc::new(delete_document_worker),- db: db.clone(),- s3_client: api_context.s3_client.clone(),- redis_client: api_context.redis_client.clone(),- sync_service_client: api_context.sync_service_client.clone(),- editing_worker_client,- properties_service: api_context.properties_service.clone(),- };-- tokio::spawn(async move {- service::delete_document_worker::run_worker(delete_worker_ctx).await;- });- }-- let server_result = api::setup_and_serve(api_context).await;-- tracing::info!("stopping event consumers");- consumer_cancellation_token.cancel();- consumer_tracker.close();- consumer_tracker.wait().await;- tracing::info!("event consumers stopped");-- tracing::info!("waiting for event broker publishes to drain");- event_broker_tracker.close();- match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {- Ok(()) => tracing::info!("event broker publishes drained"),- Err(error) => {- tracing::warn!(- error=?error,- timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),- "timed out waiting for event broker publishes to drain"- );- }- }-- server_result-}--const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);diff --git a/services/document_storage_service/src/openapi.rs b/services/document_storage_service/src/openapi.rsindex 6818634a03..5f9112c126 100644--- a/services/document_storage_service/src/openapi.rs+++ b/services/document_storage_service/src/openapi.rs@@ -1,17 +1,6 @@-#![allow(unused)]-#![recursion_limit = "256"]--mod api;-mod config;-mod model;-mod outbound;-mod service;-+use document_storage_service::ApiDoc;use utoipa::OpenApi;fn main() {- println!(- "{}",- api::swagger::ApiDoc::openapi().to_pretty_json().unwrap()- );+ println!("{}", ApiDoc::openapi().to_pretty_json().unwrap());}diff --git a/services/document_storage_service/src/server.rs b/services/document_storage_service/src/server.rsnew file mode 100644index 0000000000..c24717d2ee--- /dev/null+++ b/services/document_storage_service/src/server.rs@@ -0,0 +1,2065 @@+//! The service's composition root: reads config and wires every adapter and+//! domain service into the HTTP server and its background workers.++use crate::{+ api::{+ self, MACRO_INTERNAL_USER_ID,+ context::{+ ApiContext, AuthorizationService, DocumentStorageServiceAuthKey, TaskPropertiesAdapter,+ },+ },+ config::{+ self, CalEventTypeContentNamesKey, CalWebhookSecretKey, Config, Environment,+ MetaAccessToken, MetaPixelId, MetaTestEventCode,+ },+ outbound,+ service::{self, s3::S3},+};+use agent_session_realtime::{+ domain::service::AgentSessionLogConsumerService, outbound::log_topic_consumer::LogTopicConsumer,+};+use analytics_client::{AnalyticsClient, AnalyticsClientConfig, MetaConfig};+use anyhow::Context;+use bots::{domain::service::BotServiceImpl, outbound::pg_bots_repo::PgBotsRepo};+use cal::{+ domain::service::{CalConfig, CalEventMeta, CalWebhookServiceImpl},+ inbound::cal_webhook_router::CalWebhookRouterState,+ outbound::analytics_client::AnalyticsClientSink,+};+use calendar_events::domain::reminder_dispatch::CalendarReminderDispatchService;+use calendar_events::inbound::axum_router::CalendarRouterState;+use calendar_events::inbound::dispatch_worker::CalendarReminderDispatchWorker;+use calendar_events::outbound::notification_notifier::NotificationCalendarReminderNotifier;+use calendar_events::outbound::sqs_dispatch_queue::SqsCalendarDispatchQueue;+use call::{+ domain::service::CallServiceImpl,+ inbound::axum_router::{CallRouterState, InternalCallRouterState, WebhookRouterState},+ outbound::{+ ai_call_summarizer::AiCallSummarizer,+ livekit_rtc_client::LivekitRtcClient,+ pg_call_repo::PgCallRepo,+ pg_voice_repo::PgVoiceRepo,+ s3_recording_storage::{RecordingCloudFrontConfig, S3RecordingStorage},+ },+};+use channel_labels::{+ domain::service::ChannelLabelsServiceImpl, inbound::axum_router::ChannelLabelsRouterState,+ outbound::pg_channel_labels_repo::PgChannelLabelsRepo,+};+use channels::{+ domain::{+ list_service::ChannelListServiceImpl,+ service::ChannelServiceImpl,+ side_effects::{ChannelSideEffectService, SpawnedChannelEventDispatcher},+ },+ inbound::{axum_router::ChannelsRouterState, list_router::ChannelListRouterState},+ outbound::{+ connection_gateway_realtime::ConnectionGatewayChannelRealtimePublisher,+ contacts_dispatcher::ContactsChannelDispatcher,+ notification_sender::NotificationChannelSender,+ pg_channel_reference_share_permissions::PgChannelReferenceSharePermissions,+ pg_channels_repo::PgChannelsRepo, pg_side_effect_context::PgChannelSideEffectContext,+ },+};+use collab_surface::inbound::axum_router::CollabSurfaceRouterState;+use connection::{+ domain::service::ConnectionServiceImpl,+ outbound::connection_gateway_client::ConnectionGatewayImpl,+};+use connection_gateway_client::client::ConnectionGatewayClient;+use documents_hex::domain::models::CloudFrontConfig;+use documents_hex::domain::service::DocumentServiceImpl;+use documents_hex::inbound::axum_router::DocumentRouterState;+use documents_hex::outbound::pg_document_repo::PgDocumentRepo;+use documents_hex::outbound::s3_upload_url::S3UploadUrlAdapter;+use dynamodb_client::DynamodbClient;+use email::{+ domain::{ports::ReadonlyEmailPreviewAdapter, service::EmailServiceImpl},+ outbound::EmailPgRepo,+};+use embedding::embedding_provider::openai::TextEmbedding3Small;+use entity_registry::OwnerGrantPolicy;+use entity_registry_db_utils::OwnedEntityRegistrar;+use favorites::{+ domain::{mutation_service::FavoritesMutationServiceImpl, service::FavoritesServiceImpl},+ inbound::axum_router::FavoritesRouterState,+ outbound::pg_favorites_repo::PgFavoritesRepo,+};+use foreign_entity::{+ domain::service::ForeignEntityServiceImpl, inbound::axum_router::ForeignEntityRouterState,+ outbound::pg_foreign_entity_repo::PgForeignEntityRepo,+};+use frecency::{domain::services::FrecencyQueryServiceImpl, outbound::postgres::FrecencyPgStorage};+use github::domain::service::{+ GithubSyncConfig, GithubSyncServiceImpl, InstallationTokenConfig, InstallationTokenService,+ PullRequestIndexService,+};+use github::inbound::pull_request_index_router::PullRequestIndexRouterState;+use github::outbound::connection_gateway_realtime::ConnectionGatewayGithubRealtime;+use github::outbound::github_sync_client::GithubSyncClientImpl;+use github::outbound::pg_github_sync_repo::PgGithubSyncRepo;+use github::outbound::pull_request_diff::GithubPullRequestDiffClient;+use github_pull_requests::{+ domain::service::{+ GithubPullRequestChangesServiceImpl, GithubPullRequestChangesetStore,+ GithubPullRequestServiceImpl,+ },+ inbound::{+ axum_router::GithubPullRequestRouterState,+ changes_router::GithubPullRequestChangesRouterState,+ },+ outbound::{+ pg_github_pull_request_repo::PgGithubPullRequestRepo,+ s3_patch_store::S3GithubPullRequestPatchStore,+ },+};+use graphql_scheduled_action::ScheduledActionGraphqlContext;+use harnesses::outbound::pg_harness_repo::PgHarnessRepo;+use initiative::{+ domain::service::InitiativeServiceImpl, inbound::axum_router::InitiativeRouterState,+ outbound::PgInitiativeRepo,+};+use lexical_client::LexicalClient;+use macro_auth::middleware::decode_jwt::JwtValidationArgs;+use macro_authorization::{+ InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationServiceImpl,+ MacroAuthorizationState, PgBotAuthorizationRepo, PgBotAuthorizer, PgHarnessAuthorizationRepo,+ PgHarnessAuthorizer, PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,+};+use macro_env_var::maybe_env_vars;+use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};+#[cfg(feature = "delete_document_worker")]+use macro_service_urls::AiEditingWorkerUrl;+use macro_service_urls::{+ ConnectionGatewayUrl, LexicalServiceUrl, StaticFileServiceUrl, SyncServiceUrl,+};+use macro_sha_count_client::Redis;+use notification::domain::service::{+ NotificationReaderService, PlatformArnConfig, SqsNotificationIngress,+ WebSocketNotificationConsumerService,+};+use notification::outbound::{notification_consumer::NotificationTopicConsumer, queue::SqsQueue};+use opensearch_client::OpensearchClient;+use projects_hex::{+ domain::service::ProjectServiceImpl,+ inbound::axum_router::ProjectRouterState,+ outbound::{+ DynamoBulkUploadAdapter, PgProjectRepo, S3ProjectUploadAdapter, ShaCountAdapter,+ SqsProjectSearchIndexer,+ },+};+use properties::{+ NotificationServiceImpl, PermissionServiceImpl, PropertiesPgRepo, PropertiesServiceImpl,+};+use rate_limit::{RateLimitServiceImpl, RedisRateLimitAdapter};+use reminders::{+ domain::service::dispatch::ReminderDispatchService,+ inbound::{axum_router::RemindersRouterState, dispatch_worker::DispatchWorker},+ outbound::{+ notification_notifier::NotificationReminderNotifier, pg_reminders_repo::PgRemindersRepo,+ sqs_dispatch_queue::SqsDispatchQueue,+ },+};+use scheduled_action::domain::read_service::ScheduledActionReadServiceImpl;+use scheduled_action::outbound::pg_scheduled_action_repo::PgScheduledActionRepo;+use secretsmanager_client::SecretManager;+use soup::{+ domain::service::SoupImpl, inbound::axum_router::SoupRouterState,+ outbound::pg_soup_repo::PgSoupRepo,+};+use soup_realtime::{+ domain::service::{SoupRealtimeConsumerService, SoupRealtimeServiceImpl},+ outbound::{+ entity_access::EntityAccessExpander, kafka_publisher::KafkaSoupRealtimePublisher,+ soup_consumer::SoupTopicConsumer,+ },+};+use sqlx::postgres::PgPoolOptions;+use std::{sync::Arc, time::Duration};+use sync_service_client::SyncServiceClient;+use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};+use task_dedup::{+ TaskDedupService,+ outbound::{+ cohere::CohereReranker,+ connection_gateway::ConnectionGatewayTaskDedupNotifier,+ judge::AgentDuplicateJudge,+ postgres::{PgTaskMatchRepo, PgTaskVectorDb},+ },+};+use tokio_util::{sync::CancellationToken, task::TaskTracker};+use user_api_key::{+ domain::service::UserApiKeyServiceImpl, inbound::axum_router::UserApiKeyRouterState,+ outbound::pg_user_api_keys_repo::PgUserApiKeysRepo,+};++const SOUP_CONSUMER_RESTART_MAX_DELAY_SECS: u64 = 60;+const SOUP_CONSUMER_RESTART_ALERT_THRESHOLD: u32 = 5;++fn soup_consumer_restart_delay(consecutive_failures: u32) -> Duration {+ let exponent = consecutive_failures.saturating_sub(1).min(6);+ Duration::from_secs((1_u64 << exponent).min(SOUP_CONSUMER_RESTART_MAX_DELAY_SECS))+}++maybe_env_vars! {+ struct AppleBundleId;+ struct SnsApnsVoipPlatformArn;+}++/// Build the service from the environment and serve until shutdown.+pub async fn run() -> anyhow::Result<()> {+ let env = Environment::new_or_prod();++ let aws_config = macro_aws_config::get_macro_aws_config().await;++ let secretsmanager_client = secretsmanager_client::SecretsManager::new(+ aws_sdk_secretsmanager::Client::new(&aws_config),+ );++ // Parse our configuration from the environment, then resolve any secret-manager backed values.+ let config = Config::from_env()+ .context("expected to be able to generate config")?+ .resolve_remote_secrets(env, &secretsmanager_client)+ .await+ .context("expected to be able to resolve config secrets")?;+ let non_user_owners = config.non_user_owners()?;++ tracing::trace!("initialized config");++ let (min_connections, max_connections): (u32, u32) = match config.environment {+ Environment::Production => (50, 150),+ Environment::Develop => (25, 100),+ Environment::Local => (15, 50),+ };++ let db = PgPoolOptions::new()+ .min_connections(min_connections)+ .max_connections(max_connections)+ .connect(&config.database_url)+ .await+ .context("could not connect to db")?;++ tracing::trace!(+ min_connections,+ max_connections,+ "initialized db connection"+ );++ let readonly_db = match PgPoolOptions::new()+ .min_connections(min_connections)+ .max_connections(max_connections)+ .connect(&config.database_url_readonly)+ .await+ {+ Ok(pool) => {+ tracing::trace!("initialized readonly db connection");+ pool+ }+ Err(e) => {+ tracing::warn!(error=?e, "failed to connect to readonly db, falling back to primary");+ db.clone()+ }+ };++ let dynamo_db = aws_sdk_dynamodb::Client::new(&aws_config);++ let dynamodb_client = DynamodbClient::new_from_client(+ dynamo_db.clone(),+ Some(config.bulk_upload_requests_table.as_ref().to_string()),+ );+ tracing::trace!("initialized dynamodb client");++ let s3_client = macro_aws_config::s3_client().await;++ tracing::trace!("initialized s3 client");++ let search_event_queue = macro_queues::SearchEventQueue::new();+ let document_delete_queue = macro_queues::DocumentDeleteQueue::new();+ let contacts_queue = macro_queues::ContactsQueue::new();+ let notification_queue = macro_queues::NotificationIngressQueue::new();+ let gmail_ops_queue = macro_queues::GmailOpsQueue::new();+ let reminder_dispatch_queue = macro_queues::ReminderDispatchQueue::new();+ let calendar_reminder_dispatch_queue = macro_queues::CalendarReminderDispatchQueue::new();+ let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(&aws_config))+ .search_event_queue(&search_event_queue)+ .document_delete_queue(&document_delete_queue)+ .gmail_ops_queue(&gmail_ops_queue);+ let webhook_event_queue = webhook::outbound::SqsWebhookQueue::new(+ Arc::new(sqs_client.clone()),+ macro_queues::WebhookEventQueue::new().to_string(),+ config.webhook_queue_max_messages,+ config.webhook_queue_wait_time_seconds,+ );++ let contacts_ingress = Arc::new(contacts::domain::service::SqsContactsIngress {+ queue: contacts::outbound::ingress::SqsContactsQueue::new(+ aws_sdk_sqs::Client::new(&aws_config),+ contacts_queue.to_string(),+ ),+ });++ tracing::trace!("initialized sqs client");++ // Redis handles it own connection pool internally. Each time we use redis+ // we should be using redis_client.get_connection() to grab a specific connection+ let redis_client = redis::Client::open(config.redis_uri.to_string().as_ref())+ .expect("could not connect to redis client");++ match redis_client.get_connection().is_err() {+ true => {+ tracing::error!("unable to connect to redis");+ }+ false => {+ tracing::trace!("initialized redis connection");+ }+ }++ let dss_auth_key = DocumentStorageServiceAuthKey::new()?;++ let conn_gateway_client = ConnectionGatewayClient::new(+ config.internal_api_key.to_string(),+ ConnectionGatewayUrl::new()?.to_string(),+ );++ let sync_service_client = Arc::new(SyncServiceClient::new(+ config.sync_service_auth_key.as_ref().to_string(),+ SyncServiceUrl::new()?.to_string(),+ ));++ let lexical_client = Arc::new(LexicalClient::new(+ config.internal_api_key.to_string(),+ LexicalServiceUrl::new()?.to_string(),+ ));++ let jwt_validation_args =+ JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)+ .await?;++ let consumer_cancellation_token = CancellationToken::new();+ let consumer_tracker = TaskTracker::new();+ let event_broker_tracker = TaskTracker::new();+ let macro_event_broker = MacroEventBrokerService::new(+ KafkaEventPublisher::new(config.kafka_brokers.as_ref())+ .context("failed to create kafka event publisher")?,+ event_broker_tracker.clone(),+ );+ let bots_repo = PgBotsRepo::new(db.clone());+ // The agent API checks a newly selected MCP app against Pipedream's+ // directory, the same check the ConfigureAgent tool makes.+ let bots_service = BotServiceImpl::new(bots_repo, macro_event_broker.clone()).with_mcp_apps(+ ai_tools::PipedreamMcpAppCatalog::new(ai_tools::pipedream_client_from_env()?),+ );++ let authorization_service: AuthorizationService = MacroAuthorizationServiceImpl::new(+ MacroAuthJwtValidator::new(jwt_validation_args.clone()),+ InternalAuthConfig {+ api_key: dss_auth_key.as_ref().to_string(),+ default_user_id: Some(MACRO_INTERNAL_USER_ID.to_string()),+ },+ PgBotAuthorizer::new(PgBotAuthorizationRepo::new(db.clone())),+ PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),+ )+ .with_harness_authorizer(PgHarnessAuthorizer::new(PgHarnessAuthorizationRepo::new(+ db.clone(),+ )));+ let authorization_state = MacroAuthorizationState::new(Arc::new(authorization_service));+ let harnesses_service =+ harnesses::domain::service::HarnessServiceImpl::new(PgHarnessRepo::new(db.clone()));++ // Initialize OpenSearch client+ let opensearch_client = OpensearchClient::new(+ config.opensearch_url.to_string(),+ config.opensearch_username.to_string(),+ config.opensearch_password.to_string(),+ )+ .context("unable to create opensearch client")?;++ if let Err(e) = opensearch_client.health().await {+ tracing::error!(error=?e, "error connecting to opensearch");+ return Err(e);+ }+ tracing::trace!("initialized opensearch client");++ let frecency_storage = FrecencyPgStorage::new(db.clone());+ let frecency_service = FrecencyQueryServiceImpl::new(frecency_storage.clone());+ // DSS only reads CRM rows — no-op resolver keeps reqwest/scraper+ // out of this binary.+ let crm_service = crm::domain::service::CrmServiceImpl::new(+ crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),+ crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,+ );+ let email_service = EmailServiceImpl::new(+ EmailPgRepo::new(db.clone()),+ frecency_service.clone(),+ sqs_client.clone(),+ crm_service.clone(),+ entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(+ entity_access_management::outbound::PgRepository::new(db.clone()),+ ),+ 0,+ )+ .with_macro_event_broker(macro_event_broker.clone());+ let readonly_email_service = ReadonlyEmailPreviewAdapter(+ EmailServiceImpl::new(+ EmailPgRepo::new(readonly_db.clone()),+ frecency_service.clone(),+ sqs_client.clone(),+ crm_service.clone(),+ entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(+ entity_access_management::outbound::PgRepository::new(readonly_db.clone()),+ ),+ 0,+ )+ .with_macro_event_broker(macro_event_broker.clone()),+ );+ let system_properties_service =+ SystemPropertiesServiceImpl::new(PgSystemPropertiesRepository::new(db.clone()));+ let ingress_queue = SqsQueue::new(+ aws_sdk_sqs::Client::new(&aws_config),+ notification_queue.to_string(),+ );+ let notification_ingress_service = Arc::new(SqsNotificationIngress {+ queue: ingress_queue.clone(),+ });+ let graphql_notification_reader: Arc<ai_tools::ToolNotificationService> =+ Arc::new(NotificationReaderService {+ repository: notification::outbound::repository::DbNotificationRepository::new(+ db.clone(),+ ),+ queue: ai_tools::ToolNotificationQueue::Sqs(ingress_queue.clone()),+ sns_endpoint: ai_tools::NoOpSnsEndpointManager,+ platform_config: PlatformArnConfig {+ apns_platform_arn: String::new(),+ fcm_platform_arn: String::new(),+ apns_voip_platform_arn: String::new(),+ },+ realtime: notification::domain::ports::NoopNotificationRealtimePublisher,+ });+ tracing::trace!("initialized notification ingress service");++ let entity_access_service = Arc::new(+ entity_access::domain::service::EntityAccessServiceImpl::new(+ entity_access::outbound::PgAccessRepository::new(db.clone()),+ ),+ );++ let permission_checker = PermissionServiceImpl::new(db.clone(), entity_access_service.clone());+ let notification_service = NotificationServiceImpl::new(+ SqsNotificationIngress {+ queue: ingress_queue,+ },+ db.clone(),+ );+ let properties_service = Arc::new(+ PropertiesServiceImpl::new(+ PropertiesPgRepo::new(db.clone()),+ Some(permission_checker),+ Some(notification_service),+ )+ .with_event_broker(macro_event_broker.clone())+ .with_initiative_assignees(Arc::new(+ initiative::domain::assignees::InitiativeAssignees::new(PgInitiativeRepo::new(+ db.clone(),+ )),+ )),+ );++ // Create the channel list service used by soup.+ // Create the legacy channel list router state for routes mounted under /comms.+ let channel_list_state = ChannelListRouterState::new(+ ChannelListServiceImpl::new(+ PgChannelsRepo::new(db.clone()),+ PgChannelsRepo::new(db.clone()),+ frecency_storage.clone(),+ ),+ authorization_state.clone(),+ );++ let s3 = Arc::new(S3::new(+ s3_client,+ config.document_storage_bucket.as_ref(),+ config.docx_document_upload_bucket.as_ref(),+ config.upload_staging_bucket.as_ref(),+ ));+ let system_properties_service = Arc::new(system_properties_service);++ let owned_entity_registrar =+ OwnedEntityRegistrar::new(OwnerGrantPolicy::new(PgBotsRepo::new(db.clone())));+ let document_repo = PgDocumentRepo::new(db.clone(), owned_entity_registrar.clone());+ let cloudfront_config = CloudFrontConfig {+ distribution_url: config+ .document_storage_service_cloudfront_distribution_url+ .as_ref()+ .to_string(),+ signer_public_key_id: config+ .document_storage_service_cloudfront_signer_public_key_id+ .as_ref()+ .to_string(),+ signer_private_key: config+ .document_storage_service_cloudfront_signer_private_key+ .as_ref()+ .to_string(),+ presigned_url_expiry_seconds: config.document_storage_service_presigned_url_expiry_seconds,+ browser_cache_expiry_seconds: config+ .document_storage_service_presigned_url_browser_cache_expiry_seconds,+ };+ let s3_upload_adapter = S3UploadUrlAdapter::new(+ macro_aws_config::s3_client().await,+ config.document_storage_bucket.as_ref(),+ config.docx_document_upload_bucket.as_ref(),+ );+ let markdown_initializer =+ documents_hex::outbound::markdown_init::LexicalSyncMarkdownInitializer::new(+ lexical_client.as_ref().clone(),+ sync_service_client.as_ref().clone(),+ );++ let connection_gateway = Arc::new(ConnectionGatewayImpl::new(conn_gateway_client.clone()));++ let connection_service =+ ConnectionServiceImpl::new(entity_access_service.clone(), connection_gateway.clone());++ let entity_access_management_service =+ entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(+ entity_access_management::outbound::PgRepository::new(db.clone()),+ );++ let chat_mutation_service =+ Arc::new(chat::domain::service::ChatServiceImpl::new_without_tools(+ chat::outbound::postgres::PgChatRepo::new(db.clone(), owned_entity_registrar.clone()),+ entity_access_management_service.clone(),+ ));++ let project_service = Arc::new(ProjectServiceImpl::new(+ PgProjectRepo::new(db.clone(), owned_entity_registrar.clone()),+ S3ProjectUploadAdapter::new(+ macro_aws_config::s3_client().await,+ config.document_storage_bucket.as_ref(),+ config.docx_document_upload_bucket.as_ref(),+ config.upload_staging_bucket.as_ref(),+ ),+ DynamoBulkUploadAdapter::new(dynamodb_client.clone()),+ ShaCountAdapter::new(Redis::new(redis_client.clone())),+ entity_access_management_service.clone(),+ SqsProjectSearchIndexer::new(Arc::new(sqs_client.clone()), macro_event_broker.clone()),+ if cfg!(feature = "local") {+ Some(uuid::uuid!("d50676e2-0a12-4c62-bc07-4b1cb6d8e9bc"))+ } else {+ None+ },+ macro_event_broker.clone(),+ ));++ let document_service = Arc::new(DocumentServiceImpl::new(+ document_repo,+ cloudfront_config,+ sync_service_client.as_ref().clone(),+ s3_upload_adapter,+ TaskPropertiesAdapter {+ system_properties: system_properties_service.clone(),+ properties: properties_service.clone(),+ entity_access_service: entity_access_service.clone(),+ },+ connection_service,+ entity_access_management_service.clone(),+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),+ macro_event_broker.clone(),+ ));++ let foreign_entity_service = Arc::new(ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(+ db.clone(),+ )));++ let github_sync_service_impl = GithubSyncServiceImpl::new(+ GithubSyncConfig {+ webhook_secret: config.github_webhook_secret_key.as_ref().to_string(),+ github_sync_app_url: config.github_sync_app_url.to_string(),+ sync_app_pem: config.github_sync_app_pem_secret_key.as_ref().to_string(),+ sync_app_client_id: config.github_sync_app_client_id.to_string(),+ sync_app_client_secret: config.github_sync_app_client_secret.to_string(),+ installation_state_secret: config.github_installation_state_secret.to_string(),+ },+ document_service.clone(),+ Arc::new(GithubPullRequestServiceImpl::new(+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),+ PgGithubPullRequestRepo::new(db.clone()),+ )),+ (*notification_ingress_service).clone(),+ PgGithubSyncRepo::new(db.clone()),+ GithubSyncClientImpl::default(),+ ConnectionGatewayGithubRealtime::new(conn_gateway_client.clone()),+ );++ let github_pull_request_index_state = PullRequestIndexRouterState {+ service: Arc::new(PullRequestIndexService::new(+ InstallationTokenConfig {+ client_id: config.github_sync_app_client_id.to_string(),+ private_key_pem: config.github_sync_app_pem_secret_key.as_ref().to_string(),+ },+ PgGithubSyncRepo::new(db.clone()),+ GithubSyncClientImpl::default(),+ GithubPullRequestServiceImpl::new(+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),+ PgGithubPullRequestRepo::new(db.clone()),+ ),+ )),+ authorization_state: authorization_state.clone(),+ };++ let github_pull_request_state = GithubPullRequestRouterState::new(+ Arc::new(GithubPullRequestServiceImpl::new(+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),+ PgGithubPullRequestRepo::new(db.clone()),+ )),+ entity_access_service.clone(),+ authorization_state.clone(),+ );++ let github_pull_request_changes_state = GithubPullRequestChangesRouterState::new(+ Arc::new(GithubPullRequestChangesServiceImpl::new(+ GithubPullRequestServiceImpl::new(+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),+ PgGithubPullRequestRepo::new(db.clone()),+ ),+ GithubPullRequestChangesetStore::new(+ GithubPullRequestDiffClient::new(InstallationTokenService::new(+ InstallationTokenConfig {+ client_id: config.github_sync_app_client_id.to_string(),+ private_key_pem: config.github_sync_app_pem_secret_key.as_ref().to_string(),+ },+ PgGithubSyncRepo::new(db.clone()),+ GithubSyncClientImpl::default(),+ )),+ PgGithubPullRequestRepo::new(db.clone()),+ S3GithubPullRequestPatchStore::new(+ macro_aws_config::s3_client().await,+ config.github_pull_request_patch_bucket.to_string(),+ ),+ ),+ )),+ entity_access_service.clone(),+ authorization_state.clone(),+ );++ let foreign_entity_state = ForeignEntityRouterState::new(+ foreign_entity_service.clone(),+ entity_access_service.clone(),+ authorization_state.clone(),+ );++ // Cal.com webhooks → Meta Lead events. Both secrets are loaded here+ // (rather than on Config) to keep cal/Meta wiring colocated.+ let cal_webhook_secret = secretsmanager_client+ .get_maybe_secret_value(env, CalWebhookSecretKey::new()?)+ .await?;+ let cal_event_type_content_names_secret = secretsmanager_client+ .get_maybe_secret_value(env, CalEventTypeContentNamesKey::new()?)+ .await?;++ let analytics_client = Arc::new(AnalyticsClient::new(AnalyticsClientConfig {+ google_analytics: None,+ meta: Some(MetaConfig {+ pixel_id: MetaPixelId::new()?.as_ref().to_string(),+ access_token: MetaAccessToken::new()?.as_ref().to_string(),+ test_event_code: MetaTestEventCode::new().map(|v| v.as_ref().to_string()),+ }),+ posthog: None,+ }));++ let cal_event_type_meta: std::collections::HashMap<u64, CalEventMeta> =+ serde_json::from_str(cal_event_type_content_names_secret.as_ref())+ .context("CalEventTypeContentNames secret must be a JSON object mapping eventTypeId (u64) to { content_name: string, value: number (USD) }")?;+ let cal_webhook_service = CalWebhookServiceImpl::new(+ CalConfig {+ webhook_secret: cal_webhook_secret.as_ref().to_string(),+ event_type_meta: cal_event_type_meta,+ },+ AnalyticsClientSink::new(analytics_client.clone()),+ );+ let cal_webhook_state = CalWebhookRouterState::new(cal_webhook_service);++ // Call service (LiveKit)+ let transcription_agent_name = config+ .livekit_transcription_agent_name+ .value()+ .map(|v| v.to_owned());++ let internal_call_secret = config.internal_call_secret.value().map(|v| v.to_owned());++ anyhow::ensure!(+ transcription_agent_name.is_none() || internal_call_secret.is_some(),+ "LIVEKIT_TRANSCRIPTION_AGENT_NAME is set but INTERNAL_CALL_SECRET is missing — \+ the transcription agent will not be able to submit transcripts"+ );+ let livekit_rtc_client = LivekitRtcClient::new(+ config.livekit_server_url.as_ref(),+ config.livekit_api_key.as_ref(),+ config.livekit_api_secret.as_ref(),+ transcription_agent_name,+ );+ let call_connection_service =+ ConnectionServiceImpl::new(entity_access_service.clone(), connection_gateway.clone());+ let call_repo = PgCallRepo::new(db.clone());+ let egress_config = match (+ config::CallRecordingS3Bucket::new(),+ config::CallRecordingS3Region::new(),+ config::CallRecordingS3AccessKey::new(),+ config::CallRecordingS3Secret::new(),+ ) {+ (Some(bucket), Some(region), Some(access_key), Some(secret)) => {+ tracing::info!(bucket = bucket.as_ref(), "call recording enabled");+ Some(call::domain::models::EgressS3Config {+ bucket: bucket.as_ref().to_string(),+ region: region.as_ref().to_string(),+ access_key: access_key.as_ref().to_string(),+ secret: secret.as_ref().to_string(),+ })+ }+ _ => None,+ };+ let recording_storage = match &egress_config {+ Some(egress_config) => {+ let cloudfront_config = RecordingCloudFrontConfig {+ distribution_url: config+ .document_storage_service_cloudfront_distribution_url+ .as_ref()+ .to_string(),+ signer_public_key_id: config+ .document_storage_service_cloudfront_signer_public_key_id+ .as_ref()+ .to_string(),+ signer_private_key: config+ .document_storage_service_cloudfront_signer_private_key+ .as_ref()+ .to_string(),+ presigned_url_expiry_seconds: config::CALL_RECORDING_PRESIGNED_URL_EXPIRY_SECONDS,+ };+ Some(S3RecordingStorage::new(egress_config.bucket.clone(), cloudfront_config).await)+ }+ None => None,+ };+ let mut call_service_builder = CallServiceImpl::<_, _, _, _, _, _, AiCallSummarizer>::new(+ call_repo,+ livekit_rtc_client,+ call_connection_service,+ (*entity_access_service).clone(),+ (*notification_ingress_service).clone(),+ recording_storage,+ config.livekit_server_url.as_ref(),+ )+ .with_summarizer(AiCallSummarizer::new(+ ai_usage::pg_recorder_with_enforcement(db.clone(), config.enable_ai_usage_enforcement),+ ));+ if let Some(secret) = internal_call_secret {+ call_service_builder = call_service_builder.with_internal_call_secret(secret);+ }+ if let Some(config) = egress_config {+ call_service_builder = call_service_builder.with_egress(config);+ }+ if let Some(base_url) = config::CallRingStatusBaseUrl::new() {+ call_service_builder =+ call_service_builder.with_ring_status_base_url(base_url.as_ref().to_owned());+ }++ // VoIP push is optional: enabled only when both env vars are present.+ // APPLE_BUNDLE_ID: the app bundle ID (e.g. "com.macro.app.prod")+ // SNS_APNS_VOIP_PLATFORM_ARN: SNS platform app ARN for APNS_VOIP+ //+ // Option<VoipPushServiceImpl<...>> is used as the type parameter so the+ // type stays stable regardless of whether VoIP push is configured.+ let voip_sender = if let (Some(bundle_id), Some(voip_arn)) =+ (AppleBundleId::new(), SnsApnsVoipPlatformArn::new())+ {+ if voip_arn.is_empty() {+ tracing::warn!("voip push disabled: SNS_APNS_VOIP_PLATFORM_ARN is set but empty");+ None+ } else {+ let voip_repo =+ notification::outbound::repository::DbNotificationRepository::new(db.clone());+ let voip_mobile = notification::outbound::mobile::MobilePushAdapter {+ push_service: aws_sdk_sns::Client::new(&aws_config),+ apns_bundle_id: bundle_id.to_string(),+ voip_bundle_id: Some(format!("{}.voip", bundle_id.as_ref())),+ };+ tracing::info!(+ bundle_id = bundle_id.as_ref(),+ voip_arn = voip_arn.as_ref(),+ "voip push enabled"+ );+ Some(notification::domain::service::VoipPushServiceImpl::new(+ voip_repo,+ voip_mobile,+ ))+ }+ } else {+ tracing::info!("voip push disabled: APPLE_BUNDLE_ID or SNS_APNS_VOIP_PLATFORM_ARN not set");+ None+ };+ let call_service_builder = call_service_builder.with_voip_push_sender(voip_sender);++ let call_service = Arc::new(+ call_service_builder+ .with_voice_repo(PgVoiceRepo::new(db.clone()))+ .with_event_broker(macro_event_broker.clone()),+ );++ consumer_tracker.spawn({+ let service = call_service.clone();+ let cancellation_token = consumer_cancellation_token.clone();+ async move {+ cancellation_token+ .run_until_cancelled(call::inbound::stale_call_sweeper::run_stale_call_sweeper(+ service,+ call::inbound::stale_call_sweeper::SWEEP_INTERVAL,+ ))+ .await;+ }+ });++ let call_state = CallRouterState::new(+ call_service.clone(),+ entity_access_service.clone(),+ authorization_state.clone(),+ );+ let call_webhook_state = WebhookRouterState::new(call_service.clone());+ let call_public_rate_limiter = RateLimitServiceImpl {+ repo: RedisRateLimitAdapter {+ redis: redis_client.clone(),+ },+ };++ let webhook_repository = webhook::outbound::PgRepository::new(db.clone());+ let webhook_endpoint_scheme_policy = if matches!(env, Environment::Local) {+ webhook::domain::models::WebhookEndpointSchemePolicy::HttpAndHttps+ } else {+ webhook::domain::models::WebhookEndpointSchemePolicy::HttpsOnly+ };+ let webhook_http_client =+ webhook::outbound::ReqwestWebhookDeliveryClient::new_with_endpoint_scheme_policy(+ webhook_endpoint_scheme_policy,+ )+ .context("failed to create webhook HTTP client")?;+ let webhook_service =+ webhook::domain::service::WebhookServiceImpl::new_with_endpoint_scheme_policy(+ webhook_repository.clone(),+ webhook_http_client.clone(),+ webhook_endpoint_scheme_policy,+ macro_event_broker.clone(),+ );+ let webhook_rate_limiter = RateLimitServiceImpl {+ repo: RedisRateLimitAdapter {+ redis: redis_client.clone(),+ },+ };+ let webhook_state = webhook::inbound::axum_router::WebhookRouterState::new(+ webhook_service,+ webhook_rate_limiter,+ authorization_state.clone(),+ );++ let (webhook_stream_sender, _) =+ tokio::sync::broadcast::channel(webhook::domain::stream::WEBHOOK_STREAM_CHANNEL_CAPACITY);+ let sse_stream_service = webhook::domain::stream::WebhookEventStreamServiceImpl::new(+ webhook_stream_sender.clone(),+ entity_access_service.clone(),+ webhook_repository.clone(),+ );+ let sse_stream_state = webhook::inbound::stream_router::WebhookStreamRouterState::new(+ sse_stream_service,+ authorization_state.clone(),+ );+ consumer_tracker.spawn({+ let brokers = config.kafka_brokers.as_ref().to_string();+ let sender = webhook_stream_sender;+ let cancellation_token = consumer_cancellation_token.clone();+ async move {+ loop {+ let result = tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => break,+ result = webhook::inbound::kafka_stream_consumer::run_webhook_stream_consumer(+ &brokers,+ &sender,+ ) => result,+ };++ if cancellation_token.is_cancelled() {+ break;+ }+ if let Err(error) = result {+ tracing::error!(+ error = ?error,+ "webhook SSE Kafka consumer stopped"+ );+ tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => break,+ _ = tokio::time::sleep(Duration::from_secs(1)) => {}+ }+ }+ }+ }+ });++ let webhook_ingestion_service =+ webhook::domain::ingestion::WebhookEventIngestionServiceImpl::new(+ entity_access_service.clone(),+ webhook_repository,+ webhook_event_queue.clone(),+ );+ let webhook_delivery_repository =+ webhook::outbound::PgWebhookDeliveryRepository::new(db.clone());+ let webhook_delivery_service = webhook::domain::delivery::WebhookEventDeliveryServiceImpl::new(+ webhook_delivery_repository,+ webhook_http_client,+ );+ let webhook_worker = webhook::inbound::worker::WebhookEventWorker::new(+ webhook_event_queue,+ webhook_delivery_service,+ );+ let webhook_worker_task = tokio::spawn(async move {+ tracing::info!("starting webhook event worker");+ webhook_worker.run().await;+ });+ tokio::spawn(async move {+ match webhook_worker_task.await {+ Ok(()) => tracing::error!("webhook event worker exited unexpectedly"),+ Err(error) => {+ tracing::error!(error = ?error, "webhook event worker exited unexpectedly");+ }+ }+ });++ let webhook_consumer_brokers = config.kafka_brokers.as_ref().to_string();+ consumer_tracker.spawn({+ let cancellation_token = consumer_cancellation_token.clone();+ async move {+ loop {+ if cancellation_token.is_cancelled() {+ break;+ }++ tracing::info!("starting webhook event consumer");+ let result = webhook::inbound::kafka_consumer::run_webhook_event_consumer(+ &webhook_consumer_brokers,+ webhook_ingestion_service.clone(),+ cancellation_token.cancelled(),+ )+ .await;++ if cancellation_token.is_cancelled() {+ break;+ }++ match result {+ Ok(()) => tracing::error!("webhook event consumer exited unexpectedly"),+ Err(error) => {+ tracing::error!(error = ?error, "webhook event consumer exited unexpectedly");+ }+ }++ tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => break,+ _ = tokio::time::sleep(Duration::from_secs(5)) => {}+ }+ }+ }+ });++ let activity_consumer_brokers = config.kafka_brokers.as_ref().to_string();+ let editing_activity = Arc::new(+ documents_hex::outbound::editing_activity::RedisEditingActivityStore::new(+ redis_client.clone(),+ ),+ );+ consumer_tracker.spawn({+ let cancellation_token = consumer_cancellation_token.clone();+ let activity_repo = activity::outbound::pg_activity_repo::PgActivityRepo::new(db.clone());+ let activity_realtime = activity::domain::announcements::ActivityAnnouncements::new(+ activity::KafkaActivityRealtimePublisher::new(macro_event_broker.clone()),+ crate::service::activity::EntityAccessActivityAudience::new(+ entity_access_service.clone(),+ ),+ );+ async move {+ let consumer = activity::inbound::kafka_consumer::ActivityConsumer::<+ _,+ crate::service::activity::ActivitySourceEvent,+ _,+ _,+ >::new(+ activity_repo,+ move |event| {+ let editing_activity = editing_activity.clone();+ async move {+ crate::service::activity::ingest(event, editing_activity.as_ref()).await+ }+ },+ activity_realtime,+ );+ loop {+ if cancellation_token.is_cancelled() {+ break;+ }++ tracing::info!("starting activity consumer");+ let result = consumer+ .run(&activity_consumer_brokers, cancellation_token.cancelled())+ .await;++ if cancellation_token.is_cancelled() {+ break;+ }++ match result {+ Ok(()) => tracing::error!("activity consumer exited unexpectedly"),+ Err(error) => {+ tracing::error!(error = ?error, "activity consumer exited unexpectedly");+ }+ }++ tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => break,+ _ = tokio::time::sleep(Duration::from_secs(5)) => {}+ }+ }+ }+ });++ // Links archived calls to the CRM records of the people on them.+ let crm_call_link_brokers = config.kafka_brokers.as_ref().to_string();+ consumer_tracker.spawn({+ let cancellation_token = consumer_cancellation_token.clone();+ let consumer = crm::inbound::call_archived::CallArchivedConsumer::new(+ crm::domain::call_links::CallRecordLinker::new(+ call::domain::service::CallRecordQueryServiceImpl::new(PgCallRepo::new(+ db.clone(),+ )),+ crm::outbound::call_link::PgCallCrmLinker::new(+ db.clone(),+ (*system_properties_service).clone(),+ ),+ ),+ );+ async move {+ loop {+ if cancellation_token.is_cancelled() {+ break;+ }++ tracing::info!("starting CRM call-link consumer");+ let result = consumer+ .run(&crm_call_link_brokers, cancellation_token.cancelled())+ .await;++ if cancellation_token.is_cancelled() {+ break;+ }++ match result {+ Ok(()) => tracing::error!("CRM call-link consumer exited unexpectedly"),+ Err(error) => {+ tracing::error!(error = ?error, "CRM call-link consumer exited unexpectedly");+ }+ }++ tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => break,+ _ = tokio::time::sleep(Duration::from_secs(5)) => {}+ }+ }+ }+ });++ let call_internal_state = InternalCallRouterState::new(call_service.clone());++ // Create the SQS worker for delete document processing before config is moved.+ #[cfg(feature = "delete_document_worker")]+ let delete_document_worker = sqs_worker::SQSWorker::new(+ aws_sdk_sqs::Client::new(&aws_config),+ document_delete_queue.to_string(),+ config.queue_max_messages,+ config.queue_wait_time_seconds,+ );++ let sqs_client = Arc::new(sqs_client);+ let conn_gateway_client = Arc::new(conn_gateway_client);++ // MacroConfig reads the shared OPENAI_API_KEY from Doppler's APP_SECRETS_JSON+ // or the local environment. Validate once before constructing consumers.+ let openai_api_key = config.openai_api_key.as_ref().to_owned();+ anyhow::ensure!(+ !openai_api_key.trim().is_empty(),+ "OpenAI API key is required for task dedup embeddings and dictation",+ );+ let cohere_api_key = config.cohere_api_key.as_ref().to_owned();+ anyhow::ensure!(+ !cohere_api_key.trim().is_empty(),+ "Cohere API key is required for task dedup reranking",+ );+ let dictation_state = dictation::inbound::axum_router::DictationRouterState::new(+ dictation::domain::DictationServiceImpl::new(+ dictation::outbound::WhisperTranscriber::new(&config.openai_api_key)?,+ dictation::outbound::SymphoniaRecordingInspector,+ ai_usage::pg_recorder_with_enforcement(db.clone(), config.enable_ai_usage_enforcement),+ ),+ RateLimitServiceImpl {+ repo: RedisRateLimitAdapter {+ redis: redis_client.clone(),+ },+ },+ authorization_state.clone(),+ );+ let task_dedup_service = Arc::new(TaskDedupService::new(+ TextEmbedding3Small::new(openai_api_key),+ PgTaskVectorDb::new(db.clone()),+ CohereReranker::new(cohere_api_key),+ Arc::new(AgentDuplicateJudge::new(+ ai_usage::pg_recorder_with_enforcement(db.clone(), config.enable_ai_usage_enforcement),+ )),+ Arc::new(ConnectionGatewayTaskDedupNotifier::new(+ conn_gateway_client.clone(),+ )),+ Arc::new(PgTaskMatchRepo::new(db.clone())),+ ));+ let channels_repo = PgChannelsRepo::new(db.clone());+ // Built-in agents read the same committed-post stream as hosted and+ // external agents, for channel and document posts alike, so the channel+ // side effects no longer carry their own trigger queue.+ let (bot_trigger_sender, bot_trigger_receiver) = tokio::sync::mpsc::unbounded_channel();++ let channel_side_effects = ChannelSideEffectService::new(+ PgChannelSideEffectContext::new(db.clone()),+ ConnectionGatewayChannelRealtimePublisher::new(conn_gateway_client.clone()),+ NotificationChannelSender::new(notification_ingress_service.clone()),+ ContactsChannelDispatcher::new(contacts_ingress.clone()),+ )+ .with_macro_event_broker(macro_event_broker.clone());++ let channels_service = Arc::new(+ ChannelServiceImpl::with_dependencies(+ channels_repo,+ SpawnedChannelEventDispatcher::new(channel_side_effects.clone()),+ PgChannelReferenceSharePermissions::new(db.clone(), entity_access_service.clone()),+ )+ .with_picture_files(+ channels::outbound::static_file_pictures::StaticFileChannelPictures::new(+ static_file_service_client::StaticFileServiceClient::new(+ config.internal_api_key.to_string(),+ StaticFileServiceUrl::new()?.to_string(),+ ),+ ),+ ),+ );++ // One message implementation for channels and documents. Committed changes fan+ // out to the broker, the local agent queue, and the parent's own delivery: channel+ // messages keep their notifications, activity, and sharing side effects and send+ // the common `message_update` payload; document discussions get the common+ // payload and document comment notifications.+ let message_realtime = messages::outbound::connection_gateway::ConnectionGatewayMessages(+ conn_gateway_client.clone(),+ );+ let discussion_delivery = messages::domain::delivery::DiscussionDelivery::new(+ messages::outbound::pg_discussion_context::PgDiscussionContext(db.clone())+ .with_initiatives(+ initiative::domain::lookup::InitiativeLookup::new(+ initiative::outbound::PgInitiativeRepo::new(db.clone()),+ ),+ properties_service.clone(),+ )+ .with_crm(crm::outbound::lookup::PgCrmParentReader::new(db.clone())),+ messages::outbound::entity_access_audience::EntityAccessMessageAudience(+ (*entity_access_service).clone(),+ ),+ message_realtime.clone(),+ messages::outbound::notification_sender::MessageNotificationSender(+ notification_ingress_service.clone(),+ ),+ )+ .with_sharing(messages::outbound::pg_discussion_context::PgDiscussionContext(db.clone()));+ let annotation_service = Arc::new(messages::domain::annotations::AnnotationService::new(+ macro_db_client::annotations::repository::PgAnnotationRepository(db.clone()),+ (*entity_access_service).clone(),+ discussion_delivery.clone(),+ ));+ let channel_delivery = channels::domain::message_delivery::ChannelMessageDelivery::new(+ PgChannelsRepo::new(db.clone()),+ SpawnedChannelEventDispatcher::new(channel_side_effects.clone()),+ PgChannelReferenceSharePermissions::new(db.clone(), entity_access_service.clone()),+ message_realtime,+ );+ let message_service = Arc::new(+ messages::domain::service::MessageService::new(+ messages::outbound::pg_message_repo::PgMessageRepository::new(db.clone())+ .with_initiatives(initiative::domain::lookup::InitiativeLookup::new(+ initiative::outbound::PgInitiativeRepo::new(db.clone()),+ ))+ .with_crm(crm::outbound::lookup::PgCrmParentReader::new(db.clone())),+ messages::domain::effects::MessageEffects::new(+ messages::outbound::broker::BrokerMessagePublisher::new(macro_event_broker.clone()),+ channel_bots::outbound::conversation::LocalBotPublisher::new(bot_trigger_sender),+ messages::domain::delivery::ParentMessagePublisher::new(+ channel_delivery,+ discussion_delivery,+ ),+ ),+ )+ .with_group_recipients(channels::domain::group_mentions::ChannelGroupRecipients(+ PgChannelsRepo::new(db.clone()),+ ))+ .with_mention_extractor(lexical_mention_extractor::LexicalMentionExtractor::new(+ lexical_client.clone(),+ ))+ .with_references(+ messages::outbound::entity_access_audience::EntityAccessMessageReferences(+ (*entity_access_service).clone(),+ ),+ ),+ );+ let message_commands: Arc<dyn messages::domain::api::MessageCommands> = message_service.clone();+ let messages_state = messages::inbound::axum_router::MessagesRouterState {+ service: message_service.clone(),+ access: entity_access_service.clone(),+ authorization: authorization_state.clone(),+ };++ let teammate_dms_brokers = config.kafka_brokers.as_ref().to_string();+ consumer_tracker.spawn({+ let cancellation_token = consumer_cancellation_token.clone();+ let channels = (*channels_service).clone();+ async move {+ loop {+ if cancellation_token.is_cancelled() {+ break;+ }++ tracing::info!("starting teammate DM consumer");+ let result = channels::inbound::teammate_dms_consumer::run_teammate_dms_consumer(+ &teammate_dms_brokers,+ channels.clone(),+ cancellation_token.cancelled(),+ )+ .await;++ if cancellation_token.is_cancelled() {+ break;+ }++ match result {+ Ok(()) => tracing::error!("teammate DM consumer exited unexpectedly"),+ Err(error) => {+ tracing::error!(+ error = ?error,+ "teammate DM consumer exited unexpectedly"+ );+ }+ }++ tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => break,+ _ = tokio::time::sleep(Duration::from_secs(5)) => {}+ }+ }+ }+ });++ // Wire Macro AI to react to mentions with the classic in-channel chat+ // reply. The router posts replies through the channel service we just+ // built and runs the agent loop in-process with the channel-bot toolset:+ // a channel has no composer to finish a chat-deferred user tool in, so+ // calendar event creation executes directly and SendEmail is unavailable.+ // Agent sessions belong to a different bot entirely+ // (`bot_id::MACRO_NEW_BOT_ID`, served by the harness), so the two paths+ // can never answer the same mention.+ let mut macro_agent_tool_context = ai_tools::build_tool_service_context_from_env(+ db.clone(),+ event_broker_tracker.clone(),+ config.enable_ai_usage_enforcement,+ config.ai_pricing(),+ )+ .await+ .context("failed to build Macro agent tool context")?;+ // Wire the agent's SendChannelMessage tool to this service's own+ // side-effect pipeline so agent-posted messages share the exact instance+ // used by the HTTP API, including the in-process bot trigger sender (the+ // env builder wires an equivalent pipeline, but without bot triggers).+ macro_agent_tool_context.channel_tool_context =+ ai_tools::build_channel_tool_context_with_dispatcher(+ db.clone(),+ std::sync::Arc::new(SpawnedChannelEventDispatcher::new(channel_side_effects)),+ message_service.clone(),+ );+ let macro_agent_tools = ai_tools::tools_for(ai_tools::AiHost::ChannelBot);+ let conversation_access = Arc::new(+ channel_bots::outbound::conversation::EntityAccessConversation(+ entity_access_service.clone(),+ ),+ );+ let admission = macro_agent_tool_context.admission.clone();+ let bot_trigger_router = channel_bots::inbound::BotTriggerRouter::new_with_admission(+ message_service.clone(),+ conversation_access.clone(),+ Arc::new(channel_bots::outbound::AgentLoopResponder::new(+ macro_agent_tool_context,+ macro_agent_tools,+ )),+ Arc::new(+ channel_bots::domain::trigger_detector::MentionOrInferredDetector::new(+ message_service.clone(),+ conversation_access,+ Arc::new(channel_bots::outbound::FastModelTriggerClassifier::new(+ ai_usage::pg_recorder_with_enforcement(+ db.clone(),+ config.enable_ai_usage_enforcement,+ ),+ )),+ )+ .with_admission(admission.clone()),+ ),+ Arc::new(channel_bots::outbound::PrimaryCalendarTimeZones::new(+ Arc::new(calendar_events::domain::service::CalendarService::new(+ calendar_events::outbound::pg::PgCalendarRepository::new(readonly_db.clone()),+ )),+ )),+ Arc::new(channel_bots::outbound::LexicalCommentMarks::new(+ (*lexical_client).clone(),+ )),+ admission,+ );+ bot_trigger_router.spawn(bot_trigger_receiver);++ let channel_bot_webhook_state =+ bots::inbound::channel_webhook_router::ChannelBotWebhookRouterState::new(+ bots_service.clone(),+ message_commands.clone(),+ (*entity_access_service).clone(),+ authorization_state.clone(),+ );++ let slack_limits = slack_integration::domain::models::ImportLimits::default();+ let slack_enabled = config.slack_import_enabled;+ let slack_service = api::context::SlackService::new(+ slack_integration::outbound::pg_slack_import_repo::PgSlackImportRepo::new(+ db.clone(),+ slack_limits,+ ),+ slack_integration::outbound::s3_storage::S3ImportStorage::new(+ macro_aws_config::s3_client().await,+ config.upload_staging_bucket.as_ref().to_owned(),+ slack_limits,+ )+ .map_err(|_| anyhow::anyhow!("invalid Slack import storage configuration"))?,+ slack_integration::outbound::import_ledger::CanonicalImportLedger::new(+ import::outbound::pg_import_repo::PgImportRepo::new(db.clone()),+ ),+ slack_integration::outbound::gateway_notifier::GatewayImportNotifier::new(+ conn_gateway_client.clone(),+ ),+ api::context::SlackAdminAuthorizer(+ entity_access_service.clone(),+ entity_access::outbound::PgAccessRepository::new(db.clone()),+ ),+ Box::new(move |_| slack_enabled),+ slack_limits,+ )?;+ let slack_state = slack_integration::inbound::axum_router::SlackRouterState {+ service: Arc::new(slack_service),+ entity_access_service: entity_access_service.clone(),+ authorization_state: authorization_state.clone(),+ };++ // Held by value here and behind an `Arc` in the router state: the impl is a+ // pool handle, so cloning is cheap and `SoupImpl` needs an owned service.+ // Additional connections for guards spanning email I/O. HTTP requests and+ // dispatch share this budget, independently of the main data pool.+ let followup_lock_capacity = match config.environment {+ Environment::Production => 32,+ Environment::Develop => 16,+ Environment::Local => 8,+ };+ let email_followups = reminders::domain::email_followup::service::EmailFollowupService::new(+ PgRemindersRepo::with_followup_lock_capacity(db.clone(), followup_lock_capacity),+ email_service.clone(),+ );+ let reminders_service =+ reminders::domain::email_followup::reminder_service::EmailRemindersService::new(+ email_followups.clone(),+ )+ .with_entity_access((*entity_access_service).clone());++ let document_creator = documents_hex::domain::create::DocumentCreator::new(+ document_service.clone(),+ markdown_initializer,+ documents_hex::outbound::document_bytes_upload::ReqwestDocumentBytesUploader::default(),+ documents_hex::outbound::mention_tracker::LexicalCommsMentionTracker::new(+ db.clone(),+ lexical_client.clone(),+ ),+ );+ // Forms' ids are their surfaces' ids, so the public surface API asks the+ // forms domain which ids are taken.+ let collab_surface_service = Arc::new(+ collab_surface::outbound::pg_collab_surface_service(+ db.clone(),+ lexical_client.as_ref().clone(),+ sync_service_client.as_ref().clone(),+ config.document_permission_jwt.as_ref().to_string(),+ )+ .with_form_ids(Arc::new(+ forms::outbound::collaborative_layout::RepositoryFormIds::new(Arc::new(+ forms::outbound::pg_forms_repo::PgFormsRepo::new(db.clone()),+ )),+ )),+ );++ let initiative_service = Arc::new(+ InitiativeServiceImpl::new(+ PgInitiativeRepo::new(db.clone()),+ initiative_description::InitiativeDescriptionSurfacesAdapter::new(+ collab_surface_service.clone(),+ ),+ Arc::new(initiative::outbound::resources::ProjectResources::new(+ properties_service.clone(),+ system_properties_service.clone(),+ entity_access_service.clone(),+ )),+ )+ .with_event_publisher(Arc::new(+ initiative::outbound::event_publisher::BrokerInitiativeEventPublisher::new(+ macro_event_broker.clone(),+ ),+ )),+ );++ // Shared by the databases router and the unified entity-mutation router.+ let databases_service = Arc::new(databases::wiring::build_service(+ db.clone(),+ entity_access_service.clone(),+ databases::outbound::gateway_event_publisher::GatewayTableEventPublisher::new(+ conn_gateway_client.as_ref().clone(),+ ),+ macro_event_broker.clone(),+ ));+ let forms_service = Arc::new(forms::wiring::build_service(+ db.clone(),+ databases_service.clone(),+ entity_access_service.clone(),+ forms::outbound::gateway_event_publisher::GatewayFormEventPublisher::new(+ conn_gateway_client.as_ref().clone(),+ ),+ macro_event_broker.clone(),+ collab_surface_service.clone(),+ ));++ // Individual initiative reads preserve read-after-write consistency when a+ // newly created project opens immediately. Lists retain the replica reader.+ // Reuse Soup hydration so detail and mutation metadata include viewer history.+ let initiative_entity_soup = Arc::new(+ SoupImpl::new(+ PgSoupRepo::new(readonly_pool::ReadOnlyPool(db.clone())),+ frecency_service.clone(),+ ReadonlyEmailPreviewAdapter(email_service.clone()),+ ChannelListServiceImpl::new(+ PgChannelsRepo::new(db.clone()),+ PgChannelsRepo::new(db.clone()),+ frecency_storage.clone(),+ ),+ call::domain::service::CallRecordQueryServiceImpl::new(PgCallRepo::new(db.clone())),+ crm_service.clone(),+ GithubPullRequestServiceImpl::new(+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),+ PgGithubPullRequestRepo::new(db.clone()),+ ),+ )+ .with_agent_branches(agent_changes::outbound::postgres::PgChangesetRepo::new(+ db.clone(),+ )),+ );++ let favorites_service = Arc::new(FavoritesServiceImpl::new(PgFavoritesRepo::new(db.clone())));++ // Keep the replica-backed Soup reader alongside the primary-backed email+ // writer in SoupRouterState. REST, GraphQL lists, and realtime hydration+ // share this reader; only email mutations and their reply loader use the+ // writer service, so normal list traffic never switches to the primary.+ let soup_service = Arc::new(+ SoupImpl::new(+ PgSoupRepo::new(readonly_pool::ReadOnlyPool(readonly_db.clone())),+ frecency_service,+ readonly_email_service,+ ChannelListServiceImpl::new(+ PgChannelsRepo::new(readonly_db.clone()),+ PgChannelsRepo::new(readonly_db.clone()),+ frecency_storage.clone(),+ ),+ call::domain::service::CallRecordQueryServiceImpl::new(PgCallRepo::new(+ readonly_db.clone(),+ )),+ crm_service.clone(),+ GithubPullRequestServiceImpl::new(+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(readonly_db.clone())),+ PgGithubPullRequestRepo::new(readonly_db.clone()),+ ),+ )+ .with_favorites(favorites_service.clone())+ .with_agent_branches(agent_changes::outbound::postgres::PgChangesetRepo::new(+ readonly_db.clone(),+ )),+ );++ let websocket_notification_consumer_service =+ Arc::new(WebSocketNotificationConsumerService::new(+ NotificationTopicConsumer::<model_notifications::NotifEvent>::from_env(+ config.kafka_brokers.as_ref(),+ )+ .map_err(|error| {+ anyhow::anyhow!("failed to create WebSocket notification topic consumer: {error:?}")+ })?,+ ));+ consumer_tracker.spawn({+ let service = Arc::clone(&websocket_notification_consumer_service);+ let cancellation_token = consumer_cancellation_token.clone();+ async move {+ loop {+ let result = tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => break,+ result = service.run() => result,+ };++ if cancellation_token.is_cancelled() {+ break;+ }++ let _ = result.inspect_err(|error| {+ tracing::error!(+ error = ?error,+ "WebSocket notification consumer stopped"+ );+ });++ tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => break,+ _ = tokio::time::sleep(Duration::from_secs(1)) => {}+ }+ }+ }+ });++ let activity_realtime_service = Arc::new(activity::ActivityRealtimeConsumerService::new(+ activity::ActivityTopicConsumer::from_env(config.kafka_brokers.as_ref()).map_err(+ |error| anyhow::anyhow!("failed to create realtime activity topic consumer: {error:?}"),+ )?,+ ));+ consumer_tracker.spawn({+ let service = Arc::clone(&activity_realtime_service);+ let cancellation_token = consumer_cancellation_token.clone();+ async move {+ loop {+ let result = tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => break,+ result = service.run() => result,+ };++ if cancellation_token.is_cancelled() {+ break;+ }++ let _ = result.inspect_err(|error| {+ tracing::error!(+ error = ?error,+ "realtime activity subscription consumer stopped"+ );+ });++ tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => break,+ _ = tokio::time::sleep(Duration::from_secs(1)) => {}+ }+ }+ }+ });++ let soup_realtime_service = Arc::new(SoupRealtimeConsumerService::new(+ SoupTopicConsumer::from_env(config.kafka_brokers.as_ref()).map_err(|error| {+ anyhow::anyhow!("failed to create realtime Soup topic consumer: {error:?}")+ })?,+ ));+ consumer_tracker.spawn({+ let soup_realtime_service = Arc::clone(&soup_realtime_service);+ let cancellation_token = consumer_cancellation_token.clone();+ async move {+ loop {+ let result = tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => break,+ result = soup_realtime_service.run() => result,+ };++ if cancellation_token.is_cancelled() {+ break;+ }++ let _ = result.inspect_err(|error| {+ tracing::error!(+ error = ?error,+ "realtime Soup subscription consumer stopped"+ );+ });++ tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => break,+ _ = tokio::time::sleep(Duration::from_secs(1)) => {}+ }+ }+ }+ });++ let agent_session_log_realtime = Arc::new(AgentSessionLogConsumerService::new(+ LogTopicConsumer::from_env(config.kafka_brokers.as_ref()).map_err(|error| {+ anyhow::anyhow!("failed to create agent session log topic consumer: {error:?}")+ })?,+ ));+ consumer_tracker.spawn({+ let service = Arc::clone(&agent_session_log_realtime);+ let cancellation_token = consumer_cancellation_token.clone();+ async move {+ loop {+ let result = tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => break,+ result = service.run() => result,+ };++ if cancellation_token.is_cancelled() {+ break;+ }++ let _ = result.inspect_err(|error| {+ tracing::error!(+ error = ?error,+ "agent session log subscription consumer stopped"+ );+ });++ tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => break,+ _ = tokio::time::sleep(Duration::from_secs(1)) => {}+ }+ }+ }+ });++ consumer_tracker.spawn({+ let brokers = config.kafka_brokers.as_ref().to_string();+ let entity_access_service = entity_access_service.as_ref().clone();+ let macro_event_broker = macro_event_broker.clone();+ let cancellation_token = consumer_cancellation_token.clone();+ async move {+ let mut consecutive_failures = 0_u32;+ loop {+ if cancellation_token.is_cancelled() {+ break;+ }++ let fanout_service = SoupRealtimeServiceImpl::new(+ EntityAccessExpander::new(entity_access_service.clone()),+ KafkaSoupRealtimePublisher::new(macro_event_broker.clone()),+ );+ tracing::info!("starting realtime Soup entity consumer");+ let result = fanout_service+ .run_entity_update_consumer(&brokers, cancellation_token.cancelled())+ .await;++ if cancellation_token.is_cancelled() {+ break;+ }++ let restart_delay = match result {+ Ok(()) => {+ consecutive_failures = 0;+ tracing::error!("realtime Soup entity consumer exited unexpectedly");+ soup_consumer_restart_delay(1)+ }+ Err(error) => {+ consecutive_failures = consecutive_failures.saturating_add(1);+ let restart_delay = soup_consumer_restart_delay(consecutive_failures);+ if consecutive_failures >= SOUP_CONSUMER_RESTART_ALERT_THRESHOLD {+ tracing::error!(+ error = ?error,+ consecutive_failures,+ restart_delay_secs = restart_delay.as_secs(),+ "realtime Soup entity consumer repeatedly failed"+ );+ } else {+ tracing::warn!(+ error = ?error,+ consecutive_failures,+ restart_delay_secs = restart_delay.as_secs(),+ "realtime Soup entity consumer failed; scheduling restart"+ );+ }+ restart_delay+ }+ };++ tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => break,+ _ = tokio::time::sleep(restart_delay) => {}+ }+ }+ }+ });++ let favorites_mutation_service = Arc::new(FavoritesMutationServiceImpl::new(+ favorites_service.clone(),+ entity_access_service.clone(),+ ));+ let user_api_key_service = Arc::new(UserApiKeyServiceImpl::new(PgUserApiKeysRepo::new(+ db.clone(),+ )));+ let calendar_state = CalendarRouterState::new(+ Arc::new(calendar_events::domain::service::CalendarService::new(+ calendar_events::outbound::pg::PgCalendarRepository::new(readonly_db.clone()),+ )),+ authorization_state.clone(),+ );++ // Reminder dispatch. An EventBridge rule drops a sweep tick on this queue+ // every minute; the sweep fans one message out per due firing, onto the+ // same queue, and every task in the pool delivers them in parallel.+ let reminder_dispatch_worker = {+ let queue = SqsDispatchQueue::new(+ aws_sdk_sqs::Client::new(&aws_config),+ reminder_dispatch_queue.to_string(),+ );+ let dispatch_service = ReminderDispatchService::new(+ PgRemindersRepo::new(db.clone()),+ NotificationReminderNotifier::new((*notification_ingress_service).clone()),+ queue.clone(),+ );+ DispatchWorker::new(+ reminders::domain::email_followup::dispatch::EmailReminderDispatch::new(+ dispatch_service,+ email_followups.clone(),+ ),+ queue,+ )+ };++ consumer_tracker.spawn({+ let cancellation_token = consumer_cancellation_token.clone();+ async move {+ tracing::info!("starting reminder dispatch worker");+ reminder_dispatch_worker.run(cancellation_token).await;+ tracing::info!("reminder dispatch worker stopped");+ }+ });++ // Calendar event reminder dispatch: same sweep/deliver shape as reminders,+ // on its own queue. Behind a master switch — when off, the worker drains+ // the minutely tick so the queue neither backs up nor dead-letters.+ let calendar_reminder_dispatch_worker = {+ let queue = SqsCalendarDispatchQueue::new(+ aws_sdk_sqs::Client::new(&aws_config),+ calendar_reminder_dispatch_queue.to_string(),+ );+ let dispatch_service = CalendarReminderDispatchService::new(+ calendar_events::outbound::pg::PgCalendarRepository::new(db.clone()),+ NotificationCalendarReminderNotifier::new((*notification_ingress_service).clone()),+ queue.clone(),+ );+ CalendarReminderDispatchWorker::new(dispatch_service, queue)+ };++ consumer_tracker.spawn({+ let cancellation_token = consumer_cancellation_token.clone();+ let enabled = config.calendar_reminder_dispatch_enabled;+ async move {+ if enabled {+ tracing::info!("starting calendar reminder dispatch worker");+ calendar_reminder_dispatch_worker+ .run(cancellation_token)+ .await;+ tracing::info!("calendar reminder dispatch worker stopped");+ } else {+ tracing::info!("calendar reminder dispatch disabled; draining its queue");+ calendar_reminder_dispatch_worker+ .drain(cancellation_token)+ .await;+ }+ }+ });++ let redis_sha_client = Arc::new(Redis::new(redis_client));++ let graphql_entity_mutation_service =+ Arc::new(service::entity_mutation::DssEntityMutationService::new(+ document_service.clone(),+ chat_mutation_service,+ channels_service.clone(),+ call_service.clone(),+ Arc::new(email_service.clone()),+ project_service.clone(),+ databases_service.clone(),+ forms_service.clone(),+ entity_access_service.clone(),+ Arc::new(outbound::entity_mutation::DssEntityLifecycleAdapter::new(+ db.clone(),+ redis_sha_client.clone(),+ sqs_client.clone(),+ macro_event_broker.clone(),+ )),+ ));++ // Routine writes still use the scheduled-action service. Read from the+ // primary pool here so the GraphQL list cannot restore stale state after+ // a REST write.+ let scheduled_action_read_service = Arc::new(ScheduledActionReadServiceImpl::new(+ Arc::new(PgScheduledActionRepo::new(+ db.clone(),+ owned_entity_registrar.clone(),+ )),+ entity_access_service.clone(),+ ));++ let api_context = ApiContext {+ dictation_state,+ contacts_ingress: contacts_ingress.clone(),+ soup_router_state: SoupRouterState::from_arc(+ soup_service.clone(),+ email_service.clone(),+ entity_access_service.clone(),+ authorization_state.clone(),+ )+ .with_favorites_reader(Arc::new(+ service::soup_favorites_reader::DssSoupFavoritesReader(favorites_service.clone()),+ )),+ favorites_state: FavoritesRouterState::new(+ favorites_service.clone(),+ entity_access_service.clone(),+ authorization_state.clone(),+ ),+ favorites_service,+ favorites_mutation_service,+ channel_labels_state: ChannelLabelsRouterState::new(+ Arc::new(ChannelLabelsServiceImpl::new(PgChannelLabelsRepo::new(+ db.clone(),+ ))),+ entity_access_service.clone(),+ authorization_state.clone(),+ ),+ user_api_key_state: UserApiKeyRouterState::new(+ user_api_key_service,+ authorization_state.clone(),+ ),+ slack_state,+ reminders_state: RemindersRouterState::new(+ Arc::new(reminders_service),+ entity_access_service.clone(),+ authorization_state.clone(),+ ),+ graphql_initiative_entity_loader: graphql_initiative::InitiativeEntityLoader(+ graphql_soup::soup_item_loader(initiative_entity_soup, Arc::new(email_service.clone())),+ ),+ graphql_initiative_context: graphql_initiative::InitiativeGraphqlContext::new(+ initiative_service.clone(),+ entity_access_service.clone(),+ ),+ graphql_scheduled_action_context: ScheduledActionGraphqlContext::new(+ scheduled_action_read_service,+ ),+ initiative_state: InitiativeRouterState::new(+ initiative_service,+ entity_access_service.clone(),+ authorization_state.clone(),+ ),+ databases_state: databases::inbound::axum_router::DatabasesRouterState::new(+ databases_service,+ entity_access_service.clone(),+ authorization_state.clone(),+ ),+ forms_state: forms::inbound::axum_router::FormsRouterState::new(+ forms_service,+ entity_access_service.clone(),+ authorization_state.clone(),+ ),+ database_starter_state: databases::inbound::starter_router::DatabaseStarterRouterState::new(+ Arc::new(databases::domain::starter::DatabaseStarterServiceImpl::new(+ databases::outbound::pg_starter::PgDatabaseStarterRepo::new(+ db.clone(),+ properties::outbound::properties_pg_repo::PropertiesPgRepo::new(db.clone()),+ ),+ macro_event_broker.clone(),+ )),+ authorization_state.clone(),+ ),+ collab_surface_state: CollabSurfaceRouterState::new(+ collab_surface_service,+ entity_access_service.clone(),+ authorization_state.clone(),+ ),+ graphql_soup_schema: complete_graph::build_schema_from_arcs(+ soup_service,+ soup_realtime_service,+ websocket_notification_consumer_service,+ activity_realtime_service,+ ),+ agent_session_log_subscriptions: complete_graph::agent_session_log_subscriptions(+ agent_session_log_realtime,+ ),+ graphql_notification_reader,+ // GraphQL reads the activity log through the readonly pool; the+ // Kafka consumer's writer-pool repo above is separate on purpose.+ activity_reader: complete_graph::ActivityPortReader::new(Arc::new(+ initiative::domain::personal_activity::ProjectVisibleActivityReads::new(+ activity::outbound::pg_activity_repo::PgActivityRepo::new(readonly_db.clone()),+ entity_access_service.clone(),+ ),+ )),+ graphql_entity_mutation_service,+ github_sync_service: Arc::new(github_sync_service_impl),+ github_pull_request_index_state,+ github_pull_request_state,+ github_pull_request_changes_state,+ foreign_entity_state,+ db: db.clone(),+ readonly_db: readonly_pool::ReadOnlyPool(readonly_db.clone()),+ redis_client: redis_sha_client,+ s3_client: s3,+ dynamodb_client: Arc::new(dynamodb_client),+ dynamo_db,+ macro_event_broker: macro_event_broker.clone(),+ sqs_client: sqs_client.clone(),+ notification_ingress_service: notification_ingress_service.clone(),+ conn_gateway_client: conn_gateway_client.clone(),+ sync_service_client: sync_service_client.clone(),+ system_properties_service: system_properties_service.clone(),+ properties_service: properties_service.clone(),+ opensearch_client: Arc::new(opensearch_client),+ authorization_state: authorization_state.clone(),+ jwt_validation_args,+ dss_auth_key,+ // Shared frecency storage and legacy channel list routes.+ frecency_storage,+ channel_list_state,+ entity_access_service: entity_access_service.clone(),+ calendar_state,+ projects_state: ProjectRouterState {+ service: project_service,+ access_service: entity_access_service.clone(),+ authorization_state: authorization_state.clone(),+ non_user_owners,+ },+ documents_state: DocumentRouterState {+ service: document_service,+ access_service: entity_access_service.clone(),+ authorization_state: authorization_state.clone(),+ pool: db.clone(),+ task_dedup_service,+ lexical_client: lexical_client.clone(),+ creator: document_creator,+ document_permission_jwt_secret: config.document_permission_jwt.as_ref().to_string(),+ non_user_owners,+ },+ config: Arc::new(config),+ channel_service: channels_service.clone(),+ channels_state: ChannelsRouterState::from_arc(+ channels_service,+ (*entity_access_service).clone(),+ authorization_state.clone(),+ ),+ messages_state,+ annotation_service,+ bots_state: bots::inbound::axum_router::BotsRouterState::new(+ bots_service.clone(),+ (*entity_access_service).clone(),+ authorization_state.clone(),+ ),+ harnesses_state: harnesses::inbound::axum_router::HarnessesRouterState::new(+ harnesses_service,+ authorization_state.clone(),+ ),+ channel_bot_webhook_state,+ call_state,+ call_webhook_state,+ call_public_rate_limiter,+ webhook_state,+ sse_stream_state,+ call_internal_state,+ cal_webhook_state,+ entity_access_management_service,+ crm_state: crm::inbound::axum_router::CrmRouterState {+ service: Arc::new(crm_service),+ stage_service: Arc::new(crm::domain::stages::CrmStageServiceImpl::new(+ crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),+ crm::outbound::stage_definitions::PropertiesStageDefinitionStore::new(+ properties_service.clone(),+ ),+ )),+ entity_access_service: entity_access_service.clone(),+ authorization_state: authorization_state.clone(),+ },+ };++ #[cfg(feature = "delete_document_worker")]+ {+ let editing_worker_client = Arc::new(+ documents_hex::outbound::editing_worker_client::ReqwestEditingWorkerClient::from_url(+ AiEditingWorkerUrl::new()?.to_string(),+ )+ .with_internal_auth_key(Some(api_context.config.internal_api_key.to_string())),+ );++ // Spawn the delete document worker.+ let delete_worker_ctx = service::delete_document_worker::DeleteDocumentWorkerContext {+ worker: Arc::new(delete_document_worker),+ db: db.clone(),+ s3_client: api_context.s3_client.clone(),+ redis_client: api_context.redis_client.clone(),+ sync_service_client: api_context.sync_service_client.clone(),+ editing_worker_client,+ properties_service: api_context.properties_service.clone(),+ };++ tokio::spawn(async move {+ service::delete_document_worker::run_worker(delete_worker_ctx).await;+ });+ }++ let server_result = api::setup_and_serve(api_context).await;++ tracing::info!("stopping event consumers");+ consumer_cancellation_token.cancel();+ consumer_tracker.close();+ consumer_tracker.wait().await;+ tracing::info!("event consumers stopped");++ tracing::info!("waiting for event broker publishes to drain");+ event_broker_tracker.close();+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {+ Ok(()) => tracing::info!("event broker publishes drained"),+ Err(error) => {+ tracing::warn!(+ error=?error,+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),+ "timed out waiting for event broker publishes to drain"+ );+ }+ }++ server_result+}++const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);diff --git a/services/email_service/src/api/context.rs b/services/email_service/src/api/context.rsindex bb98d02125..309476b4fe 100644--- a/services/email_service/src/api/context.rs+++ b/services/email_service/src/api/context.rs@@ -1,3 +1,6 @@+use crate::calendar_refresh::ConnectionGatewayCalendarRefresh;+use crate::calendar_request_gate::RedisCalendarRequestGate;+use crate::calendar_tokens::CalendarTokenProviderAdapter;use axum::extract::FromRef;use calendar_events::{domain::{mutations::CalendarMutationServiceImpl, service::CalendarService},@@ -12,13 +15,10 @@ use email::{},outbound::{EmailPgRepo, GmailTokenProviderImpl},};-use email_service::calendar_refresh::ConnectionGatewayCalendarRefresh;-use email_service::calendar_request_gate::RedisCalendarRequestGate;-use email_service::calendar_tokens::CalendarTokenProviderAdapter;-use email_service::config::Config;-use email_service::outbound::email_api::GmailApi;-use email_service::util::redis::RedisClient;+use crate::config::Config;+use crate::outbound::email_api::GmailApi;+use crate::util::redis::RedisClient;use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};use entity_access_management::domain::service::EntityAccessManagementServiceImpl;use frecency::{domain::services::FrecencyQueryServiceImpl, outbound::postgres::FrecencyPgStorage};diff --git a/services/email_service/src/api/email/attachments/get_document_id.rs b/services/email_service/src/api/email/attachments/get_document_id.rsindex 664b517022..63eaec1d7f 100644--- a/services/email_service/src/api/email/attachments/get_document_id.rs+++ b/services/email_service/src/api/email/attachments/get_document_id.rs@@ -1,11 +1,11 @@use crate::api::context::ApiContext;+use crate::util::upload_attachment::{+ UploadAttachmentContext, UploadAttachmentError, upload_attachment,+};use axum::extract::{Path, State};use axum::http::StatusCode;use axum::response::{IntoResponse, Response};use axum::{Extension, Json};-use email_service::util::upload_attachment::{- UploadAttachmentContext, UploadAttachmentError, upload_attachment,-};use entity_access::domain::ports::EntityAccessService;use macro_user_id::user_id::MacroUserIdStr;use model::response::ErrorResponse;diff --git a/services/email_service/src/api/email/drafts/delete.rs b/services/email_service/src/api/email/drafts/delete.rsindex a6e80c957e..14bdf941e5 100644--- a/services/email_service/src/api/email/drafts/delete.rs+++ b/services/email_service/src/api/email/drafts/delete.rs@@ -1,9 +1,9 @@use crate::api::context::ApiContext;+use crate::util::gmail::send::cleanup_draft_attachments;use axum::Extension;use axum::extract::{Path, State};use axum::http::StatusCode;use axum::response::{IntoResponse, Response};-use email_service::util::gmail::send::cleanup_draft_attachments;use model::response::{EmptyResponse, ErrorResponse};use models_email::service::link::Link;use strum_macros::AsRefStr;diff --git a/services/email_service/src/api/email/init.rs b/services/email_service/src/api/email/init.rsindex 43b94e63bc..b0cc2e4eb8 100644--- a/services/email_service/src/api/email/init.rs+++ b/services/email_service/src/api/email/init.rs@@ -1,5 +1,6 @@use crate::api::ApiContext;use crate::api::context::{AuthorizationService, CalendarGrantService};+use crate::pubsub::publish_email_event;use crate::utils::extract_email_with_response;use anyhow::Context;use axum::{@@ -13,7 +14,6 @@ use email::domain::models::UserProvider;use email::domain::ports::EmailRepo;use email::outbound::EmailPgRepo;use email_api_client::domain::models::{EmailApiError, TokenFreshness};-use email_service::pubsub::publish_email_event;use macro_authorization::{MacroAuthorizationExtractor, UserOrInternal};use macro_db_client::in_progress_user_link::InProgressUserLink;use macro_user_id::email::EmailStr;diff --git a/services/email_service/src/api/email/messages/labels.rs b/services/email_service/src/api/email/messages/labels.rsindex 68d235437d..60976e0b25 100644--- a/services/email_service/src/api/email/messages/labels.rs+++ b/services/email_service/src/api/email/messages/labels.rs@@ -1,12 +1,12 @@use crate::api::context::{ApiContext, AuthorizationService};use crate::api::email::messages::BATCH_UPDATE_MESSAGE_LIMIT;+use crate::pubsub::publish_email_event;use anyhow::Context;use axum::Json;use axum::extract::State;use axum::http::StatusCode;use axum::response::{IntoResponse, Response};use email::domain::events::{EmailEventOrigin, EmailMacroEvent, LabelRef};-use email_service::pubsub::publish_email_event;use macro_authorization::{MacroAuthorizationExtractor, UserOrInternal};use model::response::ErrorResponse;use models_email::service;diff --git a/services/email_service/src/api/email/settings/import_signature.rs b/services/email_service/src/api/email/settings/import_signature.rsindex 082a4472ba..c471e455a0 100644--- a/services/email_service/src/api/email/settings/import_signature.rs+++ b/services/email_service/src/api/email/settings/import_signature.rs@@ -1,11 +1,11 @@use crate::api::context::ApiContext;use crate::api::email::settings::patch::UnresolvedSignatureImagesError;+use crate::util::process_pre_insert::sfs_map;use axum::extract::State;use axum::http::StatusCode;use axum::response::{IntoResponse, Response};use axum::{Extension, Json};use email_api_client::domain::models::{EmailApiError, TokenFreshness};-use email_service::util::process_pre_insert::sfs_map;use email_utils::sanitize_html_fragment;use model::response::ErrorResponse;use models_email::service::link::Link;diff --git a/services/email_service/src/api/email/settings/patch.rs b/services/email_service/src/api/email/settings/patch.rsindex a74385852d..42eb8a5968 100644--- a/services/email_service/src/api/email/settings/patch.rs+++ b/services/email_service/src/api/email/settings/patch.rs@@ -1,9 +1,9 @@use crate::api::context::ApiContext;+use crate::util::process_pre_insert::sfs_map;use axum::extract::State;use axum::http::StatusCode;use axum::response::{IntoResponse, Response};use axum::{Extension, Json};-use email_service::util::process_pre_insert::sfs_map;use email_utils::sanitize_html_fragment;use model::response::ErrorResponse;use models_email::service::link::Link;diff --git a/services/email_service/src/api/email/threads/archived.rs b/services/email_service/src/api/email/threads/archived.rsindex 5097b94a33..e70e501898 100644--- a/services/email_service/src/api/email/threads/archived.rs+++ b/services/email_service/src/api/email/threads/archived.rs@@ -1,4 +1,5 @@use crate::api::context::{ApiContext, AuthorizationService};+use crate::pubsub::publish_email_event;use anyhow::Context;use axum::Json;use axum::extract::{Path, State};@@ -6,7 +7,6 @@ use axum::http::StatusCode;use axum::response::{IntoResponse, Response};use email::domain::events::{EmailEventOrigin, EmailMacroEvent, ThreadArchivedMetadata};use email_db_client::threads::update::update_inbox_visible_status;-use email_service::pubsub::publish_email_event;use macro_authorization::{MacroAuthorizationExtractor, UserOrInternal};use model::response::{EmptyResponse, ErrorResponse};use models_email::service::label::system_labels;diff --git a/services/email_service/src/api/email/threads/seen.rs b/services/email_service/src/api/email/threads/seen.rsindex 8e7cae3098..91e6a0ce62 100644--- a/services/email_service/src/api/email/threads/seen.rs+++ b/services/email_service/src/api/email/threads/seen.rs@@ -1,11 +1,11 @@use crate::api::context::{ApiContext, AuthorizationService};+use crate::pubsub::publish_email_event;use anyhow::Context;use axum::Json;use axum::extract::{Path, State};use axum::http::StatusCode;use axum::response::{IntoResponse, Response};use email::domain::events::{EmailEventOrigin, EmailMacroEvent, ThreadReadMetadata};-use email_service::pubsub::publish_email_event;use macro_authorization::{MacroAuthorizationExtractor, UserOrInternal};use model::response::{EmptyResponse, ErrorResponse};use models_email::service::label::system_labels;diff --git a/services/email_service/src/api/gmail/webhook.rs b/services/email_service/src/api/gmail/webhook.rsindex eb0cffa500..45876c3c54 100644--- a/services/email_service/src/api/gmail/webhook.rs+++ b/services/email_service/src/api/gmail/webhook.rs@@ -1,4 +1,5 @@use crate::api::context::ApiContext;+use crate::util;use axum::extract::State;use axum::http::HeaderMap;use axum::{@@ -6,7 +7,6 @@ use axum::{http::StatusCode,response::{IntoResponse, Json, Response},};-use email_service::util;use model::response::ErrorResponse;use models_email::gmail::inbox_sync::{GmailInboxSyncPayload, GmailMessagePayload, InboxSyncOperation, InboxSyncPubsubMessage,diff --git a/services/email_service/src/lib.rs b/services/email_service/src/lib.rsindex 18dc220f69..6a28ed9745 100644--- a/services/email_service/src/lib.rs+++ b/services/email_service/src/lib.rs@@ -1,3 +1,9 @@+#![recursion_limit = "256"]+//! Email service. The service binary, the OpenAPI generator, and the backfill+//! and worker binaries all link this library, so the service compiles once and+//! caches as one rlib.++mod api;/// Durable email-backfill completion orchestration.pub mod backfill_completion_service;/// Fenced email-backfill initialization orchestration.@@ -14,4 +20,10 @@ pub mod config;/// Outbound infrastructure adapters for email provider capabilities.pub mod outbound;pub mod pubsub;+mod server;pub mod util;+mod utils;++/// The service's OpenAPI document, rendered by the `*_openapi` binary.+pub use api::swagger::ApiDoc;+pub use server::run;diff --git a/services/email_service/src/main.rs b/services/email_service/src/main.rsindex 7f5d818b4f..cd01fd815c 100644--- a/services/email_service/src/main.rs+++ b/services/email_service/src/main.rs@@ -1,278 +1,12 @@+// `run()`'s future type is deep enough to need the library's recursion limit+// wherever it is awaited.#![recursion_limit = "256"]-use crate::api::context::{ApiContext, AuthorizationService};-use anyhow::Context;-use calendar_events::{- domain::{- invitations::CalendarInvitationResolver, mutations::CalendarMutationServiceImpl,- service::CalendarService,- },- outbound::{google::GoogleCalendarClient, pg::PgCalendarRepository},-};-use document_storage_service_client::DocumentStorageServiceClient;-use email::{- domain::service::EmailServiceImpl,- inbound::axum::{- axum_impls::GmailTokenState, get_thread_router::EmailThreadRouterState,- previews_router::EmailRouterState,- },- outbound::{EmailPgRepo, GmailTokenProviderImpl},-};-use email_api_client::GmailApiClientRepository;-use email_service::calendar_refresh::ConnectionGatewayCalendarRefresh;-use email_service::calendar_request_gate::RedisCalendarRequestGate;-use email_service::calendar_tokens::CalendarTokenProviderAdapter;-use email_service::outbound::email_api::{- EmailServiceTokenSource, GmailApi, RateBudget, RedisProviderRateLimiter,-};-use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};-use frecency::{domain::services::FrecencyQueryServiceImpl, outbound::postgres::FrecencyPgStorage};-use macro_auth::middleware::decode_jwt::JwtValidationArgs;-use macro_authorization::{- InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationState,- PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,-};-use macro_entrypoint::MacroEntrypoint;-use macro_env::Environment;-use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};-use macro_service_urls::{- AuthServiceUrl, ConnectionGatewayUrl, DocumentStorageServiceUrl, StaticFileServiceUrl,-};-use sqlx::postgres::PgPoolOptions;-use static_file_service_client::StaticFileServiceClient;-use std::{sync::Arc, time::Duration};-use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};-use tokio_util::task::TaskTracker;-mod api;-mod utils;+use macro_entrypoint::MacroEntrypoint;#[tokio::main]#[tracing::instrument(err)]async fn main() -> anyhow::Result<()> {MacroEntrypoint::default().init();- let env = Environment::new_or_prod();-- let aws_config = macro_aws_config::get_macro_aws_config().await;-- let s3_client = s3_client::S3::new(macro_aws_config::s3_client().await);-- let secretsmanager_client = secretsmanager_client::SecretsManager::new(- aws_sdk_secretsmanager::Client::new(&aws_config),- );-- // Parse our configuration from the environment, then resolve any secret-manager backed values.- let config = email_service::config::Config::from_env()- .context("expected to be able to generate config")?- .resolve_remote_secrets(env, &secretsmanager_client)- .await- .context("expected to be able to resolve config secrets")?;-- // limiting to max of 200 connections (12.5% of macrodb total) in prod.- let (min_connections, max_connections): (u32, u32) = match config.environment {- Environment::Production => (3, 20),- Environment::Develop => (1, 10),- Environment::Local => (1, 10),- };-- let db = PgPoolOptions::new()- .min_connections(min_connections)- .max_connections(max_connections)- .connect(&config.macro_db_url)- .await- .context("could not connect to db")?;-- let gmail_inbox_sync_queue = macro_queues::GmailInboxSyncQueue::new();- let gmail_inbox_sync_retry_queue = macro_queues::GmailInboxSyncRetryQueue::new();- let gmail_ops_queue = macro_queues::GmailOpsQueue::new();- let backfill_queue = macro_queues::EmailBackfillQueue::new();- let email_scheduled_queue = macro_queues::EmailScheduledQueue::new();- let sfs_uploader_queue = macro_queues::SfsUploaderQueue::new();- let link_manager_queue = macro_queues::LinkManagerQueue::new();- let sqs_client = sqs_client::SQS::new(macro_aws_config::sqs_client().await)- .gmail_inbox_sync_queue(&gmail_inbox_sync_queue)- .gmail_inbox_sync_retry_queue(&gmail_inbox_sync_retry_queue)- .gmail_ops_queue(&gmail_ops_queue)- .email_backfill_queue(&backfill_queue)- .email_scheduled_queue(&email_scheduled_queue)- .sfs_uploader_queue(&sfs_uploader_queue)- .email_link_manager_queue(&link_manager_queue);-- let auth_service_client = authentication_service_client::AuthServiceClient::new(- config- .authentication_service_secret_key- .as_ref()- .to_string(),- AuthServiceUrl::new()?.to_string(),- );-- let gmail_client = gmail_client::GmailClient::new(config.gmail_gcp_queue.as_ref().to_string());- let gmail_api_repository = GmailApiClientRepository::new(gmail_client.clone());-- let redis_inner_client = redis::Client::open(config.redis_uri.as_ref())- .inspect(|client| {- client- .get_connection()- .map(|_| tracing::info!("initialized redis connection"))- .inspect_err(|e| {- tracing::error!(error=?e, "failed to connect to redis");- })- .ok();- })- .context("failed to connect to redis")?;-- let redis_client = email_service::util::redis::RedisClient::new(- redis_inner_client,- config.redis_rate_limit_reqs,- config.redis_rate_limit_reqs_backfill,- config.redis_rate_limit_window_secs,- );-- let sfs_client = StaticFileServiceClient::new(- config.internal_api_key.to_string(),- StaticFileServiceUrl::new()?.to_string(),- );-- let dss_client = DocumentStorageServiceClient::new(- config.internal_api_key.to_string(),- DocumentStorageServiceUrl::new()?.to_string(),- );-- let system_properties_service = Arc::new(SystemPropertiesServiceImpl::new(- PgSystemPropertiesRepository::new(db.clone()),- ));-- let jwt_args =- JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)- .await?;- let authorization_state = MacroAuthorizationState::new(Arc::new(AuthorizationService::new(- MacroAuthJwtValidator::new(jwt_args.clone()),- InternalAuthConfig {- api_key: config.internal_api_key.to_string(),- default_user_id: Some("macro|[email protected]".to_string()),- },- macro_authorization::NoBotAuthorizer,- PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),- )));-- let sqs_client = Arc::new(sqs_client);- let gmail_client = Arc::new(gmail_client);- // HTTP API only reads CRM rows — populate runs in the pubsub- // worker. The no-op resolver makes the unused branch explicit and- // keeps reqwest/scraper out of this binary.- let crm_service = crm::domain::service::CrmServiceImpl::new(- crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),- crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,- );- let event_broker_tracker = TaskTracker::new();- let macro_event_broker = MacroEventBrokerService::new(- KafkaEventPublisher::new(config.kafka_brokers.as_ref())- .context("failed to create kafka event publisher")?,- event_broker_tracker.clone(),- );- let email_service = EmailRouterState::new(- EmailServiceImpl::new(- EmailPgRepo::new(db.clone()),- FrecencyQueryServiceImpl::new(FrecencyPgStorage::new(db.clone())),- (*sqs_client).clone(),- crm_service,- entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(- entity_access_management::outbound::PgRepository::new(db.clone()),- ),- config.sent_undo_delay_secs,- )- .with_macro_event_broker(macro_event_broker.clone()),- );- let entity_access_service = Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(- db.clone(),- )));- let email_thread_state = EmailThreadRouterState {- service: email_service.service(),- access_service: entity_access_service.clone(),- authorization_state: authorization_state.clone(),- };- let auth_service_client = Arc::new(auth_service_client);- let redis_conn = redis_client- .inner- .get_multiplexed_async_connection()- .await- .context("failed to get multiplexed redis connection for gmail token provider")?;- let email_api = GmailApi::new(- gmail_api_repository,- EmailServiceTokenSource::new(- db.clone(),- redis_conn.clone(),- auth_service_client.as_ref().clone(),- sqs_client.as_ref().clone(),- ),- RedisProviderRateLimiter::new(redis_client.clone(), RateBudget::Live),- );- let redis_client = Arc::new(redis_client);- let gmail_token_state = GmailTokenState::new(GmailTokenProviderImpl::new(- redis_conn.clone(),- auth_service_client.clone(),- ));- let calendar_service = Arc::new(CalendarService::new(PgCalendarRepository::new(db.clone())));- let connection_gateway_client = connection_gateway_client::client::ConnectionGatewayClient::new(- config.internal_api_key.to_string(),- ConnectionGatewayUrl::new()?.to_string(),- );- let calendar_mutation_service = Arc::new(CalendarMutationServiceImpl::new(- PgCalendarRepository::new(db.clone()),- GoogleCalendarClient::with_gate(- reqwest::Client::builder()- .timeout(std::time::Duration::from_secs(30))- .build()- .context("failed to build the google calendar mutation http client")?,- RedisCalendarRequestGate::new((*redis_client).clone()),- ),- CalendarTokenProviderAdapter::new(redis_conn.clone(), auth_service_client.clone()),- macro_event_broker.clone(),- ConnectionGatewayCalendarRefresh::new(connection_gateway_client, db.clone()),- ));- let api_result = api::setup_and_serve(ApiContext {- invitation_snapshots: email::outbound::invitation_pg::InvitationPgRepository(db.clone()),- // calendar_service's sync kill switch rejects RSVP writes itself.- invitation_resolver: Arc::new(CalendarInvitationResolver::new(PgCalendarRepository::new(- db.clone(),- ))),- db,- internal_api_key: config.internal_api_key.clone(),- config: Arc::new(config),- auth_service_client,- redis_client,- sqs_client,- sfs_client: Arc::new(sfs_client),- gmail_client: gmail_client.clone(),- email_api,- s3_client: Arc::new(s3_client),- dss_client: Arc::new(dss_client),- system_properties_service,- authorization_state: authorization_state.clone(),- jwt_args,- email_service,- entity_access_service,- email_thread_state,- gmail_token_state,- macro_event_broker: Arc::new(macro_event_broker),- calendar_service,- calendar_mutation_service,- })- .await;-- tracing::info!("waiting for event broker publishes to drain");- event_broker_tracker.close();- match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {- Ok(()) => tracing::info!("event broker publishes drained"),- Err(error) => {- tracing::warn!(- error=?error,- timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),- "timed out waiting for event broker publishes to drain"- );- }- }-- api_result+ email_service::run().await}--const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);diff --git a/services/email_service/src/openapi.rs b/services/email_service/src/openapi.rsindex 029b481120..f9e57b4325 100644--- a/services/email_service/src/openapi.rs+++ b/services/email_service/src/openapi.rs@@ -1,20 +1,6 @@-#![allow(unused)]--mod api;-mod backfill_completion_service;-mod backfill_init_service;-mod backfill_outbox;-mod config;-mod outbound;-mod pubsub;-mod util;-mod utils;-+use email_service::ApiDoc;use utoipa::OpenApi;fn main() {- println!(- "{}",- api::swagger::ApiDoc::openapi().to_pretty_json().unwrap()- );+ println!("{}", ApiDoc::openapi().to_pretty_json().unwrap());}diff --git a/services/email_service/src/server.rs b/services/email_service/src/server.rsnew file mode 100644index 0000000000..6a745bc14e--- /dev/null+++ b/services/email_service/src/server.rs@@ -0,0 +1,277 @@+//! The service's composition root: reads config and wires every adapter and+//! domain service into the HTTP server.++use crate::api::{+ self,+ context::{ApiContext, AuthorizationService},+};+use crate::calendar_refresh::ConnectionGatewayCalendarRefresh;+use crate::calendar_request_gate::RedisCalendarRequestGate;+use crate::calendar_tokens::CalendarTokenProviderAdapter;+use crate::outbound::email_api::{+ EmailServiceTokenSource, GmailApi, RateBudget, RedisProviderRateLimiter,+};+use anyhow::Context;+use calendar_events::{+ domain::{+ invitations::CalendarInvitationResolver, mutations::CalendarMutationServiceImpl,+ service::CalendarService,+ },+ outbound::{google::GoogleCalendarClient, pg::PgCalendarRepository},+};+use document_storage_service_client::DocumentStorageServiceClient;+use email::{+ domain::service::EmailServiceImpl,+ inbound::axum::{+ axum_impls::GmailTokenState, get_thread_router::EmailThreadRouterState,+ previews_router::EmailRouterState,+ },+ outbound::{EmailPgRepo, GmailTokenProviderImpl},+};+use email_api_client::GmailApiClientRepository;+use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};+use frecency::{domain::services::FrecencyQueryServiceImpl, outbound::postgres::FrecencyPgStorage};+use macro_auth::middleware::decode_jwt::JwtValidationArgs;+use macro_authorization::{+ InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationState,+ PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,+};+use macro_env::Environment;+use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};+use macro_service_urls::{+ AuthServiceUrl, ConnectionGatewayUrl, DocumentStorageServiceUrl, StaticFileServiceUrl,+};+use sqlx::postgres::PgPoolOptions;+use static_file_service_client::StaticFileServiceClient;+use std::{sync::Arc, time::Duration};+use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};+use tokio_util::task::TaskTracker;++/// Build the service from the environment and serve until shutdown.+pub async fn run() -> anyhow::Result<()> {+ let env = Environment::new_or_prod();++ let aws_config = macro_aws_config::get_macro_aws_config().await;++ let s3_client = s3_client::S3::new(macro_aws_config::s3_client().await);++ let secretsmanager_client = secretsmanager_client::SecretsManager::new(+ aws_sdk_secretsmanager::Client::new(&aws_config),+ );++ // Parse our configuration from the environment, then resolve any secret-manager backed values.+ let config = crate::config::Config::from_env()+ .context("expected to be able to generate config")?+ .resolve_remote_secrets(env, &secretsmanager_client)+ .await+ .context("expected to be able to resolve config secrets")?;++ // limiting to max of 200 connections (12.5% of macrodb total) in prod.+ let (min_connections, max_connections): (u32, u32) = match config.environment {+ Environment::Production => (3, 20),+ Environment::Develop => (1, 10),+ Environment::Local => (1, 10),+ };++ let db = PgPoolOptions::new()+ .min_connections(min_connections)+ .max_connections(max_connections)+ .connect(&config.macro_db_url)+ .await+ .context("could not connect to db")?;++ let gmail_inbox_sync_queue = macro_queues::GmailInboxSyncQueue::new();+ let gmail_inbox_sync_retry_queue = macro_queues::GmailInboxSyncRetryQueue::new();+ let gmail_ops_queue = macro_queues::GmailOpsQueue::new();+ let backfill_queue = macro_queues::EmailBackfillQueue::new();+ let email_scheduled_queue = macro_queues::EmailScheduledQueue::new();+ let sfs_uploader_queue = macro_queues::SfsUploaderQueue::new();+ let link_manager_queue = macro_queues::LinkManagerQueue::new();+ let sqs_client = sqs_client::SQS::new(macro_aws_config::sqs_client().await)+ .gmail_inbox_sync_queue(&gmail_inbox_sync_queue)+ .gmail_inbox_sync_retry_queue(&gmail_inbox_sync_retry_queue)+ .gmail_ops_queue(&gmail_ops_queue)+ .email_backfill_queue(&backfill_queue)+ .email_scheduled_queue(&email_scheduled_queue)+ .sfs_uploader_queue(&sfs_uploader_queue)+ .email_link_manager_queue(&link_manager_queue);++ let auth_service_client = authentication_service_client::AuthServiceClient::new(+ config+ .authentication_service_secret_key+ .as_ref()+ .to_string(),+ AuthServiceUrl::new()?.to_string(),+ );++ let gmail_client = gmail_client::GmailClient::new(config.gmail_gcp_queue.as_ref().to_string());+ let gmail_api_repository = GmailApiClientRepository::new(gmail_client.clone());++ let redis_inner_client = redis::Client::open(config.redis_uri.as_ref())+ .inspect(|client| {+ client+ .get_connection()+ .map(|_| tracing::info!("initialized redis connection"))+ .inspect_err(|e| {+ tracing::error!(error=?e, "failed to connect to redis");+ })+ .ok();+ })+ .context("failed to connect to redis")?;++ let redis_client = crate::util::redis::RedisClient::new(+ redis_inner_client,+ config.redis_rate_limit_reqs,+ config.redis_rate_limit_reqs_backfill,+ config.redis_rate_limit_window_secs,+ );++ let sfs_client = StaticFileServiceClient::new(+ config.internal_api_key.to_string(),+ StaticFileServiceUrl::new()?.to_string(),+ );++ let dss_client = DocumentStorageServiceClient::new(+ config.internal_api_key.to_string(),+ DocumentStorageServiceUrl::new()?.to_string(),+ );++ let system_properties_service = Arc::new(SystemPropertiesServiceImpl::new(+ PgSystemPropertiesRepository::new(db.clone()),+ ));++ let jwt_args =+ JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)+ .await?;+ let authorization_state = MacroAuthorizationState::new(Arc::new(AuthorizationService::new(+ MacroAuthJwtValidator::new(jwt_args.clone()),+ InternalAuthConfig {+ api_key: config.internal_api_key.to_string(),+ default_user_id: Some("macro|[email protected]".to_string()),+ },+ macro_authorization::NoBotAuthorizer,+ PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),+ )));++ let sqs_client = Arc::new(sqs_client);+ let gmail_client = Arc::new(gmail_client);+ // HTTP API only reads CRM rows — populate runs in the pubsub+ // worker. The no-op resolver makes the unused branch explicit and+ // keeps reqwest/scraper out of this binary.+ let crm_service = crm::domain::service::CrmServiceImpl::new(+ crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),+ crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,+ );+ let event_broker_tracker = TaskTracker::new();+ let macro_event_broker = MacroEventBrokerService::new(+ KafkaEventPublisher::new(config.kafka_brokers.as_ref())+ .context("failed to create kafka event publisher")?,+ event_broker_tracker.clone(),+ );+ let email_service = EmailRouterState::new(+ EmailServiceImpl::new(+ EmailPgRepo::new(db.clone()),+ FrecencyQueryServiceImpl::new(FrecencyPgStorage::new(db.clone())),+ (*sqs_client).clone(),+ crm_service,+ entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(+ entity_access_management::outbound::PgRepository::new(db.clone()),+ ),+ config.sent_undo_delay_secs,+ )+ .with_macro_event_broker(macro_event_broker.clone()),+ );+ let entity_access_service = Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(+ db.clone(),+ )));+ let email_thread_state = EmailThreadRouterState {+ service: email_service.service(),+ access_service: entity_access_service.clone(),+ authorization_state: authorization_state.clone(),+ };+ let auth_service_client = Arc::new(auth_service_client);+ let redis_conn = redis_client+ .inner+ .get_multiplexed_async_connection()+ .await+ .context("failed to get multiplexed redis connection for gmail token provider")?;+ let email_api = GmailApi::new(+ gmail_api_repository,+ EmailServiceTokenSource::new(+ db.clone(),+ redis_conn.clone(),+ auth_service_client.as_ref().clone(),+ sqs_client.as_ref().clone(),+ ),+ RedisProviderRateLimiter::new(redis_client.clone(), RateBudget::Live),+ );+ let redis_client = Arc::new(redis_client);+ let gmail_token_state = GmailTokenState::new(GmailTokenProviderImpl::new(+ redis_conn.clone(),+ auth_service_client.clone(),+ ));+ let calendar_service = Arc::new(CalendarService::new(PgCalendarRepository::new(db.clone())));+ let connection_gateway_client = connection_gateway_client::client::ConnectionGatewayClient::new(+ config.internal_api_key.to_string(),+ ConnectionGatewayUrl::new()?.to_string(),+ );+ let calendar_mutation_service = Arc::new(CalendarMutationServiceImpl::new(+ PgCalendarRepository::new(db.clone()),+ GoogleCalendarClient::with_gate(+ reqwest::Client::builder()+ .timeout(std::time::Duration::from_secs(30))+ .build()+ .context("failed to build the google calendar mutation http client")?,+ RedisCalendarRequestGate::new((*redis_client).clone()),+ ),+ CalendarTokenProviderAdapter::new(redis_conn.clone(), auth_service_client.clone()),+ macro_event_broker.clone(),+ ConnectionGatewayCalendarRefresh::new(connection_gateway_client, db.clone()),+ ));+ let api_result = api::setup_and_serve(ApiContext {+ invitation_snapshots: email::outbound::invitation_pg::InvitationPgRepository(db.clone()),+ // calendar_service's sync kill switch rejects RSVP writes itself.+ invitation_resolver: Arc::new(CalendarInvitationResolver::new(PgCalendarRepository::new(+ db.clone(),+ ))),+ db,+ internal_api_key: config.internal_api_key.clone(),+ config: Arc::new(config),+ auth_service_client,+ redis_client,+ sqs_client,+ sfs_client: Arc::new(sfs_client),+ gmail_client: gmail_client.clone(),+ email_api,+ s3_client: Arc::new(s3_client),+ dss_client: Arc::new(dss_client),+ system_properties_service,+ authorization_state: authorization_state.clone(),+ jwt_args,+ email_service,+ entity_access_service,+ email_thread_state,+ gmail_token_state,+ macro_event_broker: Arc::new(macro_event_broker),+ calendar_service,+ calendar_mutation_service,+ })+ .await;++ tracing::info!("waiting for event broker publishes to drain");+ event_broker_tracker.close();+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {+ Ok(()) => tracing::info!("event broker publishes drained"),+ Err(error) => {+ tracing::warn!(+ error=?error,+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),+ "timed out waiting for event broker publishes to drain"+ );+ }+ }++ api_result+}++const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);diff --git a/services/organization_retention_handler/migrations/20240325143515_basic_schema_for_testing.sql b/services/organization_retention_handler/fixtures/schema.sqlsimilarity index 97%rename from services/organization_retention_handler/migrations/20240325143515_basic_schema_for_testing.sqlrename to services/organization_retention_handler/fixtures/schema.sqlindex 71a713df92..db05debb52 100644--- a/services/organization_retention_handler/migrations/20240325143515_basic_schema_for_testing.sql+++ b/services/organization_retention_handler/fixtures/schema.sql@@ -1,3 +1,9 @@+-- Test-only schema, loaded as the first fixture with `migrations = false`.+-- CI clones test databases from a pre-migrated template1 (`just setup_test_template`),+-- so start from an empty public schema.+DROP SCHEMA public CASCADE;+CREATE SCHEMA public;+-- NOTE: This is used purely to setup the testing DB-- Changes from `crates/macro_db_client/migrations` need to be reflected here as raw SQL-- Changes here WILL NOT ever make it into the dev/production db.diff --git a/services/organization_retention_handler/src/service/db/get_violating_items.rs b/services/organization_retention_handler/src/service/db/get_violating_items.rsindex e0153fbabb..abdef5e3d2 100644--- a/services/organization_retention_handler/src/service/db/get_violating_items.rs+++ b/services/organization_retention_handler/src/service/db/get_violating_items.rs@@ -85,7 +85,10 @@ pub async fn get_violating_chats(mod tests {use super::*;- #[sqlx::test(fixtures(path = "../../../fixtures", scripts("testing")))]+ #[sqlx::test(+ migrations = false,+ fixtures(path = "../../../fixtures", scripts("schema", "testing"))+ )]async fn test_get_violating_items(db: sqlx::Pool<sqlx::Postgres>) -> anyhow::Result<()> {let mut result = get_violating_items(&db, 1, 2).await?;result.sort();diff --git a/services/organization_retention_trigger/migrations/20240325143515_basic_schema_for_testing.sql b/services/organization_retention_trigger/fixtures/schema.sqlsimilarity index 97%rename from services/organization_retention_trigger/migrations/20240325143515_basic_schema_for_testing.sqlrename to services/organization_retention_trigger/fixtures/schema.sqlindex b810ed1e48..5126745f7e 100644--- a/services/organization_retention_trigger/migrations/20240325143515_basic_schema_for_testing.sql+++ b/services/organization_retention_trigger/fixtures/schema.sql@@ -1,3 +1,9 @@+-- Test-only schema, loaded as the first fixture with `migrations = false`.+-- CI clones test databases from a pre-migrated template1 (`just setup_test_template`),+-- so start from an empty public schema.+DROP SCHEMA public CASCADE;+CREATE SCHEMA public;+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";CREATE TABLE "Organization"diff --git a/services/organization_retention_trigger/src/service/db/get_organization_retention.rs b/services/organization_retention_trigger/src/service/db/get_organization_retention.rsindex 885c708401..ccb7f60231 100644--- a/services/organization_retention_trigger/src/service/db/get_organization_retention.rs+++ b/services/organization_retention_trigger/src/service/db/get_organization_retention.rs@@ -28,7 +28,10 @@ pub async fn get_organization_retention(mod tests {use super::*;- #[sqlx::test(fixtures(path = "../../../fixtures", scripts("testing")))]+ #[sqlx::test(+ migrations = false,+ fixtures(path = "../../../fixtures", scripts("schema", "testing"))+ )]async fn test_soft_delete_documents(pool: sqlx::Pool<sqlx::Postgres>) -> anyhow::Result<()> {let result = get_organization_retention(pool.clone()).await?;diff --git a/services/sha_cleanup_worker/migrations/20240325143515_basic_schema_for_testing.sql b/services/sha_cleanup_worker/fixtures/schema.sqlsimilarity index 95%rename from services/sha_cleanup_worker/migrations/20240325143515_basic_schema_for_testing.sqlrename to services/sha_cleanup_worker/fixtures/schema.sqlindex c7a1eadb4b..352fdcada9 100644--- a/services/sha_cleanup_worker/migrations/20240325143515_basic_schema_for_testing.sql+++ b/services/sha_cleanup_worker/fixtures/schema.sql@@ -1,3 +1,9 @@+-- Test-only schema, loaded as the first fixture with `migrations = false`.+-- CI clones test databases from a pre-migrated template1 (`just setup_test_template`),+-- so start from an empty public schema.+DROP SCHEMA public CASCADE;+CREATE SCHEMA public;+-- NOTE: This is used purely to setup the testing DB-- Changes from `crates/macro_db_client/migrations` need to be reflected here as raw SQL-- Changes here WILL NOT ever make it into the dev/production db.diff --git a/services/sha_cleanup_worker/src/service/db/get_sha_count.rs b/services/sha_cleanup_worker/src/service/db/get_sha_count.rsindex eb2cc4521b..57f982ba70 100644--- a/services/sha_cleanup_worker/src/service/db/get_sha_count.rs+++ b/services/sha_cleanup_worker/src/service/db/get_sha_count.rs@@ -23,7 +23,10 @@ pub(in crate::service::db) async fn get_sha_count(mod tests {use super::*;- #[sqlx::test(fixtures(path = "../../../fixtures", scripts("shas")))]+ #[sqlx::test(+ migrations = false,+ fixtures(path = "../../../fixtures", scripts("schema", "shas"))+ )]async fn test_get_sha_count(pool: sqlx::Pool<sqlx::Postgres>) -> anyhow::Result<()> {let result = get_sha_count(pool.clone(), "sha-1").await?;assert_eq!(result, 2);diff --git a/tooling/just/rust.just b/tooling/just/rust.justindex 26108b0a2d..bd63226278 100644--- a/tooling/just/rust.just+++ b/tooling/just/rust.just@@ -133,3 +133,10 @@ setup_macrodb:just crates/macro_db_client/migrate_dbinitialize_dbs: setup_macrodb++# Migrate template1 so each `#[sqlx::test]` database starts with the macrodb schema:+# Postgres clones template1 for every CREATE DATABASE, which leaves the test migrator+# nothing to apply. Every database created on this server afterwards gets the schema+# too, so only run it against a Postgres dedicated to one checkout, such as CI's.+setup_test_template:+ cd crates/macro_db_client && sqlx migrate run --database-url {{ replace(DATABASE_URL, "/macrodb", "/template1") }}diff --git a/tooling/xtask/crates/xtask_nextest_filter/Cargo.toml b/tooling/xtask/crates/xtask_nextest_filter/Cargo.tomlindex aaf2508aaf..4b1f37be1f 100644--- a/tooling/xtask/crates/xtask_nextest_filter/Cargo.toml+++ b/tooling/xtask/crates/xtask_nextest_filter/Cargo.toml@@ -14,5 +14,6 @@ anyhow = { workspace = true }determinator = { workspace = true }guppy = { workspace = true }tempfile = { workspace = true }+toml = { workspace = true }xtask_graph = { path = "../xtask_graph" }xtask_paths = { path = "../xtask_paths" }diff --git a/tooling/xtask/crates/xtask_nextest_filter/src/main.rs b/tooling/xtask/crates/xtask_nextest_filter/src/main.rsindex ab9fb572b1..9a20ec27a3 100644--- a/tooling/xtask/crates/xtask_nextest_filter/src/main.rs+++ b/tooling/xtask/crates/xtask_nextest_filter/src/main.rs@@ -4,10 +4,18 @@//! for Rust CI using [`determinator`].//!//! Input is a newline-delimited file of paths relative to the repository root-//! (typically `git diff --name-only ...`) and a Git base revision. The command-//! builds Cargo metadata for the base and current revisions, then lets-//! determinator account for path, dependency, and feature changes. Shared-//! inputs that Cargo does not know about are declared in `determinator.toml`.+//! (typically `git diff --name-only ...`) and a Git base revision. When no+//! `Cargo.toml` or `Cargo.lock` changed, the package graph is identical on+//! both sides, so the command maps paths to packages (determinator's path+//! rules) and adds their reverse dependencies from the current graph alone.+//! Otherwise it builds Cargo metadata for the base and current revisions and+//! lets determinator account for dependency and feature changes as well.+//! Shared inputs that Cargo does not know about are declared in+//! `determinator.toml`.+//!+//! A root `Cargo.toml` edit forces `all` unless it only touches+//! `workspace.members` / `workspace.dependencies`; see+//! [`root_manifest_change_forces_all`].//!//! Stdout is one of://! - `all` — every workspace package is affected.@@ -24,7 +32,10 @@ use std::process::{Command, Stdio};use anyhow::{Context, Result, bail};use determinator::Determinator;use determinator::rules::{DeterminatorRules, PathMatch};-use guppy::graph::{DependencyDirection, PackageGraph};+use guppy::PackageId;+use guppy::graph::cargo::{CargoOptions, CargoSet};+use guppy::graph::feature::{FeatureSet, StandardFeatures};+use guppy::graph::{DependencyDirection, PackageGraph, PackageMetadata};use tempfile::TempDir;#[cfg(test)]use xtask_graph::build_graph;@@ -106,13 +117,73 @@ fn run(changed_files_path: &Path, base_revision: &str) -> Result<()> {.collect();let workspace_dir = xtask_paths::workspace_root();+ if !changed_files.iter().any(|path| changes_cargo_graph(path)) {+ let graph = build_graph_at(&workspace_dir, true)?;+ println!(+ "{}",+ select_by_paths(&graph, &changed_files, &repository_rules()?)?+ );+ return Ok(());+ }+let base_worktree = BaseWorktree::new(&workspace_dir, base_revision)?;+ if changed_files+ .iter()+ .any(|path| path == Path::new(ROOT_MANIFEST))+ {+ let old_manifest = read_manifest(base_worktree.path())?;+ let new_manifest = read_manifest(&workspace_dir)?;+ if root_manifest_change_forces_all(&old_manifest, &new_manifest)? {+ println!("{}", PackageSelection::All);+ return Ok(());+ }+ }let (old_graph, new_graph) = build_graphs(base_worktree.path(), &workspace_dir)?;- let packages = compute_packages(&old_graph, &new_graph, &changed_files)?;+ let packages = compute_packages(&old_graph, &new_graph, &changed_files, &repository_rules()?)?;println!("{packages}");Ok(())}+const ROOT_MANIFEST: &str = "Cargo.toml";++fn read_manifest(dir: &Path) -> Result<String> {+ let path = dir.join(ROOT_MANIFEST);+ std::fs::read_to_string(&path).with_context(|| format!("reading {}", path.display()))+}++/// Root-manifest keys whose edits reach every affected package through the+/// old/new Cargo graphs, so determinator can scope them:+/// - `workspace.members`: an added member is a new package (determinator marks+/// it changed); a removed one changes the manifests that depended on it.+/// - `workspace.dependencies`: members inherit these specs, so a version,+/// source, or feature change shows up in the simulated builds of exactly+/// the members whose resolved dependencies changed. An unused or+/// resolution-neutral edit selects nothing, which is correct.+///+/// Everything else (`[profile.*]`, `[workspace.lints]`, `[workspace.package]`,+/// `[patch]`, `resolver`, …) changes how every package compiles without+/// changing the package graph, so determinator cannot see it.+const GRAPH_VISIBLE_WORKSPACE_KEYS: [&str; 2] = ["members", "dependencies"];++/// Whether a root `Cargo.toml` edit must run the whole workspace: true unless+/// the old and new manifests are equal once the graph-visible workspace keys+/// are removed. Compares parsed TOML, so comment and formatting edits are+/// no-ops.+fn root_manifest_change_forces_all(old: &str, new: &str) -> Result<bool> {+ let strip = |manifest: &str, revision: &str| -> Result<toml::Table> {+ let mut table: toml::Table = manifest+ .parse()+ .with_context(|| format!("parsing the {revision} root {ROOT_MANIFEST}"))?;+ if let Some(toml::Value::Table(workspace)) = table.get_mut("workspace") {+ for key in GRAPH_VISIBLE_WORKSPACE_KEYS {+ workspace.remove(key);+ }+ }+ Ok(table)+ };+ Ok(strip(old, "base")? != strip(new, "current")?)+}+fn build_graphs(base_dir: &Path, current_dir: &Path) -> Result<(PackageGraph, PackageGraph)> {Ok((build_graph_at(base_dir, true)?,@@ -120,6 +191,114 @@ fn build_graphs(base_dir: &Path, current_dir: &Path) -> Result<(PackageGraph, Pa))}+/// This repository's determinator rules (`determinator.toml`).+fn repository_rules() -> Result<DeterminatorRules> {+ DeterminatorRules::parse(DETERMINATOR_RULES).context("parsing nextest determinator rules")+}++/// Whether a changed path can alter `cargo metadata`: any manifest or+/// lockfile. Without one, the base and current package graphs are identical+/// and determinator's build-summary comparison (several seconds of feature+/// resolution per workspace member) cannot find anything.+fn changes_cargo_graph(path: &Path) -> bool {+ matches!(+ path.file_name().and_then(|name| name.to_str()),+ Some("Cargo.toml" | "Cargo.lock")+ )+}++/// Selection for a change that leaves the package graph untouched: packages+/// owning a changed path (or named by a determinator path rule) plus every+/// workspace package whose own build compiles one of them. That is the set+/// [`compute_packages`] returns when old and new graphs are identical, but+/// determinator gets there by simulating none/default/all-feature builds of+/// every member on both sides (~600 simulations, several seconds). Here only+/// the reverse dependencies of the changed packages are simulated, once.+fn select_by_paths(+ graph: &PackageGraph,+ changed_files: &[PathBuf],+ rules: &DeterminatorRules,+) -> Result<PackageSelection> {+ let mut determinator = Determinator::new(graph, graph);+ determinator+ .set_rules(rules)+ .context("applying nextest determinator rules")?;++ let mut changed = BTreeSet::new();+ for changed_file in changed_files {+ let changed_file = changed_file+ .to_str()+ .context("changed file path is not valid UTF-8")?;+ let matched = determinator.match_path(changed_file, |id| {+ changed.insert(id);+ });+ if matches!(matched, PathMatch::RuleMatchedAll) {+ return Ok(PackageSelection::All);+ }+ }++ // Every package that could build a changed one, following all edges. Cheap,+ // and a superset of the answer.+ let candidates = graph+ .query_reverse(changed.iter().copied())+ .context("querying reverse dependencies")?+ .resolve();+ let cargo_options = Determinator::default_cargo_options();+ let features_only = graph.feature_graph().resolve_none();+ let mut selected = BTreeSet::new();+ for package in candidates+ .packages(DependencyDirection::Forward)+ .filter(|package| package.in_workspace())+ {+ if changed.contains(package.id())+ || builds_any(package, &changed, &cargo_options, &features_only)?+ {+ selected.insert(package.name().to_owned());+ }+ }+ workspace_selection(graph, selected)+}++/// Whether `package`'s own test build with all its features (dev-dependencies+/// included, current platform) compiles any of `changed`. This is the one-hop+/// check determinator's reverse index makes for path changes; enabling more+/// features only adds dependencies, so the all-features build covers the+/// none and default ones it also simulates.+fn builds_any(+ package: PackageMetadata<'_>,+ changed: &BTreeSet<&PackageId>,+ cargo_options: &CargoOptions<'_>,+ features_only: &FeatureSet<'_>,+) -> Result<bool> {+ let initials = package+ .to_package_set()+ .to_feature_set(StandardFeatures::All);+ let build = CargoSet::new(initials, features_only.clone(), cargo_options)+ .with_context(|| format!("simulating the build of {}", package.name()))?;+ let target = build.target_features().to_package_set();+ let host = build.host_features().to_package_set();+ for id in changed {+ if target.contains(id)? || host.contains(id)? {+ return Ok(true);+ }+ }+ Ok(false)+}++/// `None` for an empty set, `All` when it covers the whole workspace.+fn workspace_selection(+ graph: &PackageGraph,+ packages: BTreeSet<String>,+) -> Result<PackageSelection> {+ if packages.is_empty() {+ return Ok(PackageSelection::None);+ }+ if packages.len() == graph.workspace().iter().count() {+ return Ok(PackageSelection::All);+ }+ Ok(PackageSelection::Packages(packages))+}+/// Use determinator to compare both Cargo graphs and return affected package/// names. Paths outside Cargo packages contribute nothing, preserving the CI/// behavior for Nix-only and other non-Rust changes.@@ -127,12 +306,11 @@ fn compute_packages(old_graph: &PackageGraph,new_graph: &PackageGraph,changed_files: &[PathBuf],+ rules: &DeterminatorRules,) -> Result<PackageSelection> {- let rules = DeterminatorRules::parse(DETERMINATOR_RULES)- .context("parsing nextest determinator rules")?;let mut determinator = Determinator::new(old_graph, new_graph);determinator- .set_rules(&rules)+ .set_rules(rules).context("applying nextest determinator rules")?;for changed_file in changed_files {@@ -148,21 +326,14 @@ fn compute_packages(}let affected = determinator.compute().affected_set;- let packages: BTreeSet<_> = affected- .packages(DependencyDirection::Forward)- .filter(|package| package.in_workspace())- .map(|package| package.name().to_owned())- .collect();-- if packages.is_empty() {- return Ok(PackageSelection::None);- }-- if packages.len() == new_graph.workspace().iter().count() {- return Ok(PackageSelection::All);- }-- Ok(PackageSelection::Packages(packages))+ workspace_selection(+ new_graph,+ affected+ .packages(DependencyDirection::Forward)+ .filter(|package| package.in_workspace())+ .map(|package| package.name().to_owned())+ .collect(),+ )}struct BaseWorktree {diff --git a/tooling/xtask/crates/xtask_nextest_filter/src/test.rs b/tooling/xtask/crates/xtask_nextest_filter/src/test.rsindex ded39134aa..18ffb641be 100644--- a/tooling/xtask/crates/xtask_nextest_filter/src/test.rs+++ b/tooling/xtask/crates/xtask_nextest_filter/src/test.rs@@ -4,6 +4,11 @@ fn changed_files(paths: &[&str]) -> Vec<PathBuf> {paths.iter().map(PathBuf::from).collect()}+/// Path-only selection on the real workspace with the repository's rules.+fn select_in_repo(graph: &PackageGraph, paths: &[&str]) -> PackageSelection {+ select_by_paths(graph, &changed_files(paths), &repository_rules().unwrap()).unwrap()+}+fn selected_packages(selection: &PackageSelection) -> &BTreeSet<String> {selection.packages()@@ -26,17 +31,17 @@ fn display_is_the_cli_seam() {);}+/// Path-only changes go through [`select_by_paths`], which skips+/// determinator's per-member build simulation (seconds of feature resolution+/// per call). [`compute_packages`] is covered on small synthetic workspaces+/// below, where that simulation is cheap.+////// A change in a single crate selects that crate and its reverse dependencies,/// never the whole workspace.#[test]fn crate_change_maps_to_rdeps_of_that_crate() {let graph = build_graph(false).expect("cargo metadata");- let selection = compute_packages(- &graph,- &graph,- &changed_files(&["crates/email_validator/src/lib.rs"]),- )- .unwrap();+ let selection = select_in_repo(&graph, &["crates/email_validator/src/lib.rs"]);assert_ne!(selection, PackageSelection::None);assert_ne!(selection, PackageSelection::All);assert!(@@ -50,12 +55,10 @@ fn crate_change_maps_to_rdeps_of_that_crate() {#[test]fn unmapped_files_alone_yield_none() {let graph = build_graph(false).expect("cargo metadata");- let selection = compute_packages(- &graph,+ let selection = select_in_repo(&graph,- &changed_files(&["random.json", "package.json", "docs/README.md", "justfile"]),- )- .unwrap();+ &["random.json", "package.json", "docs/README.md", "justfile"],+ );assert_eq!(selection, PackageSelection::None);}@@ -64,12 +67,7 @@ fn unmapped_files_alone_yield_none() {#[test]fn package_readme_selects_its_package() {let graph = build_graph(false).expect("cargo metadata");- let selection = compute_packages(- &graph,- &graph,- &changed_files(&["crates/webhook/README.md"]),- )- .unwrap();+ let selection = select_in_repo(&graph, &["crates/webhook/README.md"]);assert!(selected_packages(&selection).contains("webhook"),"package README must select its owner: {selection:?}"@@ -82,12 +80,7 @@ fn package_readme_selects_its_package() {fn embedded_assets_select_their_consumers() {let graph = build_graph(false).expect("cargo metadata");- let selection = compute_packages(- &graph,- &graph,- &changed_files(&["static_assets/schema.graphql"]),- )- .unwrap();+ let selection = select_in_repo(&graph, &["static_assets/schema.graphql"]);let set = selected_packages(&selection);for expected in ["cache-core",@@ -103,15 +96,13 @@ fn embedded_assets_select_their_consumers() {);}- let mixed = compute_packages(- &graph,+ let mixed = select_in_repo(&graph,- &changed_files(&[+ &["crates/email_validator/src/lib.rs","static_assets/markdown-golden.1.bin",- ]),- )- .unwrap();+ ],+ );let mixed_set = selected_packages(&mixed);assert!(mixed_set.contains("email_validator"));assert!(mixed_set.contains("documents"));@@ -180,6 +171,279 @@ fn embedded_asset_packages_match_source_references() {);}+const BASE_MANIFEST: &str = r#"+[workspace]+resolver = "2"+members = ["crates/a", "crates/b"]++[workspace.dependencies]+anyhow = "1.0.100"+serde = { version = "1.0.214", features = ["derive"] }++[profile.fig-engine-wasm]+inherits = "release"+lto = "fat"+"#;++fn manifest_forces_all(new: &str) -> bool {+ root_manifest_change_forces_all(BASE_MANIFEST, new).expect("valid manifests")+}++/// Adding or removing a member is a package-graph change determinator scopes+/// to the new package (or to the manifests that depended on a removed one).+#[test]+fn members_only_change_defers_to_determinator() {+ let new = BASE_MANIFEST.replace(+ r#"members = ["crates/a", "crates/b"]"#,+ r#"members = ["crates/a", "crates/b", "crates/c"]"#,+ );+ assert_ne!(new, BASE_MANIFEST);+ assert!(!manifest_forces_all(&new));+}++/// Workspace dependency specs reach members through inheritance, so the+/// graph diff selects exactly the members whose resolved deps changed.+#[test]+fn workspace_dependencies_only_change_defers_to_determinator() {+ let new = BASE_MANIFEST+ .replace(r#"anyhow = "1.0.100""#, r#"anyhow = "1.0.101""#)+ .replace(r#"features = ["derive"]"#, r#"features = ["derive", "rc"]"#)+ .replace(+ "[workspace.dependencies]\n",+ "[workspace.dependencies]\nhex = \"0.4.3\"\n",+ );+ assert_ne!(new, BASE_MANIFEST);+ assert!(!manifest_forces_all(&new));+}++/// Comments and formatting are not semantic edits.+#[test]+fn comment_and_formatting_change_is_not_a_change() {+ let new = BASE_MANIFEST.replace(+ "[profile.fig-engine-wasm]",+ "# Browser build for the fig engine.\n[profile.fig-engine-wasm]",+ );+ assert!(!manifest_forces_all(&new));+}++/// Profiles change compile flags for every package without touching the graph.+#[test]+fn profile_change_forces_all() {+ assert!(manifest_forces_all(+ &BASE_MANIFEST.replace(r#"lto = "fat""#, r#"lto = "thin""#)+ ));+ let added = format!("{BASE_MANIFEST}\n[profile.dev]\ndebug = \"limited\"\n");+ assert!(manifest_forces_all(&added));+}++/// Workspace lints change every member's clippy/rustc output.+#[test]+fn workspace_lints_change_forces_all() {+ let new = format!("{BASE_MANIFEST}\n[workspace.lints.rust]\nunsafe_code = \"forbid\"\n");+ assert!(manifest_forces_all(&new));+}++/// Other workspace-level settings are invisible to the graph diff as well.+#[test]+fn resolver_patch_and_package_changes_force_all() {+ assert!(manifest_forces_all(+ &BASE_MANIFEST.replace(r#"resolver = "2""#, r#"resolver = "3""#)+ ));+ assert!(manifest_forces_all(&format!(+ "{BASE_MANIFEST}\n[patch.crates-io]\nserde = {{ path = \"vendor/serde\" }}\n"+ )));+ assert!(manifest_forces_all(&format!(+ "{BASE_MANIFEST}\n[workspace.package]\nedition = \"2024\"\n"+ )));+}++/// A members edit combined with anything else still forces the full suite.+#[test]+fn mixed_members_and_profile_change_forces_all() {+ let new = BASE_MANIFEST+ .replace(+ r#"members = ["crates/a", "crates/b"]"#,+ r#"members = ["crates/a"]"#,+ )+ .replace(r#"lto = "fat""#, "lto = false");+ assert!(manifest_forces_all(&new));+}++/// The real root manifest parses, and comparing it with itself is a no-op.+#[test]+fn repository_root_manifest_is_classified() {+ let manifest = read_manifest(&xtask_paths::workspace_root()).expect("root manifest");+ assert!(!root_manifest_change_forces_all(&manifest, &manifest).unwrap());+}++/// A tiny two-crate workspace (`b` depends on `a` through+/// `workspace.dependencies`) written to a temp dir, with optional overrides.+struct SyntheticWorkspace {+ dir: TempDir,+}++impl SyntheticWorkspace {+ /// `a` plus `others`, each depending on `a` through+ /// `workspace.dependencies` with `a_features` enabled.+ fn new(members: &[&str], a_features: &str) -> Self {+ let manifests: Vec<(&str, &str)> = members+ .iter()+ .map(|member| {+ let deps = if *member == "a" {+ "[features]\nextra = []\n"+ } else {+ "[dependencies]\na = { workspace = true }\n"+ };+ (*member, deps)+ })+ .collect();+ Self::with_members(+ &manifests,+ &format!("a = {{ path = \"a\", features = [{a_features}] }}\n"),+ )+ }++ /// Members given as `(name, manifest body after [package])`.+ fn with_members(members: &[(&str, &str)], workspace_dependencies: &str) -> Self {+ let dir = tempfile::tempdir().expect("temp dir");+ let write = |path: &str, content: &str| {+ let path = dir.path().join(path);+ std::fs::create_dir_all(path.parent().unwrap()).unwrap();+ std::fs::write(path, content).unwrap();+ };+ let quoted: Vec<String> = members+ .iter()+ .map(|(name, _)| format!("{name:?}"))+ .collect();+ write(+ "Cargo.toml",+ &format!(+ "[workspace]\nresolver = \"2\"\nmembers = [{}]\n\n[workspace.dependencies]\n\+ {workspace_dependencies}",+ quoted.join(", ")+ ),+ );+ for (name, body) in members {+ write(+ &format!("{name}/Cargo.toml"),+ &format!(+ "[package]\nname = \"{name}\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n{body}"+ ),+ );+ write(&format!("{name}/src/lib.rs"), "");+ }+ Self { dir }+ }++ fn graph(&self) -> PackageGraph {+ build_graph_at(self.dir.path(), false).expect("cargo metadata for synthetic workspace")+ }+}++/// The repository's path rules name real packages, so the synthetic+/// workspaces run determinator without them.+fn no_path_rules() -> DeterminatorRules {+ DeterminatorRules::parse("use-default-rules = false").expect("empty rules")+}++/// A new member reaches determinator as a new package; nothing that existed+/// before is rebuilt.+#[test]+fn graph_diff_scopes_a_new_member_to_itself() {+ let old = SyntheticWorkspace::new(&["a", "b"], "");+ let new = SyntheticWorkspace::new(&["a", "b", "c"], "");+ let selection = compute_packages(+ &old.graph(),+ &new.graph(),+ &changed_files(&["Cargo.toml", "c/Cargo.toml", "c/src/lib.rs"]),+ &no_path_rules(),+ )+ .unwrap();+ assert_eq!(+ selection,+ PackageSelection::Packages(BTreeSet::from(["c".to_owned()]))+ );+}++/// A feature added in `workspace.dependencies` shows up in the simulated+/// build of the member that inherits it, so the root-manifest edit selects+/// that member without anyone touching its own files.+#[test]+fn graph_diff_scopes_a_workspace_dependency_change_to_inheritors() {+ let old = SyntheticWorkspace::new(&["a", "b", "c"], "");+ let new = SyntheticWorkspace::new(&["a", "b", "c"], "\"extra\"");+ let selection = compute_packages(+ &old.graph(),+ &new.graph(),+ &changed_files(&["Cargo.toml"]),+ &no_path_rules(),+ )+ .unwrap();+ assert_eq!(+ selection,+ PackageSelection::Packages(BTreeSet::from(["b".to_owned(), "c".to_owned()]))+ );+}++/// The fast path agrees with determinator on what a build contains: a+/// dev-dependency reaches only its declarer's tests (`b` dev-depends on `a`,+/// `c` uses `b`'s library), normal dependencies are transitive (`e` → `d` →+/// `a`), and an optional dependency nobody enables builds nothing (`g` uses+/// `h` without the feature that pulls in `a`).+#[test]+fn fast_path_matches_determinator_on_build_edges() {+ let workspace = SyntheticWorkspace::with_members(+ &[+ ("a", ""),+ ("b", "[dev-dependencies]\na = { path = \"../a\" }\n"),+ ("c", "[dependencies]\nb = { path = \"../b\" }\n"),+ ("d", "[dependencies]\na = { path = \"../a\" }\n"),+ ("e", "[dependencies]\nd = { path = \"../d\" }\n"),+ ("f", ""),+ ("g", "[dependencies]\nh = { path = \"../h\" }\n"),+ (+ "h",+ "[dependencies]\na = { path = \"../a\", optional = true }\n\n\+ [features]\nwith_a = [\"dep:a\"]\n",+ ),+ ],+ "",+ );+ let graph = workspace.graph();+ let expected =+ PackageSelection::Packages(BTreeSet::from(["a", "b", "d", "e", "h"].map(str::to_owned)));+ let changed = changed_files(&["a/src/lib.rs"]);+ assert_eq!(+ select_by_paths(&graph, &changed, &no_path_rules()).unwrap(),+ expected+ );+ assert_eq!(+ compute_packages(&graph, &graph, &changed, &no_path_rules()).unwrap(),+ expected+ );+}++/// Only manifests and lockfiles can change `cargo metadata`; everything else+/// takes the path-only fast path.+#[test]+fn only_manifests_and_lockfiles_need_the_graph_diff() {+ for path in [+ "Cargo.toml",+ "Cargo.lock",+ "crates/email_validator/Cargo.toml",+ ] {+ assert!(changes_cargo_graph(Path::new(path)), "{path}");+ }+ for path in [+ "crates/email_validator/src/lib.rs",+ "Cargo.toml.orig",+ "docs/Cargo.md",+ "static_assets/schema.graphql",+ ] {+ assert!(!changes_cargo_graph(Path::new(path)), "{path}");+ }+}+/// Recursively collect `.rs` files under `dir`, skipping hidden and build/// output directories.fn collect_rust_files(dir: &Path, out: &mut BTreeSet<PathBuf>) {diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows.rs b/tooling/xtask/crates/xtask_workflows/src/workflows.rsindex caf9710bb3..52cfb6a761 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows.rs@@ -34,7 +34,6 @@ mod deploy_preview;mod deploy_sync_service;mod deploy_web_app;mod deploy_website;-mod docs_check;mod ensure_daytona_snapshot;mod ios_preview;mod path_validation;@@ -325,11 +324,6 @@ const WORKFLOWS: &[WorkflowFile] = &[file_name: "sdk-check.yml",render_yaml: || render_gh_workflow(sdk_check::sdk_check)(),},- WorkflowFile {- slug: "docs_check",- file_name: "docs-check.yml",- render_yaml: || render_gh_workflow(docs_check::docs_check)(),- },];/// Write every workflow to disk.diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rsindex 3a6ba50ad1..e38d76e7f8 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage.rs@@ -5,10 +5,14 @@//! moved to Namespace profiles, and compiled objects moved off the S3/local//! volume backends onto Namespace's official remote sccache (so there are no//! AWS credentials anywhere).+//!+//! Critical path: `path-check` (no Rust compile on a warm binary cache) →+//! `test`. `check` (fmt + clippy) and `doppler-config` run alongside `test`.use gh_workflow::{Container, Env, Event, Expression, Job, Port, PullRequest, PullRequestType, Run, Step, Workflow,};+use xtask_paths::RuntimePath;use crate::workflows::{runners,@@ -34,15 +38,31 @@ pub fn code_check_cloud_storage() -> Workflow {.concurrency(vars::concurrency("code-check-cloud-storage")).add_job("path-check", path_check()).add_job("check", check())+ .add_job("doppler-config", doppler_config()).add_job("test", test()).add_job("status-check", status_check())}-/// Decide whether the rest of the workflow runs, and compute the nextest filter.+/// Same-repo PRs only: the binary cache below is shared, so an untrusted fork+/// must not be able to plant a binary under a key a later PR would execute.+const TRUSTED_CONTEXT: &str = steps::TRUSTED_NAMESPACE_SCCACHE_CONTEXT;++/// Where `path-check` keeps the release `xtask_nextest_filter` binary (keyed by+/// its source trees and `Cargo.lock`) and the target dir that builds it.+const FILTER_CACHE_DIR: RuntimePath<'static> =+ xtask_paths::runtime_path!("/home/runner/.cache/macro-nextest-filter");++/// Decide whether the rest of the workflow runs and compute the nextest+/// filter. Deliberately light: no Nix, no sccache, and a cached release+/// binary for the filter.fn path_check() -> Job {Job::default()- .runs_on(runners::Runner::Small.to_string())+ .runs_on(runners::Runner::Small.with_cache_tag(vars::NEXTEST_FILTER_CACHE_TAG)).add_output("should_run", "${{ steps.filter.outputs.should_run }}")+ .add_output(+ "doppler_candidates",+ "${{ steps.filter.outputs.doppler_candidates }}",+ ).add_output("rust_packages","${{ steps.nextest-filter.outputs.rust_packages }}",@@ -51,42 +71,75 @@ fn path_check() -> Job {"skip_tests","${{ steps.nextest-filter.outputs.skip_tests }}",)- .add_output(- "doppler_config_bins",- "${{ steps.doppler-bins.outputs.doppler_config_bins }}",- )// PR merge commits need both parents for `git merge-base`; fetching// the entire repo (every branch) is not required..add_step(steps::checkout(false, false).add_with(("fetch-depth", 2))).add_step(steps::setup_rust_light()).add_step(paths_filter())- .add_step(compute_changed_files())- .add_step(compute_doppler_bins())+ .add_step(+ compute_changed_files()+ .if_condition(Expression::new("steps.filter.outputs.should_run == 'true'")),+ )+ .add_step(+ steps::mount_path_cache_volume("Mount nextest-filter binary cache", FILTER_CACHE_DIR)+ .if_condition(Expression::new(format!(+ "steps.filter.outputs.should_run == 'true' && ({TRUSTED_CONTEXT})"+ ))),+ ).add_step(compute_nextest_filter())}-/// fmt + clippy (and Doppler-config validation).+/// fmt + clippy.fn check() -> Job {steps::gated_job().runs_on(runners::Runner::RustCi.with_cache_tag(vars::CI_CACHE_TAG))- .add_env((- "RUSTFLAGS",- "-Dwarnings -Dclippy::disallowed_methods -C link-arg=-fuse-ld=mold",- ))- .add_env(("RUSTDOCFLAGS", "-Dwarnings"))- .add_env(("SQLX_OFFLINE", "true"))+ .map(with_check_env).add_step(steps::checkout(false, false)).add_step(steps::mount_cache_volume()).add_step(steps::setup_nix()).add_step(steps::setup_dev_shell()).add_step(steps::configure_namespace_sccache(vars::CI_SCCACHE_NAME))- .add_step(validate_doppler_configs()).add_step(cargo_fmt()).add_step(cargo_clippy()).add_step(steps::show_sccache_stats()).add_step(steps::teardown_nix())}+/// The env the clippy-side jobs share, so their sccache keys stay identical.+fn with_check_env(job: Job) -> Job {+ job.add_env((+ "RUSTFLAGS",+ "-Dwarnings -Dclippy::disallowed_methods -C link-arg=-fuse-ld=mold",+ ))+ .add_env(("RUSTDOCFLAGS", "-Dwarnings"))+ .add_env(("SQLX_OFFLINE", "true"))+}++/// Live Doppler config-contract check for services whose config may have+/// changed. Runs beside `check`/`test` instead of inside `path-check`, where+/// computing the bin list compiled an xtask from scratch on every PR. A cheap+/// path filter gates it; the xtask then picks the exact binaries.+fn doppler_config() -> Job {+ steps::gated_job()+ .cond(Expression::new(+ "needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false && needs.path-check.outputs.doppler_candidates == 'true'",+ ))+ .runs_on(runners::Runner::RustCi.with_cache_tag(vars::CI_CACHE_TAG))+ .map(with_check_env)+ .add_step(steps::checkout(false, false).add_with(("fetch-depth", 2)))+ // Before the dev shell: its LD_LIBRARY_PATH points the runner's+ // git-remote-https at Nix's glibc, which aborts `git fetch`.+ .add_step(compute_changed_files())+ .add_step(steps::mount_cache_volume())+ .add_step(steps::setup_nix())+ .add_step(steps::setup_dev_shell())+ .add_step(steps::configure_namespace_sccache(vars::CI_SCCACHE_NAME))+ .add_step(compute_doppler_bins())+ .add_step(validate_doppler_configs())+ .add_step(steps::show_sccache_stats())+ .add_step(steps::teardown_nix())+}+/// cargo nextest against postgres + redis service containers. Skipped when/// the path filter selected no Rust packages (Nix-only / unmapped files) or/// when `skip_tests` is set (`.sqlx`-only diffs still clippy the snapshot).@@ -130,6 +183,7 @@ fn status_check() -> Job {.needs(vec!["path-check".to_string(),"check".to_string(),+ "doppler-config".to_string(),"test".to_string(),]).add_step(check_job_results())@@ -137,7 +191,9 @@ fn status_check() -> Job {// --- workflow-specific steps --------------------------------------------------/// Detect whether cloud-storage-relevant paths changed.+/// Detect whether cloud-storage-relevant paths changed, and whether any+/// change could affect a service's Doppler config contract (a superset of+/// what `xtask doppler-bins` checks, so the gate never hides a bin).fn paths_filter() -> Step<gh_workflow::Use> {Step::new("Filter changed paths").uses(@@ -172,18 +228,25 @@ fn paths_filter() -> Step<gh_workflow::Use> {- '.github/actions/setup-sccache/**'- '.github/services-config.json'- .github/workflows/code_check_cloud_storage.yml+ doppler_candidates:+ - '.github/services-config.json'+ - '**/src/config.rs'+ - '**/src/doppler_config.rs'+ - '**/Cargo.toml'+ - 'tooling/xtask/crates/xtask_doppler_bins/**'+ - .github/workflows/code_check_cloud_storage.yml"#},))}-/// Compute the changed-file set once (shared by the nextest filter and the-/// Doppler-bin detection) and write it to `/tmp/changed-files`. On a missing-/// merge-base we leave the list empty, which makes both downstream steps fall-/// back to "everything": run all tests, validate no Doppler bins.+/// Compute the changed-file set and write it to `/tmp/changed-files`. Stacked+/// PRs whose merge commit sits on another branch get their history deepened+/// until it reaches the base branch. If no merge-base turns up, we leave the+/// list empty, which makes downstream steps fall back to "everything": run all+/// tests, validate no Doppler bins.fn compute_changed_files() -> Step<Run> {Step::new("compute changed files").run(include_str!("scripts/compute_changed_files.sh"))- .if_condition(Expression::new("steps.filter.outputs.should_run == 'true'")).shell("bash")}@@ -193,19 +256,21 @@ fn compute_doppler_bins() -> Step<Run> {Step::new("compute affected Doppler config bins").run(include_str!("scripts/compute_doppler_bins.sh")).id("doppler-bins")- .if_condition(Expression::new("steps.filter.outputs.should_run == 'true'")).shell("bash")}/// Compute the cargo-nextest / clippy package set from the changed files, via-/// the `xtask nextest-filter` subcommand. Root Cargo/toolchain changes-/// short-circuit to the full suite; unmapped files (a top-level JSON, a Nix-/// shell tweak) select no packages.+/// the `xtask nextest-filter` subcommand.+/// Toolchain and cargo config changes short-circuit to the full suite; root+/// `Cargo.toml` edits run the full suite unless they only touch workspace+/// members or dependencies; unmapped files (a top-level JSON, a Nix shell+/// tweak) select no packages.fn compute_nextest_filter() -> Step<Run> {Step::new("compute nextest package filter").run(include_str!("scripts/compute_nextest_filter.sh")).id("nextest-filter").if_condition(Expression::new("steps.filter.outputs.should_run == 'true'"))+ .add_env(("FILTER_CACHE_DIR", FILTER_CACHE_DIR.as_str())).shell("bash")}@@ -216,11 +281,11 @@ fn validate_doppler_configs() -> Step<Run> {Step::new("validate Doppler configs").run(include_str!("scripts/validate_doppler_configs.sh")).if_condition(Expression::new(- "needs.path-check.outputs.doppler_config_bins != ''",+ "steps.doppler-bins.outputs.doppler_config_bins != ''",)).add_env(("DOPPLER_CONFIG_BINS",- "${{ needs.path-check.outputs.doppler_config_bins }}",+ "${{ steps.doppler-bins.outputs.doppler_config_bins }}",)).add_env(("DOPPLER_TOKEN", vars::DOPPLER_TOKEN))}@@ -273,9 +338,11 @@ fn configure_postgres() -> Step<Run> {.run(include_str!("scripts/configure_postgres.sh"))}-/// Set up test env files and databases.+/// Set up test env files and databases. The pre-migrated template1 spares each+/// `#[sqlx::test]` database from replaying every migration.fn prepare_tests() -> Step<Run> {- Step::new("prepare tests").run("just setup_test_envs && just initialize_dbs")+ Step::new("prepare tests")+ .run("just setup_test_envs && just initialize_dbs && just setup_test_template")}/// Run the test suite (no AWS credentials; sccache uses Namespace's remote cache).@@ -288,11 +355,13 @@ fn check_job_results() -> Step<Run> {Step::new("Check job results").run(indoc::indoc! {r#"echo "path-check: ${{ needs.path-check.result }}"echo "check: ${{ needs.check.result }}"+ echo "doppler-config: ${{ needs.doppler-config.result }}"echo "test: ${{ needs.test.result }}"# Fail if any job failed (skipped and success are both OK)if [[ "${{ needs.path-check.result }}" == "failure" ]] || \[[ "${{ needs.check.result }}" == "failure" ]] || \+ [[ "${{ needs.doppler-config.result }}" == "failure" ]] || \[[ "${{ needs.test.result }}" == "failure" ]]; thenecho "❌ One or more jobs failed"exit 1diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rsindex 9adc05b093..b307e4be73 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/code_check_cloud_storage/test.rs@@ -2,10 +2,83 @@ use super::*;#[test]fn inventory_only_changes_run_live_config_validation() {+ let with = serde_json::to_value(&paths_filter().value.with).unwrap();+ let filters = with["filters"].as_str().unwrap().to_owned();+ let doppler_filter = filters+ .split("doppler_candidates:")+ .nth(1)+ .expect("doppler gate filter");+ assert!(doppler_filter.contains(".github/services-config.json"));+ let job = serde_json::to_value(doppler_config()).unwrap();+ let condition = job["if"].as_str().unwrap();+ assert!(!condition.contains("rust_packages"), "{condition}");+ assert!(+ condition.contains("doppler_candidates == 'true'"),+ "{condition}"+ );+}++/// Every trigger `xtask doppler-bins` checks must open the gate, or a config+/// change could skip live validation.+#[test]+fn doppler_gate_covers_the_bin_triggers() {let filters = serde_json::to_string(&paths_filter().value.with).unwrap();- assert!(filters.contains(".github/services-config.json"));- let job = serde_json::to_value(check()).unwrap();- assert!(!job["if"].as_str().unwrap().contains("rust_packages"));+ for glob in [+ "**/src/config.rs",+ "**/src/doppler_config.rs",+ "**/Cargo.toml",+ ] {+ assert!(filters.contains(glob), "missing {glob}: {filters}");+ }+}++/// path-check stays compile-free on a warm cache: the Doppler bins moved to+/// their own job, and validation no longer rides on clippy.+#[test]+fn doppler_work_is_off_path_check_and_check() {+ for job in [path_check(), check()] {+ let job = serde_json::to_string(&job).unwrap();+ assert!(!job.contains("doppler-bins"), "{job}");+ assert!(!job.contains("validate Doppler configs"), "{job}");+ }+ let job = serde_json::to_string(&doppler_config()).unwrap();+ assert!(job.contains("doppler-bins") && job.contains("validate Doppler configs"));+ assert!(job.contains("steps.doppler-bins.outputs.doppler_config_bins"));+}++/// The shared binary cache is only mounted for same-repo PRs.+#[test]+fn filter_binary_cache_is_trusted_only() {+ let job = serde_json::to_value(path_check()).unwrap();+ let mount = job["steps"]+ .as_array()+ .unwrap()+ .iter()+ .find(|step| step["name"] == "Mount nextest-filter binary cache")+ .expect("binary cache mount");+ let condition = mount["if"].as_str().unwrap();+ assert!(+ condition.contains("github.event.pull_request.head.repo.full_name == github.repository"),+ "{condition}"+ );+}++/// The required status check fails when any job it gathers failed, including+/// the new Doppler job.+#[test]+fn status_check_covers_every_job() {+ let status = serde_json::to_value(status_check()).unwrap();+ let script = status["steps"][0]["run"].as_str().unwrap();+ for needed in ["path-check", "check", "doppler-config", "test"] {+ let check = format!(r#"[[ "${{{{ needs.{needed}.result }}}}" == "failure" ]]"#);+ assert!(script.contains(&check), "{check}: {script}");+ assert!(+ status["needs"]+ .as_array()+ .unwrap()+ .contains(&serde_json::json!(needed))+ );+ }}#[test]@@ -34,27 +107,43 @@ fn tests_continue_to_exclude_sync_service() {);}-#[test]-fn sqlx_only_clippies_and_skips_live_tests() {+/// The `if … fi` block starting at `marker` in the filter script.+fn filter_script_block(marker: &str) -> &'static str {let script = include_str!("../scripts/compute_nextest_filter.sh");let start = script- .find("if ! grep -qvE '^\\.sqlx/'")- .expect("sqlx-only short-circuit must remain");- let sqlx_block = script[start..]- .split("packages=\"$(cargo run")- .next()- .unwrap();+ .find(marker)+ .unwrap_or_else(|| panic!("{marker} must remain"));+ let end = script[start..].find("\nfi\n").unwrap() + start;+ &script[start..end]+}++#[test]+fn sqlx_only_clippies_and_skips_live_tests() {+ let sqlx_block = filter_script_block("if ! grep -qvE '^\\.sqlx/'");assert!(- sqlx_block.contains(r#"rust_packages=all"#),- "sqlx-only must emit all for clippy, not none: {sqlx_block}"+ sqlx_block.contains("emit all true"),+ "sqlx-only must clippy all and skip the live-Postgres suite: {sqlx_block}");assert!(- !sqlx_block.contains(r#"rust_packages=none"#),+ !sqlx_block.contains("emit none"),"sqlx-only must not skip clippy: {sqlx_block}");+}++/// Every exit goes through `emit`, so both outputs are always written once.+#[test]+fn every_exit_goes_through_emit() {+ let script = include_str!("../scripts/compute_nextest_filter.sh");+ let body = &script[script.find("emit() {").expect("emit must remain")..];+ let emit = &body[..body.find("\n}\n").unwrap()];+ assert!(+ emit.contains("rust_packages=$1") && emit.contains("skip_tests=$2"),+ "{emit}"+ );+ let after_emit = &body[body.find("\n}\n").unwrap()..];assert!(- sqlx_block.contains(r#"skip_tests=true"#),- "sqlx-only diffs must not run the live-Postgres suite: {sqlx_block}"+ !after_emit.contains(">> \"$GITHUB_OUTPUT\""),+ "outputs must only be written by emit: {after_emit}");}@@ -73,6 +162,51 @@ fn test_job_honors_skip_tests() {);}+/// Root Cargo.toml edits are classified by `xtask nextest-filter` (members and+/// workspace dependencies scope through the graph diff); only toolchain and+/// cargo config still short-circuit to the full suite in bash.+#[test]+fn root_manifest_is_classified_by_the_filter_not_forced_all() {+ let script = include_str!("../scripts/compute_nextest_filter.sh");+ let force_all = script+ .lines()+ .find(|line| line.contains(r"grep -qE '^(rust-toolchain"))+ .expect("toolchain changes must still force the full suite");+ for path in [+ "Cargo\\.toml",+ "Cross\\.toml",+ "deny\\.toml",+ "clippy\\.toml",+ ] {+ assert!(+ !force_all.contains(path),+ "{path} must not force the full suite: {force_all}"+ );+ }+ assert!(force_all.contains(r"\.cargo/"));+}++/// A clippy.toml edit re-lints everything but only skips tests when no+/// package was selected for other reasons.+#[test]+fn clippy_config_only_change_skips_tests() {+ let block = filter_script_block(r#"if [ "$clippy_config_changed" = true ]; then"#);+ assert!(block.contains("emit all true"), "{block}");+ assert!(block.contains("emit all false"), "{block}");+}++/// Stacked PRs can get a merge commit whose first parent is another branch;+/// the changed-files step deepens history before giving up on a merge-base.+#[test]+fn missing_merge_base_deepens_before_falling_back() {+ let script = include_str!("../scripts/compute_changed_files.sh");+ let deepen = script.find("--deepen").expect("must deepen the PR history");+ let fallback = script+ .find("falling back to full test suite")+ .expect("must keep the full-suite fallback");+ assert!(deepen < fallback);+}+#[test]fn tests_build_dependencies_without_debug_info() {let script = include_str!("../scripts/run_tests.sh");diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/docs_check.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/docs_check.rsdeleted file mode 100644index 38f16c93e0..0000000000--- a/tooling/xtask/crates/xtask_workflows/src/workflows/docs_check.rs+++ /dev/null@@ -1,67 +0,0 @@-//! `Docs Check` — fails a PR if the generated MCP tool reference under-//! `apps/docs/` has drifted from the Rust tool registry. The pages, their-//! nav list, and the `Tool Reference` group in `docs.json` are all written by-//! `apps/docs/scripts/generate-mcp-tool-pages.ts`, which builds the-//! `gen_tool_schemas` binary and renders from its JSON. Without this check the-//! docs site silently rots every time a tool's schema or description changes.--use gh_workflow::{Concurrency, Event, Expression, Job, PullRequest, Run, Step, Workflow};--use crate::workflows::{runners, steps, vars};--/// Build the workflow.-pub fn docs_check() -> Workflow {- Workflow::new("Docs Check")- .on(Event::default().pull_request(- PullRequest::default()- .add_branch("main")- .add_path(xtask_paths::repo_glob!("apps/docs/**"))- // Tool schemas are derived from types spread across the- // workspace, not just `crates/ai_tools`, so watch all Rust.- .add_path(xtask_paths::repo_glob!("crates/**/*.rs"))- .add_path(xtask_paths::repo_glob!("Cargo.toml"))- .add_path(xtask_paths::repo_glob!("Cargo.lock"))- .add_path(".github/workflows/docs-check.yml"),- ))- .concurrency(- Concurrency::new(Expression::new(- "${{ github.workflow }}-${{ github.ref }}-check",- ))- .cancel_in_progress(true),- )- .add_job("check-docs", check_docs())-}--/// Regenerate the tool reference and fail on drift. Shares the web CI cache-/// volume and sccache with `sdk-check`: both jobs build one small binary out of-/// the same workspace, so they warm the same cache.-fn check_docs() -> Job {- Job::default()- .name("Docs Generated Code Check")- .runs_on(runners::Runner::Mid.with_cache_tag(vars::WEB_CI_CACHE_TAG))- .add_step(steps::checkout(false, false))- .add_step(steps::mount_web_cache_volume(true))- .add_step(steps::setup_rust_sccache())- .add_step(steps::setup_bun())- .add_step(steps::configure_namespace_sccache(vars::WEB_SCCACHE_NAME))- .add_step(generate_tool_pages())- .add_step(steps::show_sccache_stats())- .add_step(verify_fresh())-}--fn generate_tool_pages() -> Step<Run> {- Step::new("Regenerate MCP tool pages")- .run("bun run generate:tools")- .working_directory("apps/docs")-}--fn verify_fresh() -> Step<Run> {- Step::new("Verify generated docs are fresh").run(indoc::indoc! {r#"- if [ -n "$(git status --porcelain -- apps/docs)" ]; then- echo "apps/docs tool reference is stale. Run 'bun run generate:tools' in apps/docs and commit the result."- git status --porcelain -- apps/docs- git diff -- apps/docs | head -200- exit 1- fi- "#})-}diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_changed_files.sh b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_changed_files.shindex a8b2ec9d7a..5b90372c6f 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_changed_files.sh+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_changed_files.sh@@ -7,8 +7,26 @@ set -euo pipefailif [ -z "${GITHUB_BASE_REF:-}" ]; thencompare_rev="$(git rev-parse HEAD~1)"else- git fetch origin "$GITHUB_BASE_REF:refs/remotes/origin/$GITHUB_BASE_REF"- if ! compare_rev="$(git merge-base "origin/${GITHUB_BASE_REF}" HEAD)"; then+ git fetch --no-tags origin "$GITHUB_BASE_REF:refs/remotes/origin/$GITHUB_BASE_REF"++ # The checkout holds only the PR merge commit and its two parents. Normally+ # the first parent is on the base branch, so the merge-base is immediate.+ # Stacked PRs can arrive with a merge commit built on another branch's tip;+ # deepen the PR side until its history reaches the base branch instead of+ # falling back to the full suite.+ compare_rev=""+ for deepen in 0 200 2000; do+ if [ "$deepen" -gt 0 ]; then+ echo "No merge-base with origin/${GITHUB_BASE_REF} yet; deepening PR history by ${deepen}"+ git fetch --no-tags --deepen="$deepen" origin "$GITHUB_SHA" || break+ fi+ if compare_rev="$(git merge-base "origin/${GITHUB_BASE_REF}" HEAD)"; then+ break+ fi+ compare_rev=""+ done++ if [ -z "$compare_rev" ]; thenecho "Unable to find merge-base for origin/${GITHUB_BASE_REF}; falling back to full test suite" >&2: > /tmp/changed-filesexit 0diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_nextest_filter.sh b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_nextest_filter.shindex 9fdd7c72a5..7f269f76f6 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_nextest_filter.sh+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/compute_nextest_filter.sh@@ -1,30 +1,74 @@set -euo pipefail-# Writes `rust_packages` for the check/test jobs:-# all — compile and run the whole workspace-# none — no Rust packages to lint/test (Nix-only, top-level JSON, docs, …)-# <names> — space-separated workspace packages (changed + reverse deps)+# Writes for the check/test jobs:+# rust_packages — all | none | space-separated packages (changed + reverse deps)+# skip_tests — true keeps clippy on rust_packages while the live-Postgres+# test job stays skipped (`.sqlx`-only or clippy.toml-only diffs)#-# `skip_tests=true` keeps clippy on a package set while the live-Postgres-# test job stays skipped (used for `.sqlx`-only diffs).-#-# Root Cargo/toolchain/config edits force `all`. Cargo.lock is handled by-# determinator's old/new graph comparison. flake.nix, the Nix shell action,-# and this workflow file used to live on the full-suite list and turned every-# dev-shell tweak into a full-suite rebuild.+# Toolchain and cargo/nextest config edits force `all`. Root Cargo.toml and+# Cargo.lock go to the xtask: determinator's old/new graph comparison scopes+# lockfile, member, and workspace-dependency changes, and the xtask returns+# `all` for any other root-manifest edit (profiles, lints, patches, …).+# Cross.toml and deny.toml configure other tools and select nothing here.+# clippy.toml changes lint results but not test outcomes (handled below).+# flake.nix, the Nix shell action, and this workflow file used to live on the+# full-suite list and turned every dev-shell tweak into a full-suite rebuild.++# The filter runs as a release binary cached per source tree: a debug build+# spends ~20 s in guppy on this runner, and compiling it took ~40 s more. The+# key covers every input of the build, so a hit is exactly what `cargo build`+# would produce. FILTER_CACHE_DIR is a Namespace cache volume for same-repo+# PRs and a plain scratch directory for forks.+filter_bin=""+ensure_filter_bin() {+ [ -n "$filter_bin" ] && return 0+ local key bin_dir staging+ key="$(git rev-parse \+ HEAD:tooling/xtask/crates/xtask_nextest_filter \+ HEAD:tooling/xtask/crates/xtask_graph \+ HEAD:tooling/xtask/crates/xtask_paths \+ HEAD:Cargo.lock \+ HEAD:rust-toolchain.toml | sha256sum | cut -c1-16)"+ bin_dir="$FILTER_CACHE_DIR/bin/$key"+ filter_bin="$bin_dir/xtask_nextest_filter"+ if [ -x "$filter_bin" ]; then+ echo "Using cached nextest-filter build $key"+ touch "$bin_dir"+ return 0+ fi++ echo "Building nextest-filter $key"+ CARGO_TARGET_DIR="$FILTER_CACHE_DIR/target" \+ cargo build --release --locked --quiet -p xtask_nextest_filter+ staging="$(mktemp -d "$FILTER_CACHE_DIR/bin/.staging.XXXXXX")"+ cp "$FILTER_CACHE_DIR/target/release/xtask_nextest_filter" "$staging/"+ # Another job may have published the same key meanwhile; either copy works.+ mv -T "$staging" "$bin_dir" 2>/dev/null || rm -rf "$staging"+ # Drop builds no PR has used for a week (every Cargo.lock change adds one).+ find "$FILTER_CACHE_DIR/bin" -mindepth 1 -maxdepth 1 -type d -mtime +7 \+ -exec rm -rf {} + || true+}+mkdir -p "$FILTER_CACHE_DIR/bin"-echo "skip_tests=false" >> "$GITHUB_OUTPUT"+emit() {+ echo "rust_packages=$1" >> "$GITHUB_OUTPUT"+ echo "skip_tests=$2" >> "$GITHUB_OUTPUT"+ exit 0+}if [ ! -s /tmp/changed-files ]; thenecho "Unknown or empty change set; running all tests"- echo "rust_packages=all" >> "$GITHUB_OUTPUT"- exit 0+ emit all falsefi-if grep -qE '^(Cargo\.toml|rust-toolchain\.toml|Cross\.toml|clippy\.toml|deny\.toml|\.cargo/.*|\.config/.*)$' /tmp/changed-files; then- echo "Workspace-level Cargo/toolchain change detected; running all tests"- echo "rust_packages=all" >> "$GITHUB_OUTPUT"- exit 0+if grep -qE '^(rust-toolchain\.toml|\.cargo/.*|\.config/.*)$' /tmp/changed-files; then+ echo "Toolchain or cargo config change detected; running all tests"+ emit all false+fi++clippy_config_changed=false+if grep -qx 'clippy\.toml' /tmp/changed-files; then+ clippy_config_changed=truefi# The test job compiles queries against live Postgres (SQLX_OFFLINE is unset@@ -33,18 +77,27 @@ fi# still compile against the cache rather than skip the check job.if ! grep -qvE '^\.sqlx/' /tmp/changed-files; thenecho "Only .sqlx changes detected; clippy all, skip live-Postgres tests"- echo "rust_packages=all" >> "$GITHUB_OUTPUT"- echo "skip_tests=true" >> "$GITHUB_OUTPUT"- exit 0+ emit all truefi-packages="$(cargo run --manifest-path tooling/xtask/Cargo.toml -- nextest-filter /tmp/changed-files "$(< /tmp/base-revision)")"+ensure_filter_bin+packages="$("$filter_bin" /tmp/changed-files "$(< /tmp/base-revision)")"++# One package list drives both clippy and tests, so a clippy.toml edit can+# only skip tests when nothing else selected a package.+if [ "$clippy_config_changed" = true ]; then+ if [ -z "$packages" ] || [ "$packages" = "none" ]; then+ echo "Only clippy.toml affects Rust; clippy all, skip live-Postgres tests"+ emit all true+ fi+ echo "clippy.toml changed alongside package changes; running all tests"+ emit all false+fiif [ -z "$packages" ] || [ "$packages" = "none" ]; thenecho "No package-specific Rust changes detected; running no tests"- echo "rust_packages=none" >> "$GITHUB_OUTPUT"- exit 0+ emit none falsefiecho "rust packages: $packages"-echo "rust_packages=$packages" >> "$GITHUB_OUTPUT"+emit "$packages" falsediff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/sdk_check.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/sdk_check.rsindex 9312f1c7a1..6bf09fc914 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/sdk_check.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/sdk_check.rs@@ -1,11 +1,13 @@use gh_workflow::{Concurrency, Event, Expression, Job, PullRequest, Push, Run, Step, Workflow};-use crate::workflows::{runners, steps, vars};+use crate::workflows::steps;#[cfg(test)]mod test;-/// Build the workflow.+/// Build the workflow. Whether `packages/sdk`'s generated layer is fresh is+/// checked by the web app workflow's `Generated Code Check`, which builds the+/// spec binaries once for every Rust-generated artifact.pub fn sdk_check() -> Workflow {Workflow::new("SDK Check").on(Event::default()@@ -13,11 +15,6 @@ pub fn sdk_check() -> Workflow {PullRequest::default().add_branch("main").add_path("packages/sdk/**")- .add_path("crates/**/*.rs")- .add_path("Cargo.toml")- .add_path("Cargo.lock")- .add_path("apps/web/scripts/generate-api-schema.ts")- .add_path("apps/web/scripts/services.ts").add_path(".github/workflows/sdk-check.yml"),)// `Release SDK` publishes from `main` without re-checking the@@ -35,12 +32,10 @@ pub fn sdk_check() -> Workflow {.cancel_in_progress(true),).add_job("check-package", check_package())- .add_job("check-sdk", check_sdk())}/// Typecheck, test, and coverage-check the package as committed. Needs nothing-/// but Bun, so it stays a cheap gate that can run on every SDK change — unlike-/// [`check_sdk`], which rebuilds the specs from Rust.+/// but Bun, so it stays a cheap gate that can run on every SDK change.fn check_package() -> Job {Job::default().name("SDK Package Check")@@ -59,45 +54,6 @@ fn install_dependencies() -> Step<Run> {.working_directory(xtask_paths::repo_dir!("packages/sdk"))}-/// Regenerate the SDK's generated layer end-to-end and fail on drift. Only-/// pull requests pay for this: it rebuilds the specs from Rust, and it shares-/// the web CI cache volume so that build hits the same sccache as the web app-/// checks.-fn check_sdk() -> Job {- Job::default()- .name("SDK Generated Code Check")- .cond(Expression::new(- "${{ github.event_name == 'pull_request' }}",- ))- .runs_on(runners::Runner::Mid.with_cache_tag(vars::WEB_CI_CACHE_TAG))- .add_step(steps::checkout(false, false))- .add_step(steps::mount_web_cache_volume(true))- .add_step(steps::setup_nix())- .add_step(steps::setup_reqs_web("Setup Prereqs", false))- .add_step(steps::configure_namespace_sccache(vars::WEB_SCCACHE_NAME))- .add_step(update_generated())- .add_step(steps::show_sccache_stats())- .add_step(verify_fresh())- .add_step(steps::teardown_nix())-}--fn update_generated() -> Step<Run> {- Step::new("Regenerate SDK code")- .run("just update-generated")- .working_directory("packages/sdk")-}--fn verify_fresh() -> Step<Run> {- Step::new("Verify generated code is fresh").run(indoc::indoc! {r#"- if [ -n "$(git status --porcelain -- packages/sdk)" ]; then- echo "packages/sdk generated code is stale. Run 'just update-generated' in packages/sdk and commit the result."- git status --porcelain -- packages/sdk- git diff -- packages/sdk | head -200- exit 1- fi- "#})-}-/// Every generated endpoint must either have a call site under `src/` or be/// hand-listed in `src/coverage/skipped.ts`; fails naming the offenders.fn check_coverage() -> Step<Run> {diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/sdk_check/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/sdk_check/test.rsindex 49f89af878..1a1484ad29 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/sdk_check/test.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/sdk_check/test.rs@@ -6,26 +6,25 @@ fn rendered_yaml() -> String {.expect("SDK check workflow should serialize")}-/// Merges to `main` publish, so the package suite has to guard them too — but-/// the expensive spec rebuild stays on pull requests.+/// Merges to `main` publish, so the package suite has to guard them too.#[test]-fn checks_the_package_on_main_and_regenerates_only_on_pull_requests() {+fn checks_the_package_on_pull_requests_and_main() {let yaml = rendered_yaml();assert!(yaml.contains("check-package"));+ assert!(yaml.contains("pull_request:"));assert!(yaml.contains("push:"));- assert!(yaml.contains("${{ github.event_name == 'pull_request' }}"));}#[test]fn the_package_job_needs_no_rust_toolchain() {let yaml = rendered_yaml();- let package_job = yaml- .split("check-sdk:")- .next()- .expect("check-package is rendered before check-sdk");- assert!(package_job.contains("bun install --frozen-lockfile"));- assert!(!package_job.contains("setup-nix"));- assert!(!package_job.contains("sccache"));+ assert!(yaml.contains("bun install --frozen-lockfile"));+ assert!(!yaml.contains("setup-nix"));+ assert!(!yaml.contains("sccache"));+ assert!(+ !yaml.contains("crates/**"),+ "Rust changes reach the SDK through the web app's Generated Code Check: {yaml}"+ );}diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rsindex 7b8be3b368..491ab982c5 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs@@ -15,7 +15,7 @@ mod test;/// Namespace's sccache setup mints a short-lived workspace credential. GitHub/// withholds repository secrets from fork PRs, but this runner-minted token is/// not a `secrets.*` value, so enforce the equivalent trust boundary here.-const TRUSTED_NAMESPACE_SCCACHE_CONTEXT: &str = concat!(+pub(crate) const TRUSTED_NAMESPACE_SCCACHE_CONTEXT: &str = concat!("(github.event_name != 'pull_request' && ","github.event_name != 'pull_request_target') || ","github.event.pull_request.head.repo.full_name == github.repository"@@ -93,18 +93,6 @@ pub fn setup_rust_light() -> Step<Use> {.add_with(("rust-cache", "false"))}-/// [`setup_rust_light`] plus sccache, for jobs that actually compile something-/// but do not need the Nix dev shell. Pair with-/// [`configure_namespace_sccache`] to point the wrapper at the remote cache.-pub fn setup_rust_sccache() -> Step<Use> {- uses_local(- "Setup Rust",- xtask_paths::repo_dir!(".github/actions/setup-rust"),- )- .add_with(("sccache", "true"))- .add_with(("rust-cache", "false"))-}-/// Install Nix on a macOS runner. The Linux [`setup_nix`] action doesn't apply/// there, so Apple jobs (DMG signing, the iOS simulator preview) install it/// with the same script instead.@@ -203,6 +191,13 @@ pub fn mount_cache_volume() -> Step<Use> {.add_with(("path", xtask_paths::runtime_path!("/nix").as_str()))}+/// Mount only `path` from the profile's cache volume, without a framework+/// mode. Unlike [`mount_cache_volume`] (`cache: rust` plus `/nix`, ~11 s on+/// `linux-rust-ci`), a plain path mounts in under a second.+pub fn mount_path_cache_volume(name: &str, path: RuntimePath<'_>) -> Step<Use> {+ nscloud_cache_action(name).add_with(("path", path.as_str()))+}+/// The pinned `nscloud-cache-action`, shared by every mount helper below./// `continue-on-error` because a cache volume is always a pure optimization — a/// failure just means cold state, never a wrong build.@@ -244,14 +239,6 @@ pub fn configure_namespace_sccache(cache_name: &str) -> Step<Run> {.if_condition(Expression::new(TRUSTED_NAMESPACE_SCCACHE_CONTEXT))}-/// Configure Namespace's remote sccache in a trusted context when-/// `additional_condition` is also true.-pub fn configure_namespace_sccache_when(cache_name: &str, additional_condition: &str) -> Step<Run> {- namespace_sccache_step(cache_name).if_condition(Expression::new(format!(- "({TRUSTED_NAMESPACE_SCCACHE_CONTEXT}) && ({additional_condition})"- )))-}-fn namespace_sccache_step(cache_name: &str) -> Step<Run> {Step::new("Configure Namespace remote sccache").run(format!(r#"set -euo pipefail@@ -306,6 +293,21 @@ pub fn mount_web_cache_volume(with_rust: bool) -> Step<Use> {})}+/// [`mount_web_cache_volume`] plus Vitest's module cache, so the web `Test`+/// job reuses transforms from earlier runs instead of redoing them all.+pub fn mount_web_test_cache_volume() -> Step<Use> {+ nscloud_cache_action("Mount Namespace cache volume")+ .add_with(("cache", "nix"))+ .add_with((+ "path",+ format!(+ "{}\n{}",+ vars::BUN_CACHE_VOLUME_DIR,+ vars::VITEST_MODULE_CACHE_VOLUME_DIR,+ ),+ ))+}+/// [`mount_web_cache_volume`] for jobs that compile the browser wasm packages/// (`just build-*`). Cargo registry/git feed wasm-pack; `.wasm-pack` is the/// downloaded `wasm-opt` the same way [`mount_wasm_cache_volume`] keeps the@@ -323,6 +325,22 @@ pub fn mount_web_build_cache_volume() -> Step<Use> {))}+/// [`mount_web_build_cache_volume`] plus [`vars::WASM_PKG_CACHE_DIR`], for the PR+/// build that reuses unchanged wasm packages. Deploy builds always rebuild them.+pub fn mount_web_build_check_cache_volume() -> Step<Use> {+ nscloud_cache_action("Mount Namespace cache volume")+ .add_with(("cache", "nix"))+ .add_with((+ "path",+ format!(+ "{}\n/home/runner/.cargo/registry\n/home/runner/.cargo/git\n{}\n{}",+ vars::BUN_CACHE_VOLUME_DIR,+ xtask_paths::runtime_path!("/home/runner/.cache/.wasm-pack").as_str(),+ vars::WASM_PKG_CACHE_DIR,+ ),+ ))+}+/// The web-app composite: Nix dev shell (bun, biome, just) + `bun install`./// Jobs that run `gen-api` follow this with [`configure_namespace_sccache`]./// Requires [`setup_nix`] first.diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/steps/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/steps/test.rsindex ac950b2555..358b2c690b 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/steps/test.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/steps/test.rs@@ -33,22 +33,6 @@ fn namespace_sccache_masks_credentials_before_exporting_them() {assert!(condition.0.contains("github.repository"));}-#[test]-fn namespace_sccache_combines_job_specific_and_trust_conditions() {- let step = configure_namespace_sccache_when("test-cache", "steps.filter.outputs.hit == 'true'");- let condition = step- .value- .if_condition- .expect("conditional Namespace sccache setup should have an if expression");-- assert!(- condition- .0- .contains("github.event.pull_request.head.repo.full_name")- );- assert!(condition.0.contains("steps.filter.outputs.hit == 'true'"));-}-#[test]fn named_dev_shell_passes_the_flake_attribute() {let step = setup_dev_shell_named("agent-daemon");diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rsindex 76daf86c2e..a9ae08c271 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs@@ -73,6 +73,12 @@ pub const WEB_CI_CACHE_TAG: &str = "web-ci";/// a different workload and runner profile.pub const WEB_SCCACHE_NAME: &str = "web-ci";+/// Namespace cache tag for `path-check`, which keeps its release+/// `xtask_nextest_filter` binary on the volume. Without a fixed tag the small+/// profile's volume is scoped per branch and PR runs never persisted, so every+/// run rebuilt the filter.+pub const NEXTEST_FILTER_CACHE_TAG: &str = "nextest-filter";+/// Namespace cache tag for the sync-service worker deploy. Its own pool: this/// job compiles for `wasm32-unknown-unknown`, so nothing in the host-target/// volumes ([`CI_CACHE_TAG`]) would serve it anyway.@@ -82,6 +88,15 @@ pub const SYNC_SERVICE_CACHE_TAG: &str = "sync-service-deploy";/// the Nix dev shell and is not available to Namespace's cache planner yet.pub const BUN_CACHE_VOLUME_DIR: &str = "/home/runner/.bun/install/cache";+/// Vitest's filesystem module cache for the web `Test` job. `apps/web`'s+/// Vitest config reads it from `VITEST_MODULE_CACHE_DIR` and keys entries by+/// the lockfile and tsconfig, so PR jobs can share one volume.+pub const VITEST_MODULE_CACHE_VOLUME_DIR: &str = "/home/runner/.cache/vitest-modules";++/// Browser wasm packages keyed by their inputs, so the PR build reuses a package+/// whose crates did not change (`apps/web/scripts/cached-wasm-build.sh`).+pub const WASM_PKG_CACHE_DIR: &str = "/home/runner/.cache/wasm-pkgs";+/// GHCR repository for the agent-harness sandbox image (the same Dockerfile/// Daytona snapshots). Pushed as `:$SHA` on PRs and `:$SHA` + `:latest` on main.pub const AGENT_HARNESS_GHCR_IMAGE: &str = "ghcr.io/macro-inc/macro-agent-harness";diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rsindex 5820f740ca..2c7c322e0b 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs@@ -20,10 +20,12 @@ fn web_runner() -> String {runners::Runner::Small.with_cache_tag(vars::WEB_CI_CACHE_TAG)}-/// Typechecking can compile the Rust binaries used by `gen-api`, and the app-/// build compiles the two browser wasm packages, so retain the mid-size-/// profile while sharing the web CI cache volume and remote sccache.-fn typecheck_runner() -> String {+/// The mid-size (16x32) profile on the web CI cache volume and remote sccache,+/// for the jobs that scale with cores: the generated-code job compiles the+/// schema binaries, the app build compiles the browser wasm packages and runs+/// Vite, typechecking runs the TS 7 compiler, and vitest runs about 1.9x faster+/// on 16 cores than on the small profile's 8.+fn mid_web_runner() -> String {runners::Runner::Mid.with_cache_tag(vars::WEB_CI_CACHE_TAG)}@@ -46,6 +48,7 @@ pub fn web_app_check_main() -> Workflow {).add_job("path-check", path_check()).add_job("typescript", typescript())+ .add_job("generated-code", generated_code()).add_job("biome-check", biome_check()).add_job("test", test()).add_job("cycles", cycles())@@ -57,30 +60,29 @@ fn path_check() -> Job {Job::default().runs_on(runners::Runner::TinyNoCache.to_string()).add_output("should_run", "${{ steps.filter.outputs.should_run }}")- .add_output("api_changed", "${{ steps.filter.outputs.api_changed }}")+ .add_output(+ "codegen_changed",+ "${{ steps.filter.outputs.codegen_changed }}",+ ).add_step(checkout("Checkout Repo", false)).add_step(paths_filter())}+/// TypeScript checks against the committed generated code. Whether that code+/// is fresh is [`generated_code`]'s job, so a Rust-only change does not start+/// this one.fn typescript() -> Job {Job::default().needs(vec!["path-check".to_string()]).cond(Expression::new(- "needs.path-check.outputs.should_run == 'true' || needs.path-check.outputs.api_changed == 'true'",+ "needs.path-check.outputs.should_run == 'true'",)).name("Typecheck")- .runs_on(typecheck_runner())+ .runs_on(mid_web_runner()).add_step(checkout("Checkout Repo", false))- .add_step(steps::mount_web_cache_volume(true))+ .add_step(steps::mount_web_cache_volume(false)).add_step(steps::setup_nix()).add_step(steps::setup_reqs_web("Setup Prereqs", false))- .add_step(steps::configure_namespace_sccache_when(- vars::WEB_SCCACHE_NAME,- "needs.path-check.outputs.api_changed == 'true'",- ))- .add_step(generate_api_types())- .add_step(check_specta_types())- .add_step(show_sccache_stats()).add_step(check_dynamic_ui_schema()).add_step(check_types()).add_step(check_collaboration_types())@@ -89,6 +91,56 @@ fn typescript() -> Job {.add_step(steps::teardown_nix())}+/// Everything generated from Rust binaries: the OpenAPI clients in `apps/web`+/// (`gen-api`), the specta wasm wire types, the SDK's specs and generated+/// layer, and the MCP tool reference in `apps/docs`. They used to be checked by+/// three jobs in three workflows, each compiling the workspace again; one job+/// on one target dir builds the binaries once. The SDK regeneration reuses the+/// specs `gen-api` just wrote rather than rerunning it.+fn generated_code() -> Job {+ Job::default()+ .needs(vec!["path-check".to_string()])+ .cond(Expression::new(+ "needs.path-check.outputs.codegen_changed == 'true'",+ ))+ .name("Generated Code Check")+ .runs_on(mid_web_runner())+ // The generators run once and are thrown away, so debug info is pure+ // cost: it made the 15 binaries 12.7 GB instead of 466 MB. Job-level,+ // so every cargo call here (the build step, gen-api, the docs and DCS+ // scripts, the specta recipes) resolves the same profile and reuses+ // one set of artifacts. No `profile.dev.package` overrides exist, so+ // this alone covers dependencies too.+ .add_env(("CARGO_PROFILE_DEV_DEBUG", "0"))+ // Link with mold, as the Rust check and test jobs do. Job-level for+ // the same reason: RUSTFLAGS is part of every unit's fingerprint.+ .add_env(("RUSTFLAGS", "-C link-arg=-fuse-ld=mold"))+ .add_step(checkout("Checkout Repo", false))+ .add_step(steps::mount_web_cache_volume(true))+ .add_step(steps::setup_nix())+ .add_step(steps::setup_reqs_web("Setup Prereqs", false))+ .add_step(steps::configure_namespace_sccache(vars::WEB_SCCACHE_NAME))+ .add_step(build_schema_binaries())+ .add_step(generate_api_types())+ .add_step(regenerate_sdk())+ .add_step(verify_fresh(+ "Verify SDK generated code is fresh",+ "packages/sdk",+ "packages/sdk generated code is stale. Run 'just update-generated' in packages/sdk and commit the result.",+ ))+ .add_step(regenerate_tool_pages())+ .add_step(verify_fresh(+ "Verify docs tool reference is fresh",+ "apps/docs",+ "apps/docs tool reference is stale. Run 'bun run generate:tools' in apps/docs and commit the result.",+ ))+ // Last: its `cargo run -p` builds resolve features per package, so+ // running them earlier would make the steps above recompile.+ .add_step(check_specta_types())+ .add_step(steps::show_sccache_stats())+ .add_step(steps::teardown_nix())+}+fn biome_check() -> Job {gated_web_job("Biome Check").add_step(checkout("Checkout Repo", true))@@ -102,8 +154,9 @@ fn biome_check() -> Job {fn test() -> Job {gated_web_job("Test")+ .runs_on(mid_web_runner()).add_step(checkout("Checkout Repo", false))- .add_step(steps::mount_web_cache_volume(false))+ .add_step(steps::mount_web_test_cache_volume()).add_step(steps::setup_nix()).add_step(steps::setup_reqs_web("Setup", true)).add_step(build_agent_fold_for_tests())@@ -128,7 +181,7 @@ fn build() -> Job {// Match preview/deploy capacity for Vite's chunk-rendering memory peak..runs_on(runners::Runner::Mid.with_cache_tag(vars::WEB_CI_CACHE_TAG)).add_step(checkout("Checkout Repo", false))- .add_step(steps::mount_web_build_cache_volume())+ .add_step(steps::mount_web_build_check_cache_volume()).add_step(steps::setup_nix()).add_step(steps::setup_reqs_web("Setup", false)).add_step(steps::configure_namespace_sccache(vars::WEB_SCCACHE_NAME))@@ -147,6 +200,7 @@ fn status_check() -> Job {.needs(vec!["path-check".to_string(),"typescript".to_string(),+ "generated-code".to_string(),"biome-check".to_string(),"test".to_string(),"cycles".to_string(),@@ -179,12 +233,14 @@ fn checkout(name: &str, full_history: bool) -> Step<Use> {fn paths_filter() -> Step<Use> {let artifact_paths = crate::workflows::web_artifact_paths::yaml_list(" ");- // `api_changed` is only the inputs to `bun run gen-api`. xtask, flake.nix,- // the Nix dev-shell action, and this workflow file do not change OpenAPI- // output, so they must not start Typecheck (which compiles the schema- // binaries). The Nix dev-shell action is also omitted from `should_run`- // so Typecheck (which is `should_run || api_changed`) stays off for a- // shell-only tweak. Workflow YAML drift is `check generated workflows`.+ // `codegen_changed` is the union of what feeds [`generated_code`]: the+ // Rust sources and manifests the schema binaries compile from, the+ // `gen-api` scripts, and the SDK and docs packages that hold generated+ // output. xtask, flake.nix, the Nix dev-shell action, and this workflow+ // file do not change generated output, so they must not start that job.+ // The Nix dev-shell action is also omitted from `should_run` so Typecheck+ // stays off for a shell-only tweak. Workflow YAML drift is `check+ // generated workflows`.Step::new("Filter changed paths").uses("dorny",@@ -195,17 +251,28 @@ fn paths_filter() -> Step<Use> {.add_with(("filters",format!(- "should_run:\n{artifact_paths} - 'services/lexical-service/**'\n - 'crates/ai_tools/src/display_results/schema.generated.json'\n - '.github/actions/setup-reqs-web/**'\napi_changed:\n - 'crates/**/*.rs'\n - 'services/**/*.rs'\n - 'Cargo.toml'\n - 'Cargo.lock'\n - 'apps/web/scripts/generate-api-schema.ts'\n - 'apps/web/scripts/services.ts'\n - '.github/actions/setup-reqs-web/**'\n"+ "should_run:\n{artifact_paths} - 'services/lexical-service/**'\n - 'crates/ai_tools/src/display_results/schema.generated.json'\n - '.github/actions/setup-reqs-web/**'\ncodegen_changed:\n - 'crates/**/*.rs'\n - 'services/**/*.rs'\n - 'Cargo.toml'\n - 'Cargo.lock'\n - 'apps/web/scripts/generate-api-schema.ts'\n - 'apps/web/scripts/services.ts'\n - 'packages/sdk/**'\n - 'apps/docs/**'\n - '.github/actions/setup-reqs-web/**'\n"),))}+/// Build every schema binary the later steps run in one cargo invocation:+/// the OpenAPI bins (`gen-api` owns the list) and `gen_tool_schemas`, which+/// `gen-api`'s DCS tool types and the docs pages both run. Without+/// `--package`, cargo resolves features across the whole workspace, so the+/// scripts' own `cargo build --bin …` calls resolve the same artifacts and+/// finish without compiling.+fn build_schema_binaries() -> Step<Run> {+ Step::new("Build schema binaries").run(indoc::indoc! {r#"+ set -euo pipefail+ read -ra openapi_bins <<< "$(cd apps/web && bun scripts/generate-api-schema.ts --print-cargo-bins)"+ SQLX_OFFLINE=true cargo build "${openapi_bins[@]}" --bin gen_tool_schemas+ "#})+}+fn generate_api_types() -> Step<Run> {Step::new("Generate API Types").run("bun run gen-api -- --check")- .if_condition(Expression::new(- "needs.path-check.outputs.api_changed == 'true'",- )).working_directory(xtask_paths::repo_dir!("apps/web"))}@@ -222,23 +289,46 @@ fn check_specta_types() -> Step<Run> {exit 1fi"#})- .if_condition(Expression::new(- "needs.path-check.outputs.api_changed == 'true'",- )).working_directory(xtask_paths::repo_dir!("apps/web"))}-fn show_sccache_stats() -> Step<Run> {- Step::new("show sccache stats")- .run("sccache --show-stats || true")- .if_condition(Expression::new(- "always() && needs.path-check.outputs.api_changed == 'true'",- ))+/// The tail of `just update-generated` in `packages/sdk`: its first line reruns+/// `gen-api`, whose specs the step before this one already wrote, so only the+/// copy into `specs/` and the client generation run here.+fn regenerate_sdk() -> Step<Run> {+ Step::new("Regenerate SDK code")+ .run(indoc::indoc! {r#"+ bun install --frozen-lockfile+ bun run sync-specs+ bun run generate+ "#})+ .working_directory(xtask_paths::repo_dir!("packages/sdk"))+}++/// Renders `apps/docs/AI/mcp/tools/` from the `gen_tool_schemas` binary, so a+/// tool's schema or description change cannot leave the docs site behind.+fn regenerate_tool_pages() -> Step<Run> {+ Step::new("Regenerate MCP tool pages")+ .run("bun run generate:tools")+ .working_directory(xtask_paths::repo_dir!("apps/docs"))}+fn verify_fresh(name: &str, dir: &str, stale_message: &str) -> Step<Run> {+ Step::new(name).run(format!(+ r#"if [ -n "$(git status --porcelain -- {dir})" ]; then+ echo "{stale_message}"+ git status --porcelain -- {dir}+ git diff -- {dir} | head -200+ exit 1+fi+"#+ ))+}++/// `type-check` runs the native TypeScript 7 compiler.fn check_types() -> Step<Run> {Step::new("Check Types")- .run("bun run --bun --silent tsc --project ./tsconfig.json")+ .run("bun run type-check").working_directory(xtask_paths::repo_dir!("apps/web"))}@@ -286,9 +376,15 @@ fn build_agent_fold_for_tests() -> Step<Run> {.timeout_minutes(15u32)}+/// `bun run test` enables Vitest's filesystem module cache; the cache volume+/// keeps it warm across jobs.fn run_tests() -> Step<Run> {Step::new("Test")- .run("bunx vitest")+ .run("bun run test")+ .add_env((+ "VITEST_MODULE_CACHE_DIR",+ vars::VITEST_MODULE_CACHE_VOLUME_DIR,+ )).working_directory(xtask_paths::repo_dir!("apps/web"))}@@ -315,6 +411,7 @@ fn collaboration_cycles_import_check() -> Step<Run> {fn run_build() -> Step<Run> {Step::new("Build").run("just build-dev")+ .add_env(("WASM_PKG_CACHE_DIR", vars::WASM_PKG_CACHE_DIR)).working_directory(xtask_paths::repo_dir!("apps/web"))}@@ -322,6 +419,7 @@ fn check_job_results() -> Step<Run> {Step::new("Check job results").run(indoc::indoc! {r#"echo "path-check: ${{ needs.path-check.result }}"echo "typescript: ${{ needs.typescript.result }}"+ echo "generated-code: ${{ needs.generated-code.result }}"echo "biome-check: ${{ needs.biome-check.result }}"echo "test: ${{ needs.test.result }}"echo "cycles: ${{ needs.cycles.result }}"@@ -330,6 +428,7 @@ fn check_job_results() -> Step<Run> {# Fail if any job failed (skipped and success are both OK)if [[ "${{ needs.path-check.result }}" == "failure" ]] || \[[ "${{ needs.typescript.result }}" == "failure" ]] || \+ [[ "${{ needs.generated-code.result }}" == "failure" ]] || \[[ "${{ needs.biome-check.result }}" == "failure" ]] || \[[ "${{ needs.test.result }}" == "failure" ]] || \[[ "${{ needs.cycles.result }}" == "failure" ]] || \diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rsindex 51f201764f..4b16e69dff 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rs@@ -1,20 +1,32 @@use super::*;-#[test]-fn nix_dev_shell_does_not_start_typecheck() {- let yaml = web_app_check_main().to_string().expect("workflow yaml");+fn filters(yaml: &str) -> (&str, &str) {let filters = yaml.split("filters: |").nth(1).and_then(|rest| rest.split("id: filter").next()).expect("path-check filters");- let should_run = filters- .split("api_changed:")- .next()- .expect("should_run block");+ filters+ .split_once("codegen_changed:")+ .expect("should_run block, then codegen_changed block")+}++/// A job's rendered YAML, from its id up to the next job's id. Slicing by id+/// keeps the job's `needs:` and `if:`, which render before its `name:`.+fn job<'a>(yaml: &'a str, id: &str, next_id: &str) -> &'a str {+ yaml.split(&format!("\n {id}:\n"))+ .nth(1)+ .and_then(|rest| rest.split(&format!("\n {next_id}:\n")).next())+ .unwrap_or_else(|| panic!("{id} job"))+}++#[test]+fn nix_dev_shell_does_not_start_typecheck() {+ let yaml = web_app_check_main().to_string().expect("workflow yaml");+ let (should_run, _) = filters(&yaml);assert!(!should_run.contains("setup-nix-dev-shell"),- "Typecheck is should_run || api_changed; nix-dev-shell must not be in should_run: {should_run}"+ "Typecheck runs on should_run; nix-dev-shell must not be in it: {should_run}");assert!(should_run.contains("setup-reqs-web"),@@ -25,11 +37,7 @@ fn nix_dev_shell_does_not_start_typecheck() {#[test]fn typecheck_runs_lexical_service_check_and_tests() {let yaml = web_app_check_main().to_string().expect("workflow yaml");- let typescript = yaml- .split("name: Typecheck")- .nth(1)- .and_then(|rest| rest.split("name: Biome Check").next())- .expect("Typecheck job");+ let typescript = job(&yaml, "typescript", "generated-code");assert!(typescript.contains("bun run check"),"Typecheck must type-check lexical-service: {typescript}"@@ -45,20 +53,125 @@ fn typecheck_runs_lexical_service_check_and_tests() {}#[test]-fn typecheck_checks_both_specta_exports_when_rust_changes() {+fn typecheck_compiles_no_rust() {+ let yaml = web_app_check_main().to_string().expect("workflow yaml");+ let typescript = job(&yaml, "typescript", "generated-code");+ for rust in ["gen-api", "cargo", "sccache", "just gen-"] {+ assert!(+ !typescript.contains(rust),+ "generated code is checked in its own job, so Typecheck must not run `{rust}`: {typescript}"+ );+ }+ assert!(+ typescript.contains("should_run == 'true'") && !typescript.contains("codegen_changed"),+ "a Rust-only change does not start Typecheck: {typescript}"+ );+}++/// The OpenAPI clients, specta types, SDK and docs tool pages all come from+/// workspace binaries, so one job builds them once and checks all four.+#[test]+fn generated_code_checks_every_rust_generated_artifact_in_one_job() {let yaml = web_app_check_main().to_string().expect("workflow yaml");- let step = yaml- .split("name: Check Specta Types")+ let generated = job(&yaml, "generated-code", "biome-check");+ // Specta runs last: its `cargo run -p` builds resolve features per+ // package and would otherwise make the docs build recompile.+ let order = [+ "Build schema binaries",+ "bun run gen-api -- --check",+ "bun run sync-specs",+ "git status --porcelain -- packages/sdk",+ "bun run generate:tools",+ "git status --porcelain -- apps/docs",+ "just gen-agent-fold-types",+ "just gen-database-sql-types",+ ];+ let positions: Vec<usize> = order+ .iter()+ .map(|needle| {+ generated+ .find(needle)+ .unwrap_or_else(|| panic!("missing `{needle}`: {generated}"))+ })+ .collect();+ assert!(+ positions.windows(2).all(|pair| pair[0] < pair[1]),+ "the binaries build first, then each artifact is checked: {generated}"+ );+ let sdk = generated+ .split("name: Regenerate SDK code").nth(1).and_then(|rest| rest.split("- name:").next())- .expect("specta types step");+ .expect("SDK regeneration step");+ assert!(+ !sdk.contains("gen-api") && !sdk.contains("update-generated"),+ "the SDK reuses the specs gen-api wrote instead of rerunning it: {sdk}"+ );+ assert!(+ generated.contains("nsc cache sccache setup --cache_name web-ci"),+ "the build uses the shared web-ci remote sccache: {generated}"+ );+}++/// Debug info only slows the throwaway generator build. Set at job level so+/// every cargo call in the job shares the profile and reuses one build.+#[test]+fn generated_code_builds_without_debug_info_for_every_step() {+ let yaml = web_app_check_main().to_string().expect("workflow yaml");+ let generated = job(&yaml, "generated-code", "biome-check");+ let (header, steps) = generated+ .split_once("\n steps:\n")+ .expect("job header, then steps");assert!(- step.contains("just gen-agent-fold-types") && step.contains("just gen-database-sql-types"),- "both wasm crates' TypeScript must be regenerated: {step}"+ header.contains("CARGO_PROFILE_DEV_DEBUG: '0'"),+ "the job env turns debug info off: {header}");assert!(- step.contains("api_changed == 'true'"),- "the exporters compile Rust, so only a Rust change runs them: {step}"+ header.contains("RUSTFLAGS: '-C link-arg=-fuse-ld=mold'"),+ "the job links with mold like the Rust jobs: {header}"+ );+ for flag in ["CARGO_PROFILE_DEV_DEBUG", "RUSTFLAGS", "--config"] {+ assert!(+ !steps.contains(flag),+ "no step may override `{flag}`, or its build stops reusing the others: {steps}"+ );+ }+}++#[test]+fn codegen_filter_covers_every_input_of_the_generated_code_job() {+ let yaml = web_app_check_main().to_string().expect("workflow yaml");+ let (_, codegen) = filters(&yaml);+ for path in [+ "'crates/**/*.rs'",+ "'services/**/*.rs'",+ "'Cargo.toml'",+ "'Cargo.lock'",+ "'apps/web/scripts/generate-api-schema.ts'",+ "'apps/web/scripts/services.ts'",+ "'packages/sdk/**'",+ "'apps/docs/**'",+ ] {+ assert!(+ codegen.contains(path),+ "codegen_changed must include {path}: {codegen}"+ );+ }+ let generated = job(&yaml, "generated-code", "biome-check");+ assert!(generated.contains("needs.path-check.outputs.codegen_changed == 'true'"));+}++#[test]+fn status_check_requires_generated_code() {+ let yaml = web_app_check_main().to_string().expect("workflow yaml");+ let status = yaml+ .split("\n status-check:\n")+ .nth(1)+ .expect("status check job");+ assert!(status.contains("- generated-code"), "{status}");+ assert!(+ status.contains("needs.generated-code.result }}\" == \"failure\""),+ "{status}");}@@ -134,7 +247,7 @@ fn test_job_runs_signup_browser_tests_after_vitest() {let wasm = test_job.find("just build-agent-fold-wasm").expect("build WASM before the test setup hook");- let vitest = test_job.find("bunx vitest").expect("vitest step");+ let vitest = test_job.find("bun run test").expect("vitest step");assert!(wasm < vitest, "cold WASM compilation precedes Vitest");let browser = test_job.find("just test-signup-browser")@@ -148,3 +261,25 @@ fn test_job_runs_signup_browser_tests_after_vitest() {"the Test job installs Playwright's Chromium: {test_job}");}++#[test]+fn test_job_keeps_vitest_module_cache_on_the_cache_volume() {+ let yaml = web_app_check_main().to_string().expect("workflow yaml");+ let start = yaml.find("name: Test\n").expect("Test job");+ let end = yaml[start..]+ .find("name: Cycles Import Check")+ .map_or(yaml.len(), |offset| start + offset);+ let test_job = &yaml[start..end];+ let mounted = format!(" {}\n", vars::VITEST_MODULE_CACHE_VOLUME_DIR);+ assert!(+ test_job.contains(&mounted),+ "the cache volume mounts the Vitest module cache: {test_job}"+ );+ assert!(+ test_job.contains(&format!(+ "VITEST_MODULE_CACHE_DIR: {}",+ vars::VITEST_MODULE_CACHE_VOLUME_DIR+ )),+ "vitest writes its module cache to the mounted path: {test_job}"+ );+}