Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
perf(local): stop just run_local redoing work it already did (#6071)
Wolf Mermelsteinecf903fbe8a9
diff --git a/.gitignore b/.gitignoreindex 42d8260ca1..9c81865a76 100644--- a/.gitignore+++ b/.gitignore@@ -23,6 +23,10 @@ run-local.fish**/.cache**/.vite**/build+# `**/build` is for JS build output. This is a Rust source module that happens+# to be named `build`, and the rule was silently dropping new files added to it+# (a new `test.rs` there is ignored, and `git status` stays clean about it).+!/tooling/xtask/crates/xtask_local/src/local/build/**/*.tsbuildinfo**/*.log**/resultdiff --git a/docs/RUNNING_LOCALLY.md b/docs/RUNNING_LOCALLY.mdindex 59bcca52d5..f16826fc85 100644--- a/docs/RUNNING_LOCALLY.md+++ b/docs/RUNNING_LOCALLY.md@@ -328,18 +328,20 @@ The app is served at `<proxy>/app/`. The bundle resolves its backend from the or### Init Snapshots-`stack up` caches the expensive infrastructure initialization. The first cold run:+`just run_local` and `stack up` both cache the expensive infrastructure initialization. The first cold run:- Migrates the database+- Creates the Kafka topics- Waits for the FusionAuth kickstart- Creates the search indices-It saves these volumes as an init snapshot. The snapshot is content-addressed and stored under `infra/local/generated/.snapshots`. Later runs restore the snapshot and skip the initialization. An input change causes a cache miss and a normal full init.+It saves these volumes as an init snapshot. The snapshot is content-addressed and stored under `infra/local/generated/.snapshots`. Later runs restore the snapshot and skip the initialization. An input change causes a cache miss and a normal full init — the key *is* the definition of clean state, so the full-delete/full-create guarantee is unchanged.Useful commands:```bash-just stack up --no-snapshot # skip the snapshot cache+just run_local --no-snapshot # skip the snapshot cache+just stack up --no-snapshot # same flag, headless```Cursor Cloud bakes the snapshot during environment install. Later `stack up` restores it.@@ -378,4 +380,14 @@ Drop, recreate, and migrate an instance database:just reset_local --instance agent-a```+### Finding out where a bring-up spent its time++Every run prints its slowest stages before the summary. To compare runs, point+`MACRO_LOCAL_TIMINGS` at a file — each run appends one JSON line of every stage+and its duration:++```bash+MACRO_LOCAL_TIMINGS=/tmp/run-local-timings.jsonl just run_local+```+For the default instance, omit `--instance`.diff --git a/services/authentication_service/Cargo.toml b/services/authentication_service/Cargo.tomlindex c4c27c97e0..2d88b1b60d 100644--- a/services/authentication_service/Cargo.toml+++ b/services/authentication_service/Cargo.toml@@ -24,6 +24,9 @@ path = "src/doppler_config.rs"[features]default = ["rate_limit"]local = []+# Local opt-out for the throttles `rate_limit` installs — additive, so local+# builds stay in the one unified `cargo` invocation. See `RATE_LIMIT_ENABLED`.+no_rate_limit = []rate_limit = []return_passwordless_code = []diff --git a/services/authentication_service/src/api/middleware/rate_limit/login_code.rs b/services/authentication_service/src/api/middleware/rate_limit/login_code.rsindex 7e4b0f0fc6..6bc13c69b3 100644--- a/services/authentication_service/src/api/middleware/rate_limit/login_code.rs+++ b/services/authentication_service/src/api/middleware/rate_limit/login_code.rs@@ -19,7 +19,7 @@ pub(in crate::api) async fn handler(req: Request,next: Next,) -> Result<Response, Response> {- if cfg!(not(feature = "rate_limit")) {+ if !super::RATE_LIMIT_ENABLED {tracing::trace!("rate limit disabled");return Ok(next.run(req).await);}diff --git a/services/authentication_service/src/api/middleware/rate_limit/mobile_welcome_email.rs b/services/authentication_service/src/api/middleware/rate_limit/mobile_welcome_email.rsindex 9979918d5b..3a461df3b4 100644--- a/services/authentication_service/src/api/middleware/rate_limit/mobile_welcome_email.rs+++ b/services/authentication_service/src/api/middleware/rate_limit/mobile_welcome_email.rs@@ -17,7 +17,7 @@ pub(in crate::api) async fn handler(req: Request,next: Next,) -> Result<Response, Response> {- if cfg!(not(feature = "rate_limit")) {+ if !super::RATE_LIMIT_ENABLED {tracing::trace!("rate limit disabled");return Ok(next.run(req).await);}diff --git a/services/authentication_service/src/api/middleware/rate_limit/mod.rs b/services/authentication_service/src/api/middleware/rate_limit/mod.rsindex c104920045..1c0722b048 100644--- a/services/authentication_service/src/api/middleware/rate_limit/mod.rs+++ b/services/authentication_service/src/api/middleware/rate_limit/mod.rs@@ -1,3 +1,16 @@pub mod login_code;pub mod mobile_welcome_email;pub mod passwordless;++/// Whether the login-code throttles apply to this build.+///+/// On by default via the `rate_limit` feature. `no_rate_limit` is the local+/// opt-out: `just run_local` needs it off (a 1-request/minute login code makes+/// local dev painful), and expressing that as an *additive* feature is what+/// lets every local service binary build in one `cargo` invocation. Turning it+/// off by dropping a default feature instead would need a package-scoped+/// `--no-default-features` build, whose different feature resolution+/// invalidates the shared dependency artifacts of every other binary — a+/// measured ~2 minutes of rebuild on every `run_local`.+pub const RATE_LIMIT_ENABLED: bool =+ cfg!(feature = "rate_limit") && !cfg!(feature = "no_rate_limit");diff --git a/services/authentication_service/src/api/middleware/rate_limit/passwordless.rs b/services/authentication_service/src/api/middleware/rate_limit/passwordless.rsindex bb739b9852..2ee408ba25 100644--- a/services/authentication_service/src/api/middleware/rate_limit/passwordless.rs+++ b/services/authentication_service/src/api/middleware/rate_limit/passwordless.rs@@ -25,7 +25,7 @@ pub(in crate::api) async fn handler(req: Request,next: Next,) -> Result<Response, Response> {- if cfg!(not(feature = "rate_limit")) {+ if !super::RATE_LIMIT_ENABLED {tracing::trace!("rate limit disabled");return Ok(next.run(req).await);}diff --git a/tooling/just/xtask.just b/tooling/just/xtask.justindex 23fbde3a93..8569190825 100644--- a/tooling/just/xtask.just+++ b/tooling/just/xtask.just@@ -10,6 +10,7 @@ xtask := "cargo run --quiet --manifest-path Cargo.toml -p xtask_local --features# just run_local --instance agent-a # isolated, concurrent instance# just run_local --no-doppler --env-file ./local.env --no-frontend --no-build# just run_local --enable-onboarding # opt into the existing onboarding v4 flag+# just run_local --no-snapshot # re-run the full DB/kickstart/index initrun_local *ARGS:{{ xtask }} run-local "$@"diff --git a/tooling/xtask/crates/xtask_local/src/local.rs b/tooling/xtask/crates/xtask_local/src/local.rsindex bbce9e81a7..a9e45a2926 100644--- a/tooling/xtask/crates/xtask_local/src/local.rs+++ b/tooling/xtask/crates/xtask_local/src/local.rs@@ -311,11 +311,21 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {}let fe_build = (share_app && !stage.is_dry_run()).then(|| {let instance = instance.clone();- std::thread::spawn(move || {- frontend::build_static(&Stage::from_env().quiet(), &instance, mode)- })+ let stage = stage.background();+ std::thread::spawn(move || frontend::build_static(&stage, &instance, mode))});+ // The init snapshot decision, exactly as `stack up` makes it (see+ // `stack::up`): hash the init-defining inputs — possible only after+ // `prepare` wrote the kickstart — and check for a stored snapshot. A hit+ // skips migrate + Kafka topics + the ~1min FusionAuth kickstart + index+ // creation; a miss runs the real init and saves one, which is how the cache+ // seeds itself. The full-delete/full-create idempotency is unchanged: the+ // key *is* the definition of clean, so any input change is a structural miss.+ let snapshot_plan = (!args.no_snapshot && !stage.is_dry_run())+ .then(|| snapshot::Plan::compute(&instance))+ .transpose()?;+// Join the background teardown before we (re)create volumes + bring infra up,// surfaced as a live spinner so it's clear what we're blocked on. It// overlapped the build above, so this is just whatever's left of stopping the@@ -337,7 +347,17 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {// start. The teardown means everything is freshly created each run, so// otherwise the services race their backends on startup (no `macrodb`,// DynamoDB/OpenSearch connection refused).- bring_up_infra(&stage, mode, &instance, &env, InfraInit::Full)?;+ match &snapshot_plan {+ Some(plan) if plan.exists() => {+ snapshot::restore(&stage, &instance, plan)?;+ bring_up_infra(&stage, mode, &instance, &env, InfraInit::FromSnapshot)?;+ }+ Some(plan) => {+ bring_up_infra(&stage, mode, &instance, &env, InfraInit::Full)?;+ snapshot::save(&stage, &instance, &env, plan)?;+ }+ None => bring_up_infra(&stage, mode, &instance, &env, InfraInit::Full)?,+ }if let Some(handle) = fe_build {stage.run_step("Building frontend (static bundle)", move || {handle@@ -386,6 +406,7 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {let shared_app_url = app_tunnel.as_ref().map(|tunnel| format!("{}/app/", tunnel.url));+ stage.print_timings("bring-up");summary::print(mode,&instance,@@ -1164,7 +1185,7 @@ fn ensure_external_resources(stage: &Stage, instance: &Instance) -> Result<()> {fn wait_http(stage: &Stage, label: &str, url: &str) -> Result<()> {let script = format!(- "for i in $(seq 1 60); do curl -fsS --max-time 3 {url} >/dev/null 2>&1 && exit 0; sleep 2; done; echo 'not ready: {url}'; exit 1"+ "for i in $(seq 1 600); do curl -fsS --max-time 3 {url} >/dev/null 2>&1 && exit 0; sleep 0.2; done; echo 'not ready: {url}'; exit 1");let mut cmd = Command::new("bash");cmd.arg("-lc").arg(script);diff --git a/tooling/xtask/crates/xtask_local/src/local/build/mod.rs b/tooling/xtask/crates/xtask_local/src/local/build.rssimilarity index 100%rename from tooling/xtask/crates/xtask_local/src/local/build/mod.rsrename to tooling/xtask/crates/xtask_local/src/local/build.rsdiff --git a/tooling/xtask/crates/xtask_local/src/local/build/runtime_image.rs b/tooling/xtask/crates/xtask_local/src/local/build/runtime_image.rsindex 9cf80e2279..ad1bca56e1 100644--- a/tooling/xtask/crates/xtask_local/src/local/build/runtime_image.rs+++ b/tooling/xtask/crates/xtask_local/src/local/build/runtime_image.rs@@ -6,30 +6,55 @@use std::process::Command;-use anyhow::Result;+use anyhow::{Context, Result};+use sha2::{Digest, Sha256};use super::super::arch::Target;use super::super::{stage::Stage, workspace_root};+#[cfg(test)]+mod test;+/// The shared runtime image tag. Not per-instance: the image has no/// instance-specific content.pub const RUNTIME_IMAGE_TAG: &str = "macro-local-runtime:dev";const DOCKERFILE_REL: &str = "docker/Dockerfile.runtime";+/// Image label carrying the content key of the inputs the image was built+/// from. A matching label is what lets a run skip the build entirely.+const KEY_LABEL: &str = "com.macro.local-runtime-key";+/// Reconcile the runtime image for the host arch through BuildKit.///-/// BuildKit's content cache makes an unchanged build cheap while still-/// invalidating the image when the Dockerfile or build context changes.-/// Existence alone is not a valid cache key because the tag survives branch-/// checkouts. `force` disables BuildKit's layer cache.+/// BuildKit's content cache makes an *unchanged* rebuild cheap in the sense+/// that no layer re-executes — but `--load` still re-exports and unpacks the+/// image every time, which measured 32s warm and 2m29s cold on a 12-core+/// Linux host. So the fast path here is not building at all: key the image on+/// its inputs, stamp that key on as a label, and skip when the loaded image+/// already carries it. Existence alone was never a valid key (the tag survives+/// branch checkouts); a content key is, and it invalidates exactly when the+/// Dockerfile or the platform changes. `force` skips the check and disables+/// BuildKit's layer cache.pub fn ensure_runtime_image(stage: &Stage, target: Target, force: bool) -> Result<()> {let ws = workspace_root();+ let key = content_key(target)?;+ if !force && current_key() == Some(key.clone()) {+ stage.note(&format!(+ "runtime image {RUNTIME_IMAGE_TAG} is up to date ({})",+ &key[..12]+ ));+ return Ok(());+ }let mut cmd = Command::new("docker");cmd.current_dir(&ws).args(["buildx", "build", "--platform", target.docker_platform]).args(["-f", DOCKERFILE_REL]).args(["-t", RUNTIME_IMAGE_TAG])+ .args(["--label", &format!("{KEY_LABEL}={key}")])+ // Nothing consumes an attestation for a local dev image, and both+ // turn the `--load` into a manifest-list export.+ .args(["--provenance=false", "--sbom=false"]).arg("--load");if force {cmd.arg("--no-cache");@@ -40,3 +65,45 @@ pub fn ensure_runtime_image(stage: &Stage, target: Target, force: bool) -> Resul&mut cmd,)}++/// Hash everything that decides the image's content: the Dockerfile and the+/// platform it is built for. The build context is the repository root, but the+/// Dockerfile copies nothing from it, so the context is not an input.+fn content_key(target: Target) -> Result<String> {+ let dockerfile = workspace_root().join(DOCKERFILE_REL);+ let mut hasher = Sha256::new();+ hasher.update(target.docker_platform.as_bytes());+ hasher.update(+ std::fs::read(&dockerfile).with_context(|| format!("reading {}", dockerfile.display()))?,+ );+ Ok(hex(&hasher.finalize()))+}++/// The key stamped on the currently loaded image, if any. Any failure (no+/// image, no daemon, no label) reads as "not current" and falls through to a+/// build, which then reports the real error.+fn current_key() -> Option<String> {+ let out = Command::new("docker")+ .args([+ "image",+ "inspect",+ RUNTIME_IMAGE_TAG,+ "--format",+ &format!("{{{{index .Config.Labels \"{KEY_LABEL}\"}}}}"),+ ])+ .output()+ .ok()?;+ if !out.status.success() {+ return None;+ }+ let key = String::from_utf8_lossy(&out.stdout).trim().to_string();+ (!key.is_empty() && key != "<no value>").then_some(key)+}++fn hex(bytes: &[u8]) -> String {+ bytes.iter().fold(String::new(), |mut s, b| {+ use std::fmt::Write;+ let _ = write!(s, "{b:02x}");+ s+ })+}diff --git a/tooling/xtask/crates/xtask_local/src/local/build/runtime_image/test.rs b/tooling/xtask/crates/xtask_local/src/local/build/runtime_image/test.rsnew file mode 100644index 0000000000..9262fe2050--- /dev/null+++ b/tooling/xtask/crates/xtask_local/src/local/build/runtime_image/test.rs@@ -0,0 +1,31 @@+use super::*;++fn target(arch: &'static str) -> Target {+ match arch {+ "x86_64" => Target {+ triple: "x86_64-unknown-linux-gnu",+ docker_platform: "linux/amd64",+ },+ _ => Target {+ triple: "aarch64-unknown-linux-gnu",+ docker_platform: "linux/arm64",+ },+ }+}++#[test]+fn content_key_is_deterministic() {+ let key = content_key(target("x86_64")).expect("hashes the tracked Dockerfile");+ assert_eq!(key, content_key(target("x86_64")).unwrap());+ assert_eq!(key.len(), 64);+}++/// Volumes and binaries are arch-specific, so an image built for one platform+/// must never satisfy the other's key.+#[test]+fn content_key_is_platform_specific() {+ assert_ne!(+ content_key(target("x86_64")).unwrap(),+ content_key(target("aarch64")).unwrap()+ );+}diff --git a/tooling/xtask/crates/xtask_local/src/local/build/zigbuild.rs b/tooling/xtask/crates/xtask_local/src/local/build/zigbuild.rsindex 3e6bbb4680..d4c52777a6 100644--- a/tooling/xtask/crates/xtask_local/src/local/build/zigbuild.rs+++ b/tooling/xtask/crates/xtask_local/src/local/build/zigbuild.rs@@ -5,20 +5,31 @@//! `AWS_LC_SYS_CMAKE_BUILDER=1` (aws-lc-sys's cc-builder rejects the zig C//! compiler), `SQLX_OFFLINE`, and writable zig cache dirs.-use std::path::Path;+use std::path::{Path, PathBuf};use std::process::Command;use anyhow::{Context, Result, bail};use super::super::arch::Target;-use super::super::inventory::RUST_SERVICES;+use super::super::inventory::{RUST_SERVICES, RustService};use super::super::{stage::Stage, workspace_root};+#[cfg(test)]+mod test;+/// Build every non-opt-in service binary (debug profile) for `target`.///-/// Most bins build together in one invocation; bins that need-/// `--no-default-features` (e.g. `authentication_service`, to drop `rate_limit`)-/// build in their own invocation since `--no-default-features` is package-wide.+/// One unified invocation builds as many bins as possible, with each service's+/// extra features passed package-qualified (`<package>/<feature>`) so they+/// compose instead of forcing a separate build. Services that must drop a+/// *default* feature still need their own package-scoped invocation, and that+/// is expensive in a way that is invisible at the call site: a package-scoped+/// build resolves features over a different package selection, so it+/// invalidates shared dependency artifacts the unified build produced, and the+/// *next* run pays to rebuild them. Measured on a 12-core Linux host: with the+/// extra invocations, a `run_local` with zero source changes rebuilt 230 units+/// in 2m06s; unified, the same no-op build is 16s. Keep+/// [`RustService::no_default_features`] as close to empty as the features allow.// xtask is host tooling; reading CARGO_BUILD_JOBS from the process env is correct.#[allow(clippy::disallowed_methods)]pub fn run(stage: &Stage, target: Target) -> Result<()> {@@ -36,45 +47,99 @@ pub fn run(stage: &Stage, target: Target) -> Result<()> {.to_string()});- // Group 1: default-feature bins, all in one invocation.- let default_bins: Vec<&str> = RUST_SERVICES- .iter()- .filter(|s| !s.is_opt_in() && !s.no_default_features)- .map(|s| s.cargo_bin)- .collect();let mut cmd = base_command(&ws, &zig_cache, &jobs, target);- for bin in &default_bins {- cmd.arg("--bin").arg(bin);- }+ cmd.args(unified_args());stage.run(&format!("Building service binaries ({})", target.triple),&mut cmd,)?;- // Group 2: each --no-default-features bin in its own package-scoped invocation.- for svc in RUST_SERVICES- .iter()- .filter(|s| !s.is_opt_in() && s.no_default_features)- {+ // Each --no-default-features bin in its own package-scoped invocation, and+ // in its own target dir: that is what keeps its different feature+ // resolution from invalidating the unified build's shared artifacts (and+ // vice versa). Each dir then only ever sees one resolution, so both stay+ // incrementally fresh. Costs one cold build per dir, once.+ for svc in isolated_services() {+ let target_dir = isolated_target_dir(&ws, svc);let mut cmd = base_command(&ws, &zig_cache, &jobs, target);- cmd.arg("-p")- .arg(svc.package)- .arg("--no-default-features")- .arg("--bin")- .arg(svc.cargo_bin);- let features = svc.build_features();- if !features.is_empty() {- cmd.arg("--features").arg(features.join(","));- }+ cmd.args(isolated_args(svc, &target_dir));stage.run(&format!("Building {} (no default features)", svc.cargo_bin),&mut cmd,)?;+ // The stack mounts one directory at `/app/out`, so land the binary+ // beside the unified ones.+ let built = target_dir+ .join(target.triple)+ .join("debug")+ .join(svc.cargo_bin);+ let dest = ws.join(target.debug_dir()).join(svc.cargo_bin);+ std::fs::copy(&built, &dest)+ .with_context(|| format!("copying {} to {}", built.display(), dest.display()))?;}Ok(())}+/// Where a `--no-default-features` service builds. Under `target/` so it is+/// covered by the same ignore rules and cleaned by `cargo clean`'s neighbors.+fn isolated_target_dir(ws: &Path, svc: &RustService) -> PathBuf {+ ws.join("target/local-isolated").join(svc.cargo_bin)+}++/// The services built together in the unified invocation.+fn unified_services() -> impl Iterator<Item = &'static RustService> {+ RUST_SERVICES+ .iter()+ .filter(|s| !s.is_opt_in() && !s.no_default_features)+}++/// The services that need their own package-scoped, own-target-dir build.+fn isolated_services() -> impl Iterator<Item = &'static RustService> {+ RUST_SERVICES+ .iter()+ .filter(|s| !s.is_opt_in() && s.no_default_features)+}++/// `--bin` per service plus one `--features` carrying every service's local+/// features, package-qualified so a feature only reaches the package that+/// declares it.+fn unified_args() -> Vec<String> {+ let mut args: Vec<String> = unified_services()+ .flat_map(|svc| ["--bin".to_owned(), svc.cargo_bin.to_owned()])+ .collect();+ let features: Vec<String> = unified_services()+ .flat_map(|svc| {+ svc.local_features()+ .iter()+ .map(|f| format!("{}/{f}", svc.package))+ })+ .collect();+ if !features.is_empty() {+ args.push("--features".to_owned());+ args.push(features.join(","));+ }+ args+}++fn isolated_args(svc: &RustService, target_dir: &Path) -> Vec<String> {+ let mut args = vec![+ "--target-dir".to_owned(),+ target_dir.display().to_string(),+ "-p".to_owned(),+ svc.package.to_owned(),+ "--no-default-features".to_owned(),+ "--bin".to_owned(),+ svc.cargo_bin.to_owned(),+ ];+ let features = svc.local_features();+ if !features.is_empty() {+ args.push("--features".to_owned());+ args.push(features.join(","));+ }+ args+}+/// A `cargo zigbuild` command with the shared env (mirrors flake.nix/// `lambdaCommonArgs`).fn base_command(ws: &Path, zig_cache: &Path, jobs: &str, target: Target) -> Command {@@ -98,37 +163,15 @@ fn base_command(ws: &Path, zig_cache: &Path, jobs: &str, target: Target) -> Comm.env("ZIG_LOCAL_CACHE_DIR", zig_cache).env("RUSTC_WRAPPER", "sccache").env("XDG_CACHE_HOME", zig_cache);- if let Some(flag) = curl_include_flag() {- // First-found wins in cc's env chain (CFLAGS_<target> before CFLAGS),- // so set the target-specific var too in case something else exports a- // plain CFLAGS that would otherwise shadow ours — and vice versa.- let var = format!("CFLAGS_{}", target.triple.replace('-', "_"));- cmd.env(var, &flag).env("CFLAGS", &flag);- }+ // No CFLAGS here. The dev shell already exports target-scoped+ // `CFLAGS_<triple>`/`CXXFLAGS_<triple>` pointing at curl.dev for exactly+ // this cross build (see `nix/cloud-storage.nix`). Setting our own spelling+ // of the same include path only differed textually — and cargo fingerprints+ // these env vars, so the two spellings alternating rebuilt aws-lc-sys and+ // everything downstream.cmd}-/// librdkafka 2.12's bundled cmake build needs curl *headers* even with-/// `WITH_CURL=0` (its config header always defines `WITH_OAUTHBEARER_OIDC` via-/// `#cmakedefine01`, and rdkafka_conf.c guards the curl include with `#ifdef`-/// instead of `#if`). Host builds get them from the Nix cc-wrapper, which-/// reads `NIX_CFLAGS_COMPILE` — but `zig cc` is a raw clang that ignores it,-/// so the dev shell's `curl.dev` never reaches the cross cmake build. Mine-/// the wrapper flags for an include dir that actually carries `curl/curl.h`-/// and hand it to cc/cmake explicitly; `-idirafter` keeps it at the lowest-/// search priority so it cannot shadow zig's own libc headers.-// Reading the ambient Nix shell env is the point here; there is no config-// layer this could come from.-#[allow(clippy::disallowed_methods)]-fn curl_include_flag() -> Option<String> {- let flags = std::env::var("NIX_CFLAGS_COMPILE").ok()?;- flags- .split_whitespace()- .filter(|tok| tok.starts_with('/'))- .find(|dir| Path::new(dir).join("curl/curl.h").is_file())- .map(|dir| format!("-idirafter {dir}"))-}-/// Fail with an actionable hint if the rust-std for `target` is not installed./// The repo's `rustup` is a no-op shim, so the only fix is adding the triple to/// `rust-toolchain.toml` and re-entering `nix develop`.diff --git a/tooling/xtask/crates/xtask_local/src/local/build/zigbuild/test.rs b/tooling/xtask/crates/xtask_local/src/local/build/zigbuild/test.rsnew file mode 100644index 0000000000..1753c35ae5--- /dev/null+++ b/tooling/xtask/crates/xtask_local/src/local/build/zigbuild/test.rs@@ -0,0 +1,70 @@+use super::*;++/// The whole point of the unified invocation: every service that *can* build+/// together does. A service silently gaining `no_default_features` costs a+/// package-scoped build and, without its own target dir, a full minute of+/// rebuild on the next `run_local`.+#[test]+fn only_services_that_drop_default_features_build_separately() {+ let isolated: Vec<&str> = isolated_services().map(|s| s.cargo_bin).collect();+ assert_eq!(isolated, vec!["search_processing_service"]);+ assert!(unified_services().any(|s| s.cargo_bin == "authentication_service"));+}++#[test]+fn unified_args_pass_local_features_package_qualified() {+ let args = unified_args();+ let features = args+ .iter()+ .position(|a| a == "--features")+ .map(|i| args[i + 1].clone())+ .expect("unified build passes features");+ for feature in features.split(',') {+ assert!(+ feature.contains('/'),+ "{feature} must be package-qualified or it applies to the wrong package"+ );+ }+ assert!(features.contains("authentication_service/no_rate_limit"));+ assert!(features.contains("authentication_service/return_passwordless_code"));+}++#[test]+fn unified_args_build_every_unified_service_bin() {+ let args = unified_args();+ for svc in unified_services() {+ assert!(+ args.windows(2)+ .any(|w| w[0] == "--bin" && w[1] == svc.cargo_bin),+ "{} is missing from the unified build",+ svc.cargo_bin+ );+ }+}++/// Isolation is the load-bearing part: sharing a target dir with the unified+/// build is exactly what invalidates it.+#[test]+fn isolated_args_use_their_own_target_dir() {+ let svc = isolated_services().next().expect("one isolated service");+ let dir = isolated_target_dir(Path::new("/ws"), svc);+ let args = isolated_args(svc, &dir);+ let target_dir = args+ .iter()+ .position(|a| a == "--target-dir")+ .map(|i| args[i + 1].clone())+ .expect("isolated build sets --target-dir");+ assert_eq!(+ target_dir,+ "/ws/target/local-isolated/search_processing_service"+ );+ assert!(args.iter().any(|a| a == "--no-default-features"));+}++#[test]+fn isolated_target_dirs_do_not_collide() {+ let dirs: std::collections::BTreeSet<PathBuf> = isolated_services()+ .map(|svc| isolated_target_dir(Path::new("/ws"), svc))+ .collect();+ assert_eq!(dirs.len(), isolated_services().count());+}diff --git a/tooling/xtask/crates/xtask_local/src/local/cli.rs b/tooling/xtask/crates/xtask_local/src/local/cli.rsindex a3d767a3c7..67b223f047 100644--- a/tooling/xtask/crates/xtask_local/src/local/cli.rs+++ b/tooling/xtask/crates/xtask_local/src/local/cli.rs@@ -121,6 +121,10 @@ pub struct RunArgs {/// Stream subprocess output and show per-step timings.#[arg(long, short)]pub verbose: bool,+ /// Neither restore from nor save an init snapshot — always run the full+ /// DB migrate / FusionAuth kickstart / index creation.+ #[arg(long)]+ pub no_snapshot: bool,/// Which local OTLP collector to start and wire services to. Defaults to/// the LGTM stack (Grafana/Tempo/Loki); pass `off` to run without tracing/// — see `docker/docker-compose.yml` for what each backend does.diff --git a/tooling/xtask/crates/xtask_local/src/local/cli/test.rs b/tooling/xtask/crates/xtask_local/src/local/cli/test.rsindex 0cfd543c09..147c9d4bee 100644--- a/tooling/xtask/crates/xtask_local/src/local/cli/test.rs+++ b/tooling/xtask/crates/xtask_local/src/local/cli/test.rs@@ -90,6 +90,21 @@ fn stack_has_no_snapshot_verb() {}}+#[test]+fn run_local_and_stack_up_accept_no_snapshot() {+ let cli = Cli::try_parse_from(["cargo-x", "run-local", "--no-snapshot"]).unwrap();+ let Cmd::RunLocal(args) = cli.command else {+ panic!("expected run-local command");+ };+ assert!(args.no_snapshot);++ let cli = Cli::try_parse_from(["cargo-x", "stack", "up", "--no-snapshot"]).unwrap();+ let Cmd::Stack(StackCmd::Up(args)) = cli.command else {+ panic!("expected stack up");+ };+ assert!(args.run.no_snapshot);+}+#[test]fn seed_scenario_accepts_instance_and_trailing_scenario_arguments() {let cli = Cli::try_parse_from([diff --git a/tooling/xtask/crates/xtask_local/src/local/e2e.rs b/tooling/xtask/crates/xtask_local/src/local/e2e.rsindex a0ddf455a4..b379d7a148 100644--- a/tooling/xtask/crates/xtask_local/src/local/e2e.rs+++ b/tooling/xtask/crates/xtask_local/src/local/e2e.rs@@ -128,6 +128,7 @@ pub fn run(args: &LocalE2eArgs) -> Result<()> {traces: TracesBackend::Off,with_chrome: false,verbose: false,+ no_snapshot: false,with_cf_tunnel: false,},..stack::UpArgs::default()diff --git a/tooling/xtask/crates/xtask_local/src/local/frontend.rs b/tooling/xtask/crates/xtask_local/src/local/frontend.rsindex 9d5f5f1845..a3c56aa68f 100644--- a/tooling/xtask/crates/xtask_local/src/local/frontend.rs+++ b/tooling/xtask/crates/xtask_local/src/local/frontend.rs@@ -154,7 +154,7 @@ fn dev_env(pub fn wait_backend_ready(stage: &Stage, instance: &Instance) -> Result<()> {let url = format!("{}/auth/health", proxy::url(instance));let script = format!(- "for i in $(seq 1 60); do curl -fsS --max-time 3 {url} >/dev/null 2>&1 && exit 0; sleep 2; done; echo 'backend not ready'; exit 1"+ "for i in $(seq 1 600); do curl -fsS --max-time 3 {url} >/dev/null 2>&1 && exit 0; sleep 0.2; done; echo 'backend not ready'; exit 1");let mut cmd = Command::new("bash");cmd.arg("-lc").arg(script);diff --git a/tooling/xtask/crates/xtask_local/src/local/fusionauth.rs b/tooling/xtask/crates/xtask_local/src/local/fusionauth.rsindex 5b190856e6..18bfc24d7e 100644--- a/tooling/xtask/crates/xtask_local/src/local/fusionauth.rs+++ b/tooling/xtask/crates/xtask_local/src/local/fusionauth.rs@@ -77,7 +77,7 @@ pub fn wait_ready(stage: &Stage, instance: &Instance) -> Result<()> {// maintenance-mode 302 (e.g. after a boot-time DB connect failure) would// otherwise pass as ready and every later login would 500.let script = format!(- "for i in $(seq 1 120); do [ \"$(curl -sS -o /dev/null -w '%{{http_code}}' --max-time 3 -H 'Authorization: {key}' {url} 2>/dev/null)\" = 200 ] && exit 0; sleep 2; done; \+ "for i in $(seq 1 480); do [ \"$(curl -sS -o /dev/null -w '%{{http_code}}' --max-time 3 -H 'Authorization: {key}' {url} 2>/dev/null)\" = 200 ] && exit 0; sleep 0.5; done; \echo 'timed out waiting for the FusionAuth kickstart (a 302 here means maintenance mode: FusionAuth could not reach its db)'; exit 1",key = identity::FUSIONAUTH_API_KEY,);diff --git a/tooling/xtask/crates/xtask_local/src/local/inventory.rs b/tooling/xtask/crates/xtask_local/src/local/inventory.rsindex fd50875393..082e67df92 100644--- a/tooling/xtask/crates/xtask_local/src/local/inventory.rs+++ b/tooling/xtask/crates/xtask_local/src/local/inventory.rs@@ -41,10 +41,20 @@ pub struct RustService {/// Opt-in services (e.g. `search_processing_service`) only start with an/// explicit profile and are never converted to the runtime image.pub opt_in: bool,- /// Built locally with `--no-default-features` (plus any [`Self::build_features`]).- /// `authentication_service` drops the default `rate_limit` feature (removing- /// the 1-request/minute login-code throttle that makes local dev painful);- /// `search_processing_service` drops the default amd64-only `pdf` feature.+ /// Built locally in its own `cargo` invocation with `--no-default-features`+ /// (plus [`Self::local_features`]), because the feature it must drop is a+ /// *default* one and cargo can only drop defaults per-package.+ ///+ /// Keep this at zero services if you possibly can. A package-scoped+ /// invocation resolves features over a different package selection than the+ /// unified build, which invalidates the shared dependency artifacts the+ /// unified build just produced — measured at 230 units and ~2 minutes of+ /// rebuild on the *next* `run_local`, forever. Prefer an additive opt-out+ /// feature (see `authentication_service`'s `no_rate_limit`), which composes+ /// into the one unified invocation.+ ///+ /// `search_processing_service` genuinely cannot: its `pdf` default feature+ /// pulls an optional dependency, and no additive feature can un-pull one.pub no_default_features: bool,}@@ -60,15 +70,19 @@ impl RustService {self.opt_in}- /// Cargo features to re-enable alongside `--no-default-features` for this- /// service's local build (empty = none). Only `search_processing_service`- /// needs this: `authentication_service` opts into local passwordless code- /// responses, and dropping `search_processing_service` default features- /// removes the amd64-only `pdf` feature, but its bin still requires- /// `processing`/`service`.- pub fn build_features(&self) -> &'static [&'static str] {+ /// Cargo features this service's local build enables (empty = none).+ ///+ /// For a unified-build service these are passed package-qualified+ /// (`<package>/<feature>`) so they compose with every other binary in the+ /// same invocation. For a [`Self::no_default_features`] service they are+ /// the features re-enabled alongside the dropped defaults.+ ///+ /// `authentication_service` opts into local passwordless code responses and+ /// out of the login-code throttles; `search_processing_service` loses+ /// `processing`/`service` with its defaults and its bin requires them.+ pub fn local_features(&self) -> &'static [&'static str] {match self.cargo_bin {- "authentication_service" => &["return_passwordless_code"],+ "authentication_service" => &["return_passwordless_code", "no_rate_limit"],"search_processing_service" => &["processing", "service"],_ => &[],}@@ -86,7 +100,9 @@ pub const RUST_SERVICES: &[RustService] = &[is_websocket: false,modes: &[Mode::Local, Mode::Dev],opt_in: false,- no_default_features: true,+ // Joins the unified invocation: the local behavior it needs+ // (`no_rate_limit`, `return_passwordless_code`) is additive.+ no_default_features: false,},RustService {compose_name: "connection_gateway",diff --git a/tooling/xtask/crates/xtask_local/src/local/stack.rs b/tooling/xtask/crates/xtask_local/src/local/stack.rsindex 50aa3a2b0c..0d17eb05bf 100644--- a/tooling/xtask/crates/xtask_local/src/local/stack.rs+++ b/tooling/xtask/crates/xtask_local/src/local/stack.rs@@ -25,10 +25,6 @@ use super::{Mode, arch, env_layer, frontend, mailpit, proxy, sdk_webhook, snapshpub struct UpArgs {#[command(flatten)]pub run: RunArgs,- /// Neither restore from nor save an init snapshot — always run the full- /// migrate/kickstart/index init.- #[arg(long)]- pub no_snapshot: bool,/// Stop after the infra bring-up + init (and the snapshot save/restore):/// no app services, proxy, or frontend. This is the CI bake mode — the/// app services need the Doppler-sourced env (AWS endpoints, shared@@ -192,16 +188,15 @@ pub fn up(mode: Mode, args: &UpArgs) -> Result<Instance> {}let fe_build = (static_frontend && !stage.is_dry_run()).then(|| {let instance = instance.clone();- std::thread::spawn(move || {- frontend::build_static(&Stage::from_env().quiet(), &instance, mode)- })+ let stage = stage.background();+ std::thread::spawn(move || frontend::build_static(&stage, &instance, mode))});// The init snapshot decision: hash the init-defining inputs (possible only// after `prepare` wrote the kickstart) and check for a stored snapshot.// Restores skip migrate/kickstart/index-init; a cold init saves one for// next time — that's how the cache seeds itself.- let snapshot_plan = (!args.no_snapshot && !stage.is_dry_run())+ let snapshot_plan = (!args.run.no_snapshot && !stage.is_dry_run()).then(|| snapshot::Plan::compute(&instance)).transpose()?;@@ -293,6 +288,7 @@ pub fn up(mode: Mode, args: &UpArgs) -> Result<Instance> {} else {mailpit::direct_ui_url(&instance)};+ stage.print_timings("bring-up");summary::print(mode, &instance, &env, &frontend_url, &mailpit_url, None);stage.note(&format!(" headless: `just stack status`, `just stack update`, `just stack down`{}",@@ -330,11 +326,11 @@ fn bootstrap_from_update(args: &UpdateArgs) -> Result<()> {no_frontend: false,enable_onboarding: false,verbose: args.verbose,+ no_snapshot: false,traces: super::cli::TracesBackend::default(),with_chrome: false,with_cf_tunnel: false,},- no_snapshot: false,infra_only: false,json: args.json,};diff --git a/tooling/xtask/crates/xtask_local/src/local/stage.rs b/tooling/xtask/crates/xtask_local/src/local/stage.rsindex 81b121482d..e35551d1fe 100644--- a/tooling/xtask/crates/xtask_local/src/local/stage.rs+++ b/tooling/xtask/crates/xtask_local/src/local/stage.rs@@ -10,8 +10,9 @@//! cannot deadlock by filling a pipe buffer while we wait. `indicatif` only//! animates — it never reads the child's pipes.-use std::io::{self, Read};+use std::io::{self, Read, Write};use std::process::{Command, Stdio};+use std::sync::{Arc, Mutex};use std::time::{Duration, Instant};use anyhow::{Result, bail};@@ -102,6 +103,10 @@ pub struct Stage {/// notes are suppressed (commands still run + capture). Used to fold a/// multi-step action under a single parent spinner.quiet: bool,+ /// Every resolved stage's label + elapsed, in completion order. Shared with+ /// [`Stage::quiet`] children so folded sub-steps are recorded too, and with+ /// the threads that run the background lanes.+ timings: Arc<Mutex<Vec<(String, Duration)>>>,}impl Stage {@@ -116,6 +121,7 @@ impl Stage {verbose: env_flag("MACRO_LOCAL_VERBOSE"),dry_run: env_flag("MACRO_LOCAL_DRY_RUN"),quiet: false,+ timings: Arc::new(Mutex::new(Vec::new())),}}@@ -152,6 +158,85 @@ impl Stage {verbose: self.verbose,dry_run: self.dry_run,quiet: true,+ timings: Arc::clone(&self.timings),+ }+ }++ /// A handle that records into the same timing table from another thread+ /// (the background teardown / frontend-build lanes). Output stays quiet:+ /// a lane that races the foreground stages cannot share the spinner.+ pub fn background(&self) -> Stage {+ self.quiet()+ }++ fn record(&self, label: &str, elapsed: Duration) {+ if let Ok(mut timings) = self.timings.lock() {+ timings.push((label.to_owned(), elapsed));+ }+ }++ /// Print the slowest stages of the run, and append a JSON line to+ /// `MACRO_LOCAL_TIMINGS` when that env var names a file. This is how a+ /// change to the bring-up is shown to be faster rather than argued to be.+ // xtask is host tooling; reading the process environment directly is correct.+ #[allow(clippy::disallowed_methods)]+ pub fn print_timings(&self, title: &str) {+ let Ok(timings) = self.timings.lock() else {+ return;+ };+ if timings.is_empty() {+ return;+ }+ const SUM_LABEL: &str = "sum of stages (lanes overlap)";+ let total: Duration = timings.iter().map(|(_, d)| *d).sum();+ let mut slowest: Vec<&(String, Duration)> = timings.iter().collect();+ slowest.sort_by_key(|(_, d)| std::cmp::Reverse(*d));+ let shown = &slowest[..slowest.len().min(10)];+ // Labels run past the stage-line column ("Building search_processing…"),+ // so size this table to its own longest row rather than clipping.+ let width = shown+ .iter()+ .map(|(label, _)| label.chars().count())+ .chain([SUM_LABEL.len(), WIDTH])+ .max()+ .unwrap_or(WIDTH);++ self.section(&format!("{title} — slowest stages"));+ for (label, elapsed) in shown {+ println!(+ " {label:<width$} {:>8}",+ Style::new().bold().apply_to(format_elapsed(*elapsed))+ );+ }+ println!(+ " {:<width$} {:>8}",+ Style::new().dim().apply_to(SUM_LABEL),+ Style::new().dim().apply_to(format_elapsed(total)),+ );++ if let Some(path) = std::env::var_os("MACRO_LOCAL_TIMINGS") {+ let entries: Vec<String> = timings+ .iter()+ .map(|(label, d)| {+ format!(+ "{{\"label\":{},\"seconds\":{:.3}}}",+ serde_json::Value::from(label.as_str()),+ d.as_secs_f64()+ )+ })+ .collect();+ let line = format!("{{\"stages\":[{}]}}\n", entries.join(","));+ if let Err(e) = std::fs::OpenOptions::new()+ .create(true)+ .append(true)+ .open(&path)+ .and_then(|mut f| f.write_all(line.as_bytes()))+ {+ self.note(&format!(+ "could not write MACRO_LOCAL_TIMINGS to {}: {e}",+ std::path::Path::new(&path).display()+ ));+ }}}@@ -284,6 +369,7 @@ impl Stage {let mut captured = out_handle.join().unwrap_or_default();captured.extend(err_handle.join().unwrap_or_default());if self.quiet {+ self.record(label, start.elapsed());// Silent on success; on failure bubble the captured output up in the// error so the parent stage can show it after its spinner clears.return match status {@@ -315,11 +401,15 @@ impl Stage {return Ok(());}if self.quiet {- return f();+ let start = Instant::now();+ let result = f();+ self.record(label, start.elapsed());+ return result;}let start = Instant::now();let spinner = self.spinner(label);let result = f();+ self.record(label, start.elapsed());let elapsed = format_elapsed(start.elapsed());match result {Ok(()) => {@@ -359,6 +449,7 @@ impl Stage {program: &str,args: &[String],) -> Result<()> {+ self.record(label, start.elapsed());let elapsed = format_elapsed(start.elapsed());match outcome {Ok((status, _captured)) if status.success() => {diff --git a/tooling/xtask/crates/xtask_local/src/local/stage/test.rs b/tooling/xtask/crates/xtask_local/src/local/stage/test.rsindex 36819d45af..20cf79b073 100644--- a/tooling/xtask/crates/xtask_local/src/local/stage/test.rs+++ b/tooling/xtask/crates/xtask_local/src/local/stage/test.rs@@ -10,3 +10,41 @@ fn spinner_draw_tracks_narrow_terminal_widths() {assert_eq!(spinner_draw_width(40), 40);assert_eq!(spinner_draw_width(0), 1);}++#[test]+fn stages_record_their_elapsed_time() {+ let stage = Stage::from_env();+ stage.run_step("work", || Ok(())).unwrap();+ let timings = stage.timings.lock().unwrap();+ assert_eq!(timings.len(), 1);+ assert_eq!(timings[0].0, "work");+}++/// Sub-steps folded under a parent spinner, and the background lanes, must land+/// in the same table — otherwise the summary omits exactly the slow parts that+/// run off the main thread.+#[test]+fn quiet_and_background_children_record_into_the_parent() {+ let stage = Stage::from_env();+ stage.quiet().run_step("folded", || Ok(())).unwrap();+ let background = stage.background();+ std::thread::spawn(move || background.run_step("lane", || Ok(())))+ .join()+ .unwrap()+ .unwrap();+ let recorded: Vec<String> = stage+ .timings+ .lock()+ .unwrap()+ .iter()+ .map(|(label, _)| label.clone())+ .collect();+ assert_eq!(recorded, vec!["folded", "lane"]);+}++#[test]+fn a_failed_stage_is_still_recorded() {+ let stage = Stage::from_env();+ let _ = stage.run_step("doomed", || anyhow::bail!("no"));+ assert_eq!(stage.timings.lock().unwrap()[0].0, "doomed");+}