Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

perf(local): stop just run_local redoing work it already did (#6071)

Wolf Mermelsteinecf903fbe8a9
diff --git a/.gitignore b/.gitignore
index 42d8260ca1..9c81865a76 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,6 +23,10 @@ run-local.fish
**/.cache
**/.vite
**/build
+# `**/build` is for JS build output. This is a Rust source module that happens
+# to be named `build`, and the rule was silently dropping new files added to it
+# (a new `test.rs` there is ignored, and `git status` stays clean about it).
+!/tooling/xtask/crates/xtask_local/src/local/build/
**/*.tsbuildinfo
**/*.log
**/result
diff --git a/docs/RUNNING_LOCALLY.md b/docs/RUNNING_LOCALLY.md
index 59bcca52d5..f16826fc85 100644
--- a/docs/RUNNING_LOCALLY.md
+++ b/docs/RUNNING_LOCALLY.md
@@ -328,18 +328,20 @@ The app is served at `<proxy>/app/`. The bundle resolves its backend from the or
### Init Snapshots
-`stack up` caches the expensive infrastructure initialization. The first cold run:
+`just run_local` and `stack up` both cache the expensive infrastructure initialization. The first cold run:
- Migrates the database
+- Creates the Kafka topics
- Waits for the FusionAuth kickstart
- Creates the search indices
-It saves these volumes as an init snapshot. The snapshot is content-addressed and stored under `infra/local/generated/.snapshots`. Later runs restore the snapshot and skip the initialization. An input change causes a cache miss and a normal full init.
+It saves these volumes as an init snapshot. The snapshot is content-addressed and stored under `infra/local/generated/.snapshots`. Later runs restore the snapshot and skip the initialization. An input change causes a cache miss and a normal full init — the key *is* the definition of clean state, so the full-delete/full-create guarantee is unchanged.
Useful commands:
```bash
-just stack up --no-snapshot # skip the snapshot cache
+just run_local --no-snapshot # skip the snapshot cache
+just stack up --no-snapshot # same flag, headless
```
Cursor Cloud bakes the snapshot during environment install. Later `stack up` restores it.
@@ -378,4 +380,14 @@ Drop, recreate, and migrate an instance database:
just reset_local --instance agent-a
```
+### Finding out where a bring-up spent its time
+
+Every run prints its slowest stages before the summary. To compare runs, point
+`MACRO_LOCAL_TIMINGS` at a file — each run appends one JSON line of every stage
+and its duration:
+
+```bash
+MACRO_LOCAL_TIMINGS=/tmp/run-local-timings.jsonl just run_local
+```
+
For the default instance, omit `--instance`.
diff --git a/services/authentication_service/Cargo.toml b/services/authentication_service/Cargo.toml
index c4c27c97e0..2d88b1b60d 100644
--- a/services/authentication_service/Cargo.toml
+++ b/services/authentication_service/Cargo.toml
@@ -24,6 +24,9 @@ path = "src/doppler_config.rs"
[features]
default = ["rate_limit"]
local = []
+# Local opt-out for the throttles `rate_limit` installs — additive, so local
+# builds stay in the one unified `cargo` invocation. See `RATE_LIMIT_ENABLED`.
+no_rate_limit = []
rate_limit = []
return_passwordless_code = []
diff --git a/services/authentication_service/src/api/middleware/rate_limit/login_code.rs b/services/authentication_service/src/api/middleware/rate_limit/login_code.rs
index 7e4b0f0fc6..6bc13c69b3 100644
--- a/services/authentication_service/src/api/middleware/rate_limit/login_code.rs
+++ b/services/authentication_service/src/api/middleware/rate_limit/login_code.rs
@@ -19,7 +19,7 @@ pub(in crate::api) async fn handler(
req: Request,
next: Next,
) -> Result<Response, Response> {
- if cfg!(not(feature = "rate_limit")) {
+ if !super::RATE_LIMIT_ENABLED {
tracing::trace!("rate limit disabled");
return Ok(next.run(req).await);
}
diff --git a/services/authentication_service/src/api/middleware/rate_limit/mobile_welcome_email.rs b/services/authentication_service/src/api/middleware/rate_limit/mobile_welcome_email.rs
index 9979918d5b..3a461df3b4 100644
--- a/services/authentication_service/src/api/middleware/rate_limit/mobile_welcome_email.rs
+++ b/services/authentication_service/src/api/middleware/rate_limit/mobile_welcome_email.rs
@@ -17,7 +17,7 @@ pub(in crate::api) async fn handler(
req: Request,
next: Next,
) -> Result<Response, Response> {
- if cfg!(not(feature = "rate_limit")) {
+ if !super::RATE_LIMIT_ENABLED {
tracing::trace!("rate limit disabled");
return Ok(next.run(req).await);
}
diff --git a/services/authentication_service/src/api/middleware/rate_limit/mod.rs b/services/authentication_service/src/api/middleware/rate_limit/mod.rs
index c104920045..1c0722b048 100644
--- a/services/authentication_service/src/api/middleware/rate_limit/mod.rs
+++ b/services/authentication_service/src/api/middleware/rate_limit/mod.rs
@@ -1,3 +1,16 @@
pub mod login_code;
pub mod mobile_welcome_email;
pub mod passwordless;
+
+/// Whether the login-code throttles apply to this build.
+///
+/// On by default via the `rate_limit` feature. `no_rate_limit` is the local
+/// opt-out: `just run_local` needs it off (a 1-request/minute login code makes
+/// local dev painful), and expressing that as an *additive* feature is what
+/// lets every local service binary build in one `cargo` invocation. Turning it
+/// off by dropping a default feature instead would need a package-scoped
+/// `--no-default-features` build, whose different feature resolution
+/// invalidates the shared dependency artifacts of every other binary — a
+/// measured ~2 minutes of rebuild on every `run_local`.
+pub const RATE_LIMIT_ENABLED: bool =
+ cfg!(feature = "rate_limit") && !cfg!(feature = "no_rate_limit");
diff --git a/services/authentication_service/src/api/middleware/rate_limit/passwordless.rs b/services/authentication_service/src/api/middleware/rate_limit/passwordless.rs
index bb739b9852..2ee408ba25 100644
--- a/services/authentication_service/src/api/middleware/rate_limit/passwordless.rs
+++ b/services/authentication_service/src/api/middleware/rate_limit/passwordless.rs
@@ -25,7 +25,7 @@ pub(in crate::api) async fn handler(
req: Request,
next: Next,
) -> Result<Response, Response> {
- if cfg!(not(feature = "rate_limit")) {
+ if !super::RATE_LIMIT_ENABLED {
tracing::trace!("rate limit disabled");
return Ok(next.run(req).await);
}
diff --git a/tooling/just/xtask.just b/tooling/just/xtask.just
index 23fbde3a93..8569190825 100644
--- a/tooling/just/xtask.just
+++ b/tooling/just/xtask.just
@@ -10,6 +10,7 @@ xtask := "cargo run --quiet --manifest-path Cargo.toml -p xtask_local --features
# just run_local --instance agent-a # isolated, concurrent instance
# just run_local --no-doppler --env-file ./local.env --no-frontend --no-build
# just run_local --enable-onboarding # opt into the existing onboarding v4 flag
+# just run_local --no-snapshot # re-run the full DB/kickstart/index init
run_local *ARGS:
{{ xtask }} run-local "$@"
diff --git a/tooling/xtask/crates/xtask_local/src/local.rs b/tooling/xtask/crates/xtask_local/src/local.rs
index bbce9e81a7..a9e45a2926 100644
--- a/tooling/xtask/crates/xtask_local/src/local.rs
+++ b/tooling/xtask/crates/xtask_local/src/local.rs
@@ -311,11 +311,21 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {
}
let fe_build = (share_app && !stage.is_dry_run()).then(|| {
let instance = instance.clone();
- std::thread::spawn(move || {
- frontend::build_static(&Stage::from_env().quiet(), &instance, mode)
- })
+ let stage = stage.background();
+ std::thread::spawn(move || frontend::build_static(&stage, &instance, mode))
});
+ // The init snapshot decision, exactly as `stack up` makes it (see
+ // `stack::up`): hash the init-defining inputs — possible only after
+ // `prepare` wrote the kickstart — and check for a stored snapshot. A hit
+ // skips migrate + Kafka topics + the ~1min FusionAuth kickstart + index
+ // creation; a miss runs the real init and saves one, which is how the cache
+ // seeds itself. The full-delete/full-create idempotency is unchanged: the
+ // key *is* the definition of clean, so any input change is a structural miss.
+ let snapshot_plan = (!args.no_snapshot && !stage.is_dry_run())
+ .then(|| snapshot::Plan::compute(&instance))
+ .transpose()?;
+
// Join the background teardown before we (re)create volumes + bring infra up,
// surfaced as a live spinner so it's clear what we're blocked on. It
// overlapped the build above, so this is just whatever's left of stopping the
@@ -337,7 +347,17 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {
// start. The teardown means everything is freshly created each run, so
// otherwise the services race their backends on startup (no `macrodb`,
// DynamoDB/OpenSearch connection refused).
- bring_up_infra(&stage, mode, &instance, &env, InfraInit::Full)?;
+ match &snapshot_plan {
+ Some(plan) if plan.exists() => {
+ snapshot::restore(&stage, &instance, plan)?;
+ bring_up_infra(&stage, mode, &instance, &env, InfraInit::FromSnapshot)?;
+ }
+ Some(plan) => {
+ bring_up_infra(&stage, mode, &instance, &env, InfraInit::Full)?;
+ snapshot::save(&stage, &instance, &env, plan)?;
+ }
+ None => bring_up_infra(&stage, mode, &instance, &env, InfraInit::Full)?,
+ }
if let Some(handle) = fe_build {
stage.run_step("Building frontend (static bundle)", move || {
handle
@@ -386,6 +406,7 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {
let shared_app_url = app_tunnel
.as_ref()
.map(|tunnel| format!("{}/app/", tunnel.url));
+ stage.print_timings("bring-up");
summary::print(
mode,
&instance,
@@ -1164,7 +1185,7 @@ fn ensure_external_resources(stage: &Stage, instance: &Instance) -> Result<()> {
fn wait_http(stage: &Stage, label: &str, url: &str) -> Result<()> {
let script = format!(
- "for i in $(seq 1 60); do curl -fsS --max-time 3 {url} >/dev/null 2>&1 && exit 0; sleep 2; done; echo 'not ready: {url}'; exit 1"
+ "for i in $(seq 1 600); do curl -fsS --max-time 3 {url} >/dev/null 2>&1 && exit 0; sleep 0.2; done; echo 'not ready: {url}'; exit 1"
);
let mut cmd = Command::new("bash");
cmd.arg("-lc").arg(script);
diff --git a/tooling/xtask/crates/xtask_local/src/local/build/mod.rs b/tooling/xtask/crates/xtask_local/src/local/build.rs
similarity index 100%
rename from tooling/xtask/crates/xtask_local/src/local/build/mod.rs
rename to tooling/xtask/crates/xtask_local/src/local/build.rs
diff --git a/tooling/xtask/crates/xtask_local/src/local/build/runtime_image.rs b/tooling/xtask/crates/xtask_local/src/local/build/runtime_image.rs
index 9cf80e2279..ad1bca56e1 100644
--- a/tooling/xtask/crates/xtask_local/src/local/build/runtime_image.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/build/runtime_image.rs
@@ -6,30 +6,55 @@
use std::process::Command;
-use anyhow::Result;
+use anyhow::{Context, Result};
+use sha2::{Digest, Sha256};
use super::super::arch::Target;
use super::super::{stage::Stage, workspace_root};
+#[cfg(test)]
+mod test;
+
/// The shared runtime image tag. Not per-instance: the image has no
/// instance-specific content.
pub const RUNTIME_IMAGE_TAG: &str = "macro-local-runtime:dev";
const DOCKERFILE_REL: &str = "docker/Dockerfile.runtime";
+/// Image label carrying the content key of the inputs the image was built
+/// from. A matching label is what lets a run skip the build entirely.
+const KEY_LABEL: &str = "com.macro.local-runtime-key";
+
/// Reconcile the runtime image for the host arch through BuildKit.
///
-/// BuildKit's content cache makes an unchanged build cheap while still
-/// invalidating the image when the Dockerfile or build context changes.
-/// Existence alone is not a valid cache key because the tag survives branch
-/// checkouts. `force` disables BuildKit's layer cache.
+/// BuildKit's content cache makes an *unchanged* rebuild cheap in the sense
+/// that no layer re-executes — but `--load` still re-exports and unpacks the
+/// image every time, which measured 32s warm and 2m29s cold on a 12-core
+/// Linux host. So the fast path here is not building at all: key the image on
+/// its inputs, stamp that key on as a label, and skip when the loaded image
+/// already carries it. Existence alone was never a valid key (the tag survives
+/// branch checkouts); a content key is, and it invalidates exactly when the
+/// Dockerfile or the platform changes. `force` skips the check and disables
+/// BuildKit's layer cache.
pub fn ensure_runtime_image(stage: &Stage, target: Target, force: bool) -> Result<()> {
let ws = workspace_root();
+ let key = content_key(target)?;
+ if !force && current_key() == Some(key.clone()) {
+ stage.note(&format!(
+ "runtime image {RUNTIME_IMAGE_TAG} is up to date ({})",
+ &key[..12]
+ ));
+ return Ok(());
+ }
let mut cmd = Command::new("docker");
cmd.current_dir(&ws)
.args(["buildx", "build", "--platform", target.docker_platform])
.args(["-f", DOCKERFILE_REL])
.args(["-t", RUNTIME_IMAGE_TAG])
+ .args(["--label", &format!("{KEY_LABEL}={key}")])
+ // Nothing consumes an attestation for a local dev image, and both
+ // turn the `--load` into a manifest-list export.
+ .args(["--provenance=false", "--sbom=false"])
.arg("--load");
if force {
cmd.arg("--no-cache");
@@ -40,3 +65,45 @@ pub fn ensure_runtime_image(stage: &Stage, target: Target, force: bool) -> Resul
&mut cmd,
)
}
+
+/// Hash everything that decides the image's content: the Dockerfile and the
+/// platform it is built for. The build context is the repository root, but the
+/// Dockerfile copies nothing from it, so the context is not an input.
+fn content_key(target: Target) -> Result<String> {
+ let dockerfile = workspace_root().join(DOCKERFILE_REL);
+ let mut hasher = Sha256::new();
+ hasher.update(target.docker_platform.as_bytes());
+ hasher.update(
+ std::fs::read(&dockerfile).with_context(|| format!("reading {}", dockerfile.display()))?,
+ );
+ Ok(hex(&hasher.finalize()))
+}
+
+/// The key stamped on the currently loaded image, if any. Any failure (no
+/// image, no daemon, no label) reads as "not current" and falls through to a
+/// build, which then reports the real error.
+fn current_key() -> Option<String> {
+ let out = Command::new("docker")
+ .args([
+ "image",
+ "inspect",
+ RUNTIME_IMAGE_TAG,
+ "--format",
+ &format!("{{{{index .Config.Labels \"{KEY_LABEL}\"}}}}"),
+ ])
+ .output()
+ .ok()?;
+ if !out.status.success() {
+ return None;
+ }
+ let key = String::from_utf8_lossy(&out.stdout).trim().to_string();
+ (!key.is_empty() && key != "<no value>").then_some(key)
+}
+
+fn hex(bytes: &[u8]) -> String {
+ bytes.iter().fold(String::new(), |mut s, b| {
+ use std::fmt::Write;
+ let _ = write!(s, "{b:02x}");
+ s
+ })
+}
diff --git a/tooling/xtask/crates/xtask_local/src/local/build/runtime_image/test.rs b/tooling/xtask/crates/xtask_local/src/local/build/runtime_image/test.rs
new file mode 100644
index 0000000000..9262fe2050
--- /dev/null
+++ b/tooling/xtask/crates/xtask_local/src/local/build/runtime_image/test.rs
@@ -0,0 +1,31 @@
+use super::*;
+
+fn target(arch: &'static str) -> Target {
+ match arch {
+ "x86_64" => Target {
+ triple: "x86_64-unknown-linux-gnu",
+ docker_platform: "linux/amd64",
+ },
+ _ => Target {
+ triple: "aarch64-unknown-linux-gnu",
+ docker_platform: "linux/arm64",
+ },
+ }
+}
+
+#[test]
+fn content_key_is_deterministic() {
+ let key = content_key(target("x86_64")).expect("hashes the tracked Dockerfile");
+ assert_eq!(key, content_key(target("x86_64")).unwrap());
+ assert_eq!(key.len(), 64);
+}
+
+/// Volumes and binaries are arch-specific, so an image built for one platform
+/// must never satisfy the other's key.
+#[test]
+fn content_key_is_platform_specific() {
+ assert_ne!(
+ content_key(target("x86_64")).unwrap(),
+ content_key(target("aarch64")).unwrap()
+ );
+}
diff --git a/tooling/xtask/crates/xtask_local/src/local/build/zigbuild.rs b/tooling/xtask/crates/xtask_local/src/local/build/zigbuild.rs
index 3e6bbb4680..d4c52777a6 100644
--- a/tooling/xtask/crates/xtask_local/src/local/build/zigbuild.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/build/zigbuild.rs
@@ -5,20 +5,31 @@
//! `AWS_LC_SYS_CMAKE_BUILDER=1` (aws-lc-sys's cc-builder rejects the zig C
//! compiler), `SQLX_OFFLINE`, and writable zig cache dirs.
-use std::path::Path;
+use std::path::{Path, PathBuf};
use std::process::Command;
use anyhow::{Context, Result, bail};
use super::super::arch::Target;
-use super::super::inventory::RUST_SERVICES;
+use super::super::inventory::{RUST_SERVICES, RustService};
use super::super::{stage::Stage, workspace_root};
+#[cfg(test)]
+mod test;
+
/// Build every non-opt-in service binary (debug profile) for `target`.
///
-/// Most bins build together in one invocation; bins that need
-/// `--no-default-features` (e.g. `authentication_service`, to drop `rate_limit`)
-/// build in their own invocation since `--no-default-features` is package-wide.
+/// One unified invocation builds as many bins as possible, with each service's
+/// extra features passed package-qualified (`<package>/<feature>`) so they
+/// compose instead of forcing a separate build. Services that must drop a
+/// *default* feature still need their own package-scoped invocation, and that
+/// is expensive in a way that is invisible at the call site: a package-scoped
+/// build resolves features over a different package selection, so it
+/// invalidates shared dependency artifacts the unified build produced, and the
+/// *next* run pays to rebuild them. Measured on a 12-core Linux host: with the
+/// extra invocations, a `run_local` with zero source changes rebuilt 230 units
+/// in 2m06s; unified, the same no-op build is 16s. Keep
+/// [`RustService::no_default_features`] as close to empty as the features allow.
// xtask is host tooling; reading CARGO_BUILD_JOBS from the process env is correct.
#[allow(clippy::disallowed_methods)]
pub fn run(stage: &Stage, target: Target) -> Result<()> {
@@ -36,45 +47,99 @@ pub fn run(stage: &Stage, target: Target) -> Result<()> {
.to_string()
});
- // Group 1: default-feature bins, all in one invocation.
- let default_bins: Vec<&str> = RUST_SERVICES
- .iter()
- .filter(|s| !s.is_opt_in() && !s.no_default_features)
- .map(|s| s.cargo_bin)
- .collect();
let mut cmd = base_command(&ws, &zig_cache, &jobs, target);
- for bin in &default_bins {
- cmd.arg("--bin").arg(bin);
- }
+ cmd.args(unified_args());
stage.run(
&format!("Building service binaries ({})", target.triple),
&mut cmd,
)?;
- // Group 2: each --no-default-features bin in its own package-scoped invocation.
- for svc in RUST_SERVICES
- .iter()
- .filter(|s| !s.is_opt_in() && s.no_default_features)
- {
+ // Each --no-default-features bin in its own package-scoped invocation, and
+ // in its own target dir: that is what keeps its different feature
+ // resolution from invalidating the unified build's shared artifacts (and
+ // vice versa). Each dir then only ever sees one resolution, so both stay
+ // incrementally fresh. Costs one cold build per dir, once.
+ for svc in isolated_services() {
+ let target_dir = isolated_target_dir(&ws, svc);
let mut cmd = base_command(&ws, &zig_cache, &jobs, target);
- cmd.arg("-p")
- .arg(svc.package)
- .arg("--no-default-features")
- .arg("--bin")
- .arg(svc.cargo_bin);
- let features = svc.build_features();
- if !features.is_empty() {
- cmd.arg("--features").arg(features.join(","));
- }
+ cmd.args(isolated_args(svc, &target_dir));
stage.run(
&format!("Building {} (no default features)", svc.cargo_bin),
&mut cmd,
)?;
+ // The stack mounts one directory at `/app/out`, so land the binary
+ // beside the unified ones.
+ let built = target_dir
+ .join(target.triple)
+ .join("debug")
+ .join(svc.cargo_bin);
+ let dest = ws.join(target.debug_dir()).join(svc.cargo_bin);
+ std::fs::copy(&built, &dest)
+ .with_context(|| format!("copying {} to {}", built.display(), dest.display()))?;
}
Ok(())
}
+/// Where a `--no-default-features` service builds. Under `target/` so it is
+/// covered by the same ignore rules and cleaned by `cargo clean`'s neighbors.
+fn isolated_target_dir(ws: &Path, svc: &RustService) -> PathBuf {
+ ws.join("target/local-isolated").join(svc.cargo_bin)
+}
+
+/// The services built together in the unified invocation.
+fn unified_services() -> impl Iterator<Item = &'static RustService> {
+ RUST_SERVICES
+ .iter()
+ .filter(|s| !s.is_opt_in() && !s.no_default_features)
+}
+
+/// The services that need their own package-scoped, own-target-dir build.
+fn isolated_services() -> impl Iterator<Item = &'static RustService> {
+ RUST_SERVICES
+ .iter()
+ .filter(|s| !s.is_opt_in() && s.no_default_features)
+}
+
+/// `--bin` per service plus one `--features` carrying every service's local
+/// features, package-qualified so a feature only reaches the package that
+/// declares it.
+fn unified_args() -> Vec<String> {
+ let mut args: Vec<String> = unified_services()
+ .flat_map(|svc| ["--bin".to_owned(), svc.cargo_bin.to_owned()])
+ .collect();
+ let features: Vec<String> = unified_services()
+ .flat_map(|svc| {
+ svc.local_features()
+ .iter()
+ .map(|f| format!("{}/{f}", svc.package))
+ })
+ .collect();
+ if !features.is_empty() {
+ args.push("--features".to_owned());
+ args.push(features.join(","));
+ }
+ args
+}
+
+fn isolated_args(svc: &RustService, target_dir: &Path) -> Vec<String> {
+ let mut args = vec![
+ "--target-dir".to_owned(),
+ target_dir.display().to_string(),
+ "-p".to_owned(),
+ svc.package.to_owned(),
+ "--no-default-features".to_owned(),
+ "--bin".to_owned(),
+ svc.cargo_bin.to_owned(),
+ ];
+ let features = svc.local_features();
+ if !features.is_empty() {
+ args.push("--features".to_owned());
+ args.push(features.join(","));
+ }
+ args
+}
+
/// A `cargo zigbuild` command with the shared env (mirrors flake.nix
/// `lambdaCommonArgs`).
fn base_command(ws: &Path, zig_cache: &Path, jobs: &str, target: Target) -> Command {
@@ -98,37 +163,15 @@ fn base_command(ws: &Path, zig_cache: &Path, jobs: &str, target: Target) -> Comm
.env("ZIG_LOCAL_CACHE_DIR", zig_cache)
.env("RUSTC_WRAPPER", "sccache")
.env("XDG_CACHE_HOME", zig_cache);
- if let Some(flag) = curl_include_flag() {
- // First-found wins in cc's env chain (CFLAGS_<target> before CFLAGS),
- // so set the target-specific var too in case something else exports a
- // plain CFLAGS that would otherwise shadow ours — and vice versa.
- let var = format!("CFLAGS_{}", target.triple.replace('-', "_"));
- cmd.env(var, &flag).env("CFLAGS", &flag);
- }
+ // No CFLAGS here. The dev shell already exports target-scoped
+ // `CFLAGS_<triple>`/`CXXFLAGS_<triple>` pointing at curl.dev for exactly
+ // this cross build (see `nix/cloud-storage.nix`). Setting our own spelling
+ // of the same include path only differed textually — and cargo fingerprints
+ // these env vars, so the two spellings alternating rebuilt aws-lc-sys and
+ // everything downstream.
cmd
}
-/// librdkafka 2.12's bundled cmake build needs curl *headers* even with
-/// `WITH_CURL=0` (its config header always defines `WITH_OAUTHBEARER_OIDC` via
-/// `#cmakedefine01`, and rdkafka_conf.c guards the curl include with `#ifdef`
-/// instead of `#if`). Host builds get them from the Nix cc-wrapper, which
-/// reads `NIX_CFLAGS_COMPILE` — but `zig cc` is a raw clang that ignores it,
-/// so the dev shell's `curl.dev` never reaches the cross cmake build. Mine
-/// the wrapper flags for an include dir that actually carries `curl/curl.h`
-/// and hand it to cc/cmake explicitly; `-idirafter` keeps it at the lowest
-/// search priority so it cannot shadow zig's own libc headers.
-// Reading the ambient Nix shell env is the point here; there is no config
-// layer this could come from.
-#[allow(clippy::disallowed_methods)]
-fn curl_include_flag() -> Option<String> {
- let flags = std::env::var("NIX_CFLAGS_COMPILE").ok()?;
- flags
- .split_whitespace()
- .filter(|tok| tok.starts_with('/'))
- .find(|dir| Path::new(dir).join("curl/curl.h").is_file())
- .map(|dir| format!("-idirafter {dir}"))
-}
-
/// Fail with an actionable hint if the rust-std for `target` is not installed.
/// The repo's `rustup` is a no-op shim, so the only fix is adding the triple to
/// `rust-toolchain.toml` and re-entering `nix develop`.
diff --git a/tooling/xtask/crates/xtask_local/src/local/build/zigbuild/test.rs b/tooling/xtask/crates/xtask_local/src/local/build/zigbuild/test.rs
new file mode 100644
index 0000000000..1753c35ae5
--- /dev/null
+++ b/tooling/xtask/crates/xtask_local/src/local/build/zigbuild/test.rs
@@ -0,0 +1,70 @@
+use super::*;
+
+/// The whole point of the unified invocation: every service that *can* build
+/// together does. A service silently gaining `no_default_features` costs a
+/// package-scoped build and, without its own target dir, a full minute of
+/// rebuild on the next `run_local`.
+#[test]
+fn only_services_that_drop_default_features_build_separately() {
+ let isolated: Vec<&str> = isolated_services().map(|s| s.cargo_bin).collect();
+ assert_eq!(isolated, vec!["search_processing_service"]);
+ assert!(unified_services().any(|s| s.cargo_bin == "authentication_service"));
+}
+
+#[test]
+fn unified_args_pass_local_features_package_qualified() {
+ let args = unified_args();
+ let features = args
+ .iter()
+ .position(|a| a == "--features")
+ .map(|i| args[i + 1].clone())
+ .expect("unified build passes features");
+ for feature in features.split(',') {
+ assert!(
+ feature.contains('/'),
+ "{feature} must be package-qualified or it applies to the wrong package"
+ );
+ }
+ assert!(features.contains("authentication_service/no_rate_limit"));
+ assert!(features.contains("authentication_service/return_passwordless_code"));
+}
+
+#[test]
+fn unified_args_build_every_unified_service_bin() {
+ let args = unified_args();
+ for svc in unified_services() {
+ assert!(
+ args.windows(2)
+ .any(|w| w[0] == "--bin" && w[1] == svc.cargo_bin),
+ "{} is missing from the unified build",
+ svc.cargo_bin
+ );
+ }
+}
+
+/// Isolation is the load-bearing part: sharing a target dir with the unified
+/// build is exactly what invalidates it.
+#[test]
+fn isolated_args_use_their_own_target_dir() {
+ let svc = isolated_services().next().expect("one isolated service");
+ let dir = isolated_target_dir(Path::new("/ws"), svc);
+ let args = isolated_args(svc, &dir);
+ let target_dir = args
+ .iter()
+ .position(|a| a == "--target-dir")
+ .map(|i| args[i + 1].clone())
+ .expect("isolated build sets --target-dir");
+ assert_eq!(
+ target_dir,
+ "/ws/target/local-isolated/search_processing_service"
+ );
+ assert!(args.iter().any(|a| a == "--no-default-features"));
+}
+
+#[test]
+fn isolated_target_dirs_do_not_collide() {
+ let dirs: std::collections::BTreeSet<PathBuf> = isolated_services()
+ .map(|svc| isolated_target_dir(Path::new("/ws"), svc))
+ .collect();
+ assert_eq!(dirs.len(), isolated_services().count());
+}
diff --git a/tooling/xtask/crates/xtask_local/src/local/cli.rs b/tooling/xtask/crates/xtask_local/src/local/cli.rs
index a3d767a3c7..67b223f047 100644
--- a/tooling/xtask/crates/xtask_local/src/local/cli.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/cli.rs
@@ -121,6 +121,10 @@ pub struct RunArgs {
/// Stream subprocess output and show per-step timings.
#[arg(long, short)]
pub verbose: bool,
+ /// Neither restore from nor save an init snapshot — always run the full
+ /// DB migrate / FusionAuth kickstart / index creation.
+ #[arg(long)]
+ pub no_snapshot: bool,
/// Which local OTLP collector to start and wire services to. Defaults to
/// the LGTM stack (Grafana/Tempo/Loki); pass `off` to run without tracing
/// — see `docker/docker-compose.yml` for what each backend does.
diff --git a/tooling/xtask/crates/xtask_local/src/local/cli/test.rs b/tooling/xtask/crates/xtask_local/src/local/cli/test.rs
index 0cfd543c09..147c9d4bee 100644
--- a/tooling/xtask/crates/xtask_local/src/local/cli/test.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/cli/test.rs
@@ -90,6 +90,21 @@ fn stack_has_no_snapshot_verb() {
}
}
+#[test]
+fn run_local_and_stack_up_accept_no_snapshot() {
+ let cli = Cli::try_parse_from(["cargo-x", "run-local", "--no-snapshot"]).unwrap();
+ let Cmd::RunLocal(args) = cli.command else {
+ panic!("expected run-local command");
+ };
+ assert!(args.no_snapshot);
+
+ let cli = Cli::try_parse_from(["cargo-x", "stack", "up", "--no-snapshot"]).unwrap();
+ let Cmd::Stack(StackCmd::Up(args)) = cli.command else {
+ panic!("expected stack up");
+ };
+ assert!(args.run.no_snapshot);
+}
+
#[test]
fn seed_scenario_accepts_instance_and_trailing_scenario_arguments() {
let cli = Cli::try_parse_from([
diff --git a/tooling/xtask/crates/xtask_local/src/local/e2e.rs b/tooling/xtask/crates/xtask_local/src/local/e2e.rs
index a0ddf455a4..b379d7a148 100644
--- a/tooling/xtask/crates/xtask_local/src/local/e2e.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/e2e.rs
@@ -128,6 +128,7 @@ pub fn run(args: &LocalE2eArgs) -> Result<()> {
traces: TracesBackend::Off,
with_chrome: false,
verbose: false,
+ no_snapshot: false,
with_cf_tunnel: false,
},
..stack::UpArgs::default()
diff --git a/tooling/xtask/crates/xtask_local/src/local/frontend.rs b/tooling/xtask/crates/xtask_local/src/local/frontend.rs
index 9d5f5f1845..a3c56aa68f 100644
--- a/tooling/xtask/crates/xtask_local/src/local/frontend.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/frontend.rs
@@ -154,7 +154,7 @@ fn dev_env(
pub fn wait_backend_ready(stage: &Stage, instance: &Instance) -> Result<()> {
let url = format!("{}/auth/health", proxy::url(instance));
let script = format!(
- "for i in $(seq 1 60); do curl -fsS --max-time 3 {url} >/dev/null 2>&1 && exit 0; sleep 2; done; echo 'backend not ready'; exit 1"
+ "for i in $(seq 1 600); do curl -fsS --max-time 3 {url} >/dev/null 2>&1 && exit 0; sleep 0.2; done; echo 'backend not ready'; exit 1"
);
let mut cmd = Command::new("bash");
cmd.arg("-lc").arg(script);
diff --git a/tooling/xtask/crates/xtask_local/src/local/fusionauth.rs b/tooling/xtask/crates/xtask_local/src/local/fusionauth.rs
index 5b190856e6..18bfc24d7e 100644
--- a/tooling/xtask/crates/xtask_local/src/local/fusionauth.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/fusionauth.rs
@@ -77,7 +77,7 @@ pub fn wait_ready(stage: &Stage, instance: &Instance) -> Result<()> {
// maintenance-mode 302 (e.g. after a boot-time DB connect failure) would
// otherwise pass as ready and every later login would 500.
let script = format!(
- "for i in $(seq 1 120); do [ \"$(curl -sS -o /dev/null -w '%{{http_code}}' --max-time 3 -H 'Authorization: {key}' {url} 2>/dev/null)\" = 200 ] && exit 0; sleep 2; done; \
+ "for i in $(seq 1 480); do [ \"$(curl -sS -o /dev/null -w '%{{http_code}}' --max-time 3 -H 'Authorization: {key}' {url} 2>/dev/null)\" = 200 ] && exit 0; sleep 0.5; done; \
echo 'timed out waiting for the FusionAuth kickstart (a 302 here means maintenance mode: FusionAuth could not reach its db)'; exit 1",
key = identity::FUSIONAUTH_API_KEY,
);
diff --git a/tooling/xtask/crates/xtask_local/src/local/inventory.rs b/tooling/xtask/crates/xtask_local/src/local/inventory.rs
index fd50875393..082e67df92 100644
--- a/tooling/xtask/crates/xtask_local/src/local/inventory.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/inventory.rs
@@ -41,10 +41,20 @@ pub struct RustService {
/// Opt-in services (e.g. `search_processing_service`) only start with an
/// explicit profile and are never converted to the runtime image.
pub opt_in: bool,
- /// Built locally with `--no-default-features` (plus any [`Self::build_features`]).
- /// `authentication_service` drops the default `rate_limit` feature (removing
- /// the 1-request/minute login-code throttle that makes local dev painful);
- /// `search_processing_service` drops the default amd64-only `pdf` feature.
+ /// Built locally in its own `cargo` invocation with `--no-default-features`
+ /// (plus [`Self::local_features`]), because the feature it must drop is a
+ /// *default* one and cargo can only drop defaults per-package.
+ ///
+ /// Keep this at zero services if you possibly can. A package-scoped
+ /// invocation resolves features over a different package selection than the
+ /// unified build, which invalidates the shared dependency artifacts the
+ /// unified build just produced — measured at 230 units and ~2 minutes of
+ /// rebuild on the *next* `run_local`, forever. Prefer an additive opt-out
+ /// feature (see `authentication_service`'s `no_rate_limit`), which composes
+ /// into the one unified invocation.
+ ///
+ /// `search_processing_service` genuinely cannot: its `pdf` default feature
+ /// pulls an optional dependency, and no additive feature can un-pull one.
pub no_default_features: bool,
}
@@ -60,15 +70,19 @@ impl RustService {
self.opt_in
}
- /// Cargo features to re-enable alongside `--no-default-features` for this
- /// service's local build (empty = none). Only `search_processing_service`
- /// needs this: `authentication_service` opts into local passwordless code
- /// responses, and dropping `search_processing_service` default features
- /// removes the amd64-only `pdf` feature, but its bin still requires
- /// `processing`/`service`.
- pub fn build_features(&self) -> &'static [&'static str] {
+ /// Cargo features this service's local build enables (empty = none).
+ ///
+ /// For a unified-build service these are passed package-qualified
+ /// (`<package>/<feature>`) so they compose with every other binary in the
+ /// same invocation. For a [`Self::no_default_features`] service they are
+ /// the features re-enabled alongside the dropped defaults.
+ ///
+ /// `authentication_service` opts into local passwordless code responses and
+ /// out of the login-code throttles; `search_processing_service` loses
+ /// `processing`/`service` with its defaults and its bin requires them.
+ pub fn local_features(&self) -> &'static [&'static str] {
match self.cargo_bin {
- "authentication_service" => &["return_passwordless_code"],
+ "authentication_service" => &["return_passwordless_code", "no_rate_limit"],
"search_processing_service" => &["processing", "service"],
_ => &[],
}
@@ -86,7 +100,9 @@ pub const RUST_SERVICES: &[RustService] = &[
is_websocket: false,
modes: &[Mode::Local, Mode::Dev],
opt_in: false,
- no_default_features: true,
+ // Joins the unified invocation: the local behavior it needs
+ // (`no_rate_limit`, `return_passwordless_code`) is additive.
+ no_default_features: false,
},
RustService {
compose_name: "connection_gateway",
diff --git a/tooling/xtask/crates/xtask_local/src/local/stack.rs b/tooling/xtask/crates/xtask_local/src/local/stack.rs
index 50aa3a2b0c..0d17eb05bf 100644
--- a/tooling/xtask/crates/xtask_local/src/local/stack.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/stack.rs
@@ -25,10 +25,6 @@ use super::{Mode, arch, env_layer, frontend, mailpit, proxy, sdk_webhook, snapsh
pub struct UpArgs {
#[command(flatten)]
pub run: RunArgs,
- /// Neither restore from nor save an init snapshot — always run the full
- /// migrate/kickstart/index init.
- #[arg(long)]
- pub no_snapshot: bool,
/// Stop after the infra bring-up + init (and the snapshot save/restore):
/// no app services, proxy, or frontend. This is the CI bake mode — the
/// app services need the Doppler-sourced env (AWS endpoints, shared
@@ -192,16 +188,15 @@ pub fn up(mode: Mode, args: &UpArgs) -> Result<Instance> {
}
let fe_build = (static_frontend && !stage.is_dry_run()).then(|| {
let instance = instance.clone();
- std::thread::spawn(move || {
- frontend::build_static(&Stage::from_env().quiet(), &instance, mode)
- })
+ let stage = stage.background();
+ std::thread::spawn(move || frontend::build_static(&stage, &instance, mode))
});
// The init snapshot decision: hash the init-defining inputs (possible only
// after `prepare` wrote the kickstart) and check for a stored snapshot.
// Restores skip migrate/kickstart/index-init; a cold init saves one for
// next time — that's how the cache seeds itself.
- let snapshot_plan = (!args.no_snapshot && !stage.is_dry_run())
+ let snapshot_plan = (!args.run.no_snapshot && !stage.is_dry_run())
.then(|| snapshot::Plan::compute(&instance))
.transpose()?;
@@ -293,6 +288,7 @@ pub fn up(mode: Mode, args: &UpArgs) -> Result<Instance> {
} else {
mailpit::direct_ui_url(&instance)
};
+ stage.print_timings("bring-up");
summary::print(mode, &instance, &env, &frontend_url, &mailpit_url, None);
stage.note(&format!(
" headless: `just stack status`, `just stack update`, `just stack down`{}",
@@ -330,11 +326,11 @@ fn bootstrap_from_update(args: &UpdateArgs) -> Result<()> {
no_frontend: false,
enable_onboarding: false,
verbose: args.verbose,
+ no_snapshot: false,
traces: super::cli::TracesBackend::default(),
with_chrome: false,
with_cf_tunnel: false,
},
- no_snapshot: false,
infra_only: false,
json: args.json,
};
diff --git a/tooling/xtask/crates/xtask_local/src/local/stage.rs b/tooling/xtask/crates/xtask_local/src/local/stage.rs
index 81b121482d..e35551d1fe 100644
--- a/tooling/xtask/crates/xtask_local/src/local/stage.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/stage.rs
@@ -10,8 +10,9 @@
//! cannot deadlock by filling a pipe buffer while we wait. `indicatif` only
//! animates — it never reads the child's pipes.
-use std::io::{self, Read};
+use std::io::{self, Read, Write};
use std::process::{Command, Stdio};
+use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use anyhow::{Result, bail};
@@ -102,6 +103,10 @@ pub struct Stage {
/// notes are suppressed (commands still run + capture). Used to fold a
/// multi-step action under a single parent spinner.
quiet: bool,
+ /// Every resolved stage's label + elapsed, in completion order. Shared with
+ /// [`Stage::quiet`] children so folded sub-steps are recorded too, and with
+ /// the threads that run the background lanes.
+ timings: Arc<Mutex<Vec<(String, Duration)>>>,
}
impl Stage {
@@ -116,6 +121,7 @@ impl Stage {
verbose: env_flag("MACRO_LOCAL_VERBOSE"),
dry_run: env_flag("MACRO_LOCAL_DRY_RUN"),
quiet: false,
+ timings: Arc::new(Mutex::new(Vec::new())),
}
}
@@ -152,6 +158,85 @@ impl Stage {
verbose: self.verbose,
dry_run: self.dry_run,
quiet: true,
+ timings: Arc::clone(&self.timings),
+ }
+ }
+
+ /// A handle that records into the same timing table from another thread
+ /// (the background teardown / frontend-build lanes). Output stays quiet:
+ /// a lane that races the foreground stages cannot share the spinner.
+ pub fn background(&self) -> Stage {
+ self.quiet()
+ }
+
+ fn record(&self, label: &str, elapsed: Duration) {
+ if let Ok(mut timings) = self.timings.lock() {
+ timings.push((label.to_owned(), elapsed));
+ }
+ }
+
+ /// Print the slowest stages of the run, and append a JSON line to
+ /// `MACRO_LOCAL_TIMINGS` when that env var names a file. This is how a
+ /// change to the bring-up is shown to be faster rather than argued to be.
+ // xtask is host tooling; reading the process environment directly is correct.
+ #[allow(clippy::disallowed_methods)]
+ pub fn print_timings(&self, title: &str) {
+ let Ok(timings) = self.timings.lock() else {
+ return;
+ };
+ if timings.is_empty() {
+ return;
+ }
+ const SUM_LABEL: &str = "sum of stages (lanes overlap)";
+ let total: Duration = timings.iter().map(|(_, d)| *d).sum();
+ let mut slowest: Vec<&(String, Duration)> = timings.iter().collect();
+ slowest.sort_by_key(|(_, d)| std::cmp::Reverse(*d));
+ let shown = &slowest[..slowest.len().min(10)];
+ // Labels run past the stage-line column ("Building search_processing…"),
+ // so size this table to its own longest row rather than clipping.
+ let width = shown
+ .iter()
+ .map(|(label, _)| label.chars().count())
+ .chain([SUM_LABEL.len(), WIDTH])
+ .max()
+ .unwrap_or(WIDTH);
+
+ self.section(&format!("{title} — slowest stages"));
+ for (label, elapsed) in shown {
+ println!(
+ " {label:<width$} {:>8}",
+ Style::new().bold().apply_to(format_elapsed(*elapsed))
+ );
+ }
+ println!(
+ " {:<width$} {:>8}",
+ Style::new().dim().apply_to(SUM_LABEL),
+ Style::new().dim().apply_to(format_elapsed(total)),
+ );
+
+ if let Some(path) = std::env::var_os("MACRO_LOCAL_TIMINGS") {
+ let entries: Vec<String> = timings
+ .iter()
+ .map(|(label, d)| {
+ format!(
+ "{{\"label\":{},\"seconds\":{:.3}}}",
+ serde_json::Value::from(label.as_str()),
+ d.as_secs_f64()
+ )
+ })
+ .collect();
+ let line = format!("{{\"stages\":[{}]}}\n", entries.join(","));
+ if let Err(e) = std::fs::OpenOptions::new()
+ .create(true)
+ .append(true)
+ .open(&path)
+ .and_then(|mut f| f.write_all(line.as_bytes()))
+ {
+ self.note(&format!(
+ "could not write MACRO_LOCAL_TIMINGS to {}: {e}",
+ std::path::Path::new(&path).display()
+ ));
+ }
}
}
@@ -284,6 +369,7 @@ impl Stage {
let mut captured = out_handle.join().unwrap_or_default();
captured.extend(err_handle.join().unwrap_or_default());
if self.quiet {
+ self.record(label, start.elapsed());
// Silent on success; on failure bubble the captured output up in the
// error so the parent stage can show it after its spinner clears.
return match status {
@@ -315,11 +401,15 @@ impl Stage {
return Ok(());
}
if self.quiet {
- return f();
+ let start = Instant::now();
+ let result = f();
+ self.record(label, start.elapsed());
+ return result;
}
let start = Instant::now();
let spinner = self.spinner(label);
let result = f();
+ self.record(label, start.elapsed());
let elapsed = format_elapsed(start.elapsed());
match result {
Ok(()) => {
@@ -359,6 +449,7 @@ impl Stage {
program: &str,
args: &[String],
) -> Result<()> {
+ self.record(label, start.elapsed());
let elapsed = format_elapsed(start.elapsed());
match outcome {
Ok((status, _captured)) if status.success() => {
diff --git a/tooling/xtask/crates/xtask_local/src/local/stage/test.rs b/tooling/xtask/crates/xtask_local/src/local/stage/test.rs
index 36819d45af..20cf79b073 100644
--- a/tooling/xtask/crates/xtask_local/src/local/stage/test.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/stage/test.rs
@@ -10,3 +10,41 @@ fn spinner_draw_tracks_narrow_terminal_widths() {
assert_eq!(spinner_draw_width(40), 40);
assert_eq!(spinner_draw_width(0), 1);
}
+
+#[test]
+fn stages_record_their_elapsed_time() {
+ let stage = Stage::from_env();
+ stage.run_step("work", || Ok(())).unwrap();
+ let timings = stage.timings.lock().unwrap();
+ assert_eq!(timings.len(), 1);
+ assert_eq!(timings[0].0, "work");
+}
+
+/// Sub-steps folded under a parent spinner, and the background lanes, must land
+/// in the same table — otherwise the summary omits exactly the slow parts that
+/// run off the main thread.
+#[test]
+fn quiet_and_background_children_record_into_the_parent() {
+ let stage = Stage::from_env();
+ stage.quiet().run_step("folded", || Ok(())).unwrap();
+ let background = stage.background();
+ std::thread::spawn(move || background.run_step("lane", || Ok(())))
+ .join()
+ .unwrap()
+ .unwrap();
+ let recorded: Vec<String> = stage
+ .timings
+ .lock()
+ .unwrap()
+ .iter()
+ .map(|(label, _)| label.clone())
+ .collect();
+ assert_eq!(recorded, vec!["folded", "lane"]);
+}
+
+#[test]
+fn a_failed_stage_is_still_recorded() {
+ let stage = Stage::from_env();
+ let _ = stage.run_step("doomed", || anyhow::bail!("no"));
+ assert_eq!(stage.timings.lock().unwrap()[0].0, "doomed");
+}