Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(agent_harness): local Docker sandboxes for just run_local (#5826)

Wolf Mermelsteined0e93e92362
diff --git a/.github/workflows/ensure_daytona_snapshot.yml b/.github/workflows/ensure_daytona_snapshot.yml
index 40309f157b..4c60502430 100644
--- a/.github/workflows/ensure_daytona_snapshot.yml
+++ b/.github/workflows/ensure_daytona_snapshot.yml
@@ -2,6 +2,19 @@
# Source: tooling/xtask/crates/xtask_workflows/src/workflows/ensure_daytona_snapshot.rs
name: Ensure Daytona Snapshot
'on':
+ pull_request:
+ types:
+ - opened
+ - synchronize
+ - reopened
+ branches:
+ - main
+ paths:
+ - crates/agent_harness/container/**
+ - crates/agent_harness/justfile
+ - nix/cloud-storage.nix
+ - tooling/xtask/crates/xtask_workflows/src/workflows/ensure_daytona_snapshot.rs
+ - '.github/workflows/ensure_daytona_snapshot.yml'
push:
branches:
- main
@@ -14,8 +27,10 @@ name: Ensure Daytona Snapshot
workflow_dispatch: {}
permissions:
contents: read
+ packages: write
jobs:
ensure-snapshot:
+ if: github.event_name != 'pull_request'
name: Ensure macro-agent-harness snapshot
runs-on: namespace-profile-linux-small
steps:
@@ -45,6 +60,41 @@ jobs:
shell: bash
env:
DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}
+ publish-image:
+ if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+ name: Publish sandbox image to GHCR
+ runs-on: namespace-profile-linux-mid
+ timeout-minutes: 90
+ steps:
+ - name: Checkout
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+ with:
+ clean: 'false'
+ persist-credentials: 'false'
+ - name: Set up Namespace Docker builder
+ uses: namespacelabs/nscloud-setup-buildx-action@d059ed7184f0bc7c8b27e8810cea153d02bcc6dd
+ - name: Log in to GHCR
+ run: |
+ set -euo pipefail
+ echo "$GITHUB_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin
+ shell: bash
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GITHUB_ACTOR: ${{ github.actor }}
+ - name: Build and push sandbox image
+ run: |
+ set -euo pipefail
+ image=ghcr.io/macro-inc/macro-agent-harness
+ tags=(--tag "$image:$GITHUB_SHA")
+ if [ "${GITHUB_REF:-}" = "refs/heads/main" ]; then
+ tags+=(--tag "$image:latest")
+ fi
+ docker buildx build \
+ --platform linux/amd64 \
+ "${tags[@]}" \
+ --push \
+ crates/agent_harness/container
+ shell: bash
concurrency:
- group: ${{ github.workflow }}
+ group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: false
diff --git a/.github/workflows/preview-fly.yml b/.github/workflows/preview-fly.yml
index f7fc03d9bb..632159d6cf 100644
--- a/.github/workflows/preview-fly.yml
+++ b/.github/workflows/preview-fly.yml
@@ -21,7 +21,7 @@ jobs:
TEMPLATE_APP: macro-preview-template
MACRO_STACK_SNAPSHOT_DIR: /home/runner/.cache/macro-preview-snapshots
CARGO_INCREMENTAL: '0'
- timeout-minutes: 60
+ timeout-minutes: 90
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
@@ -93,6 +93,10 @@ jobs:
cd apps/web
export MODE=development NODE_ENV=production
export VITE_LOCAL_SERVERS=ALL VITE_LOCAL_BACKEND_ORIGIN=same-origin
+ # Same ceiling as `just -f apps/web/justfile build-dev`: vite's
+ # chunk-render peaks at node's default ~4GB heap and aborts with
+ # "Ineffective mark-compacts near heap limit".
+ export NODE_OPTIONS=--max-old-space-size=6144
bun run --bun build
LANE
@@ -152,7 +156,10 @@ jobs:
cat > "$lanes/images.sh" <<'LANE'
set -euo pipefail
docker compose --project-directory . -p macro -f docker/docker-compose.yml build \
- search sync_service websocket_service lexical_service ai_editing_worker
+ search sync_service websocket_service lexical_service ai_editing_worker analytics_proxy
+ # Not a compose service, so `config --images` never lists it.
+ # BuildKit cache on this runner is the freshness check.
+ docker build --tag "$LOCAL_SANDBOX_IMAGE" crates/agent_harness/container
# The image BUILD above is fatal; the premirror below is a pure
# optimization with an authoritative fallback (the deploy step's
# mirror loop), so its failure only costs a slower push later.
@@ -172,7 +179,7 @@ jobs:
flyctl auth docker
registry="registry.fly.io/$APP_NAME"
for img in $(docker compose --project-directory . -p macro \
- -f docker/docker-compose.yml config --images | sort -u) alpine:3; do
+ -f docker/docker-compose.yml config --images | sort -u) alpine:3 "$LOCAL_SANDBOX_IMAGE"; do
if ! docker image inspect "$img" >/dev/null 2>&1 && ! docker pull "$img"; then
echo "premirror: skipping $img (not local, not pullable)"
continue
@@ -303,6 +310,7 @@ jobs:
fi
env:
FLY_ORG: ${{ vars.FLY_ORG || secrets.FLY_ORG }}
+ LOCAL_SANDBOX_IMAGE: macro-agent-harness:latest
- name: Dump stack diagnostics
if: failure()
run: |
@@ -529,10 +537,17 @@ jobs:
docker image inspect alpine:3 >/dev/null 2>&1 || docker pull alpine:3
mkdir -p preview-ctx/preload
: > preview-ctx/preload/manifest.txt
- for img in $images alpine:3; do
+ for img in $images alpine:3 "$LOCAL_SANDBOX_IMAGE"; do
# Images the bake didn't leave in the daemon (snapshot cache hit,
# or app-layer-only images like proxy/mailpit) get pulled here.
- docker image inspect "$img" >/dev/null 2>&1 || docker pull "$img"
+ # Compose `config --images` also lists locally-built tags
+ # (analytics_proxy, sdk-webhook-relay) that are not on Docker Hub;
+ # premirror already skips those — do the same so a missing local
+ # tag cannot fail the deploy.
+ if ! docker image inspect "$img" >/dev/null 2>&1 && ! docker pull "$img"; then
+ echo "mirror: skipping $img (not local, not pullable)"
+ continue
+ fi
id=$(docker image inspect -f '{{.Id}}' "$img")
# Content-addressed tag: same image content = same ref, so a
# redeploy's push is skippable by a manifest existence check and
@@ -653,6 +668,7 @@ jobs:
env:
FLY_ORG: ${{ vars.FLY_ORG || secrets.FLY_ORG }}
DOPPLER_PREVIEW_TOKEN: ${{ secrets.DOPPLER_PREVIEW_TOKEN }}
+ LOCAL_SANDBOX_IMAGE: macro-agent-harness:latest
- name: Dump Fly diagnostics
if: failure()
run: |
diff --git a/crates/agent_harness/Cargo.toml b/crates/agent_harness/Cargo.toml
index 93d71e3c3c..3c682582b1 100644
--- a/crates/agent_harness/Cargo.toml
+++ b/crates/agent_harness/Cargo.toml
@@ -32,7 +32,13 @@ serde = { workspace = true }
sqlx = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
-tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "time"] }
+tokio = { workspace = true, features = [
+ "macros",
+ "process",
+ "rt-multi-thread",
+ "sync",
+ "time",
+] }
tokio-util = { workspace = true, features = ["rt"] }
tokio-tungstenite = { workspace = true }
tracing = { workspace = true }
diff --git a/crates/agent_harness/container/Dockerfile b/crates/agent_harness/container/Dockerfile
index b1231d21ed..cb3563a6a6 100644
--- a/crates/agent_harness/container/Dockerfile
+++ b/crates/agent_harness/container/Dockerfile
@@ -33,6 +33,8 @@ RUN printf '%s\n' \
WORKDIR /env
COPY flake.nix flake.lock* ./
+# The flake exposes x86_64-linux and aarch64-linux. `nix develop` uses the
+# container's arch, so an unpinned `docker build` on Apple Silicon is native.
RUN nix develop --profile /env/.base-root path:/env --command true && \
nix print-dev-env path:/env > /env/dev-env.sh
diff --git a/crates/agent_harness/container/flake.nix b/crates/agent_harness/container/flake.nix
index 913214c212..d89ced7d83 100644
--- a/crates/agent_harness/container/flake.nix
+++ b/crates/agent_harness/container/flake.nix
@@ -5,12 +5,18 @@
outputs = { self, nixpkgs }:
let
- system = "x86_64-linux";
- pkgs = nixpkgs.legacyPackages.${system};
+ # Linux only: the image always runs in Docker, including on macOS.
+ # Unpinned `docker build` then bakes native amd64 or Apple Silicon / ARM
+ # Linux. GHCR / Daytona / Fly stay linux/amd64 and do not use this path.
+ forEachSystem = nixpkgs.lib.genAttrs [ "x86_64-linux" "aarch64-linux" ];
in
{
- devShells.${system}.default = pkgs.mkShell {
- packages = [ pkgs.nodejs pkgs.git pkgs.gh pkgs.opencode pkgs.github-mcp-server ];
- };
+ devShells = forEachSystem (system:
+ let pkgs = nixpkgs.legacyPackages.${system};
+ in {
+ default = pkgs.mkShell {
+ packages = [ pkgs.nodejs pkgs.git pkgs.gh pkgs.opencode pkgs.github-mcp-server ];
+ };
+ });
};
}
diff --git a/crates/agent_harness/justfile b/crates/agent_harness/justfile
index 3fde87d9c1..f9799e4a67 100644
--- a/crates/agent_harness/justfile
+++ b/crates/agent_harness/justfile
@@ -5,10 +5,18 @@
#
# Override the prompt with `just boot-daytona "your prompt"`.
#
-# Both recipes skip the image build when the image already exists, which is
+# `build-local` builds the same `container/` as a local Docker tag so
+# `just run_local` (which sets DEV_DANGEROUS_LOCAL_CONTAINERS) can spawn
+# sandboxes without a Daytona account. `run_local` / `stack up` already
+# `docker build` the tag with no `--platform` (BuildKit cache no-ops when
+# `container/` is unchanged); this recipe is the same unpinned build, so
+# Apple Silicon / ARM Linux bake native `aarch64-linux`.
+#
+# All recipes skip the image build when the image already exists, which is
# wrong whenever `container/` has changed - a stale image is reused silently and
# the failure shows up much later, inside the sandbox. Use `just rebuild-daytona`
-# / `rebuild-namespace` after touching the Dockerfile or the sidecar.
+# / `rebuild-namespace` / `build-local --force` after touching the Dockerfile
+# or the sidecar.
set dotenv-load := true
# What the image is called on both providers: a Daytona snapshot, and a
@@ -109,3 +117,18 @@ boot-namespace prompt=default_prompt:
fi
echo "booting $NAMESPACE_IMAGE_REF"
cargo run -p agent_harness --bin boot_agent -- namespace {{ quote(prompt) }}
+
+# Build the image for the local Docker provider if missing
+build-local force="":
+ #!/usr/bin/env bash
+ set -euo pipefail
+ # The same `container/` the Daytona snapshot is built from, so a local
+ # stack exercises the deployed image rather than an approximation of it.
+ # Slow on a cold build: the image bakes two nix dev shells.
+ tag={{image_name}}:latest
+ if [ -z "{{force}}" ] && docker image inspect "$tag" >/dev/null 2>&1; then
+ echo "image $tag already exists, skipping build (pass --force to rebuild)"
+ exit 0
+ fi
+ echo "building $tag (native platform; flake has x86_64-linux and aarch64-linux)"
+ docker build --tag "$tag" container
diff --git a/crates/agent_harness/src/outbound/containers.rs b/crates/agent_harness/src/outbound/containers.rs
new file mode 100644
index 0000000000..0b688e5757
--- /dev/null
+++ b/crates/agent_harness/src/outbound/containers.rs
@@ -0,0 +1,86 @@
+//! Which sandbox provider a deployment runs on.
+//!
+//! [`ContainerManager`] is a type parameter, so the choice between providers is
+//! a type-level one, and a composition root cannot make it from a config
+//! flag without something to hold both. This is that something: one enum
+//! that is a `ContainerManager` whichever arm it holds.
+//!
+//! Both arms happen to split into the same halves — every provider dials a
+//! sidecar over a websocket — so the transport needs no enum of its own, and
+//! only the manager does.
+
+use agent_runtime_protocol::domain::ports::Transport;
+use agent_runtime_protocol::domain::schema::v0::{ToRuntimeMessage, ToServerMessage};
+use agent_session::domain::model::AgentSessionId;
+
+use crate::domain::error::Result;
+use crate::domain::model::SpawnContainer;
+use crate::domain::ports::ContainerManager;
+use crate::outbound::daytona::{DaytonaContainer, DaytonaContainerManager};
+use crate::outbound::local::LocalContainerManager;
+use crate::outbound::sidecar::{SidecarSender, SidecarTransport};
+
+/// The provider a deployment hands sandboxes out through.
+#[derive(Clone)]
+pub enum HarnessContainers {
+ /// Daytona sandboxes: what a deployed harness runs on.
+ Daytona(DaytonaContainerManager),
+ /// Containers on the local Docker daemon: what a developer runs on.
+ Local(LocalContainerManager),
+}
+
+/// A sandbox from either provider.
+pub enum HarnessContainer {
+ /// A Daytona sandbox.
+ Daytona(DaytonaContainer),
+ /// A local Docker container.
+ Local(SidecarTransport),
+}
+
+impl HarnessContainers {
+ /// Stop everything the provider still owns, returning the number that
+ /// refused to stop.
+ pub async fn shutdown_all(&self) -> usize {
+ match self {
+ Self::Daytona(manager) => manager.shutdown_all().await,
+ Self::Local(manager) => manager.shutdown_all().await,
+ }
+ }
+}
+
+impl ContainerManager for HarnessContainers {
+ type Transport = HarnessContainer;
+
+ async fn spawn(&self, command: SpawnContainer) -> Result<Self::Transport> {
+ match self {
+ Self::Daytona(manager) => manager.spawn(command).await.map(HarnessContainer::Daytona),
+ Self::Local(manager) => manager.spawn(command).await.map(HarnessContainer::Local),
+ }
+ }
+
+ async fn resume(&self, session: AgentSessionId) -> Result<Self::Transport> {
+ match self {
+ Self::Daytona(manager) => manager.resume(session).await.map(HarnessContainer::Daytona),
+ Self::Local(manager) => manager.resume(session).await.map(HarnessContainer::Local),
+ }
+ }
+
+ async fn teardown(&self, session: AgentSessionId) -> Result<()> {
+ match self {
+ Self::Daytona(manager) => manager.teardown(session).await,
+ Self::Local(manager) => manager.teardown(session).await,
+ }
+ }
+}
+
+impl Transport<ToRuntimeMessage, ToServerMessage> for HarnessContainer {
+ type Sender = SidecarSender;
+ type Receiver = tokio::sync::mpsc::UnboundedReceiver<ToServerMessage>;
+
+ fn split(self) -> (Self::Sender, Self::Receiver) {
+ match self {
+ Self::Daytona(container) => container.split(),
+ Self::Local(transport) => transport.split(),
+ }
+ }
+}
diff --git a/crates/agent_harness/src/outbound/daytona/manager.rs b/crates/agent_harness/src/outbound/daytona/manager.rs
index a44b0d44a4..fdfe1326b3 100644
--- a/crates/agent_harness/src/outbound/daytona/manager.rs
+++ b/crates/agent_harness/src/outbound/daytona/manager.rs
@@ -16,7 +16,7 @@ use crate::domain::error::{HarnessError, Result};
use crate::domain::model::SpawnContainer;
use crate::domain::ports::ContainerManager;
use crate::outbound::managed_containers::ManagedContainers;
-use crate::outbound::provision;
+use crate::outbound::provision::{self, SESSION_LABEL};
use crate::outbound::sidecar::SidecarTransport;
const LOG_FETCH_TIMEOUT: Duration = Duration::from_secs(15);
@@ -24,7 +24,6 @@ const IDLE_TIMEOUT: Duration = Duration::from_secs(5 * 60);
const REAP_INTERVAL: Duration = Duration::from_secs(1);
const STOP_TIMEOUT: Duration = Duration::from_secs(30);
const STOP_CONCURRENCY: usize = 10;
-const SESSION_LABEL: &str = "macro.agent_session_id";
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
struct DaytonaSandboxId(String);
diff --git a/crates/agent_harness/src/outbound/local/docker.rs b/crates/agent_harness/src/outbound/local/docker.rs
new file mode 100644
index 0000000000..60f0739bbb
--- /dev/null
+++ b/crates/agent_harness/src/outbound/local/docker.rs
@@ -0,0 +1,244 @@
+//! A thin wrapper over the `docker` CLI.
+//!
+//! The CLI rather than the daemon's HTTP API on purpose: this provider exists
+//! for local development, where `docker` is already on PATH and already
+//! authenticated, and the verbs below are the whole of what a sandbox needs.
+//! Reaching for an API client would add a dependency to buy nothing.
+//!
+//! Every verb builds its argument list in a pure function so the wiring can be
+//! asserted without a daemon — the part that is easy to get wrong is the
+//! argument order, not the process spawn.
+
+use std::process::Stdio;
+use std::time::Duration;
+
+use super::errors::LocalError;
+
+#[cfg(test)]
+mod test;
+
+/// One container's identity, as both docker and a dialer need it.
+#[derive(Debug, Clone)]
+pub struct ContainerRef {
+ /// The name docker knows it by, which is also its DNS name on a network.
+ pub name: String,
+}
+
+/// What a container should be created with.
+#[derive(Debug, Clone)]
+pub struct RunSpec {
+ /// Image to run.
+ pub image: String,
+ /// Container name, deterministic per session.
+ pub name: String,
+ /// Labels, including the session label providers look containers up by.
+ pub labels: Vec<(String, String)>,
+ /// Environment handed to the sandbox.
+ pub env: Vec<(String, String)>,
+ /// Docker network the sandbox joins so the harness can dial it by name.
+ pub network: String,
+}
+
+/// Drives the local Docker daemon.
+#[derive(Debug, Clone)]
+pub struct Docker {
+ binary: String,
+}
+
+impl Docker {
+ /// Wrap a `docker`-compatible binary (`docker`, `podman`, a wrapper).
+ #[must_use]
+ pub fn new(binary: impl Into<String>) -> Self {
+ Self {
+ binary: binary.into(),
+ }
+ }
+
+ /// Whether `image` is present on the daemon.
+ pub async fn has_image(&self, image: &str) -> Result<bool, LocalError> {
+ let output = self.output(&image_inspect_args(image)).await?;
+ Ok(output.status.success())
+ }
+
+ /// Create and start a container, returning what to call it.
+ pub async fn run(&self, spec: &RunSpec) -> Result<ContainerRef, LocalError> {
+ self.run_checked("run", &run_args(spec)).await?;
+ Ok(ContainerRef {
+ name: spec.name.clone(),
+ })
+ }
+
+ /// Run one command inside a container, returning its combined output.
+ ///
+ /// A non-zero exit is returned as output plus status rather than an error:
+ /// the readiness recipe's failure is the caller's to report, together with
+ /// what the recipe said, and only the caller knows which of its own steps
+ /// is allowed to fail.
+ pub async fn exec(
+ &self,
+ container: &ContainerRef,
+ command: &str,
+ timeout: Duration,
+ ) -> Result<(i32, String), LocalError> {
+ let args = exec_args(&container.name, command);
+ let output = tokio::time::timeout(timeout, self.output(&args))
+ .await
+ .map_err(|_| LocalError::ExecTimeout {
+ container: container.name.clone(),
+ seconds: timeout.as_secs(),
+ })??;
+
+ let mut combined = String::from_utf8_lossy(&output.stdout).into_owned();
+ combined.push_str(&String::from_utf8_lossy(&output.stderr));
+ Ok((output.status.code().unwrap_or(-1), combined))
+ }
+
+ /// The container carrying `value` for `label`, if one exists.
+ pub async fn find_by_label(
+ &self,
+ label: &str,
+ value: &str,
+ ) -> Result<Option<ContainerRef>, LocalError> {
+ let args = find_by_label_args(label, value);
+ let output = self.run_checked("ps", &args).await?;
+ Ok(output
+ .lines()
+ .map(str::trim)
+ .find(|line| !line.is_empty())
+ .map(|name| ContainerRef {
+ name: name.to_owned(),
+ }))
+ }
+
+ /// Every container carrying `label`, running or not.
+ pub async fn find_all_by_label_key(
+ &self,
+ label: &str,
+ ) -> Result<Vec<ContainerRef>, LocalError> {
+ let args = find_all_by_label_key_args(label);
+ let output = self.run_checked("ps", &args).await?;
+ Ok(output
+ .lines()
+ .map(str::trim)
+ .filter(|line| !line.is_empty())
+ .map(|name| ContainerRef {
+ name: name.to_owned(),
+ })
+ .collect())
+ }
+
+ /// Start a container that exists but is stopped.
+ pub async fn start(&self, container: &ContainerRef) -> Result<(), LocalError> {
+ self.run_checked("start", &["start".to_owned(), container.name.clone()])
+ .await?;
+ Ok(())
+ }
+
+ /// Remove a container for good, running or not.
+ pub async fn remove(&self, container: &ContainerRef) -> Result<(), LocalError> {
+ self.run_checked(
+ "rm",
+ &["rm".to_owned(), "-f".to_owned(), container.name.clone()],
+ )
+ .await?;
+ Ok(())
+ }
+
+ /// Run a docker subcommand and fail on any non-zero exit.
+ async fn run_checked(&self, command: &str, args: &[String]) -> Result<String, LocalError> {
+ let output = self.output(args).await?;
+ if !output.status.success() {
+ return Err(LocalError::Command {
+ command: command.to_owned(),
+ status: output.status.code().unwrap_or(-1),
+ stderr: String::from_utf8_lossy(&output.stderr).trim().to_owned(),
+ });
+ }
+ Ok(String::from_utf8_lossy(&output.stdout).into_owned())
+ }
+
+ async fn output(&self, args: &[String]) -> Result<std::process::Output, LocalError> {
+ tokio::process::Command::new(&self.binary)
+ .args(args)
+ .stdin(Stdio::null())
+ .output()
+ .await
+ .map_err(|source| LocalError::Spawn {
+ binary: self.binary.clone(),
+ source,
+ })
+ }
+}
+
+/// `docker run` for one sandbox.
+///
+/// `sleep infinity` replaces the image's own `CMD`: the image's command is an
+/// interactive `bash`, which exits at once without a TTY, and the container has
+/// to outlive its own entrypoint for `docker exec` to have anything to enter.
+fn run_args(spec: &RunSpec) -> Vec<String> {
+ let mut args = vec![
+ "run".to_owned(),
+ "--detach".to_owned(),
+ "--name".to_owned(),
+ spec.name.clone(),
+ ];
+
+ for (key, value) in &spec.labels {
+ args.push("--label".to_owned());
+ args.push(format!("{key}={value}"));
+ }
+ for (key, value) in &spec.env {
+ args.push("--env".to_owned());
+ args.push(format!("{key}={value}"));
+ }
+
+ args.push("--network".to_owned());
+ args.push(spec.network.clone());
+
+ args.push(spec.image.clone());
+ args.push("sleep".to_owned());
+ args.push("infinity".to_owned());
+ args
+}
+
+/// `docker exec` running one shell command.
+///
+/// A login shell, matching the Daytona provider: the image puts its baked nix
+/// dev shell on `PATH` through `BASH_ENV`, which a non-login shell would skip.
+fn exec_args(name: &str, command: &str) -> Vec<String> {
+ vec![
+ "exec".to_owned(),
+ name.to_owned(),
+ "bash".to_owned(),
+ "-lc".to_owned(),
+ command.to_owned(),
+ ]
+}
+
+fn find_by_label_args(label: &str, value: &str) -> Vec<String> {
+ vec![
+ "ps".to_owned(),
+ // Stopped containers count: a resume is precisely the case where the
+ // container exists and is not running.
+ "--all".to_owned(),
+ "--filter".to_owned(),
+ format!("label={label}={value}"),
+ "--format".to_owned(),
+ "{{.Names}}".to_owned(),
+ ]
+}
+
+fn find_all_by_label_key_args(label: &str) -> Vec<String> {
+ vec![
+ "ps".to_owned(),
+ "--all".to_owned(),
+ "--filter".to_owned(),
+ format!("label={label}"),
+ "--format".to_owned(),
+ "{{.Names}}".to_owned(),
+ ]
+}
+
+fn image_inspect_args(image: &str) -> Vec<String> {
+ vec!["image".to_owned(), "inspect".to_owned(), image.to_owned()]
+}
diff --git a/crates/agent_harness/src/outbound/local/docker/test.rs b/crates/agent_harness/src/outbound/local/docker/test.rs
new file mode 100644
index 0000000000..190191c3b5
--- /dev/null
+++ b/crates/agent_harness/src/outbound/local/docker/test.rs
@@ -0,0 +1,87 @@
+use super::*;
+
+fn spec() -> RunSpec {
+ RunSpec {
+ image: "macro-agent-harness:latest".to_owned(),
+ name: "macro-agent-abc".to_owned(),
+ labels: vec![("macro.agent_session_id".to_owned(), "abc".to_owned())],
+ env: vec![(
+ "REPO_URL".to_owned(),
+ "https://github.com/macro-inc/macro".to_owned(),
+ )],
+ network: "macro_services".to_owned(),
+ }
+}
+
+/// The container has to outlive the image's own `CMD`, or there is nothing
+/// left for `docker exec` to enter.
+#[test]
+fn run_detaches_and_keeps_the_container_alive() {
+ let args = run_args(&spec());
+
+ assert_eq!(args[0], "run");
+ assert!(args.contains(&"--detach".to_owned()));
+ assert_eq!(
+ &args[args.len() - 3..],
+ ["macro-agent-harness:latest", "sleep", "infinity"]
+ );
+}
+
+#[test]
+fn run_passes_labels_and_environment_as_key_equals_value() {
+ let args = run_args(&spec());
+
+ let label = args.iter().position(|arg| arg == "--label").unwrap();
+ assert_eq!(args[label + 1], "macro.agent_session_id=abc");
+ let env = args.iter().position(|arg| arg == "--env").unwrap();
+ assert_eq!(args[env + 1], "REPO_URL=https://github.com/macro-inc/macro");
+}
+
+/// The harness is itself a container, so a published host port would be on the
+/// host's loopback, which is not this process. Sandboxes join the Compose
+/// network and are dialed by name instead.
+#[test]
+fn a_sandbox_joins_the_compose_network_and_publishes_nothing() {
+ let args = run_args(&spec());
+
+ let network = args.iter().position(|arg| arg == "--network").unwrap();
+ assert_eq!(args[network + 1], "macro_services");
+ assert!(!args.contains(&"--publish".to_owned()));
+}
+
+/// A login shell, so the image's baked nix dev shell is on `PATH`.
+#[test]
+fn exec_runs_the_command_through_a_login_shell() {
+ assert_eq!(
+ exec_args("macro-agent-abc", "echo hi"),
+ ["exec", "macro-agent-abc", "bash", "-lc", "echo hi"]
+ );
+}
+
+/// A resume looks up a container that exists and is *not* running, so the
+/// lookup must not be limited to running ones.
+#[test]
+fn finding_by_label_includes_stopped_containers() {
+ let args = find_by_label_args("macro.agent_session_id", "abc");
+
+ assert!(args.contains(&"--all".to_owned()));
+ assert!(args.contains(&"label=macro.agent_session_id=abc".to_owned()));
+}
+
+/// Shutdown has to see stopped sandboxes too, or a Ctrl-C after idle leaves
+/// them behind.
+#[test]
+fn listing_by_label_key_includes_stopped_containers() {
+ let args = find_all_by_label_key_args("macro.agent_session_id");
+
+ assert!(args.contains(&"--all".to_owned()));
+ assert!(args.contains(&"label=macro.agent_session_id".to_owned()));
+}
+
+#[test]
+fn image_inspect_names_the_image() {
+ assert_eq!(
+ image_inspect_args("macro-agent-harness:latest"),
+ ["image", "inspect", "macro-agent-harness:latest"]
+ );
+}
diff --git a/crates/agent_harness/src/outbound/local/errors.rs b/crates/agent_harness/src/outbound/local/errors.rs
new file mode 100644
index 0000000000..b54729f7db
--- /dev/null
+++ b/crates/agent_harness/src/outbound/local/errors.rs
@@ -0,0 +1,76 @@
+//! Failures the local Docker provider can produce.
+
+use thiserror::Error;
+
+/// Something went wrong driving the local container runtime.
+#[derive(Debug, Error)]
+pub enum LocalError {
+ /// The `docker` binary could not be run at all.
+ #[error("could not run `{binary}`: {source}")]
+ Spawn {
+ /// The binary we tried to run.
+ binary: String,
+ /// Why it could not be run.
+ #[source]
+ source: std::io::Error,
+ },
+
+ /// Docker ran and refused.
+ #[error("`docker {command}` failed with status {status}: {stderr}")]
+ Command {
+ /// The subcommand, for a message that says which step broke.
+ command: String,
+ /// Exit status docker reported.
+ status: i32,
+ /// Whatever docker said about it.
+ stderr: String,
+ },
+
+ /// A command in a container outlived its timeout.
+ #[error("`docker exec` in {container} did not finish within {seconds}s")]
+ ExecTimeout {
+ /// Container the command was running in.
+ container: String,
+ /// How long it was given.
+ seconds: u64,
+ },
+
+ /// The readiness recipe ran and failed inside the container.
+ #[error("readiness recipe failed in {container} with status {status}: {output}")]
+ ReadinessRecipe {
+ /// Container the recipe ran in.
+ container: String,
+ /// Exit status of the recipe.
+ status: i32,
+ /// Combined output, for the person reading the log.
+ output: String,
+ },
+
+ /// The sandbox image is not on the daemon.
+ #[error(
+ "sandbox image `{image}` is missing; build it with `just -f crates/agent_harness/justfile build-local`"
+ )]
+ ImageMissing {
+ /// Image the local provider was configured to run.
+ image: String,
+ },
+
+ /// The sidecar never answered its readiness probe.
+ #[error("sidecar in {container} was not ready within {seconds}s")]
+ NotReady {
+ /// Container whose sidecar stayed silent.
+ container: String,
+ /// How long we waited.
+ seconds: u64,
+ },
+
+ /// The sidecar socket could not be dialed.
+ #[error("could not dial the sidecar at {url}: {source}")]
+ WebSocketConnect {
+ /// Address we dialed.
+ url: String,
+ /// Why the dial failed.
+ #[source]
+ source: tokio_tungstenite::tungstenite::Error,
+ },
+}
diff --git a/crates/agent_harness/src/outbound/local/manager.rs b/crates/agent_harness/src/outbound/local/manager.rs
new file mode 100644
index 0000000000..a728544fdf
--- /dev/null
+++ b/crates/agent_harness/src/outbound/local/manager.rs
@@ -0,0 +1,290 @@
+use std::time::Duration;
+
+use agent_session::domain::model::AgentSessionId;
+
+use super::docker::{ContainerRef, Docker, RunSpec};
+use super::errors::LocalError;
+use crate::domain::error::{HarnessError, Result};
+use crate::domain::model::SpawnContainer;
+use crate::domain::ports::ContainerManager;
+use crate::outbound::daytona::GithubToken;
+use crate::outbound::provision::{self, SESSION_LABEL};
+use crate::outbound::sidecar::SidecarTransport;
+
+#[cfg(test)]
+mod test;
+
+/// How long to keep asking the sidecar whether it is up.
+const PING_INTERVAL: Duration = Duration::from_millis(250);
+
+/// Settings for the local Docker-backed provider.
+pub struct LocalSettings {
+ /// The `docker`-compatible binary to drive.
+ pub docker_binary: String,
+ /// Image sandboxes are created from.
+ pub image: String,
+ /// Compose network the sandbox joins so this service can dial it by name.
+ pub network: String,
+ /// Token with read access to the repository cloned into sandboxes.
+ pub github_token: GithubToken,
+}
+
+/// Hands out containers on the local Docker daemon.
+///
+/// The development counterpart to the Daytona provider, and deliberately the
+/// same shape: it runs the same image, execs the same readiness recipe, and
+/// dials the same sidecar, so what it exercises is what a deployed sandbox
+/// does.
+///
+/// What it does not have is Daytona's idle reaper. A local sandbox costs
+/// nothing per hour; `teardown` and [`Self::shutdown_all`] still remove
+/// containers so nothing outlives a deleted session or a stopped stack.
+#[derive(Clone)]
+pub struct LocalContainerManager {
+ docker: Docker,
+ image: String,
+ network: String,
+ github_token: GithubToken,
+}
+
+impl LocalContainerManager {
+ /// Build the manager from its settings.
+ #[must_use]
+ pub fn new(settings: LocalSettings) -> Self {
+ let LocalSettings {
+ docker_binary,
+ image,
+ network,
+ github_token,
+ } = settings;
+ Self {
+ docker: Docker::new(docker_binary),
+ image,
+ network,
+ github_token,
+ }
+ }
+
+ /// Stop every sandbox this provider still owns, returning how many refused.
+ ///
+ /// Compose does not reap siblings created over the mounted Docker socket,
+ /// so without this a `just run_local` Ctrl-C leaves `macro-agent-*`
+ /// containers on the network.
+ pub async fn shutdown_all(&self) -> usize {
+ let containers = match self.docker.find_all_by_label_key(SESSION_LABEL).await {
+ Ok(containers) => containers,
+ Err(error) => {
+ tracing::error!(error = ?error, "failed to list local sandboxes for shutdown");
+ return 1;
+ }
+ };
+
+ let mut failures = 0;
+ for container in containers {
+ if let Err(error) = self.docker.remove(&container).await {
+ tracing::error!(
+ error = ?error,
+ container = %container.name,
+ "failed to remove a local sandbox on shutdown"
+ );
+ failures += 1;
+ }
+ }
+ failures
+ }
+
+ /// Run the readiness recipe, then dial the sidecar behind it.
+ async fn bring_up(&self, container: &ContainerRef) -> Result<SidecarTransport> {
+ let (status, output) = self
+ .docker
+ .exec(
+ container,
+ &provision::ensure_ready_command(),
+ provision::ENSURE_TIMEOUT,
+ )
+ .await
+ .map_err(unavailable)?;
+ if status != 0 {
+ return Err(unavailable(LocalError::ReadinessRecipe {
+ container: container.name.clone(),
+ status,
+ output,
+ }));
+ }
+ tracing::info!(container = %container.name, %output, "readiness recipe finished");
+
+ let address = sidecar_address(container);
+ self.wait_for_ping(container, &address).await?;
+
+ let url = format!("ws://{address}");
+ let (socket, _) = tokio_tungstenite::connect_async(&url)
+ .await
+ .map_err(|source| {
+ unavailable(LocalError::WebSocketConnect {
+ url: url.clone(),
+ source,
+ })
+ })?;
+ Ok(SidecarTransport::connect(socket))
+ }
+
+ /// Poll the sidecar's `/ping` until it answers or we give up.
+ ///
+ /// The recipe backgrounds the sidecar with `nohup`, so it has returned
+ /// before the sidecar is listening; dialing the websocket immediately would
+ /// race it.
+ async fn wait_for_ping(&self, container: &ContainerRef, address: &str) -> Result<()> {
+ let client = reqwest::Client::new();
+ let url = format!("http://{address}/ping");
+ let deadline = tokio::time::Instant::now() + provision::PING_TIMEOUT;
+
+ while tokio::time::Instant::now() < deadline {
+ if let Ok(response) = client.get(&url).send().await
+ && response.status().is_success()
+ {
+ return Ok(());
+ }
+ tokio::time::sleep(PING_INTERVAL).await;
+ }
+
+ // Say what the sidecar said before it failed to come up; without this
+ // the only symptom is a timeout with no cause attached.
+ if let Ok((_status, log)) = self
+ .docker
+ .exec(
+ container,
+ &format!("tail -50 {} 2>&1 || true", provision::SIDECAR_LOG),
+ Duration::from_secs(15),
+ )
+ .await
+ {
+ tracing::error!(container = %container.name, sidecar_log = %log, "sidecar log");
+ }
+
+ Err(unavailable(LocalError::NotReady {
+ container: container.name.clone(),
+ seconds: provision::PING_TIMEOUT.as_secs(),
+ }))
+ }
+
+ async fn find(&self, session: AgentSessionId) -> Result<Option<ContainerRef>> {
+ self.docker
+ .find_by_label(SESSION_LABEL, &session.to_string())
+ .await
+ .map_err(unavailable)
+ }
+
+ /// Remove a container that never came up, so a retry is not blocked by the
+ /// name it already took.
+ async fn discard(&self, container: &ContainerRef) {
+ let _ = self.docker.remove(container).await.inspect_err(|error| {
+ tracing::error!(
+ error = ?error,
+ container = %container.name,
+ "failed to remove a container that never came up"
+ );
+ });
+ }
+}
+
+impl ContainerManager for LocalContainerManager {
+ type Transport = SidecarTransport;
+
+ #[tracing::instrument(err, skip(self))]
+ async fn spawn(&self, command: SpawnContainer) -> Result<Self::Transport> {
+ let SpawnContainer {
+ session_id,
+ repo_url,
+ } = command;
+
+ if !self
+ .docker
+ .has_image(&self.image)
+ .await
+ .map_err(unavailable)?
+ {
+ return Err(unavailable(LocalError::ImageMissing {
+ image: self.image.clone(),
+ }));
+ }
+
+ // A previous run of the same session leaves a container holding this
+ // name, and docker refuses to reuse it. Spawning is the point at which
+ // the old one is definitively stale.
+ if let Some(existing) = self.find(session_id).await? {
+ tracing::warn!(
+ container = %existing.name,
+ session = %session_id,
+ "removing a stale container for a session being spawned"
+ );
+ self.discard(&existing).await;
+ }
+
+ let spec = RunSpec {
+ image: self.image.clone(),
+ name: container_name(session_id),
+ labels: vec![(SESSION_LABEL.to_owned(), session_id.to_string())],
+ env: sandbox_env(repo_url, &self.github_token),
+ network: self.network.clone(),
+ };
+ let container = self.docker.run(&spec).await.map_err(unavailable)?;
+ tracing::info!(container = %container.name, session = %session_id, "container created");
+
+ match self.bring_up(&container).await {
+ Ok(transport) => Ok(transport),
+ Err(error) => {
+ self.discard(&container).await;
+ Err(error)
+ }
+ }
+ }
+
+ #[tracing::instrument(err, skip(self))]
+ async fn resume(&self, session: AgentSessionId) -> Result<Self::Transport> {
+ let container = self.find(session).await?.ok_or_else(|| {
+ HarnessError::Container(format!("session {session} has no container to resume"))
+ })?;
+
+ // Idempotent: docker starts a stopped container and says nothing about
+ // one already running.
+ self.docker.start(&container).await.map_err(unavailable)?;
+ self.bring_up(&container).await
+ }
+
+ #[tracing::instrument(err, skip(self))]
+ async fn teardown(&self, session: AgentSessionId) -> Result<()> {
+ let Some(container) = self.find(session).await? else {
+ // Already the state the caller asked for.
+ return Ok(());
+ };
+ self.docker.remove(&container).await.map_err(unavailable)?;
+ tracing::info!(container = %container.name, %session, "container removed");
+ Ok(())
+ }
+}
+
+/// The container name for a session.
+///
+/// Deterministic so a session has at most one container and `docker ps` reads
+/// legibly, and prefixed so a developer can tell Macro's sandboxes apart from
+/// the rest of their daemon.
+fn container_name(session: AgentSessionId) -> String {
+ format!("macro-agent-{session}")
+}
+
+/// The sidecar keeps its own port; on a shared network the container name is
+/// its DNS name.
+fn sidecar_address(container: &ContainerRef) -> String {
+ format!("{}:{}", container.name, provision::SIDECAR_PORT)
+}
+
+fn sandbox_env(repo_url: String, github_token: &GithubToken) -> Vec<(String, String)> {
+ vec![
+ ("REPO_URL".to_owned(), repo_url),
+ ("GITHUB_TOKEN".to_owned(), github_token.expose().to_owned()),
+ ]
+}
+
+fn unavailable(error: LocalError) -> HarnessError {
+ HarnessError::Container(error.to_string())
+}
diff --git a/crates/agent_harness/src/outbound/local/manager/test.rs b/crates/agent_harness/src/outbound/local/manager/test.rs
new file mode 100644
index 0000000000..36ef342eca
--- /dev/null
+++ b/crates/agent_harness/src/outbound/local/manager/test.rs
@@ -0,0 +1,39 @@
+use super::*;
+use crate::outbound::daytona::GithubToken;
+
+/// One container per session, so a resume finds exactly one and `docker ps`
+/// says which session it belongs to.
+#[test]
+fn a_session_names_one_container() {
+ let session = AgentSessionId::TEST_A;
+
+ assert_eq!(container_name(session), format!("macro-agent-{session}"));
+}
+
+/// On a shared network the sidecar keeps its own port and the container name
+/// is its DNS name.
+#[test]
+fn a_sidecar_is_dialed_by_container_name() {
+ assert_eq!(
+ sidecar_address(&ContainerRef {
+ name: "macro-agent-abc".to_owned(),
+ }),
+ format!("macro-agent-abc:{}", provision::SIDECAR_PORT)
+ );
+}
+
+/// Same clone credentials Daytona injects, so the readiness recipe is
+/// exercised against the same environment a deployed sandbox sees.
+#[test]
+fn sandbox_env_carries_the_repo_and_github_token() {
+ let env = sandbox_env(
+ "https://github.com/macro-inc/macro".to_owned(),
+ &GithubToken::new("test-token".to_owned()),
+ );
+
+ assert!(env.contains(&(
+ "REPO_URL".to_owned(),
+ "https://github.com/macro-inc/macro".to_owned()
+ )));
+ assert!(env.contains(&("GITHUB_TOKEN".to_owned(), "test-token".to_owned())));
+}
diff --git a/crates/agent_harness/src/outbound/local/mod.rs b/crates/agent_harness/src/outbound/local/mod.rs
new file mode 100644
index 0000000000..307c556d6a
--- /dev/null
+++ b/crates/agent_harness/src/outbound/local/mod.rs
@@ -0,0 +1,9 @@
+//! Sandboxes on the local Docker daemon, for developing against the real
+//! image without a Daytona account.
+
+mod docker;
+mod errors;
+mod manager;
+
+pub use errors::LocalError;
+pub use manager::{LocalContainerManager, LocalSettings};
diff --git a/crates/agent_harness/src/outbound/mod.rs b/crates/agent_harness/src/outbound/mod.rs
index faba70fdcb..4c187ed161 100644
--- a/crates/agent_harness/src/outbound/mod.rs
+++ b/crates/agent_harness/src/outbound/mod.rs
@@ -2,7 +2,9 @@
//! ports are satisfied by.
pub mod channel_announcer;
+pub mod containers;
pub mod daytona;
+pub mod local;
pub(crate) mod managed_containers;
pub mod namespace;
pub(crate) mod provision;
diff --git a/crates/agent_harness/src/outbound/provision.rs b/crates/agent_harness/src/outbound/provision.rs
index b0e9e9de4b..65afd1fef8 100644
--- a/crates/agent_harness/src/outbound/provision.rs
+++ b/crates/agent_harness/src/outbound/provision.rs
@@ -14,6 +14,13 @@ pub const SIDECAR_LOG: &str = "/tmp/acp-sidecar.log";
/// Port exposed by the ACP sidecar.
pub const SIDECAR_PORT: u16 = 8700;
+/// Provider-side label carrying the session a container belongs to.
+///
+/// Shared by every provider because it is what `resume` and `teardown` look a
+/// container up by: the harness knows only the session id, and the container is
+/// whatever the provider tagged with it.
+pub const SESSION_LABEL: &str = "macro.agent_session_id";
+
/// Readiness recipe baked alongside the harness container.
const ENSURE_READY_SCRIPT: &str = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev
index f18f1054bb..9610028901 100644
--- a/docker/Dockerfile.dev
+++ b/docker/Dockerfile.dev
@@ -45,8 +45,11 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=shared \
# Base runtime image
FROM debian:trixie-slim AS base
+# docker-cli, for the agent harness's local container provider: it drives the
+# host's daemon over the mounted socket, so sandboxes are this container's
+# siblings. The client tarball only - the daemon is the host's.
RUN apt-get update && apt-get install -y --no-install-recommends \
- ca-certificates dumb-init curl \
+ ca-certificates dumb-init curl docker-cli \
&& apt-get clean && rm -rf /var/lib/apt/lists/*
WORKDIR /app
ENV PORT=8080
diff --git a/docker/Dockerfile.runtime b/docker/Dockerfile.runtime
index 50d225b59b..7b4c1c59cf 100644
--- a/docker/Dockerfile.runtime
+++ b/docker/Dockerfile.runtime
@@ -8,8 +8,11 @@
# rebuilding when Rust code changes. This is the `base` stage of docker/Dockerfile.dev
# with the in-Docker `cargo build` and binary COPY removed.
FROM debian:trixie-slim
+# docker-cli, for the agent harness's local container provider: it drives the
+# host's daemon over the socket the compose file mounts, so sandboxes are this
+# container's siblings. The client only - the daemon is the host's.
RUN apt-get update && apt-get install -y --no-install-recommends \
- ca-certificates dumb-init curl \
+ ca-certificates dumb-init curl docker-cli \
&& apt-get clean && rm -rf /var/lib/apt/lists/*
WORKDIR /app
ENV PORT=8080
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index 247f1c8f49..95226b3641 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -300,8 +300,9 @@ services:
# Channel messages -> agent session trigger events -> sandboxed coding agent
# sessions, plus the session control routes, which have to be served from the
# process that owns the live transports they act on.
- # Managed sandboxes require DAYTONA_API_KEY and GITHUB_TOKEN; trigger
- # processing and external sessions run without them.
+ # Daytona-backed sandboxes require DAYTONA_API_KEY; `just run_local` sets
+ # DEV_DANGEROUS_LOCAL_CONTAINERS and runs sandboxes on the host Docker daemon
+ # instead. Trigger processing and external sessions run without either.
# ============================================================================
agent_harness_service:
<<: [*common-env, *rust-services-image]
@@ -313,6 +314,11 @@ services:
# In-network gateway address; the local default (http://localhost:8082)
# would point at this container itself.
- OVERRIDE_CONNECTION_GATEWAY_URL=http://connection-gateway:8080
+ volumes:
+ # The local container provider drives the host's Docker daemon, so
+ # sandboxes are siblings of this container rather than children. Harmless
+ # under the Daytona provider, which never opens it.
+ - /var/run/docker.sock:/var/run/docker.sock
depends_on:
- postgres
- kafka
diff --git a/services/agent_harness_service/src/config.rs b/services/agent_harness_service/src/config.rs
index acbd8408a7..ed22ba1124 100644
--- a/services/agent_harness_service/src/config.rs
+++ b/services/agent_harness_service/src/config.rs
@@ -41,6 +41,27 @@ pub struct Config {
/// unaffected.
#[macro_config_default(String::new())]
pub github_token: String,
+ /// Run sandboxes on the local Docker daemon instead of Daytona.
+ ///
+ /// Default off: a deployed harness must keep using Daytona even if this
+ /// binary is started with a copied local env file. `just run_local` sets
+ /// it. Refused at boot unless `ENVIRONMENT=local`, because this path
+ /// drives the host Docker daemon over a mounted socket.
+ #[macro_config_default(false)]
+ pub dev_dangerous_local_containers: bool,
+ /// `docker`-compatible binary the local provider drives.
+ #[macro_config_default(String::from("docker"))]
+ pub local_container_docker_binary: String,
+ /// Image the local provider creates sandboxes from.
+ #[macro_config_default(String::from("macro-agent-harness:latest"))]
+ pub local_container_image: String,
+ /// Compose network local sandboxes join so this service can dial them.
+ ///
+ /// Required when `dev_dangerous_local_containers` is on: the harness is
+ /// itself a container, so the address that works is one on a network both
+ /// share. `just run_local` sets `{project}_services`.
+ #[macro_config_default(String::new())]
+ pub local_container_network: String,
/// The bot this deployment answers for.
///
/// Configuration rather than a constant: `@claude` and `@codex` are separate
diff --git a/services/agent_harness_service/src/main.rs b/services/agent_harness_service/src/main.rs
index 84fb699156..4c52100215 100644
--- a/services/agent_harness_service/src/main.rs
+++ b/services/agent_harness_service/src/main.rs
@@ -1,8 +1,8 @@
//! Composition root for the agent harness service.
//!
//! The hexagon lives in `crates/agent_harness`; this binary is the shell
-//! around it: it builds the Postgres repositories, the Daytona container
-//! manager, and a channel service with the full side-effect stack for
+//! around it: it builds the Postgres repositories, the container
+//! manager (Daytona, or local Docker when opted in), and a channel service with the full side-effect stack for
//! announcements, derives agent triggers from `macro.channels`, then drives
//! the orchestrator from the resulting `macro.agent_sessions` events.
@@ -19,10 +19,12 @@ use agent_harness::domain::service::AgentHarnessService;
use agent_harness::inbound::kafka::{RoutedTrigger, route_agent_trigger};
use agent_harness::inbound::runtime_gateway::RuntimeGatewayState;
use agent_harness::outbound::channel_announcer::ChannelAnnouncer;
+use agent_harness::outbound::containers::HarnessContainers;
use agent_harness::outbound::daytona::{
DaytonaApiKey as DaytonaApiKeySecret, DaytonaContainerManager, DaytonaSettings,
GithubToken as GithubTokenSecret, Snapshot,
};
+use agent_harness::outbound::local::{LocalContainerManager, LocalSettings};
use agent_harness::outbound::runtime_registry::RuntimeRegistry;
use agent_session::domain::ports::NoOpRealtime;
use agent_session::domain::service::AgentSessionServiceImpl;
@@ -43,7 +45,7 @@ use channels::outbound::contacts_dispatcher::ContactsChannelDispatcher;
use channels::outbound::notification_sender::NotificationChannelSender;
use channels::outbound::pg_channels_repo::PgChannelsRepo;
use channels::outbound::pg_side_effect_context::PgChannelSideEffectContext;
-use config::Config;
+use config::{Config, Environment};
use connection_gateway_client::ConnectionGatewayClient;
use kafka_util::{GroupName, KafkaEventConsumer};
use lexical_client::LexicalClient;
@@ -91,13 +93,6 @@ async fn main() -> anyhow::Result<()> {
agent_harness::install_tls_provider();
let config = Config::from_env()?;
let bot_id = BotId::new_from_uuid(config.harness_bot_id);
- // Credential-less boot is deliberate: external sessions need neither.
- // A managed spawn without them fails at spawn time instead, loudly.
- if config.daytona_api_key.trim().is_empty() || config.github_token.trim().is_empty() {
- tracing::warn!(
- "DAYTONA_API_KEY and/or GITHUB_TOKEN are unset: managed sandboxes are unarmed; external agent sessions are unaffected"
- );
- }
let pool = PgPoolOptions::new()
.min_connections(1)
@@ -128,13 +123,43 @@ async fn main() -> anyhow::Result<()> {
),
);
- // Containers: Daytona sandboxes.
- let containers = DaytonaContainerManager::new(DaytonaSettings {
- api_url: config.daytona_api_url.clone(),
- api_key: DaytonaApiKeySecret::new(config.daytona_api_key.clone()),
- snapshot: Snapshot::new(config.daytona_snapshot.clone()),
- github_token: GithubTokenSecret::new(config.github_token.clone()),
- });
+ // Containers: local Docker when a developer has opted in, Daytona otherwise.
+ let containers = if config.dev_dangerous_local_containers {
+ if !matches!(config.environment, Environment::Local) {
+ anyhow::bail!("DEV_DANGEROUS_LOCAL_CONTAINERS is only allowed when ENVIRONMENT=local");
+ }
+ if config.github_token.trim().is_empty() {
+ tracing::warn!(
+ "GITHUB_TOKEN is unset: local sandboxes cannot clone private repositories; external agent sessions are unaffected"
+ );
+ }
+ let network = config.local_container_network.trim();
+ if network.is_empty() {
+ anyhow::bail!(
+ "LOCAL_CONTAINER_NETWORK is required when DEV_DANGEROUS_LOCAL_CONTAINERS is set"
+ );
+ }
+ HarnessContainers::Local(LocalContainerManager::new(LocalSettings {
+ docker_binary: config.local_container_docker_binary.clone(),
+ image: config.local_container_image.clone(),
+ network: network.to_owned(),
+ github_token: GithubTokenSecret::new(config.github_token.clone()),
+ }))
+ } else {
+ // Credential-less boot is deliberate: external sessions need neither.
+ // A managed spawn without them fails at spawn time instead, loudly.
+ if config.daytona_api_key.trim().is_empty() || config.github_token.trim().is_empty() {
+ tracing::warn!(
+ "DAYTONA_API_KEY and/or GITHUB_TOKEN are unset: managed sandboxes are unarmed; external agent sessions are unaffected"
+ );
+ }
+ HarnessContainers::Daytona(DaytonaContainerManager::new(DaytonaSettings {
+ api_url: config.daytona_api_url.clone(),
+ api_key: DaytonaApiKeySecret::new(config.daytona_api_key.clone()),
+ snapshot: Snapshot::new(config.daytona_snapshot.clone()),
+ github_token: GithubTokenSecret::new(config.github_token.clone()),
+ }))
+ };
let container_shutdown = containers.clone();
let aws_config = macro_aws_config::get_macro_aws_config().await;
@@ -390,7 +415,7 @@ async fn main() -> anyhow::Result<()> {
let stop_failures = container_shutdown.shutdown_all().await;
if stop_failures > 0 {
- tracing::error!(stop_failures, "some Daytona sandboxes failed to stop");
+ tracing::error!(stop_failures, "some sandboxes failed to stop");
}
match run_error {
diff --git a/tooling/xtask/crates/xtask_local/src/local/inventory.rs b/tooling/xtask/crates/xtask_local/src/local/inventory.rs
index 814c295de9..6fcf04dfec 100644
--- a/tooling/xtask/crates/xtask_local/src/local/inventory.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/inventory.rs
@@ -251,8 +251,9 @@ pub const RUST_SERVICES: &[RustService] = &[
host_port: Some(Port::AgentHarness),
path_prefix: Some("/agent-harness"),
is_websocket: false,
- // Runs without sandbox credentials so channel triggers and external
- // sessions remain available; managed spawns fail loudly when unarmed.
+ // Local stacks default to DEV_DANGEROUS_LOCAL_CONTAINERS, so managed
+ // sandboxes run on the host Docker daemon. Daytona is opt-in via
+ // DEV_DANGEROUS_LOCAL_CONTAINERS=false DAYTONA_API_KEY=... just run_local.
modes: &[Mode::Local],
opt_in: false,
no_default_features: false,
diff --git a/tooling/xtask/crates/xtask_local/src/local/local_env.rs b/tooling/xtask/crates/xtask_local/src/local/local_env.rs
index fbb186cab0..671eb52623 100644
--- a/tooling/xtask/crates/xtask_local/src/local/local_env.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/local_env.rs
@@ -57,7 +57,7 @@ impl LocalEnv {
storage: StorageEnv::local(),
queues: QueueEnv::local(),
mail: MailEnv::local(),
- agent_harness: AgentHarnessEnv::local(),
+ agent_harness: AgentHarnessEnv::local(instance.project_name()),
service_auth: ServiceAuthEnv::for_instance(name),
fusionauth: FusionAuthEnv::for_instance(instance),
boot_stubs: BootStubEnv,
@@ -249,23 +249,35 @@ impl MailEnv {
}
}
-/// The agent harness: which bot it answers for, and the Daytona sandbox
-/// credentials.
+/// The agent harness: which bot it answers for, and where its sandboxes come
+/// from.
///
-/// The bot id and snapshot name are deterministic (the bot is seeded by
-/// migration). The two secrets are seeded empty so the process-env overlay can
-/// replace them; the harness refuses to start unless both are supplied.
+/// Local stacks run sandboxes on the developer's own Docker daemon, so no
+/// Daytona account is involved. `DAYTONA_API_KEY` is seeded empty so Doppler
+/// cannot bill Daytona, while still leaving the key in the map for
+/// `DAYTONA_API_KEY=... DEV_DANGEROUS_LOCAL_CONTAINERS=false just run_local`.
+/// `GITHUB_TOKEN` is left to Doppler / process env so a local sandbox can
+/// still clone.
struct AgentHarnessEnv {
bot_id: &'static str,
snapshot: &'static str,
+ /// Image the local provider runs. `run_local` / `stack up` `docker build`
+ /// `crates/agent_harness/container` to this tag (BuildKit cache is the
+ /// freshness check).
+ image: &'static str,
+ /// Network sandboxes join, so this service can dial their sidecars.
+ network: String,
}
impl AgentHarnessEnv {
- fn local() -> Self {
+ fn local(project_name: &str) -> Self {
AgentHarnessEnv {
// bot_id::MACRO_CODER_BOT_ID, seeded by the bots_has_agent migration.
bot_id: "00000000-0000-0000-0000-00000000a9e7",
snapshot: "macro-agent-harness",
+ image: super::sandbox_image::DEFAULT_LOCAL_TAG,
+ // Compose names a network `<project>_<network>`.
+ network: format!("{project_name}_services"),
}
}
@@ -273,7 +285,9 @@ impl AgentHarnessEnv {
env.insert("HARNESS_BOT_ID".into(), self.bot_id.into());
env.insert("DAYTONA_SNAPSHOT".into(), self.snapshot.into());
env.insert("DAYTONA_API_KEY".into(), String::new());
- env.insert("GITHUB_TOKEN".into(), String::new());
+ env.insert("DEV_DANGEROUS_LOCAL_CONTAINERS".into(), "true".into());
+ env.insert("LOCAL_CONTAINER_IMAGE".into(), self.image.into());
+ env.insert("LOCAL_CONTAINER_NETWORK".into(), self.network.clone());
}
}
@@ -464,6 +478,11 @@ impl BootStubEnv {
"GITHUB_IDP_ID".into(),
"99999999-9999-4999-8999-999999999999".into(),
);
+ // Clone token for local sandboxes. Empty here so `--no-doppler` still
+ // has the key for `GITHUB_TOKEN=... just run_local`; Doppler overlays
+ // a real token when present. Not in `to_env`, so a local stack does
+ // not wipe Doppler's value the way it wipes `DAYTONA_API_KEY`.
+ env.insert("GITHUB_TOKEN".into(), String::new());
env.insert("STRIPE_SECRET_KEY".into(), "local-stripe-secret".into());
env.insert("STRIPE_PRICE_ID".into(), "local-stripe-price".into());
env.insert(
diff --git a/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs b/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs
index 103249b331..9f8c00ddee 100644
--- a/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs
@@ -87,6 +87,12 @@ fn emits_required_keys() {
"CAL_EVENT_TYPE_CONTENT_NAMES_KEY",
"META_PIXEL_ID",
"META_ACCESS_TOKEN",
+ "GITHUB_TOKEN",
+ "DEV_DANGEROUS_LOCAL_CONTAINERS",
+ "LOCAL_CONTAINER_IMAGE",
+ "LOCAL_CONTAINER_NETWORK",
+ "DAYTONA_API_KEY",
+ "HARNESS_BOT_ID",
] {
assert!(
env.contains_key(key),
@@ -256,3 +262,46 @@ fn fusionauth_public_url_uses_the_instance_host_port() {
Some(named_public_url.as_str())
);
}
+
+/// A local stack runs sandboxes on the developer's own daemon, so it needs no
+/// Daytona account to exercise the sandbox path.
+#[test]
+fn the_agent_harness_uses_local_containers_and_wipes_daytona() {
+ let env = local_env();
+
+ assert_eq!(
+ env.get("DEV_DANGEROUS_LOCAL_CONTAINERS")
+ .map(String::as_str),
+ Some("true")
+ );
+ assert_eq!(env.get("DAYTONA_API_KEY").map(String::as_str), Some(""));
+ // Present so `GITHUB_TOKEN=... just run_local` overlays, but empty in a
+ // no-Doppler stack. Doppler would supply a real token above this stub.
+ assert_eq!(env.get("GITHUB_TOKEN").map(String::as_str), Some(""));
+}
+
+/// Sandboxes and the harness are both containers, so they reach each other on a
+/// shared Compose network and never over the host's loopback. The network name
+/// has to track the instance, or a named stack's sandboxes join the wrong one.
+#[test]
+fn local_sandboxes_join_the_instances_compose_network() {
+ let named = Instance::derive(Some("2508"), None).unwrap();
+ let default_env =
+ LocalEnv::for_instance(Mode::Local, &Instance::derive(None, None).unwrap(), true).to_env();
+ let named_env = LocalEnv::for_instance(Mode::Local, &named, true).to_env();
+
+ assert_eq!(
+ default_env
+ .get("LOCAL_CONTAINER_NETWORK")
+ .map(String::as_str),
+ Some("macro_services")
+ );
+ assert_eq!(
+ named_env.get("LOCAL_CONTAINER_NETWORK").map(String::as_str),
+ Some(format!("{}_services", named.project_name()).as_str())
+ );
+ assert_eq!(
+ default_env.get("LOCAL_CONTAINER_IMAGE").map(String::as_str),
+ Some("macro-agent-harness:latest")
+ );
+}
diff --git a/tooling/xtask/crates/xtask_local/src/local/mod.rs b/tooling/xtask/crates/xtask_local/src/local/mod.rs
index 654534bf1d..012bd1786a 100644
--- a/tooling/xtask/crates/xtask_local/src/local/mod.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/mod.rs
@@ -36,6 +36,7 @@ pub mod opensearch;
pub mod portmap;
pub mod proxy;
pub mod resources;
+pub mod sandbox_image;
pub mod sdk_webhook;
pub mod seed_env;
pub mod snapshot;
@@ -239,7 +240,7 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {
// Foreground: resolve env, build binaries + runtime image, generate the
// compose override / Caddyfile / kickstart. None of this touches the volumes
// or containers the teardown is removing, so it's safe to overlap.
- let (env, target) = prepare(&stage, mode, &instance, args, false, false)?;
+ let (env, target) = prepare(&stage, mode, &instance, args, false, false, false)?;
// Join the background teardown before we (re)create volumes + bring infra up,
// surfaced as a live spinner so it's clear what we're blocked on. It
@@ -496,7 +497,9 @@ fn interact(
/// Caddyfile / kickstart. Deliberately does NOT create the external
/// networks/volumes — that's done after the background teardown joins, since
/// teardown removes them. `static_frontend` wires the proxy to serve the staged
-/// app bundle (headless `stack up`). Returns the resolved env + build target.
+/// app bundle (headless `stack up`). `infra_only` skips zigbuild: bake never
+/// starts Rust services and runs in parallel with the cargo lane. Returns the
+/// resolved env + build target.
fn prepare(
stage: &Stage,
mode: Mode,
@@ -504,6 +507,7 @@ fn prepare(
args: &cli::RunArgs,
static_frontend: bool,
pull_app_images: bool,
+ infra_only: bool,
) -> Result<(env_layer::ResolvedEnv, arch::Target)> {
let env = env_layer::resolve(
mode,
@@ -513,18 +517,25 @@ fn prepare(
static_frontend,
)?;
stage.note(&format!("env: {}", env_layer::summarize(&env.merged)));
+ sandbox_image::ensure(stage, &env.merged, args.build.no_build)?;
// Build the runtime image (idempotent) and the service binaries.
let target = arch::detect()?;
build::ensure_runtime_image(stage, target, false)?;
- let binaries = build::resolve(
- stage,
- target,
- &build::BuildOptions {
- no_build: args.build.no_build,
- binaries_dir: args.build.binaries_dir.clone(),
- },
- )?;
+ let binaries = if infra_only {
+ // Compose still emits `/app/out` mounts, but infra-only never starts
+ // those services. Bake runs this in parallel with zigbuild.
+ build::BinariesDir::TargetDir(workspace_root().join(target.debug_dir()))
+ } else {
+ build::resolve(
+ stage,
+ target,
+ &build::BuildOptions {
+ no_build: args.build.no_build,
+ binaries_dir: args.build.binaries_dir.clone(),
+ },
+ )?
+ };
// Generate the override + the Caddyfile (the frontend reaches the services
// through the proxy in every mode), and — for the self-contained local
diff --git a/tooling/xtask/crates/xtask_local/src/local/sandbox_image.rs b/tooling/xtask/crates/xtask_local/src/local/sandbox_image.rs
new file mode 100644
index 0000000000..c8bc5edc82
--- /dev/null
+++ b/tooling/xtask/crates/xtask_local/src/local/sandbox_image.rs
@@ -0,0 +1,92 @@
+//! Ensure the local agent-harness sandbox image exists before compose starts
+//! the harness.
+//!
+//! When local sandboxes are on, `run_local` / `stack up` always `docker build`
+//! the tag with no `--platform`, so the daemon's native arch wins. The sandbox
+//! flake exposes `x86_64-linux` and `aarch64-linux`; Apple Silicon and ARM
+//! Linux therefore bake natively instead of qemu. BuildKit cache is the
+//! freshness check: unchanged `container/` is a no-op rebuild, a Dockerfile or
+//! flake change pays the two nix shells. `--no-build` (Fly preview VM boot, CI
+//! snapshot bake) skips that: the image is already on the daemon from preload /
+//! the images lane, and the Fly VM's staged repo does not even include
+//! `crates/agent_harness/container`.
+
+use std::collections::BTreeMap;
+use std::path::Path;
+use std::process::Command;
+
+use anyhow::Result;
+
+use super::stage::Stage;
+
+#[cfg(test)]
+mod test;
+
+/// Local Docker tag `just run_local` / Fly previews load. Keep in sync with
+/// `xtask_workflows::workflows::vars::AGENT_HARNESS_LOCAL_IMAGE`.
+pub const DEFAULT_LOCAL_TAG: &str = "macro-agent-harness:latest";
+
+/// Build context matching `just -f crates/agent_harness/justfile build-local`.
+pub const CONTEXT_REL: &str = "crates/agent_harness/container";
+
+/// What [`ensure`] will do for a resolved env, before talking to Docker.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) struct EnsurePlan {
+ /// Tag the harness is configured to run.
+ pub tag: String,
+}
+
+impl EnsurePlan {
+ /// `None` when local containers are off, so stack-up skips this entirely.
+ pub fn from_env(env: &BTreeMap<String, String>) -> Option<Self> {
+ if env
+ .get("DEV_DANGEROUS_LOCAL_CONTAINERS")
+ .map(String::as_str)
+ != Some("true")
+ {
+ return None;
+ }
+ let tag = env
+ .get("LOCAL_CONTAINER_IMAGE")
+ .map(String::as_str)
+ .filter(|s| !s.is_empty())
+ .unwrap_or(DEFAULT_LOCAL_TAG)
+ .to_owned();
+ Some(Self { tag })
+ }
+}
+
+/// Unpinned on purpose: `--platform` would force qemu on Apple Silicon.
+pub(crate) fn build_args(tag: &str, context: &Path) -> Vec<String> {
+ vec![
+ "build".to_owned(),
+ "--tag".to_owned(),
+ tag.to_owned(),
+ context.display().to_string(),
+ ]
+}
+
+/// `docker build` the sandbox image when local sandboxes are on.
+///
+/// `no_build` skips the invocation so Fly / `--no-build` stack-up can use a
+/// preloaded tag. Dry-run notes the plan and does not invoke Docker.
+pub fn ensure(stage: &Stage, env: &BTreeMap<String, String>, no_build: bool) -> Result<()> {
+ let Some(plan) = EnsurePlan::from_env(env) else {
+ return Ok(());
+ };
+ if no_build {
+ stage.note(&format!(
+ "sandbox image: skipping build (--no-build); using {}",
+ plan.tag
+ ));
+ return Ok(());
+ }
+ if stage.is_dry_run() {
+ stage.note(&format!("sandbox image: would build {}", plan.tag));
+ return Ok(());
+ }
+ let context = super::repo_root().join(CONTEXT_REL);
+ let mut build = Command::new("docker");
+ build.args(build_args(&plan.tag, &context));
+ stage.run(&format!("Building sandbox image {}", plan.tag), &mut build)
+}
diff --git a/tooling/xtask/crates/xtask_local/src/local/sandbox_image/test.rs b/tooling/xtask/crates/xtask_local/src/local/sandbox_image/test.rs
new file mode 100644
index 0000000000..40c08331b5
--- /dev/null
+++ b/tooling/xtask/crates/xtask_local/src/local/sandbox_image/test.rs
@@ -0,0 +1,69 @@
+use super::*;
+use std::collections::BTreeMap;
+use std::path::Path;
+
+fn env_with(pairs: &[(&str, &str)]) -> BTreeMap<String, String> {
+ pairs
+ .iter()
+ .map(|(k, v)| ((*k).to_owned(), (*v).to_owned()))
+ .collect()
+}
+
+#[test]
+fn skips_when_the_flag_is_off() {
+ assert_eq!(EnsurePlan::from_env(&BTreeMap::new()), None);
+ assert_eq!(
+ EnsurePlan::from_env(&env_with(&[("DEV_DANGEROUS_LOCAL_CONTAINERS", "false")])),
+ None
+ );
+}
+
+#[test]
+fn skips_truthy_non_true_values() {
+ assert_eq!(
+ EnsurePlan::from_env(&env_with(&[("DEV_DANGEROUS_LOCAL_CONTAINERS", "1")])),
+ None
+ );
+}
+
+#[test]
+fn builds_the_configured_tag_when_local_containers_are_on() {
+ assert_eq!(
+ EnsurePlan::from_env(&env_with(&[("DEV_DANGEROUS_LOCAL_CONTAINERS", "true")])),
+ Some(EnsurePlan {
+ tag: DEFAULT_LOCAL_TAG.to_owned(),
+ })
+ );
+ assert_eq!(
+ EnsurePlan::from_env(&env_with(&[
+ ("DEV_DANGEROUS_LOCAL_CONTAINERS", "true"),
+ ("LOCAL_CONTAINER_IMAGE", "my-sandbox:dev"),
+ ])),
+ Some(EnsurePlan {
+ tag: "my-sandbox:dev".to_owned(),
+ })
+ );
+}
+
+#[test]
+fn docker_build_args_match_the_cli() {
+ let args = build_args(DEFAULT_LOCAL_TAG, Path::new(CONTEXT_REL));
+ assert_eq!(args, ["build", "--tag", DEFAULT_LOCAL_TAG, CONTEXT_REL]);
+ assert!(
+ !args.iter().any(|arg| arg == "--platform"),
+ "pinning a platform would qemu Apple Silicon: {args:?}"
+ );
+}
+
+#[test]
+fn no_build_still_plans_the_tag() {
+ // `--no-build` is an ensure() argument (Fly VM / CI bake), not an env flag.
+ // The plan is unchanged so stack-up still documents which preloaded tag it
+ // expects; docker is simply not invoked.
+ assert_eq!(
+ EnsurePlan::from_env(&env_with(&[("DEV_DANGEROUS_LOCAL_CONTAINERS", "true")])),
+ Some(EnsurePlan {
+ tag: DEFAULT_LOCAL_TAG.to_owned(),
+ })
+ );
+}
diff --git a/tooling/xtask/crates/xtask_local/src/local/stack.rs b/tooling/xtask/crates/xtask_local/src/local/stack.rs
index f6dc0f4aa4..a11791b3b5 100644
--- a/tooling/xtask/crates/xtask_local/src/local/stack.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/stack.rs
@@ -180,23 +180,16 @@ pub fn up(mode: Mode, args: &UpArgs) -> Result<Instance> {
});
let static_frontend = !args.run.no_frontend && !args.infra_only;
+ let infra_only = args.infra_only;
let (env, target) = super::prepare(
&stage,
mode,
&instance,
&args.run,
static_frontend,
- args.infra_only,
+ infra_only,
+ infra_only,
)?;
- let configured_binaries = args
- .run
- .build
- .binaries_dir
- .clone()
- .unwrap_or_else(|| super::workspace_root().join(target.debug_dir()));
- let active_binaries = super::build::BinariesDir::classify(&configured_binaries)?
- .host_dir()
- .to_path_buf();
// Build + stage the frontend bundle in the background: it's pure host-side
// work, independent of Docker until the proxy container mounts the staged
@@ -273,6 +266,15 @@ pub fn up(mode: Mode, args: &UpArgs) -> Result<Instance> {
}
return Ok(instance);
}
+ let configured_binaries = args
+ .run
+ .build
+ .binaries_dir
+ .clone()
+ .unwrap_or_else(|| super::workspace_root().join(target.debug_dir()));
+ let active_binaries = super::build::BinariesDir::classify(&configured_binaries)?
+ .host_dir()
+ .to_path_buf();
super::bring_up_app(&stage, mode, &instance, &env)?;
let _sdk_webhook_tunnel = (mode == Mode::Local && !stage.is_dry_run())
.then(|| sdk_webhook::start(&instance))
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/ensure_daytona_snapshot.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/ensure_daytona_snapshot.rs
index d3993f0da6..953f468f02 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/ensure_daytona_snapshot.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/ensure_daytona_snapshot.rs
@@ -1,48 +1,79 @@
//! `Ensure Daytona Snapshot` — creates the managed agent harness snapshot when
-//! it is missing. Generated into `ensure_daytona_snapshot.yml`.
+//! it is missing, and publishes the same image to GHCR. Generated into
+//! `ensure_daytona_snapshot.yml`.
+//!
+//! Local stacks and Fly previews `docker build` this image themselves (BuildKit
+//! cache is the freshness check). GHCR is the published copy for `main`
+//! (`:latest`) and for verifying the image on PRs (`:$SHA` only).
use gh_workflow::{
- Concurrency, Event, Expression, Job, Level, Permissions, Push, Step, Workflow, WorkflowDispatch,
+ Concurrency, Event, Expression, Job, Level, Permissions, PullRequest, PullRequestType, Push,
+ Step, Workflow, WorkflowDispatch,
};
use crate::workflows::{runners, steps, vars};
+#[cfg(test)]
+mod test;
+
+fn image_source_push() -> Push {
+ let mut push = Push::default().add_branch("main");
+ for path in image_source_paths() {
+ push = push.add_path(path);
+ }
+ push
+}
+
+fn image_source_pull_request() -> PullRequest {
+ let mut pr = PullRequest::default()
+ .add_branch("main")
+ .add_type(PullRequestType::Opened)
+ .add_type(PullRequestType::Synchronize)
+ .add_type(PullRequestType::Reopened);
+ for path in image_source_paths() {
+ pr = pr.add_path(path);
+ }
+ pr
+}
+
+fn image_source_paths() -> [xtask_paths::RepoGlob<'static>; 5] {
+ [
+ xtask_paths::repo_glob!("crates/agent_harness/container/**"),
+ xtask_paths::repo_glob!("crates/agent_harness/justfile"),
+ xtask_paths::repo_glob!("nix/cloud-storage.nix"),
+ xtask_paths::repo_glob!(
+ "tooling/xtask/crates/xtask_workflows/src/workflows/ensure_daytona_snapshot.rs"
+ ),
+ xtask_paths::repo_glob!(".github/workflows/ensure_daytona_snapshot.yml"),
+ ]
+}
+
/// Build the workflow.
pub fn ensure_daytona_snapshot() -> Workflow {
Workflow::new("Ensure Daytona Snapshot")
- .permissions(Permissions {
- contents: Some(Level::Read),
- ..Default::default()
- })
+ .permissions(
+ Permissions::default()
+ .contents(Level::Read)
+ .packages(Level::Write),
+ )
.on(Event::default()
- .push(
- Push::default()
- .add_branch("main")
- .add_path(xtask_paths::repo_glob!(
- "crates/agent_harness/container/**"
- ))
- .add_path(xtask_paths::repo_glob!(
- "crates/agent_harness/justfile"
- ))
- .add_path(xtask_paths::repo_glob!("nix/cloud-storage.nix"))
- .add_path(xtask_paths::repo_glob!(
- "tooling/xtask/crates/xtask_workflows/src/workflows/ensure_daytona_snapshot.rs"
- ))
- .add_path(xtask_paths::repo_glob!(
- ".github/workflows/ensure_daytona_snapshot.yml"
- )),
- )
+ .push(image_source_push())
+ .pull_request(image_source_pull_request())
.workflow_dispatch(WorkflowDispatch::default()))
.concurrency(
- Concurrency::new(Expression::new("${{ github.workflow }}")).cancel_in_progress(false),
+ Concurrency::new(Expression::new("${{ github.workflow }}-${{ github.ref }}"))
+ .cancel_in_progress(false),
)
.add_job("ensure-snapshot", ensure_snapshot())
+ .add_job("publish-image", publish_image())
}
fn ensure_snapshot() -> Job {
Job::default()
.name("Ensure macro-agent-harness snapshot")
.runs_on(runners::Runner::Small.to_string())
+ // PRs only need the GHCR publish; Daytona snapshot create is main-only.
+ .cond(Expression::new("github.event_name != 'pull_request'"))
.add_step(steps::checkout(false, false))
.add_step(steps::mount_nix_cache_volume())
.add_step(steps::setup_nix())
@@ -62,3 +93,51 @@ fn ensure_snapshot() -> Job {
.add_env(("DAYTONA_API_KEY", vars::DAYTONA_API_KEY)),
)
}
+
+/// GHCR publish. Runs on Mid with Namespace remote buildx: the Dockerfile
+/// bakes two nix shells, which is too large for the Small Daytona job's local
+/// daemon. linux/amd64 only — Daytona snapshots and Fly preview machines are
+/// amd64. Local `docker build` is unpinned so Apple Silicon / ARM Linux bake
+/// native `aarch64-linux` from the same flake. PRs push `:$SHA` only so they
+/// cannot clobber `:latest`.
+fn publish_image() -> Job {
+ let image = vars::AGENT_HARNESS_GHCR_IMAGE;
+ Job::default()
+ .name("Publish sandbox image to GHCR")
+ .runs_on(runners::Runner::Mid.to_string())
+ .timeout_minutes(90u32)
+ .cond(Expression::new(
+ "github.event_name != 'pull_request' || \
+ github.event.pull_request.head.repo.full_name == github.repository",
+ ))
+ .add_step(steps::checkout(false, false))
+ .add_step(steps::setup_namespace_buildx())
+ .add_step(
+ Step::new("Log in to GHCR")
+ .run(indoc::indoc! {r#"
+ set -euo pipefail
+ echo "$GITHUB_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin
+ "#})
+ .shell("bash")
+ .add_env(("GITHUB_TOKEN", "${{ secrets.GITHUB_TOKEN }}"))
+ .add_env(("GITHUB_ACTOR", "${{ github.actor }}")),
+ )
+ .add_step(
+ Step::new("Build and push sandbox image")
+ .run(format!(
+ r#"set -euo pipefail
+image={image}
+tags=(--tag "$image:$GITHUB_SHA")
+if [ "${{GITHUB_REF:-}}" = "refs/heads/main" ]; then
+ tags+=(--tag "$image:latest")
+fi
+docker buildx build \
+ --platform linux/amd64 \
+ "${{tags[@]}}" \
+ --push \
+ crates/agent_harness/container
+"#
+ ))
+ .shell("bash"),
+ )
+}
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/ensure_daytona_snapshot/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/ensure_daytona_snapshot/test.rs
new file mode 100644
index 0000000000..188bd0d415
--- /dev/null
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/ensure_daytona_snapshot/test.rs
@@ -0,0 +1,76 @@
+use super::*;
+
+fn rendered() -> String {
+ ensure_daytona_snapshot()
+ .to_string()
+ .expect("workflow should serialize")
+}
+
+#[test]
+fn workflow_can_write_packages_and_still_reads_contents() {
+ let yaml = rendered();
+ assert!(yaml.contains("packages: write"), "{yaml}");
+ assert!(yaml.contains("contents: read"), "{yaml}");
+}
+
+#[test]
+fn publish_job_pushes_amd64_ghcr_tags() {
+ let yaml = rendered();
+ assert!(
+ yaml.contains(
+ "namespacelabs/nscloud-setup-buildx-action@d059ed7184f0bc7c8b27e8810cea153d02bcc6dd"
+ ),
+ "{yaml}"
+ );
+ assert!(
+ yaml.contains(&format!("image={}", vars::AGENT_HARNESS_GHCR_IMAGE)),
+ "{yaml}"
+ );
+ assert!(yaml.contains("--platform linux/amd64 \\"), "{yaml}");
+ assert!(
+ !yaml.contains("linux/arm64"),
+ "GHCR stays amd64; local stacks bake native: {yaml}"
+ );
+ assert!(yaml.contains(r#"--tag "$image:$GITHUB_SHA""#), "{yaml}");
+ assert!(yaml.contains(r#"tags+=(--tag "$image:latest")"#), "{yaml}");
+ assert!(
+ yaml.contains("refs/heads/main"),
+ "latest must only be tagged on main: {yaml}"
+ );
+ assert!(yaml.contains("crates/agent_harness/container"), "{yaml}");
+}
+
+#[test]
+fn login_reads_token_from_env_not_the_script_body() {
+ let yaml = rendered();
+ assert!(
+ yaml.contains("echo \"$GITHUB_TOKEN\" | docker login ghcr.io"),
+ "{yaml}"
+ );
+ assert!(
+ !yaml.contains("${{ github.token }}"),
+ "token must not be interpolated into a run script: {yaml}"
+ );
+}
+
+#[test]
+fn daytona_job_stays_on_the_small_runner_and_skips_prs() {
+ let yaml = rendered();
+ assert!(yaml.contains("namespace-profile-linux-small"), "{yaml}");
+ assert!(yaml.contains("namespace-profile-linux-mid"), "{yaml}");
+ assert!(yaml.contains("ensure-daytona"), "{yaml}");
+ assert!(
+ yaml.contains("github.event_name != 'pull_request'"),
+ "{yaml}"
+ );
+}
+
+#[test]
+fn publish_runs_on_same_repo_pull_requests() {
+ let yaml = rendered();
+ assert!(yaml.contains("pull_request:"), "{yaml}");
+ assert!(
+ yaml.contains("github.event.pull_request.head.repo.full_name == github.repository"),
+ "{yaml}"
+ );
+}
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly.rs
index f6bb7dd840..bdd408fe3e 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly.rs
@@ -25,6 +25,9 @@ use gh_workflow::{
use crate::workflows::{runners, steps, vars};
+#[cfg(test)]
+mod test;
+
/// The per-PR Fly app name; also the preview hostname (`<app>.fly.dev`).
const APP_NAME: &str = "macro-pr-${{ github.event.pull_request.number }}";
@@ -83,10 +86,12 @@ fn deploy() -> Job {
// sccache keys, and their volumes never carry a cargo target dir).
.runs_on(runners::Runner::Mid.with_cache_tag(vars::PREVIEW_CACHE_TAG))
// Backstop against hangs: worst honest case is ~15 min cold builds +
- // ~6 min cold mirror push + the 30 min flyctl wait cap. Anything past
- // an hour is a stuck step, not a slow deploy (a hung log fetch once
- // burned 2h26m of runner before someone cancelled it by hand).
- .timeout_minutes(60u32)
+ // a cold `crates/agent_harness/container` bake (two nix shells) +
+ // ~6 min cold mirror push + the 30 min flyctl wait cap. 90m leaves
+ // room for that extra image; anything past that is a stuck step (a
+ // hung log fetch once burned 2h26m of runner before someone cancelled
+ // it by hand).
+ .timeout_minutes(90u32)
.permissions(
Permissions::default()
.contents(Level::Read)
@@ -121,13 +126,7 @@ fn deploy() -> Job {
// Namespace remote builder: persistent BuildKit layer cache across
// runs, same as the deploy workflows use. The aux-image builds and the
// preview-image build both go through it.
- .add_step(
- Step::new("Set up Namespace Docker builder").uses(
- "namespacelabs",
- "nscloud-setup-buildx-action",
- "d059ed7184f0bc7c8b27e8810cea153d02bcc6dd",
- ), // v0.0.23
- )
+ .add_step(steps::setup_namespace_buildx())
.add_step(steps::setup_nix())
.add_step(steps::setup_reqs_web("Setup dev shell + web deps", false))
.add_step(steps::configure_namespace_sccache(
@@ -222,6 +221,10 @@ fn build_artifacts() -> Step<Run> {
cd apps/web
export MODE=development NODE_ENV=production
export VITE_LOCAL_SERVERS=ALL VITE_LOCAL_BACKEND_ORIGIN=same-origin
+ # Same ceiling as `just -f apps/web/justfile build-dev`: vite's
+ # chunk-render peaks at node's default ~4GB heap and aborts with
+ # "Ineffective mark-compacts near heap limit".
+ export NODE_OPTIONS=--max-old-space-size=6144
bun run --bun build
LANE
@@ -281,7 +284,10 @@ fn build_artifacts() -> Step<Run> {
cat > "$lanes/images.sh" <<'LANE'
set -euo pipefail
docker compose --project-directory . -p macro -f docker/docker-compose.yml build \
- search sync_service websocket_service lexical_service ai_editing_worker
+ search sync_service websocket_service lexical_service ai_editing_worker analytics_proxy
+ # Not a compose service, so `config --images` never lists it.
+ # BuildKit cache on this runner is the freshness check.
+ docker build --tag "$LOCAL_SANDBOX_IMAGE" crates/agent_harness/container
# The image BUILD above is fatal; the premirror below is a pure
# optimization with an authoritative fallback (the deploy step's
# mirror loop), so its failure only costs a slower push later.
@@ -301,7 +307,7 @@ fn build_artifacts() -> Step<Run> {
flyctl auth docker
registry="registry.fly.io/$APP_NAME"
for img in $(docker compose --project-directory . -p macro \
- -f docker/docker-compose.yml config --images | sort -u) alpine:3; do
+ -f docker/docker-compose.yml config --images | sort -u) alpine:3 "$LOCAL_SANDBOX_IMAGE"; do
if ! docker image inspect "$img" >/dev/null 2>&1 && ! docker pull "$img"; then
echo "premirror: skipping $img (not local, not pullable)"
continue
@@ -432,6 +438,7 @@ fn build_artifacts() -> Step<Run> {
fi
"#})
.add_env(("FLY_ORG", "${{ vars.FLY_ORG || secrets.FLY_ORG }}"))
+ .add_env(("LOCAL_SANDBOX_IMAGE", vars::AGENT_HARNESS_LOCAL_IMAGE))
}
/// When the bake dies (typically the backend health gate), the answer is in
@@ -684,10 +691,17 @@ fn deploy_to_fly() -> Step<Run> {
docker image inspect alpine:3 >/dev/null 2>&1 || docker pull alpine:3
mkdir -p preview-ctx/preload
: > preview-ctx/preload/manifest.txt
- for img in $images alpine:3; do
+ for img in $images alpine:3 "$LOCAL_SANDBOX_IMAGE"; do
# Images the bake didn't leave in the daemon (snapshot cache hit,
# or app-layer-only images like proxy/mailpit) get pulled here.
- docker image inspect "$img" >/dev/null 2>&1 || docker pull "$img"
+ # Compose `config --images` also lists locally-built tags
+ # (analytics_proxy, sdk-webhook-relay) that are not on Docker Hub;
+ # premirror already skips those — do the same so a missing local
+ # tag cannot fail the deploy.
+ if ! docker image inspect "$img" >/dev/null 2>&1 && ! docker pull "$img"; then
+ echo "mirror: skipping $img (not local, not pullable)"
+ continue
+ fi
id=$(docker image inspect -f '{{.Id}}' "$img")
# Content-addressed tag: same image content = same ref, so a
# redeploy's push is skippable by a manifest existence check and
@@ -810,6 +824,7 @@ fn deploy_to_fly() -> Step<Run> {
// it's not sensitive, but people reasonably reach for secrets first.
.add_env(("FLY_ORG", "${{ vars.FLY_ORG || secrets.FLY_ORG }}"))
.add_env(("DOPPLER_PREVIEW_TOKEN", vars::DOPPLER_PREVIEW_TOKEN))
+ .add_env(("LOCAL_SANDBOX_IMAGE", vars::AGENT_HARNESS_LOCAL_IMAGE))
}
/// After a healthy deploy this PR's volume is, by construction, a fully
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly/test.rs
new file mode 100644
index 0000000000..d497198130
--- /dev/null
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/preview_fly/test.rs
@@ -0,0 +1,87 @@
+use super::*;
+
+fn rendered() -> String {
+ preview_fly()
+ .to_string()
+ .expect("workflow should serialize")
+}
+
+#[test]
+fn deploy_does_not_need_ghcr_packages() {
+ let yaml = rendered();
+ assert!(
+ !yaml.contains("packages: read"),
+ "preview builds the sandbox image locally: {yaml}"
+ );
+}
+
+#[test]
+fn both_image_loops_include_the_sandbox_tag() {
+ let yaml = rendered();
+ let tag = vars::AGENT_HARNESS_LOCAL_IMAGE;
+ assert!(
+ yaml.contains(&format!("LOCAL_SANDBOX_IMAGE: {tag}")),
+ "sandbox tag should be injected as step env: {yaml}"
+ );
+ assert!(
+ yaml.contains("alpine:3 \"$LOCAL_SANDBOX_IMAGE\""),
+ "premirror loop should append the sandbox tag: {yaml}"
+ );
+ assert!(
+ yaml.contains("$images alpine:3 \"$LOCAL_SANDBOX_IMAGE\""),
+ "deploy loop should append the sandbox tag: {yaml}"
+ );
+}
+
+#[test]
+fn deploy_timeout_covers_a_cold_sandbox_image_bake() {
+ let yaml = rendered();
+ assert!(
+ yaml.contains("timeout-minutes: 90"),
+ "sandbox image bake can exceed the old 60m backstop: {yaml}"
+ );
+}
+
+#[test]
+fn images_lane_always_builds_the_sandbox_image() {
+ let yaml = rendered();
+ assert!(
+ yaml.contains("docker build --tag \"$LOCAL_SANDBOX_IMAGE\" crates/agent_harness/container"),
+ "{yaml}"
+ );
+ assert!(
+ !yaml.contains("PREVIEW_BASE_SHA"),
+ "git-diff vs base is unnecessary when we always build: {yaml}"
+ );
+ assert!(
+ !yaml.contains("SANDBOX_GHCR_IMAGE"),
+ "preview should not pull GHCR: {yaml}"
+ );
+}
+
+#[test]
+fn web_lane_raises_the_node_heap() {
+ let yaml = rendered();
+ assert!(
+ yaml.contains("NODE_OPTIONS=--max-old-space-size=6144"),
+ "vite OOM is the documented CI flap without this: {yaml}"
+ );
+}
+
+#[test]
+fn images_lane_builds_analytics_proxy() {
+ let yaml = rendered();
+ assert!(
+ yaml.contains("ai_editing_worker analytics_proxy"),
+ "analytics_proxy is a default compose service; omitting it makes deploy pull docker.io/library/macro-analytics_proxy: {yaml}"
+ );
+}
+
+#[test]
+fn deploy_mirror_skips_unpullable_local_tags() {
+ let yaml = rendered();
+ assert!(
+ yaml.contains("mirror: skipping $img (not local, not pullable)"),
+ "deploy must skip compose-built tags the way premirror already does: {yaml}"
+ );
+}
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs
index fbdf73dbad..635c30ae36 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/steps.rs
@@ -48,6 +48,17 @@ pub(crate) fn uses_local(name: &str, path: RepoDir<'_>) -> Step<Use> {
step
}
+/// Namespace remote BuildKit, pinned. Multi-arch `docker buildx --push` and the
+/// Fly preview image builds both go through it so nix-in-Docker work does not
+/// land on a Small runner's local daemon.
+pub fn setup_namespace_buildx() -> Step<Use> {
+ Step::new("Set up Namespace Docker builder").uses(
+ "namespacelabs",
+ "nscloud-setup-buildx-action",
+ "d059ed7184f0bc7c8b27e8810cea153d02bcc6dd",
+ ) // v0.0.23
+}
+
/// `actions/checkout`, pinned. `full_history` fetches the full history, which
/// the path-filter diff in `path-check` needs. `persist_credentials` controls
/// whether checkout leaves the token in git config for later steps.
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs
index 03cb5f67af..f63828e6de 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs
@@ -105,6 +105,14 @@ pub const BUN_CACHE_VOLUME_DIR: &str = "/home/runner/.bun/install/cache";
/// cache-volume miss does not force another infra bake.
pub const PREVIEW_SNAPSHOT_VOLUME_DIR: &str = "/home/runner/.cache/macro-preview-snapshots";
+/// GHCR repository for the agent-harness sandbox image (the same Dockerfile
+/// Daytona snapshots). Pushed as `:$SHA` on PRs and `:$SHA` + `:latest` on main.
+pub const AGENT_HARNESS_GHCR_IMAGE: &str = "ghcr.io/macro-inc/macro-agent-harness";
+
+/// Local Docker tag `just run_local` / Fly previews load. Keep in sync with
+/// `xtask_local::local::sandbox_image::DEFAULT_LOCAL_TAG`.
+pub const AGENT_HARNESS_LOCAL_IMAGE: &str = "macro-agent-harness:latest";
+
/// The repo-wide env block (mirrors the original top-level `env:`). Defaults the
/// linker to `lld`; the heavy jobs override `RUSTFLAGS` to use `mold`.
pub fn with_global_env(workflow: Workflow) -> Workflow {