Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(agents): rotate in-memory session credentials on cross-replica resume (#7944)

teob5bd9a3fb69d
diff --git a/Cargo.lock b/Cargo.lock
index 292e17e153..fff26b46a2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -405,6 +405,7 @@ dependencies = [
name = "agent_harness_service"
version = "0.1.0"
dependencies = [
+ "agent",
"agent-client-protocol",
"agent_changes",
"agent_egress",
@@ -419,6 +420,7 @@ dependencies = [
"ai_toolset",
"ai_usage",
"anyhow",
+ "async-trait",
"authentication_service_client",
"aws-sdk-kms",
"aws-sdk-secretsmanager",
diff --git a/crates/agent_inmem/src/outbound/manager.rs b/crates/agent_inmem/src/outbound/manager.rs
index f6769c3db1..33be2df9c0 100644
--- a/crates/agent_inmem/src/outbound/manager.rs
+++ b/crates/agent_inmem/src/outbound/manager.rs
@@ -77,9 +77,7 @@ pub struct InMemAgentManager {
live: DashMap<AgentSessionId, LiveAgent>,
/// Each live session's egress token. An in-process session has no
/// container environment to read it back from, so the manager is the
- /// one place it survives spawn; a resume after a restart finds nothing
- /// here and advertises no servers, exactly like a sandbox whose
- /// container is gone.
+ /// one place it survives spawn or resume.
tokens: DashMap<AgentSessionId, String>,
}
@@ -137,8 +135,9 @@ impl InMemAgentManager {
/// resume after a restart) gets its conversation rebuilt from the durable
/// frame log first.
///
- /// `session_token` is the egress token minted at spawn; a resume passes
- /// `None` and keeps whatever spawn stored.
+ /// `session_token` is the egress token minted for this attach; `None`
+ /// keeps the stored one. A token it replaces is no longer accepted, so
+ /// the MCP sessions opened with it are released.
#[must_use]
pub async fn attach(
&self,
@@ -147,8 +146,11 @@ impl InMemAgentManager {
) -> ServerChannel {
// A replaced agent must die before its successor serves the session.
self.live.remove(&facts.id);
- if let Some(token) = session_token {
- self.tokens.insert(facts.id, token);
+ if let Some(token) = session_token
+ && let Some(replaced) = self.tokens.insert(facts.id, token.clone())
+ && replaced != token
+ {
+ self.mcp.release_dyn(&replaced);
}
if !self.store.contains_key(&facts.id) {
// Loaded before taking the entry so the read never blocks the
diff --git a/crates/agent_inmem/src/outbound/manager/test.rs b/crates/agent_inmem/src/outbound/manager/test.rs
index b0da15fb8f..e6df24c2ea 100644
--- a/crates/agent_inmem/src/outbound/manager/test.rs
+++ b/crates/agent_inmem/src/outbound/manager/test.rs
@@ -801,6 +801,32 @@ async fn teardown_releases_pooled_mcp_sessions_for_the_egress_token() {
);
}
+/// A resume that rotates the credential leaves the old token unaccepted, so
+/// the MCP sessions pooled under it are released; the same token again is
+/// still in use and keeps them.
+#[tokio::test]
+async fn attaching_with_a_new_token_releases_the_replaced_one() {
+ let repo = InMemoryAgentSessionRepo::new();
+ let mcp = Arc::new(RecordingRelease {
+ released: std::sync::Mutex::new(Vec::new()),
+ });
+ let manager = InMemAgentManager::new(
+ Arc::new(ScriptedEngine::new(Vec::new())),
+ Arc::new(LogFrameSource::new(repo.clone())),
+ mcp.clone(),
+ Arc::new(crate::testing::TestModelAccess::paid()),
+ );
+ let id = AgentSessionId::new();
+ let _spawned = manager.attach(facts(id), Some("spawned".to_owned())).await;
+ let _same = manager.attach(facts(id), Some("spawned".to_owned())).await;
+ assert!(mcp.released.lock().expect("lock").is_empty());
+
+ let _rotated = manager.attach(facts(id), Some("rotated".to_owned())).await;
+
+ assert_eq!(manager.session_token(id).as_deref(), Some("rotated"));
+ assert_eq!(mcp.released.lock().expect("lock").as_slice(), ["spawned"]);
+}
+
/// Records [`McpToolConnector::release`] so teardown can be seen doing it.
struct RecordingRelease {
released: std::sync::Mutex<Vec<String>>,
diff --git a/services/agent_harness_service/Cargo.toml b/services/agent_harness_service/Cargo.toml
index 7b195caba0..923060e73c 100644
--- a/services/agent_harness_service/Cargo.toml
+++ b/services/agent_harness_service/Cargo.toml
@@ -121,7 +121,9 @@ tracing = { workspace = true }
workspace-hack = { version = "0.1", path = "../../crates/workspace-hack" }
[dev-dependencies]
+agent = { path = "../../crates/agent" }
agent-client-protocol = { workspace = true }
agent_session = { path = "../../crates/agent_session", features = ["test-utils"] }
+async-trait = { workspace = true }
harness_id = { path = "../../crates/harness_id", features = ["test-utils"] }
harnesses = { path = "../../crates/harnesses", default-features = false, features = ["test-utils"] }
diff --git a/services/agent_harness_service/src/containers.rs b/services/agent_harness_service/src/containers.rs
index ca67001d15..51877fd080 100644
--- a/services/agent_harness_service/src/containers.rs
+++ b/services/agent_harness_service/src/containers.rs
@@ -7,6 +7,7 @@
//! container is asked for - decides which runtime a session's transport comes
//! from.
+use agent_egress::domain::model::SessionToken;
use agent_harness::domain::error::{HarnessError, Result};
use agent_harness::domain::model::{AgentKind, SpawnContainer};
use agent_harness::domain::ports::ContainerManager;
@@ -18,7 +19,8 @@ use agent_inmem::outbound::manager::{InMemAgentManager, SessionFacts};
use agent_runtime_protocol::domain::connection::ServerChannel;
use agent_runtime_protocol::domain::ports::{Transport, TransportError, TransportSender};
use agent_runtime_protocol::domain::schema::v0::{ToRuntimeMessage, ToServerMessage};
-use agent_session::domain::model::{AgentSessionId, SandboxSize};
+use agent_session::domain::error::AgentSessionError;
+use agent_session::domain::model::{AgentSessionId, SandboxSize, SessionManagement};
use agent_session::domain::service::AgentSessionService;
#[cfg(test)]
@@ -200,14 +202,48 @@ where
}
}
+ /// An in-process session's raw token lives only in the process that was
+ /// handed it, and the row keeps only its hash. Another replica may have
+ /// resumed the session since, rotating the hash past any token this
+ /// process still holds, so every resume rotates the credential. A session
+ /// another live replica manages is refused before rotating: its claim
+ /// would refuse this attach anyway, and the rotation would revoke the
+ /// credential its live agent holds.
async fn resume(
&self,
session: AgentSessionId,
) -> Result<agent_session::domain::connection::RuntimeAttachment<Self::Transport>> {
match self.route(session).await? {
- Route::InMem(facts) => Ok(agent_session::domain::connection::RuntimeAttachment::solo(
- RoutedTransport::InMem(self.inmem().manager.attach(*facts, None).await),
- )),
+ Route::InMem(facts) => {
+ let management = self
+ .sessions
+ .management(session)
+ .await
+ .map_err(HarnessError::Session)?;
+ if let SessionManagement::Peer(manager) = management {
+ tracing::info!(
+ %session,
+ peer = %manager.replica,
+ "not rotating the credential of a session another replica manages"
+ );
+ return Err(HarnessError::Session(AgentSessionError::ManagedElsewhere(
+ session,
+ )));
+ }
+ let token = SessionToken::mint();
+ self.sessions
+ .set_egress_token_hash(session, &token.hash())
+ .await
+ .map_err(HarnessError::Session)?;
+ Ok(agent_session::domain::connection::RuntimeAttachment::solo(
+ RoutedTransport::InMem(
+ self.inmem()
+ .manager
+ .attach(*facts, Some(token.as_str().to_owned()))
+ .await,
+ ),
+ ))
+ }
Route::Sandbox => self
.sandbox
.resume(session)
@@ -217,9 +253,8 @@ where
}
/// An in-process session has no container environment to read a token
- /// back from, so the manager remembers the one spawn handed it; a
- /// session this process never spawned yields none. Sandboxes delegate to
- /// their provider.
+ /// back from, so the manager remembers the one spawn or [`Self::resume`]
+ /// handed it. Sandboxes delegate to their provider.
async fn session_token(&self, session: AgentSessionId) -> Result<Option<String>> {
match self.route(session).await? {
Route::InMem(_) => Ok(self.inmem().manager.session_token(session)),
diff --git a/services/agent_harness_service/src/containers/test.rs b/services/agent_harness_service/src/containers/test.rs
index 76a8045ab6..1e0df2f847 100644
--- a/services/agent_harness_service/src/containers/test.rs
+++ b/services/agent_harness_service/src/containers/test.rs
@@ -1,9 +1,15 @@
use super::*;
use agent_fold::domain::service::FoldedMessageService;
+use agent_harness::domain::model::SandboxEgress;
use agent_harness::outbound::daytona::AnthropicApiKey;
use agent_harness::outbound::local::{LocalContainerManager, LocalSettings};
-use agent_session::domain::model::ReplicaId;
-use agent_session::domain::ports::NoOpRealtime;
+use agent_harness::testing::helpers::egress::test_egress;
+use agent_inmem::domain::engine::{TurnEngine, TurnRequest};
+use agent_inmem::domain::mcp::NoMcpServers;
+use agent_inmem::domain::model_access::{InMemModelAccess, ModelAccess, ModelAccessError};
+use agent_inmem::outbound::log_frames::LogFrameSource;
+use agent_session::domain::model::{ClaimOutcome, ReplicaId};
+use agent_session::domain::ports::{AgentSessionRepo, NoOpRealtime, SessionOwnership};
use agent_session::domain::ports::{
NoOpAgentSessionNameGenerator, NoOpTurnObserver, NoopLifecyclePublisher,
};
@@ -31,13 +37,22 @@ fn sessions_with_harness(
bot: BotId,
harness: &str,
) -> (AgentSessionId, impl AgentSessionService + use<>) {
+ let (id, repo) = repo_with_session(bot, harness);
+ (id, service_over(repo))
+}
+
+fn repo_with_session(bot: BotId, harness: &str) -> (AgentSessionId, InMemoryAgentSessionRepo) {
let repo = InMemoryAgentSessionRepo::new();
let id = AgentSessionId::new();
let mut session = test_agent_session(id);
session.bot_id = bot;
session.harness = harness.to_owned();
repo.insert_session(session);
- let service = AgentSessionServiceImpl::new(
+ (id, repo)
+}
+
+fn service_over(repo: InMemoryAgentSessionRepo) -> impl AgentSessionService + use<> {
+ AgentSessionServiceImpl::new(
repo.clone(),
FoldedMessageService::new(repo),
NoOpRealtime,
@@ -45,8 +60,63 @@ fn sessions_with_harness(
Arc::new(NoOpTurnObserver),
Arc::new(NoopLifecyclePublisher),
ReplicaId::mint(),
- );
- (id, service)
+ )
+}
+
+/// Never asked to run a turn: these tests only attach sessions.
+struct IdleEngine;
+
+impl TurnEngine for IdleEngine {
+ fn supported_models(&self) -> &[&str] {
+ &[]
+ }
+
+ fn run_turn(
+ &self,
+ _request: TurnRequest,
+ ) -> tokio::sync::mpsc::Receiver<Result<agent::StreamPart, agent::AgentError>> {
+ tokio::sync::mpsc::channel(1).1
+ }
+}
+
+struct FreeModels;
+
+#[async_trait::async_trait]
+impl InMemModelAccess for FreeModels {
+ async fn access(&self, _owner: &model_owner::Owner) -> Result<ModelAccess, ModelAccessError> {
+ Ok(ModelAccess::Free)
+ }
+}
+
+fn in_process_runtime(repo: InMemoryAgentSessionRepo) -> InMemRuntime {
+ InMemRuntime {
+ manager: InMemAgentManager::new(
+ Arc::new(IdleEngine),
+ Arc::new(LogFrameSource::new(repo)),
+ Arc::new(NoMcpServers),
+ Arc::new(FreeModels),
+ ),
+ }
+}
+
+fn in_process_containers(
+ repo: &InMemoryAgentSessionRepo,
+) -> RoutedContainers<impl AgentSessionService + use<>> {
+ RoutedContainers::new(
+ unreachable_sandbox(),
+ Some(in_process_runtime(repo.clone())),
+ service_over(repo.clone()),
+ )
+}
+
+async fn session_for_token(
+ repo: &InMemoryAgentSessionRepo,
+ token: &SessionToken,
+) -> Option<AgentSessionId> {
+ repo.find_by_egress_token_hash(&token.hash())
+ .await
+ .expect("token lookup")
+ .map(|session| session.id)
}
#[tokio::test]
@@ -109,6 +179,104 @@ async fn routes_an_owned_bot_to_the_sandbox() {
));
}
+/// A replica that replaced the one which spawned an in-process session has
+/// no way to read its token back, so resuming there rotates the credential:
+/// the session's MCP servers are advertised under a token the proxy accepts,
+/// and the token that died with the old process no longer is.
+#[tokio::test]
+async fn resuming_an_in_process_session_held_elsewhere_rotates_its_credential() {
+ let (id, repo) = repo_with_session(bot_id::MACRO_NEW_BOT_ID, "opencode");
+ let lost = SessionToken::new("held by the replaced replica");
+ repo.set_egress_token_hash(id, &lost.hash())
+ .await
+ .expect("spawn's hash");
+ let containers = in_process_containers(&repo);
+
+ let _attachment = containers.resume(id).await.expect("resumes");
+
+ let token = containers
+ .session_token(id)
+ .await
+ .expect("token lookup")
+ .expect("a resumed in-process session holds a credential");
+ assert_eq!(
+ session_for_token(&repo, &SessionToken::new(token)).await,
+ Some(id)
+ );
+ assert_eq!(session_for_token(&repo, &lost).await, None);
+}
+
+/// A session can move to another replica and back while the first stays up.
+/// The token the first still holds was revoked when the second rotated, so
+/// resuming on the first again must not reuse it.
+#[tokio::test]
+async fn resuming_where_a_revoked_token_is_still_held_does_not_reuse_it() {
+ let (id, repo) = repo_with_session(bot_id::MACRO_NEW_BOT_ID, "opencode");
+ let spawned = SessionToken::new("held by the first replica");
+ repo.set_egress_token_hash(id, &spawned.hash())
+ .await
+ .expect("spawn's hash");
+ let first = in_process_containers(&repo);
+ let second = in_process_containers(&repo);
+ let _spawned = first
+ .spawn(SpawnContainer {
+ session_id: id,
+ kind: AgentKind::InMemory,
+ size: SandboxSize::Default,
+ egress: SandboxEgress {
+ session_token: spawned.as_str().to_owned(),
+ ..test_egress()
+ },
+ })
+ .await
+ .expect("spawns");
+ let _moved = second.resume(id).await.expect("resumes on the second");
+
+ let _returned = first.resume(id).await.expect("resumes on the first again");
+
+ let token = first
+ .session_token(id)
+ .await
+ .expect("token lookup")
+ .expect("a resumed in-process session holds a credential");
+ assert_ne!(token, spawned.as_str());
+ assert_eq!(
+ session_for_token(&repo, &SessionToken::new(token)).await,
+ Some(id)
+ );
+}
+
+/// Another live replica's claim would refuse this attach, and rotating first
+/// would revoke the credential that replica's agent is using.
+#[tokio::test]
+async fn resuming_a_session_a_live_peer_manages_is_refused_without_rotating() {
+ let (id, repo) = repo_with_session(bot_id::MACRO_NEW_BOT_ID, "opencode");
+ let current = SessionToken::new("held by the managing replica");
+ repo.set_egress_token_hash(id, &current.hash())
+ .await
+ .expect("spawn's hash");
+ let claimed = repo
+ .claim(id, ReplicaId::mint())
+ .await
+ .expect("peer claims");
+ assert!(matches!(claimed, ClaimOutcome::Claimed(_)));
+ let containers = in_process_containers(&repo);
+
+ let Err(error) = containers.resume(id).await else {
+ panic!("a session a live peer manages must not resume here");
+ };
+
+ assert!(matches!(
+ error,
+ HarnessError::Session(AgentSessionError::ManagedElsewhere(session)) if session == id
+ ));
+ assert_eq!(session_for_token(&repo, &current).await, Some(id));
+ assert_eq!(
+ containers.session_token(id).await.expect("token lookup"),
+ None
+ );
+}
+
/// The in-process runtime skips Macro's own server by the name the harness
/// gives it, and restates that name rather than importing it. This is the one
/// crate that sees both, so it is where they are held equal.