Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(agents): rotate in-memory session credentials on cross-replica resume (#7944)
teob5bd9a3fb69d
diff --git a/Cargo.lock b/Cargo.lockindex 292e17e153..fff26b46a2 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -405,6 +405,7 @@ dependencies = [name = "agent_harness_service"version = "0.1.0"dependencies = [+ "agent","agent-client-protocol","agent_changes","agent_egress",@@ -419,6 +420,7 @@ dependencies = ["ai_toolset","ai_usage","anyhow",+ "async-trait","authentication_service_client","aws-sdk-kms","aws-sdk-secretsmanager",diff --git a/crates/agent_inmem/src/outbound/manager.rs b/crates/agent_inmem/src/outbound/manager.rsindex f6769c3db1..33be2df9c0 100644--- a/crates/agent_inmem/src/outbound/manager.rs+++ b/crates/agent_inmem/src/outbound/manager.rs@@ -77,9 +77,7 @@ pub struct InMemAgentManager {live: DashMap<AgentSessionId, LiveAgent>,/// Each live session's egress token. An in-process session has no/// container environment to read it back from, so the manager is the- /// one place it survives spawn; a resume after a restart finds nothing- /// here and advertises no servers, exactly like a sandbox whose- /// container is gone.+ /// one place it survives spawn or resume.tokens: DashMap<AgentSessionId, String>,}@@ -137,8 +135,9 @@ impl InMemAgentManager {/// resume after a restart) gets its conversation rebuilt from the durable/// frame log first.///- /// `session_token` is the egress token minted at spawn; a resume passes- /// `None` and keeps whatever spawn stored.+ /// `session_token` is the egress token minted for this attach; `None`+ /// keeps the stored one. A token it replaces is no longer accepted, so+ /// the MCP sessions opened with it are released.#[must_use]pub async fn attach(&self,@@ -147,8 +146,11 @@ impl InMemAgentManager {) -> ServerChannel {// A replaced agent must die before its successor serves the session.self.live.remove(&facts.id);- if let Some(token) = session_token {- self.tokens.insert(facts.id, token);+ if let Some(token) = session_token+ && let Some(replaced) = self.tokens.insert(facts.id, token.clone())+ && replaced != token+ {+ self.mcp.release_dyn(&replaced);}if !self.store.contains_key(&facts.id) {// Loaded before taking the entry so the read never blocks thediff --git a/crates/agent_inmem/src/outbound/manager/test.rs b/crates/agent_inmem/src/outbound/manager/test.rsindex b0da15fb8f..e6df24c2ea 100644--- a/crates/agent_inmem/src/outbound/manager/test.rs+++ b/crates/agent_inmem/src/outbound/manager/test.rs@@ -801,6 +801,32 @@ async fn teardown_releases_pooled_mcp_sessions_for_the_egress_token() {);}+/// A resume that rotates the credential leaves the old token unaccepted, so+/// the MCP sessions pooled under it are released; the same token again is+/// still in use and keeps them.+#[tokio::test]+async fn attaching_with_a_new_token_releases_the_replaced_one() {+ let repo = InMemoryAgentSessionRepo::new();+ let mcp = Arc::new(RecordingRelease {+ released: std::sync::Mutex::new(Vec::new()),+ });+ let manager = InMemAgentManager::new(+ Arc::new(ScriptedEngine::new(Vec::new())),+ Arc::new(LogFrameSource::new(repo.clone())),+ mcp.clone(),+ Arc::new(crate::testing::TestModelAccess::paid()),+ );+ let id = AgentSessionId::new();+ let _spawned = manager.attach(facts(id), Some("spawned".to_owned())).await;+ let _same = manager.attach(facts(id), Some("spawned".to_owned())).await;+ assert!(mcp.released.lock().expect("lock").is_empty());++ let _rotated = manager.attach(facts(id), Some("rotated".to_owned())).await;++ assert_eq!(manager.session_token(id).as_deref(), Some("rotated"));+ assert_eq!(mcp.released.lock().expect("lock").as_slice(), ["spawned"]);+}+/// Records [`McpToolConnector::release`] so teardown can be seen doing it.struct RecordingRelease {released: std::sync::Mutex<Vec<String>>,diff --git a/services/agent_harness_service/Cargo.toml b/services/agent_harness_service/Cargo.tomlindex 7b195caba0..923060e73c 100644--- a/services/agent_harness_service/Cargo.toml+++ b/services/agent_harness_service/Cargo.toml@@ -121,7 +121,9 @@ tracing = { workspace = true }workspace-hack = { version = "0.1", path = "../../crates/workspace-hack" }[dev-dependencies]+agent = { path = "../../crates/agent" }agent-client-protocol = { workspace = true }agent_session = { path = "../../crates/agent_session", features = ["test-utils"] }+async-trait = { workspace = true }harness_id = { path = "../../crates/harness_id", features = ["test-utils"] }harnesses = { path = "../../crates/harnesses", default-features = false, features = ["test-utils"] }diff --git a/services/agent_harness_service/src/containers.rs b/services/agent_harness_service/src/containers.rsindex ca67001d15..51877fd080 100644--- a/services/agent_harness_service/src/containers.rs+++ b/services/agent_harness_service/src/containers.rs@@ -7,6 +7,7 @@//! container is asked for - decides which runtime a session's transport comes//! from.+use agent_egress::domain::model::SessionToken;use agent_harness::domain::error::{HarnessError, Result};use agent_harness::domain::model::{AgentKind, SpawnContainer};use agent_harness::domain::ports::ContainerManager;@@ -18,7 +19,8 @@ use agent_inmem::outbound::manager::{InMemAgentManager, SessionFacts};use agent_runtime_protocol::domain::connection::ServerChannel;use agent_runtime_protocol::domain::ports::{Transport, TransportError, TransportSender};use agent_runtime_protocol::domain::schema::v0::{ToRuntimeMessage, ToServerMessage};-use agent_session::domain::model::{AgentSessionId, SandboxSize};+use agent_session::domain::error::AgentSessionError;+use agent_session::domain::model::{AgentSessionId, SandboxSize, SessionManagement};use agent_session::domain::service::AgentSessionService;#[cfg(test)]@@ -200,14 +202,48 @@ where}}+ /// An in-process session's raw token lives only in the process that was+ /// handed it, and the row keeps only its hash. Another replica may have+ /// resumed the session since, rotating the hash past any token this+ /// process still holds, so every resume rotates the credential. A session+ /// another live replica manages is refused before rotating: its claim+ /// would refuse this attach anyway, and the rotation would revoke the+ /// credential its live agent holds.async fn resume(&self,session: AgentSessionId,) -> Result<agent_session::domain::connection::RuntimeAttachment<Self::Transport>> {match self.route(session).await? {- Route::InMem(facts) => Ok(agent_session::domain::connection::RuntimeAttachment::solo(- RoutedTransport::InMem(self.inmem().manager.attach(*facts, None).await),- )),+ Route::InMem(facts) => {+ let management = self+ .sessions+ .management(session)+ .await+ .map_err(HarnessError::Session)?;+ if let SessionManagement::Peer(manager) = management {+ tracing::info!(+ %session,+ peer = %manager.replica,+ "not rotating the credential of a session another replica manages"+ );+ return Err(HarnessError::Session(AgentSessionError::ManagedElsewhere(+ session,+ )));+ }+ let token = SessionToken::mint();+ self.sessions+ .set_egress_token_hash(session, &token.hash())+ .await+ .map_err(HarnessError::Session)?;+ Ok(agent_session::domain::connection::RuntimeAttachment::solo(+ RoutedTransport::InMem(+ self.inmem()+ .manager+ .attach(*facts, Some(token.as_str().to_owned()))+ .await,+ ),+ ))+ }Route::Sandbox => self.sandbox.resume(session)@@ -217,9 +253,8 @@ where}/// An in-process session has no container environment to read a token- /// back from, so the manager remembers the one spawn handed it; a- /// session this process never spawned yields none. Sandboxes delegate to- /// their provider.+ /// back from, so the manager remembers the one spawn or [`Self::resume`]+ /// handed it. Sandboxes delegate to their provider.async fn session_token(&self, session: AgentSessionId) -> Result<Option<String>> {match self.route(session).await? {Route::InMem(_) => Ok(self.inmem().manager.session_token(session)),diff --git a/services/agent_harness_service/src/containers/test.rs b/services/agent_harness_service/src/containers/test.rsindex 76a8045ab6..1e0df2f847 100644--- a/services/agent_harness_service/src/containers/test.rs+++ b/services/agent_harness_service/src/containers/test.rs@@ -1,9 +1,15 @@use super::*;use agent_fold::domain::service::FoldedMessageService;+use agent_harness::domain::model::SandboxEgress;use agent_harness::outbound::daytona::AnthropicApiKey;use agent_harness::outbound::local::{LocalContainerManager, LocalSettings};-use agent_session::domain::model::ReplicaId;-use agent_session::domain::ports::NoOpRealtime;+use agent_harness::testing::helpers::egress::test_egress;+use agent_inmem::domain::engine::{TurnEngine, TurnRequest};+use agent_inmem::domain::mcp::NoMcpServers;+use agent_inmem::domain::model_access::{InMemModelAccess, ModelAccess, ModelAccessError};+use agent_inmem::outbound::log_frames::LogFrameSource;+use agent_session::domain::model::{ClaimOutcome, ReplicaId};+use agent_session::domain::ports::{AgentSessionRepo, NoOpRealtime, SessionOwnership};use agent_session::domain::ports::{NoOpAgentSessionNameGenerator, NoOpTurnObserver, NoopLifecyclePublisher,};@@ -31,13 +37,22 @@ fn sessions_with_harness(bot: BotId,harness: &str,) -> (AgentSessionId, impl AgentSessionService + use<>) {+ let (id, repo) = repo_with_session(bot, harness);+ (id, service_over(repo))+}++fn repo_with_session(bot: BotId, harness: &str) -> (AgentSessionId, InMemoryAgentSessionRepo) {let repo = InMemoryAgentSessionRepo::new();let id = AgentSessionId::new();let mut session = test_agent_session(id);session.bot_id = bot;session.harness = harness.to_owned();repo.insert_session(session);- let service = AgentSessionServiceImpl::new(+ (id, repo)+}++fn service_over(repo: InMemoryAgentSessionRepo) -> impl AgentSessionService + use<> {+ AgentSessionServiceImpl::new(repo.clone(),FoldedMessageService::new(repo),NoOpRealtime,@@ -45,8 +60,63 @@ fn sessions_with_harness(Arc::new(NoOpTurnObserver),Arc::new(NoopLifecyclePublisher),ReplicaId::mint(),- );- (id, service)+ )+}++/// Never asked to run a turn: these tests only attach sessions.+struct IdleEngine;++impl TurnEngine for IdleEngine {+ fn supported_models(&self) -> &[&str] {+ &[]+ }++ fn run_turn(+ &self,+ _request: TurnRequest,+ ) -> tokio::sync::mpsc::Receiver<Result<agent::StreamPart, agent::AgentError>> {+ tokio::sync::mpsc::channel(1).1+ }+}++struct FreeModels;++#[async_trait::async_trait]+impl InMemModelAccess for FreeModels {+ async fn access(&self, _owner: &model_owner::Owner) -> Result<ModelAccess, ModelAccessError> {+ Ok(ModelAccess::Free)+ }+}++fn in_process_runtime(repo: InMemoryAgentSessionRepo) -> InMemRuntime {+ InMemRuntime {+ manager: InMemAgentManager::new(+ Arc::new(IdleEngine),+ Arc::new(LogFrameSource::new(repo)),+ Arc::new(NoMcpServers),+ Arc::new(FreeModels),+ ),+ }+}++fn in_process_containers(+ repo: &InMemoryAgentSessionRepo,+) -> RoutedContainers<impl AgentSessionService + use<>> {+ RoutedContainers::new(+ unreachable_sandbox(),+ Some(in_process_runtime(repo.clone())),+ service_over(repo.clone()),+ )+}++async fn session_for_token(+ repo: &InMemoryAgentSessionRepo,+ token: &SessionToken,+) -> Option<AgentSessionId> {+ repo.find_by_egress_token_hash(&token.hash())+ .await+ .expect("token lookup")+ .map(|session| session.id)}#[tokio::test]@@ -109,6 +179,104 @@ async fn routes_an_owned_bot_to_the_sandbox() {));}+/// A replica that replaced the one which spawned an in-process session has+/// no way to read its token back, so resuming there rotates the credential:+/// the session's MCP servers are advertised under a token the proxy accepts,+/// and the token that died with the old process no longer is.+#[tokio::test]+async fn resuming_an_in_process_session_held_elsewhere_rotates_its_credential() {+ let (id, repo) = repo_with_session(bot_id::MACRO_NEW_BOT_ID, "opencode");+ let lost = SessionToken::new("held by the replaced replica");+ repo.set_egress_token_hash(id, &lost.hash())+ .await+ .expect("spawn's hash");+ let containers = in_process_containers(&repo);++ let _attachment = containers.resume(id).await.expect("resumes");++ let token = containers+ .session_token(id)+ .await+ .expect("token lookup")+ .expect("a resumed in-process session holds a credential");+ assert_eq!(+ session_for_token(&repo, &SessionToken::new(token)).await,+ Some(id)+ );+ assert_eq!(session_for_token(&repo, &lost).await, None);+}++/// A session can move to another replica and back while the first stays up.+/// The token the first still holds was revoked when the second rotated, so+/// resuming on the first again must not reuse it.+#[tokio::test]+async fn resuming_where_a_revoked_token_is_still_held_does_not_reuse_it() {+ let (id, repo) = repo_with_session(bot_id::MACRO_NEW_BOT_ID, "opencode");+ let spawned = SessionToken::new("held by the first replica");+ repo.set_egress_token_hash(id, &spawned.hash())+ .await+ .expect("spawn's hash");+ let first = in_process_containers(&repo);+ let second = in_process_containers(&repo);+ let _spawned = first+ .spawn(SpawnContainer {+ session_id: id,+ kind: AgentKind::InMemory,+ size: SandboxSize::Default,+ egress: SandboxEgress {+ session_token: spawned.as_str().to_owned(),+ ..test_egress()+ },+ })+ .await+ .expect("spawns");+ let _moved = second.resume(id).await.expect("resumes on the second");++ let _returned = first.resume(id).await.expect("resumes on the first again");++ let token = first+ .session_token(id)+ .await+ .expect("token lookup")+ .expect("a resumed in-process session holds a credential");+ assert_ne!(token, spawned.as_str());+ assert_eq!(+ session_for_token(&repo, &SessionToken::new(token)).await,+ Some(id)+ );+}++/// Another live replica's claim would refuse this attach, and rotating first+/// would revoke the credential that replica's agent is using.+#[tokio::test]+async fn resuming_a_session_a_live_peer_manages_is_refused_without_rotating() {+ let (id, repo) = repo_with_session(bot_id::MACRO_NEW_BOT_ID, "opencode");+ let current = SessionToken::new("held by the managing replica");+ repo.set_egress_token_hash(id, ¤t.hash())+ .await+ .expect("spawn's hash");+ let claimed = repo+ .claim(id, ReplicaId::mint())+ .await+ .expect("peer claims");+ assert!(matches!(claimed, ClaimOutcome::Claimed(_)));+ let containers = in_process_containers(&repo);++ let Err(error) = containers.resume(id).await else {+ panic!("a session a live peer manages must not resume here");+ };++ assert!(matches!(+ error,+ HarnessError::Session(AgentSessionError::ManagedElsewhere(session)) if session == id+ ));+ assert_eq!(session_for_token(&repo, ¤t).await, Some(id));+ assert_eq!(+ containers.session_token(id).await.expect("token lookup"),+ None+ );+}+/// The in-process runtime skips Macro's own server by the name the harness/// gives it, and restates that name rather than importing it. This is the one/// crate that sees both, so it is where they are held equal.