Enterprise agent platform for building, deploying, and operating AI teammates.
"""Slice 2 of #757: AgentService cache eviction on scope-fingerprint mismatch.
``get_agent_for_task`` resolves the ExecutionScope per call (same place the
owner is resolved) and compares its fingerprint against the one the cached
instance was built under. These tests drive the real ``get_agent_for_task``
through the resolver path and pin:
* a scope change between turns evicts and rebuilds (never silently reuses
the old scope's namespace), without destroying the same-owner workspace,
* an A -> B -> A flap is logged as a probable resolver bug,
* unscoped behavior is unchanged (cached instance reused, no eviction),
* the resolved scope reaches sandbox acquisition: the rebuilt agent records
a scope-suffixed sandbox key.
"""
from __future__ import annotations
import logging
from collections import deque
from contextlib import ExitStack
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from tests.shared.execution_scope import register_scope_resolver
from xagent.core.execution_scope import (
ExecutionScope,
ExecutionScopeAuthorityError,
ExecutionScopeContext,
scope_fingerprint,
set_execution_scope_snapshot_loader,
)
from xagent.web.models.agent import AgentStatus
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.user import User
from xagent.web.services.agent_service_manager import AgentServiceManager
from xagent.web.services.llm_utils import AgentRuntimeFields
from xagent.web.services.task_setup_snapshot import (
RuntimeUserFields,
TaskSetupSnapshot,
_TaskFields,
)
SCOPE_A = ExecutionScope(sandbox_key_suffix="tenant-a")
SCOPE_B = ExecutionScope(sandbox_key_suffix="tenant-b")
def _make_user() -> User:
return User(id=1, username="scope-fp-user", password_hash="hash", is_admin=False)
def _make_task_row() -> Task:
return Task(
id=42,
user_id=1,
title="scope-fp task",
description="x",
status=TaskStatus.PENDING,
agent_id=7,
agent_type="standard",
)
def _build_snapshot() -> TaskSetupSnapshot:
return TaskSetupSnapshot(
task=_TaskFields(
id=42,
user_id=1,
status=TaskStatus.PENDING,
agent_id=7,
agent_config=None,
model_name=None,
compact_model_name=None,
execution_mode="flash",
agent_type="standard",
),
runtime_user=RuntimeUserFields(id=1, is_admin=False),
has_reconstructable_history=False,
task_pattern="single_call",
task_llm=None,
task_fast_llm=None,
task_vision_llm=None,
task_compact_llm=None,
agent=AgentRuntimeFields(
id=7,
name="scope-fp-agent",
status=AgentStatus.PUBLISHED,
instructions="be terse",
),
agent_config={
"llms": (None, None, None, None),
"execution_mode": "flash",
"instructions": "be terse",
"skills": [],
"knowledge_bases": [],
"tool_categories": ["basic"],
},
excluded_agent_id=7,
)
def _build_db_mock(task_row: Task) -> MagicMock:
db = MagicMock()
db.query.return_value.filter.return_value.first.return_value = task_row
return db
def _common_patches(
manager: AgentServiceManager, *, sandbox_manager: Any = None
) -> list[Any]:
# Environments without API keys have no default LLM; the build must get
# past LLM resolution to reach the cache/sandbox logic under test.
manager._default_llm = MagicMock()
return [
patch.object(manager, "_load_persisted_conversation_history"),
patch.object(manager, "_load_persisted_execution_context", new=AsyncMock()),
patch(
"xagent.web.services.agent_service_manager.create_task_tracer",
return_value=MagicMock(),
),
patch(
"xagent.web.services.agent_service_manager.create_default_tools",
new=AsyncMock(return_value=([], MagicMock())),
),
patch(
"xagent.web.sandbox_manager.get_sandbox_manager",
return_value=sandbox_manager,
),
patch("xagent.web.services.agent_service_manager.AgentService"),
]
async def _call(manager: AgentServiceManager, **kwargs: Any) -> None:
try:
await manager.get_agent_for_task(task_id=42, **kwargs)
except Exception:
# Downstream stubs may raise after the cache decision under test;
# the assertions below inspect the cache maps directly.
pass
@pytest.mark.asyncio
async def test_scope_change_between_turns_evicts_and_rebuilds() -> None:
register_scope_resolver(lambda task_id: SCOPE_B)
manager = AgentServiceManager()
stale_agent = MagicMock()
manager._agents[42] = stale_agent
manager._agent_owner_ids[42] = 1
manager._agent_sandbox_keys[42] = "user:1:tenant-a"
manager._agent_scope_fingerprints[42] = scope_fingerprint(SCOPE_A)
with ExitStack() as stack:
for p in _common_patches(manager):
stack.enter_context(p)
await _call(
manager,
db=_build_db_mock(_make_task_row()),
user=_make_user(),
task_setup_snapshot=_build_snapshot(),
)
# The stale-scope instance is gone and the rebuild recorded the new
# fingerprint; turn 2 must not execute in scope A's namespace.
assert manager._agents.get(42) is not stale_agent
assert manager._agent_scope_fingerprints.get(42) == scope_fingerprint(SCOPE_B)
# Same owner: the workspace survives a scope reassignment.
stale_agent.cleanup_workspace.assert_not_called()
@pytest.mark.asyncio
async def test_isolate_external_dirs_only_change_evicts_and_rebuilds() -> None:
"""(#296) isolate_external_dirs is baked into the
cached AgentService's allowed_external_dirs at build time, not read
fresh -- an isolate_external_dirs-only change (same sandbox suffix,
workspace segments, mount, memory dimensions) must still evict the
cache, or the stale allowed-dirs list keeps being enforced."""
scope_shared = ExecutionScope(sandbox_key_suffix="tenant-a")
scope_isolated = ExecutionScope(
sandbox_key_suffix="tenant-a", isolate_external_dirs=True
)
assert scope_fingerprint(scope_shared) != scope_fingerprint(scope_isolated)
register_scope_resolver(
lambda task_id: scope_isolated,
)
manager = AgentServiceManager()
stale_agent = MagicMock()
manager._agents[42] = stale_agent
manager._agent_owner_ids[42] = 1
manager._agent_sandbox_keys[42] = "user:1:tenant-a"
manager._agent_scope_fingerprints[42] = scope_fingerprint(scope_shared)
with ExitStack() as stack:
for p in _common_patches(manager):
stack.enter_context(p)
await _call(
manager,
db=_build_db_mock(_make_task_row()),
user=_make_user(),
task_setup_snapshot=_build_snapshot(),
)
assert manager._agents.get(42) is not stale_agent
assert manager._agent_scope_fingerprints.get(42) == scope_fingerprint(
scope_isolated
)
@pytest.mark.asyncio
async def test_scope_flap_is_logged_as_probable_resolver_bug(caplog) -> None:
"""A -> B -> A: the resolver returning the fingerprint that a previous
rebuild evicted means it flaps between values — every turn would evict
and rebuild, silently defeating the cache."""
register_scope_resolver(lambda task_id: SCOPE_A)
manager = AgentServiceManager()
manager._agents[42] = MagicMock()
manager._agent_owner_ids[42] = 1
manager._agent_scope_fingerprints[42] = scope_fingerprint(SCOPE_B)
# A previous scope-mismatch rebuild evicted fingerprint A.
manager._agent_evicted_scope_fingerprints[42] = deque(
[scope_fingerprint(SCOPE_A)], maxlen=4
)
with ExitStack() as stack:
for p in _common_patches(manager):
stack.enter_context(p)
with caplog.at_level(logging.WARNING):
await _call(
manager,
db=_build_db_mock(_make_task_row()),
user=_make_user(),
task_setup_snapshot=_build_snapshot(),
)
assert any("probable resolver bug" in r.message for r in caplog.records)
@pytest.mark.asyncio
async def test_multi_scope_cycle_is_also_flagged_as_resolver_bug(caplog) -> None:
"""A resolver cycling through 3+ scopes (A -> B -> C -> A -> ...)
defeats the cache every turn just like a period-2 flap; the bounded
recently-evicted memory must flag it too (Roger's approval-round
finding on #789)."""
cycle = [SCOPE_A, SCOPE_B, ExecutionScope(sandbox_key_suffix="tenant-c")]
turn = {"n": 0}
def cycling_resolver(task_id):
return cycle[turn["n"] % len(cycle)]
register_scope_resolver(cycling_resolver)
manager = AgentServiceManager()
with ExitStack() as stack:
for p in _common_patches(manager):
stack.enter_context(p)
with caplog.at_level(logging.WARNING):
for _ in range(4): # A, B, C, then back to A
await _call(
manager,
db=_build_db_mock(_make_task_row()),
user=_make_user(),
task_setup_snapshot=_build_snapshot(),
)
turn["n"] += 1
assert any("probable resolver bug" in r.message for r in caplog.records)
@pytest.mark.asyncio
async def test_activated_turn_scope_is_reused_without_re_resolving() -> None:
"""One turn resolves once: inside an activated turn context,
get_agent_for_task consumes the contextvar scope instead of a fresh
loader/resolver round-trip (Roger's approval-round finding on #789)."""
resolver_calls: list[str] = []
def resolver(task_id):
resolver_calls.append(task_id)
return SCOPE_A
register_scope_resolver(resolver)
manager = AgentServiceManager()
with ExitStack() as stack:
for p in _common_patches(manager):
stack.enter_context(p)
with ExecutionScopeContext(SCOPE_A): # the turn's activation
await _call(
manager,
db=_build_db_mock(_make_task_row()),
user=_make_user(),
task_setup_snapshot=_build_snapshot(),
)
assert resolver_calls == []
assert manager._agent_scope_fingerprints.get(42) == scope_fingerprint(SCOPE_A)
@pytest.mark.asyncio
async def test_unscoped_cached_agent_is_reused_without_eviction() -> None:
"""No resolver -> fingerprint None on both sides -> today's behavior:
the cached instance is returned untouched."""
manager = AgentServiceManager()
cached_agent = MagicMock()
manager._agents[42] = cached_agent
manager._agent_owner_ids[42] = 1
manager._agent_scope_fingerprints[42] = None
result = await manager.get_agent_for_task(
task_id=42,
db=_build_db_mock(_make_task_row()),
user=_make_user(),
task_setup_snapshot=_build_snapshot(),
)
assert result is cached_agent
cached_agent.cleanup_workspace.assert_not_called()
@pytest.mark.asyncio
async def test_stable_scope_does_not_evict_between_turns() -> None:
"""An idempotent resolver returning an equal scope every turn keeps the
cache warm — equality is by fingerprint value, not object identity."""
register_scope_resolver(
lambda task_id: ExecutionScope(sandbox_key_suffix="tenant-a"),
)
manager = AgentServiceManager()
cached_agent = MagicMock()
manager._agents[42] = cached_agent
manager._agent_owner_ids[42] = 1
manager._agent_scope_fingerprints[42] = scope_fingerprint(
ExecutionScope(sandbox_key_suffix="tenant-a")
)
result = await manager.get_agent_for_task(
task_id=42,
db=_build_db_mock(_make_task_row()),
user=_make_user(),
task_setup_snapshot=_build_snapshot(),
)
assert result is cached_agent
class _ScopeTrackingToolConfig:
"""Minimal ``WebToolConfig`` stand-in: records every scope it is handed
without exercising the real swap/cache-drop logic (that logic is pinned
separately in ``tests/web/tools/test_web_tool_config_session_factory.py``).
"""
def __init__(self, scope: ExecutionScope) -> None:
self.scope = scope
self.set_calls: list[ExecutionScope] = []
def set_execution_scope(self, scope: ExecutionScope) -> bool:
self.set_calls.append(scope)
self.scope = scope
return True
class _CachedAgentWithToolConfig:
def __init__(self, tool_config: _ScopeTrackingToolConfig) -> None:
self.tool_config = tool_config
self.invalidate_calls = 0
def invalidate_tools(self) -> None:
self.invalidate_calls += 1
def revoke_memory(self, **kwargs: Any) -> None:
pass
def apply_memory_policy(self, **kwargs: Any) -> None:
pass
def cleanup_workspace(self) -> None:
pass
@pytest.mark.asyncio
async def test_cached_agent_service_is_resynced_with_the_turn_execution_scope() -> None:
"""A same-fingerprint scope change keeps the cached AgentService (no
rebuild), but the resolved scope for THIS turn must still reach the
cached tool config -- otherwise a resolver that hands back a scope
object carrying turn-varying data outside the namespace fingerprint
(here: ``strict_memory_isolation``, which ``scope_fingerprint`` doesn't
cover) would leave the OAuth resolver hook reading the first turn's
scope object forever.
Also covers the ``connector_runtime_turn_id=None`` case: unlike
``_sync_connector_runtime_turn``, the execution-scope resync must not be
gated on a turn id being present.
"""
tool_config = _ScopeTrackingToolConfig(SCOPE_A)
cached_agent = _CachedAgentWithToolConfig(tool_config)
manager = AgentServiceManager()
manager._agents[42] = cached_agent
manager._agent_owner_ids[42] = 1
manager._agent_scope_fingerprints[42] = scope_fingerprint(SCOPE_A)
scope_b = ExecutionScope(
sandbox_key_suffix="tenant-a", strict_memory_isolation=True
)
assert scope_fingerprint(scope_b) == scope_fingerprint(SCOPE_A)
assert scope_b != SCOPE_A
result = await manager.get_agent_for_task(
task_id=42,
db=_build_db_mock(_make_task_row()),
user=_make_user(),
task_setup_snapshot=_build_snapshot(),
connector_runtime_turn_id=None,
resolved_execution_scope=scope_b,
)
assert result is cached_agent
assert tool_config.scope is scope_b
assert tool_config.set_calls == [scope_b]
assert cached_agent.invalidate_calls == 1
@pytest.mark.asyncio
async def test_resolver_scope_reaches_sandbox_key_on_build() -> None:
"""End-to-end through the resolver path: a fresh build under a scoped
resolver acquires the scoped container family and records the
scope-suffixed key for execution-time attach."""
register_scope_resolver(lambda task_id: SCOPE_A)
manager = AgentServiceManager()
fake_sandbox_manager = MagicMock()
fake_sandbox_manager.get_or_create_lease_provider = AsyncMock(
return_value=AsyncMock()
)
with ExitStack() as stack:
for p in _common_patches(manager, sandbox_manager=fake_sandbox_manager):
stack.enter_context(p)
await _call(
manager,
db=_build_db_mock(_make_task_row()),
user=_make_user(),
task_setup_snapshot=_build_snapshot(),
)
fake_sandbox_manager.get_or_create_lease_provider.assert_awaited_once()
lifecycle_args = fake_sandbox_manager.get_or_create_lease_provider.await_args.args
assert lifecycle_args == ("user", "1:tenant-a")
assert manager._agent_sandbox_keys.get(42) == "user:1:tenant-a"
assert manager._agent_scope_fingerprints.get(42) == scope_fingerprint(SCOPE_A)
@pytest.mark.asyncio
async def test_resolver_scope_reaches_workspace_paths_on_build() -> None:
"""Slice 3: the resolved scope's workspace_segments reach the agent
build — the AgentService is constructed with a scoped
workspace_base_dir and carries the segments, and two scopes produce
disjoint base dirs. Driven through the resolver path."""
from xagent.config import get_uploads_dir
from xagent.core.workspace import scoped_user_root
scope = ExecutionScope(workspace_segments=("tenant-a",))
register_scope_resolver(lambda task_id: scope)
manager = AgentServiceManager()
with ExitStack() as stack:
# _common_patches's last entry patches AgentService anonymously;
# patch it here instead to keep a handle on the mock's call kwargs.
for p in _common_patches(manager)[:-1]:
stack.enter_context(p)
agent_service_mock = stack.enter_context(
patch("xagent.web.services.agent_service_manager.AgentService")
)
await _call(
manager,
db=_build_db_mock(_make_task_row()),
user=_make_user(),
task_setup_snapshot=_build_snapshot(),
)
kwargs = agent_service_mock.call_args.kwargs
expected_base = str(scoped_user_root(get_uploads_dir(), 1, ("tenant-a",)))
assert kwargs["workspace_base_dir"] == expected_base
assert kwargs["scope_segments"] == ("tenant-a",)
# Default sharing: the user-level upload dir stays in the allowlist.
assert (
str(scoped_user_root(get_uploads_dir(), 1)) in kwargs["allowed_external_dirs"]
)
@pytest.mark.asyncio
async def test_unscoped_build_uses_legacy_workspace_base_dir() -> None:
"""No resolver -> AgentService gets the byte-identical legacy base dir
and empty scope segments."""
from xagent.config import get_uploads_dir
from xagent.core.workspace import scoped_user_root
manager = AgentServiceManager()
manager._default_llm = MagicMock()
with ExitStack() as stack:
for p in _common_patches(manager)[:-1]:
stack.enter_context(p)
agent_service_mock = stack.enter_context(
patch("xagent.web.services.agent_service_manager.AgentService")
)
await _call(
manager,
db=_build_db_mock(_make_task_row()),
user=_make_user(),
task_setup_snapshot=_build_snapshot(),
)
kwargs = agent_service_mock.call_args.kwargs
assert kwargs["workspace_base_dir"] == str(scoped_user_root(get_uploads_dir(), 1))
assert kwargs["scope_segments"] == ()
@pytest.mark.asyncio
async def test_unscoped_build_records_legacy_key() -> None:
"""No resolver -> the build records the byte-identical legacy key."""
manager = AgentServiceManager()
fake_sandbox_manager = MagicMock()
fake_sandbox_manager.get_or_create_lease_provider = AsyncMock(
return_value=AsyncMock()
)
with ExitStack() as stack:
for p in _common_patches(manager, sandbox_manager=fake_sandbox_manager):
stack.enter_context(p)
await _call(
manager,
db=_build_db_mock(_make_task_row()),
user=_make_user(),
task_setup_snapshot=_build_snapshot(),
)
lifecycle_args = fake_sandbox_manager.get_or_create_lease_provider.await_args.args
assert lifecycle_args == ("user", "1")
assert manager._agent_sandbox_keys.get(42) == "user:1"
assert manager._agent_scope_fingerprints.get(42) is None
@pytest.mark.asyncio
async def test_turn_face_authority_mismatch_fails_the_turn() -> None:
"""``get_agent_for_task`` is a turn-face consumer of
``resolve_execution_scope``: unlike the off-turn consumers
(``ManagedFileRef``'s construction, the pause/resume handlers), a
namespace-affecting authority mismatch here must propagate and fail the
turn rather than being downgraded to a warning.
``websocket._scope_segments_for_task`` is off-turn and still resolves
fail-closed, so what decides this is whether a namespace is being chosen
for new bytes, not whether a turn is running."""
register_scope_resolver(
lambda task_id: ExecutionScope(sandbox_key_suffix="from-resolver"),
)
set_execution_scope_snapshot_loader(
lambda task_id: ExecutionScope(sandbox_key_suffix="from-snapshot")
)
manager = AgentServiceManager()
manager._default_llm = MagicMock()
with pytest.raises(ExecutionScopeAuthorityError):
await manager.get_agent_for_task(
task_id=42,
db=None,
user=_make_user(),
task_owner_user_id=1,
task_setup_snapshot=_build_snapshot(),
)