Enterprise agent platform for building, deploying, and operating AI teammates.
"""Auto binding, default resolution and lifecycle regressions against a real DB,
plus smoke tests against the installed xrouter-llm catalog."""
import asyncio
from types import SimpleNamespace
import pytest
from fastapi import FastAPI, HTTPException
from fastapi.testclient import TestClient
from sqlalchemy import create_engine, event
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import StaticPool
from xagent.core.model.chat.basic.router import (
_MODALITY_ABILITIES,
RouterLLM,
_ResolvedRouterLLM,
)
from xagent.web import models
from xagent.web.api import admin_users, agents
from xagent.web.api import model as model_api
from xagent.web.api import workforces
from xagent.web.auth_dependencies import get_current_user
from xagent.web.models import database
from xagent.web.models.auto_model import AutoModelCandidate, AutoModelConfig
from xagent.web.models.model import Model
from xagent.web.models.user import User, UserDefaultModel, UserModel
from xagent.web.schemas.model import AutoModelConfigUpdate, ModelUpdate
from xagent.web.services import (
auto_model_service,
builder_chat_runtime,
llm_utils,
model_service,
)
from xagent.web.services.auto_model_service import (
AutoModelConfigurationError,
AutoModelService,
)
from xagent.web.services.client_error_messages import CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE
from xagent.web.services.llm_utils import (
AutoModelUnavailableError,
UserAwareModelStorage,
)
@pytest.fixture
def env(monkeypatch, tmp_path):
monkeypatch.setenv("XAGENT_STORAGE_ROOT", str(tmp_path))
engine = create_engine(
"sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool
)
@event.listens_for(engine, "connect")
def foreign_keys(conn, _):
conn.execute("PRAGMA foreign_keys=ON")
models.Base.metadata.create_all(engine)
factory = sessionmaker(bind=engine, expire_on_commit=False)
db = factory()
monkeypatch.setattr(database, "_SessionLocal", factory)
monkeypatch.setattr(admin_users, "get_session_local", lambda: factory)
monkeypatch.setattr(builder_chat_runtime, "get_session_local", lambda: factory)
monkeypatch.setattr(model_service, "_visible_user_ids_hook", None)
profiles = {
"text-a": SimpleNamespace(input_modalities=("text",)),
"text-b": SimpleNamespace(input_modalities=("text",)),
"image-a": SimpleNamespace(input_modalities=("text", "image")),
}
catalog = SimpleNamespace(
known_model_ids=lambda: tuple(profiles), get=profiles.__getitem__
)
monkeypatch.setattr(
auto_model_service, "load_router_profile_catalog", lambda: catalog
)
owner = User(username="owner", password_hash="unused", is_admin=True)
consumer = User(username="consumer", password_hash="unused", is_admin=False)
db.add_all([owner, consumer])
db.commit()
yield SimpleNamespace(db=db, factory=factory, owner=owner, consumer=consumer)
db.close()
engine.dispose()
def target(env, owner, *, name="saved", abilities=None, shared=False):
row = Model(
model_id=name,
category="llm",
model_provider="openai",
model_name="gpt-4",
api_key="fake-test-key",
base_url="https://example.invalid/v1",
abilities=abilities or ["chat"],
is_active=True,
)
env.db.add(row)
env.db.flush()
env.db.add(
UserModel(
user_id=owner.id,
model_id=row.id,
is_owner=True,
can_edit=True,
can_delete=True,
is_shared=shared,
)
)
env.db.commit()
return row
def configure(env, user, bindings, *, default=False):
return AutoModelService(env.db).upsert_config(
user_id=user.id,
request=AutoModelConfigUpdate(
candidates=[
{"routing_model_id": profile, "target_model_id": t.id}
for profile, t in bindings
],
fallback_model_id=bindings[0][1].id,
set_as_default=default,
),
)
@pytest.fixture
def llm_utils_warnings(monkeypatch):
"""Record the warnings llm_utils logs, bypassing logging configuration.
Replacing the module's ``logger`` binding keeps the assertion independent
of whichever logging setup ran first in the test process.
"""
real = llm_utils.logger
warnings: list[str] = []
class RecordingLogger:
def warning(self, msg, *args, **kwargs):
warnings.append(msg % args if args else msg)
def __getattr__(self, name):
return getattr(real, name)
monkeypatch.setattr(llm_utils, "logger", RecordingLogger())
return warnings
def skipped_candidates(warnings):
return [w for w in warnings if w.startswith("Skipping Auto candidate ")]
def test_candidate_relationship_reads_persisted_position_order(env):
first = target(env, env.consumer, name="first")
second = target(env, env.consumer, name="second")
config = configure(
env,
env.consumer,
[("text-a", first), ("text-b", second)],
)
assert config.mode == "routing"
assert [candidate.position for candidate in config.candidates] == [0, 1]
config.candidates[0].position = -1
env.db.flush()
config.candidates[1].position = 0
env.db.flush()
config.candidates[0].position = 1
env.db.commit()
env.db.expire_all()
loaded = env.db.get(AutoModelConfig, config.id)
assert loaded is not None
assert [candidate.routing_model_id for candidate in loaded.candidates] == [
"text-b",
"text-a",
]
def test_create_all_schema_allows_same_profile_for_distinct_targets(env):
first = target(env, env.consumer, name="first")
second = target(env, env.consumer, name="second")
config = configure(env, env.consumer, [("text-a", first)])
env.db.add(
AutoModelCandidate(
config_id=config.id,
routing_model_id="text-a",
target_model_id=second.id,
position=1,
)
)
env.db.commit()
env.db.expire_all()
loaded = env.db.get(AutoModelConfig, config.id)
assert loaded is not None
assert [
(candidate.routing_model_id, candidate.target_model_id, candidate.position)
for candidate in loaded.candidates
] == [
("text-a", first.id, 0),
("text-a", second.id, 1),
]
def broken_default(env, slot="general"):
t = target(env, env.consumer)
cfg = configure(env, env.consumer, [("text-a", t)])
env.db.add(
UserDefaultModel(
user_id=env.consumer.id, model_id=cfg.router_model_id, config_type=slot
)
)
t.is_active = False
env.db.commit()
return cfg
@pytest.mark.parametrize(
"profile,abilities", [("image-a", ["chat"]), ("text-a", ["chat", "vision"])]
)
def test_rejects_profile_capability_mismatch_without_saving(env, profile, abilities):
t = target(env, env.consumer, abilities=abilities)
with pytest.raises(AutoModelConfigurationError, match="input modalities"):
configure(env, env.consumer, [(profile, t)])
assert env.db.query(AutoModelConfig).count() == 0
def test_legacy_mismatched_binding_fails_before_provider_call(env, llm_utils_warnings):
t = target(env, env.consumer)
cfg = configure(env, env.consumer, [("text-a", t)])
cfg.candidates[0].routing_model_id = "image-a"
env.db.commit()
with pytest.raises(AutoModelUnavailableError, match="no active configured"):
UserAwareModelStorage(env.db).get_llm_by_id(
cfg.router_model.model_id, env.consumer.id
)
skipped = skipped_candidates(llm_utils_warnings)
assert len(skipped) == 1
assert "'image-a'" in skipped[0] and "input modalities" in skipped[0]
@pytest.mark.parametrize("removed_index", [0, 1])
def test_routing_skips_candidate_whose_profile_left_the_catalog(
env, llm_utils_warnings, removed_index
):
first = target(env, env.consumer, name="first")
second = target(env, env.consumer, name="second")
cfg = configure(
env, env.consumer, [("text-a", first), ("text-b", second)], default=True
)
cfg.candidates[removed_index].routing_model_id = "removed-profile"
env.db.commit()
remaining = ("text-a", "text-b")[1 - removed_index]
llm = UserAwareModelStorage(env.db).get_llm_by_id(
cfg.router_model.model_id, env.consumer.id
)
assert isinstance(llm, RouterLLM)
assert llm._candidate_models == (remaining,)
# The first candidate is the saved fallback; skipping it clears the fallback.
assert llm._fallback_model == (None if removed_index == 0 else remaining)
assert (
llm._downstream_resolver(remaining).model_id
== (("first", "second")[1 - removed_index])
)
skipped = skipped_candidates(llm_utils_warnings)
assert len(skipped) == 1
assert "'removed-profile'" in skipped[0] and "xrouter-llm catalog" in skipped[0]
default = model_service.get_default_model(env.consumer.id)
assert isinstance(default, RouterLLM)
assert default._candidate_models == (remaining,)
@pytest.mark.parametrize("mismatched_index", [0, 1])
def test_routing_skips_candidate_whose_profile_no_longer_matches_its_target(
env, llm_utils_warnings, mismatched_index
):
# Saving rejects this binding, but it can still exist, for example after
# an xrouter-llm release changes a profile's input modalities.
first = target(env, env.consumer, name="first")
second = target(env, env.consumer, name="second")
cfg = configure(
env, env.consumer, [("text-a", first), ("text-b", second)], default=True
)
cfg.candidates[mismatched_index].routing_model_id = "image-a"
env.db.commit()
remaining = ("text-a", "text-b")[1 - mismatched_index]
llm = UserAwareModelStorage(env.db).get_llm_by_id(
cfg.router_model.model_id, env.consumer.id
)
assert isinstance(llm, RouterLLM)
assert llm._candidate_models == (remaining,)
# The first candidate is the saved fallback; skipping it clears the fallback.
assert llm._fallback_model == (None if mismatched_index == 0 else remaining)
assert (
llm._downstream_resolver(remaining).model_id
== ("first", "second")[1 - mismatched_index]
)
skipped = skipped_candidates(llm_utils_warnings)
assert len(skipped) == 1
assert "'image-a'" in skipped[0] and "input modalities" in skipped[0]
default = model_service.get_default_model(env.consumer.id)
assert isinstance(default, RouterLLM)
assert default._candidate_models == (remaining,)
@pytest.mark.parametrize(
"stale_profiles",
[
("removed-profile", "another-removed-profile"),
("image-a", "image-a"),
("removed-profile", "image-a"),
],
ids=["unknown", "mismatched", "mixed"],
)
def test_routing_with_only_stale_bindings_stays_unavailable(
env, llm_utils_warnings, stale_profiles
):
first = target(env, env.consumer, name="first")
second = target(env, env.consumer, name="second")
cfg = configure(
env, env.consumer, [("text-a", first), ("text-b", second)], default=True
)
cfg.candidates[0].routing_model_id = stale_profiles[0]
cfg.candidates[1].routing_model_id = stale_profiles[1]
env.db.commit()
with pytest.raises(AutoModelUnavailableError, match="no active configured"):
UserAwareModelStorage(env.db).get_llm_by_id(
cfg.router_model.model_id, env.consumer.id
)
assert len(skipped_candidates(llm_utils_warnings)) == 2
with pytest.raises(AutoModelUnavailableError):
model_service.get_default_model(env.consumer.id)
def test_routing_maps_missing_router_dependency_to_unavailable(env, monkeypatch):
t = target(env, env.consumer)
cfg = configure(env, env.consumer, [("text-a", t)], default=True)
def missing_dependency():
raise auto_model_service.AutoModelDependencyError(
"Auto model configuration needs xrouter-llm>=0.3.6"
)
monkeypatch.setattr(
auto_model_service, "load_router_profile_catalog", missing_dependency
)
with pytest.raises(AutoModelUnavailableError, match="xrouter-llm"):
UserAwareModelStorage(env.db).get_llm_by_id(
cfg.router_model.model_id, env.consumer.id
)
with pytest.raises(AutoModelUnavailableError):
model_service.get_default_model(env.consumer.id)
def test_unknown_profile_has_its_own_configuration_error_type():
profiles = {"image-a": SimpleNamespace(input_modalities=("text", "image"))}
catalog = SimpleNamespace(
known_model_ids=lambda: tuple(profiles), get=profiles.__getitem__
)
unknown_error = auto_model_service.UnknownAutoProfileError
with pytest.raises(unknown_error, match="xrouter-llm catalog") as unknown:
auto_model_service.validate_candidate_modalities(
catalog, "removed-profile", ["chat"]
)
assert isinstance(unknown.value, AutoModelConfigurationError)
# A mismatch must stay the parent type, or ability edits would skip it.
with pytest.raises(
AutoModelConfigurationError, match="input modalities"
) as mismatch:
auto_model_service.validate_candidate_modalities(catalog, "image-a", ["chat"])
assert not isinstance(mismatch.value, unknown_error)
def test_sequential_keeps_candidate_whose_profile_left_the_catalog(env):
first = target(env, env.consumer, name="first")
second = target(env, env.consumer, name="second")
cfg = configure(env, env.consumer, [("text-a", first), ("text-b", second)])
cfg.mode = "sequential_failover"
cfg.fallback_model_id = None
cfg.candidates[0].routing_model_id = "removed-profile"
env.db.commit()
llm = UserAwareModelStorage(env.db).get_llm_by_id(
cfg.router_model.model_id, env.consumer.id
)
assert isinstance(llm, RouterLLM)
assert llm._candidate_models == tuple(
f"candidate:{candidate.id}" for candidate in cfg.candidates
)
def test_configured_wrapper_never_adds_profile_only_abilities():
downstream = SimpleNamespace(abilities=["chat"])
router = RouterLLM(
candidate_models=["image-a"], downstream_resolver=lambda _: downstream
)
resolved = _ResolvedRouterLLM(
router=router,
downstream=downstream,
selected_model="image-a",
context_window=None,
input_modalities=("image",),
)
assert resolved.abilities == ["chat"]
@pytest.mark.parametrize(
"changes",
[
{"model_name": "gpt-4.1"},
{"model_provider": "deepseek", "model_name": "deepseek-v4-flash"},
{"base_url": "https://another.invalid/v1"},
{"abilities": ["chat", "vision"]},
],
)
def test_bound_candidate_identity_and_modalities_cannot_change(env, changes):
t = target(env, env.consumer)
configure(env, env.consumer, [("text-a", t)])
with pytest.raises(HTTPException) as error:
asyncio.run(
model_api.update_model(
t.model_id, ModelUpdate(**changes), db=env.db, user=env.consumer
)
)
assert error.value.status_code == 409
env.db.rollback()
assert t.model_name == "gpt-4" and t.abilities == ["chat"]
def test_owner_identity_change_prunes_external_binding(env):
t = target(env, env.owner, shared=True)
cfg = configure(env, env.consumer, [("text-a", t)])
asyncio.run(
model_api.update_model(
t.model_id, ModelUpdate(model_name="gpt-4.1"), db=env.db, user=env.owner
)
)
env.db.expire_all()
assert env.db.query(AutoModelCandidate).filter_by(config_id=cfg.id).count() == 0
assert env.db.get(AutoModelConfig, cfg.id).fallback_model_id is None
@pytest.mark.parametrize("configured_compact", [False, True])
def test_missing_compact_does_not_resolve_broken_unrelated_general(
env, configured_compact
):
broken_default(env)
healthy = target(env, env.consumer, name="healthy")
if configured_compact:
env.db.add(
UserDefaultModel(
user_id=env.consumer.id, model_id=healthy.id, config_type="compact"
)
)
env.db.commit()
resolved = UserAwareModelStorage(env.db).resolve_llms_from_names(
[healthy.model_id] * 3 + [None], env.consumer.id
)
assert all(llm.model_id == healthy.model_id for llm in resolved)
builder = builder_chat_runtime._load_builder_chat_runtime_inputs_sync(
user_id=env.consumer.id,
requested_file_ids=[],
model_name=healthy.model_id,
compact_model_name=None,
)
assert builder.llm.model_id == builder.compact_llm.model_id == healthy.model_id
@pytest.mark.parametrize(
"slot,getter",
[
("general", "get_default_model"),
("small_fast", "get_fast_model"),
("visual", "get_default_vision_model"),
("compact", "get_compact_model"),
],
)
def test_default_getters_preserve_auto_configuration_error(env, slot, getter):
broken_default(env, slot)
with pytest.raises(AutoModelUnavailableError):
UserAwareModelStorage(env.db).get_configured_defaults(env.consumer.id)
kwargs = {"db": env.db} if slot == "visual" else {}
with pytest.raises(AutoModelUnavailableError):
getattr(model_service, getter)(env.consumer.id, **kwargs)
@pytest.mark.parametrize("surface", ["optimize", "workforce"])
def test_http_entrypoints_map_actual_broken_auto_to_safe_409(env, surface):
broken_default(env)
app = FastAPI()
app.include_router(agents.router if surface == "optimize" else workforces.router)
def db_override():
yield env.db
app.dependency_overrides[agents.get_db] = db_override
app.dependency_overrides[workforces.get_db] = db_override
app.dependency_overrides[get_current_user] = lambda: env.consumer
with TestClient(app) as client:
if surface == "optimize":
response = client.post(
"/api/agents/optimize-instructions",
json={"instructions": "Write a clear summary."},
)
else:
response = client.post(
"/api/workforces/from-prompt", json={"prompt": "Write a clear summary."}
)
assert response.status_code == 409
assert response.json()["detail"] == CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE
@pytest.mark.parametrize("retained_grant", [False, True])
def test_admin_delete_prunes_only_inaccessible_bindings(env, retained_grant):
t = target(env, env.owner, shared=True, abilities=["chat", "tool_calling"])
cfg = configure(env, env.consumer, [("text-a", t)])
ids = env.owner.id, env.consumer.id, t.id, cfg.id, cfg.router_model_id
if retained_grant:
env.db.add(UserModel(user_id=env.consumer.id, model_id=t.id, is_owner=True))
env.db.commit()
env.db.close()
assert admin_users._delete_user_rows_sync(user_id=ids[0]) is not None
with env.factory() as db:
assert db.get(User, ids[0]) is None
assert db.get(Model, ids[2]) is not None
assert db.query(AutoModelCandidate).filter_by(config_id=ids[3]).count() == int(
retained_grant
)
assert db.get(AutoModelConfig, ids[3]).fallback_model_id == (
ids[2] if retained_grant else None
)
assert db.get(Model, ids[4]).abilities == (
["chat", "tool_calling"] if retained_grant else ["chat"]
)
def test_compatible_ability_edit_refreshes_auto_without_locking_credentials(env):
t = target(env, env.consumer, abilities=["chat", "tool_calling"])
cfg = configure(env, env.consumer, [("text-a", t)])
asyncio.run(
model_api.update_model(
t.model_id,
ModelUpdate(
abilities=["chat"], api_key="new-fake-key", base_url=t.base_url
),
db=env.db,
user=env.consumer,
)
)
env.db.expire_all()
assert env.db.get(Model, cfg.router_model_id).abilities == ["chat"]
assert t.api_key == "new-fake-key"
def test_pruning_candidate_recalculates_remaining_abilities(env):
shared = target(env, env.owner, name="shared", shared=True)
own = target(env, env.consumer, name="own", abilities=["chat", "tool_calling"])
cfg = configure(env, env.consumer, [("text-a", shared), ("text-b", own)])
assert "tool_calling" not in cfg.router_model.abilities
asyncio.run(
model_api.update_model(
shared.model_id,
ModelUpdate(share_with_users=False),
db=env.db,
user=env.owner,
)
)
env.db.expire_all()
assert env.db.get(Model, cfg.router_model_id).abilities == ["chat", "tool_calling"]
def test_editing_model_bound_to_unknown_profile_is_not_blocked(env):
t = target(env, env.consumer)
cfg = configure(env, env.consumer, [("text-a", t)])
cfg.candidates[0].routing_model_id = "removed-profile"
env.db.commit()
# Routing skips this binding while its profile is not in the installed
# catalog, so the edit is not checked against it.
asyncio.run(
model_api.update_model(
t.model_id,
ModelUpdate(abilities=["chat", "vision"]),
db=env.db,
user=env.consumer,
)
)
env.db.expire_all()
assert env.db.get(Model, t.id).abilities == ["chat", "vision"]
with pytest.raises(AutoModelUnavailableError, match="no active configured"):
UserAwareModelStorage(env.db).get_llm_by_id(
cfg.router_model.model_id, env.consumer.id
)
@pytest.mark.parametrize("unknown_first", [True, False])
def test_ability_edit_skips_only_the_unknown_profile_binding(env, unknown_first):
# Two users bind the same model. A shared model's abilities cannot change
# at all, so the consumer reaches the owner's model through a direct,
# unshared grant row instead.
t = target(env, env.owner)
env.db.add(UserModel(user_id=env.consumer.id, model_id=t.id, is_owner=True))
env.db.commit()
users = [env.consumer, env.owner] if unknown_first else [env.owner, env.consumer]
configs = {user.id: configure(env, user, [("text-a", t)]) for user in users}
configs[env.consumer.id].candidates[0].routing_model_id = "removed-profile"
env.db.commit()
# update_model walks the bindings in this query's order; pin it so each
# case really puts the unknown binding where its name says.
order = [
candidate.routing_model_id
for candidate in env.db.query(AutoModelCandidate)
.filter(AutoModelCandidate.target_model_id == t.id)
.all()
]
assert order == (
["removed-profile", "text-a"]
if unknown_first
else ["text-a", "removed-profile"]
)
with pytest.raises(HTTPException) as error:
asyncio.run(
model_api.update_model(
t.model_id,
ModelUpdate(abilities=["chat", "vision"]),
db=env.db,
user=env.owner,
)
)
assert error.value.status_code == 409
assert "input modalities" in str(error.value.detail)
env.db.rollback()
assert t.abilities == ["chat"]
def _bundled_auto_router_models() -> tuple[str, ...]:
pytest.importorskip("xrouter_llm")
from xrouter_llm import default_routers_dir
from xrouter_llm.serving import load_router_configs
configs = load_router_configs(default_routers_dir())
return tuple(configs[auto_model_service.AUTO_ROUTER_CONFIG_NAME].models)
def test_bundled_auto_router_profiles_pass_candidate_validation():
"""Every profile in xrouter-llm's own ``auto`` router config can be bound.
Uses the installed catalog, not the stub the ``env`` fixture injects.
"""
auto_models = _bundled_auto_router_models()
catalog = auto_model_service.load_router_profile_catalog()
known = set(catalog.known_model_ids())
assert auto_models
assert [profile_id for profile_id in auto_models if profile_id not in known] == []
for profile_id in auto_models:
modalities = set(catalog.get(profile_id).input_modalities)
abilities = [
ability
for modality, ability in _MODALITY_ABILITIES.items()
if modality in modalities
]
auto_model_service.validate_candidate_modalities(catalog, profile_id, abilities)
def test_bundled_catalog_has_the_profiles_saved_auto_configs_rebind_across():
"""Pin the catalog changes at the current xrouter-llm floor that saved Auto
configurations have to be rebound across; update together with that floor.
"""
auto_models = set(_bundled_auto_router_models())
catalog = auto_model_service.load_router_profile_catalog()
known = set(catalog.known_model_ids())
removed = {"deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro-0813"}
added = {
"anthropic/claude-fable-5.1",
"anthropic/claude-opus-5.5",
"anthropic/claude-sonnet-5.5",
"deepseek/deepseek-v4.1-flash",
"google/gemini-3.8-flash",
"openai/gpt-6-astra",
"openai/gpt-6-luna",
"openai/gpt-6.1-sol",
"qwen/qwen3.8-max-0902",
"qwen/qwen3.8-omni-flash",
"x-ai/grok-4.7",
"xiaomi/mimo-v2.6-flash",
"xiaomi/mimo-v2.6-pro",
"z-ai/glm-5.3",
"z-ai/glm-5.3-flashx",
}
assert known.isdisjoint(removed)
assert added <= known
assert set(catalog.get("deepseek/deepseek-v4.1-flash").input_modalities) == {
"text",
"image",
}
assert auto_models.isdisjoint(
removed | {"google/gemini-3.5-flash", "google/gemini-3.7-flash"}
)
assert {
"deepseek/deepseek-v4.1-flash",
"google/gemini-3.8-flash",
"openai/gpt-6-astra",
"qwen/qwen3.8-max-0902",
"qwen/qwen3.8-omni-flash",
"z-ai/glm-5.3",
} <= auto_models