Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

25.2 KB
Raw Blame
"""Integration tests for the /v1/workforces/{id}/runs SDK endpoint (#949).

Covers workforce-bound API-key run creation, the shared
/v1/chat/tasks/{task_id} surface reached through a workforce key,
owner-type isolation (agent key vs workforce key), idempotency replay,
and usage metering.

The background-execution kickoff is mocked (``_schedule_bg``) so the
suite exercises HTTP shape + DB rows + the orchestrator's atomic claim
without spinning up an LLM -- same approach as test_tasks.py.
"""

from unittest.mock import MagicMock, patch

import pytest

from xagent.web.models.agent import Agent, AgentStatus
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.user import User
from xagent.web.models.workforce import Workforce
from xagent.web.services import task_start

from ..conftest import (
    _admin_headers,
    _direct_db_session,
    _install_one_slot_queue_pool,
    client,
)

pytestmark = pytest.mark.usefixtures("_test_db")


@pytest.fixture(autouse=True)
def mock_schedule_bg():
    """Stub the lease-aware bg scheduler so the orchestrator's atomic
    claim + transcript persist still runs against the real DB; only the
    asyncio.create_task / agent execution is skipped."""
    with patch(
        "xagent.web.services.task_orchestrator._schedule_bg",
        new=MagicMock(),
    ) as mocked:
        yield mocked


def _bearer(full_key: str) -> dict[str, str]:
    return {"Authorization": f"Bearer {full_key}"}


def _user_id(username: str = "admin") -> int:
    db = _direct_db_session()
    try:
        return int(db.query(User.id).filter(User.username == username).scalar())
    finally:
        db.close()


def _create_published_agent(user_id: int, name: str) -> int:
    db = _direct_db_session()
    try:
        agent = Agent(
            user_id=user_id,
            name=name,
            description=f"{name} description",
            instructions=f"{name} instructions",
            execution_mode="balanced",
            status=AgentStatus.PUBLISHED,
        )
        db.add(agent)
        db.commit()
        db.refresh(agent)
        return int(agent.id)
    finally:
        db.close()


def _create_active_workforce(
    headers: dict[str, str], name: str = "SDK Workforce", username: str = "admin"
) -> int:
    """Create + publish an active workforce; return its id."""
    owner_id = _user_id(username)
    manager_agent_id = _create_published_agent(owner_id, f"{name} Manager")
    worker_agent_id = _create_published_agent(owner_id, f"{name} Worker")
    resp = client.post(
        "/api/workforces",
        headers=headers,
        json={
            "name": name,
            "description": "Coordinates SDK tests",
            "manager_agent_id": manager_agent_id,
            "workers": [
                {
                    "source_type": "existing",
                    "agent_id": worker_agent_id,
                    "alias": "worker-1",
                    "assignment_instructions": "Handle everything",
                    "enabled": True,
                    "sort_order": 1,
                }
            ],
        },
    )
    assert resp.status_code == 200, resp.text
    workforce_id = int(resp.json()["id"])
    published = client.post(f"/api/workforces/{workforce_id}/publish", headers=headers)
    assert published.status_code == 200, published.text
    return workforce_id


def _create_draft_workforce(
    headers: dict[str, str], name: str = "Draft Workforce"
) -> int:
    """Create a workforce WITHOUT publishing it (status stays 'draft')."""
    owner_id = _user_id()
    manager_agent_id = _create_published_agent(owner_id, f"{name} Manager")
    worker_agent_id = _create_published_agent(owner_id, f"{name} Worker")
    resp = client.post(
        "/api/workforces",
        headers=headers,
        json={
            "name": name,
            "description": "Draft for SDK tests",
            "manager_agent_id": manager_agent_id,
            "workers": [
                {
                    "source_type": "existing",
                    "agent_id": worker_agent_id,
                    "alias": "worker-1",
                    "assignment_instructions": "Handle everything",
                    "enabled": True,
                    "sort_order": 1,
                }
            ],
        },
    )
    assert resp.status_code == 200, resp.text
    return int(resp.json()["id"])


def _set_workforce_status(workforce_id: int, status: str) -> None:
    db = _direct_db_session()
    try:
        db.query(Workforce).filter(Workforce.id == workforce_id).update(
            {"status": status}
        )
        db.commit()
    finally:
        db.close()


def _create_workforce_key(headers: dict[str, str], workforce_id: int) -> str:
    resp = client.post(
        "/api/agent-api-keys",
        headers=headers,
        json={"workforce_id": workforce_id, "label": "sdk"},
    )
    assert resp.status_code == 200, resp.text
    return resp.json()["full_key"]


def _create_agent_key(headers: dict[str, str]) -> tuple[int, str]:
    agent_resp = client.post(
        "/api/agents",
        headers=headers,
        json={
            "name": "v1 wf test agent",
            "description": "test",
            "instructions": "you are a test agent",
            "execution_mode": "balanced",
        },
    )
    assert agent_resp.status_code == 200, agent_resp.text
    agent_id = agent_resp.json()["id"]
    key_resp = client.post(f"/api/agents/{agent_id}/api-key", headers=headers)
    assert key_resp.status_code == 200, key_resp.text
    return agent_id, key_resp.json()["full_key"]


def _manager_agent_id(workforce_id: int) -> int:
    db = _direct_db_session()
    try:
        wf = db.query(Workforce).filter(Workforce.id == workforce_id).one()
        return int(wf.manager_agent_id)
    finally:
        db.close()


# ===== POST /v1/workforces/{id}/runs =====


def test_create_run_forwards_timezone_to_the_opening_turn(mock_schedule_bg):
    """The workforce SDK opener starts its turn inside this HTTP request and
    has no websocket, so the create body is its only way to declare a zone."""
    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers)
    full_key = _create_workforce_key(headers, workforce_id)

    resp = client.post(
        f"/v1/workforces/{workforce_id}/runs",
        headers=_bearer(full_key),
        json={
            "message": {"role": "user", "content": "how many shifts tomorrow?"},
            "timezone": "Australia/Melbourne",
        },
    )

    assert resp.status_code == 202, resp.text
    assert mock_schedule_bg.call_args.kwargs["context"] == {
        "timezone": "Australia/Melbourne"
    }


def test_create_run_without_timezone_sends_no_context(mock_schedule_bg):
    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers)
    full_key = _create_workforce_key(headers, workforce_id)

    resp = client.post(
        f"/v1/workforces/{workforce_id}/runs",
        headers=_bearer(full_key),
        json={"message": {"role": "user", "content": "coordinate the work"}},
    )

    assert resp.status_code == 202, resp.text
    assert mock_schedule_bg.call_args.kwargs["context"] is None


def test_create_run_happy_path():
    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers)
    full_key = _create_workforce_key(headers, workforce_id)

    resp = client.post(
        f"/v1/workforces/{workforce_id}/runs",
        headers=_bearer(full_key),
        json={"message": {"role": "user", "content": "coordinate the work"}},
    )
    assert resp.status_code == 202, resp.text
    body = resp.json()
    assert body["workforce_id"] == workforce_id
    assert body["agent_id"] == _manager_agent_id(workforce_id)
    assert body["created"] is True
    assert body["status"] in ("pending", "running")
    assert body["run_id"]
    assert body["control_state"] == "running"
    task_id = body["task_id"]

    # DB: manager-agent task created with source='sdk', bound to a run.
    db = _direct_db_session()
    try:
        task = db.query(Task).filter(Task.id == task_id).one()
        assert task.source == "sdk"
        assert task.is_visible is False
        assert int(task.agent_id) == _manager_agent_id(workforce_id)
        from xagent.web.models.workforce import WorkforceRun

        run = (
            db.query(WorkforceRun)
            .filter(WorkforceRun.id == body["workforce_run_id"])
            .one()
        )
        assert int(run.task_id) == task_id
        assert int(run.workforce_id) == workforce_id
    finally:
        db.close()


def test_create_run_uses_one_connection_at_a_time(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The /v1 adapter must not hold a request Session across run startup."""

    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers, name="Single Slot Workforce")
    full_key = _create_workforce_key(headers, workforce_id)
    engine = _install_one_slot_queue_pool(monkeypatch)

    try:
        response = client.post(
            f"/v1/workforces/{workforce_id}/runs",
            headers=_bearer(full_key),
            json={"message": {"role": "user", "content": "single-slot run"}},
        )
        assert response.status_code == 202, response.text
        assert engine.pool.checkedout() == 0
    finally:
        engine.dispose()


def test_bound_task_reachable_via_chat_tasks_with_workforce_key():
    """The run's task_id is pollable / has steps through the shared
    /v1/chat/tasks surface using the same workforce key."""
    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers)
    full_key = _create_workforce_key(headers, workforce_id)

    run = client.post(
        f"/v1/workforces/{workforce_id}/runs",
        headers=_bearer(full_key),
        json={"message": {"role": "user", "content": "go"}},
    ).json()
    task_id = run["task_id"]

    got = client.get(f"/v1/chat/tasks/{task_id}", headers=_bearer(full_key))
    assert got.status_code == 200, got.text
    body = got.json()
    assert body["task_id"] == task_id
    assert body["workforce_id"] == workforce_id
    assert body["agent_id"] == _manager_agent_id(workforce_id)

    steps = client.get(f"/v1/chat/tasks/{task_id}/steps", headers=_bearer(full_key))
    assert steps.status_code == 200, steps.text
    assert steps.json()["task_id"] == task_id


def test_agent_key_cannot_create_workforce_run():
    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers)
    _agent_id, agent_key = _create_agent_key(headers)

    resp = client.post(
        f"/v1/workforces/{workforce_id}/runs",
        headers=_bearer(agent_key),
        json={"message": {"role": "user", "content": "go"}},
    )
    # get_workforce_from_api_key rejects an agent-bound key with the same
    # opaque 401 as any other auth failure.
    assert resp.status_code == 401, resp.text
    assert resp.json()["error"]["code"] == "invalid_api_key"


def test_workforce_key_cannot_create_agent_task():
    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers)
    full_key = _create_workforce_key(headers, workforce_id)
    manager_id = _manager_agent_id(workforce_id)

    resp = client.post(
        "/v1/chat/tasks",
        headers=_bearer(full_key),
        json={
            "agent_id": manager_id,
            "message": {"role": "user", "content": "direct"},
        },
    )
    # A workforce key resolves to no agent, so agent_id never matches.
    assert resp.status_code == 404, resp.text
    assert resp.json()["error"]["code"] == "agent_not_found"


def test_path_workforce_id_mismatch_returns_404():
    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers)
    other_workforce_id = _create_active_workforce(headers, name="Other Workforce")
    full_key = _create_workforce_key(headers, workforce_id)

    resp = client.post(
        f"/v1/workforces/{other_workforce_id}/runs",
        headers=_bearer(full_key),
        json={"message": {"role": "user", "content": "go"}},
    )
    assert resp.status_code == 404, resp.text
    assert resp.json()["error"]["code"] == "workforce_not_found"


def test_missing_authorization_returns_401():
    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers)
    resp = client.post(
        f"/v1/workforces/{workforce_id}/runs",
        json={"message": {"role": "user", "content": "go"}},
    )
    assert resp.status_code == 401, resp.text
    assert resp.json()["error"]["code"] == "invalid_api_key"


def test_empty_message_returns_422():
    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers)
    full_key = _create_workforce_key(headers, workforce_id)
    resp = client.post(
        f"/v1/workforces/{workforce_id}/runs",
        headers=_bearer(full_key),
        json={"message": {"role": "user", "content": ""}},
    )
    assert resp.status_code == 422, resp.text
    assert resp.json()["error"]["code"] == "invalid_input"


def test_idempotency_replay_returns_same_run_without_double_metering():
    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers)
    full_key = _create_workforce_key(headers, workforce_id)

    first = client.post(
        f"/v1/workforces/{workforce_id}/runs",
        headers=_bearer(full_key),
        json={
            "message": {"role": "user", "content": "go"},
            "idempotency_key": "abc-123",
        },
    )
    assert first.status_code == 202, first.text
    assert first.json()["created"] is True
    first_run_id = first.json()["workforce_run_id"]

    second = client.post(
        f"/v1/workforces/{workforce_id}/runs",
        headers=_bearer(full_key),
        json={
            "message": {"role": "user", "content": "go again"},
            "idempotency_key": "abc-123",
        },
    )
    assert second.status_code == 202, second.text
    assert second.json()["created"] is False
    assert second.json()["workforce_run_id"] == first_run_id

    # Only the real creation counted; the replay must not double-bill.
    stats = client.get("/api/agent-api-keys/stats", headers=headers).json()
    assert stats["calls_this_month"] == 1


def test_run_creation_records_key_usage():
    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers)
    full_key = _create_workforce_key(headers, workforce_id)

    before = client.get("/api/agent-api-keys/stats", headers=headers).json()
    assert before["calls_this_month"] == 0

    client.post(
        f"/v1/workforces/{workforce_id}/runs",
        headers=_bearer(full_key),
        json={"message": {"role": "user", "content": "go"}},
    )
    after = client.get("/api/agent-api-keys/stats", headers=headers).json()
    assert after["calls_this_month"] == 1


def test_append_message_through_workforce_key():
    """After the first turn ends, a workforce key can append a next turn
    to the manager task through /v1/chat/tasks/{id}/messages."""
    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers)
    full_key = _create_workforce_key(headers, workforce_id)

    run = client.post(
        f"/v1/workforces/{workforce_id}/runs",
        headers=_bearer(full_key),
        json={"message": {"role": "user", "content": "first"}},
    ).json()
    task_id = run["task_id"]

    # Drive the task to a terminal state so append is allowed (append
    # requires COMPLETED/FAILED; RUNNING/PENDING both 409). Also project the
    # same terminal status onto WorkforceRun.status, matching what
    # sync_workforce_run_status does in production at the end of a real
    # turn (task_orchestrator.py) -- leaving it at its creation-time value
    # would make this test pass regardless of whether
    # ensure_workforce_turn_allowed's own run-status check is correct, since
    # a "pending"/"running" run never exercises that check either way.
    from xagent.web.models.workforce import WorkforceRun

    db = _direct_db_session()
    try:
        task = db.query(Task).filter(Task.id == task_id).one()
        task.status = TaskStatus.COMPLETED
        agent_config = dict(task.agent_config or {})
        workforce_run_id = agent_config.get("workforce_run_id")
        if isinstance(workforce_run_id, int):
            db.query(WorkforceRun).filter(WorkforceRun.id == workforce_run_id).update(
                {"status": "completed"}
            )
        db.commit()
    finally:
        db.close()

    resp = client.post(
        f"/v1/chat/tasks/{task_id}/messages",
        headers=_bearer(full_key),
        json={
            "workforce_id": workforce_id,
            "message": {"role": "user", "content": "next"},
        },
    )
    assert resp.status_code == 202, resp.text
    body = resp.json()
    assert body["task_id"] == task_id
    assert body["workforce_id"] == workforce_id


def test_append_rejects_prompt_context_for_a_workforce_key():
    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers)
    full_key = _create_workforce_key(headers, workforce_id)
    run = client.post(
        f"/v1/workforces/{workforce_id}/runs",
        headers=_bearer(full_key),
        json={"message": {"role": "user", "content": "first"}},
    ).json()
    task_id = run["task_id"]
    db = _direct_db_session()
    try:
        task = db.query(Task).filter(Task.id == task_id).one()
        task.status = TaskStatus.COMPLETED
        db.commit()
    finally:
        db.close()

    resp = client.post(
        f"/v1/chat/tasks/{task_id}/messages",
        headers=_bearer(full_key),
        json={
            "workforce_id": workforce_id,
            "message": {"role": "user", "content": "next"},
            "prompt_context": {"date": "2026-10-20"},
        },
    )

    assert resp.status_code == 422, resp.text
    assert resp.json()["error"]["code"] == "invalid_input"
    db = _direct_db_session()
    try:
        task = db.query(Task).filter(Task.id == task_id).one()
        assert "prompt_context" not in (task.agent_config or {})
    finally:
        db.close()


def test_append_rejects_mismatched_workforce_id():
    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers)
    full_key = _create_workforce_key(headers, workforce_id)

    run = client.post(
        f"/v1/workforces/{workforce_id}/runs",
        headers=_bearer(full_key),
        json={"message": {"role": "user", "content": "first"}},
    ).json()
    task_id = run["task_id"]

    db = _direct_db_session()
    try:
        task = db.query(Task).filter(Task.id == task_id).one()
        task.status = TaskStatus.COMPLETED
        db.commit()
    finally:
        db.close()

    resp = client.post(
        f"/v1/chat/tasks/{task_id}/messages",
        headers=_bearer(full_key),
        json={"workforce_id": 999999, "message": {"role": "user", "content": "next"}},
    )
    assert resp.status_code == 404, resp.text
    assert resp.json()["error"]["code"] == "workforce_not_found"


def test_cross_workforce_task_isolation():
    """A workforce key cannot read a task belonging to another workforce."""
    headers = _admin_headers()
    wf_a = _create_active_workforce(headers, name="Workforce A")
    wf_b = _create_active_workforce(headers, name="Workforce B")
    key_a = _create_workforce_key(headers, wf_a)
    key_b = _create_workforce_key(headers, wf_b)

    run_a = client.post(
        f"/v1/workforces/{wf_a}/runs",
        headers=_bearer(key_a),
        json={"message": {"role": "user", "content": "a"}},
    ).json()
    task_a = run_a["task_id"]

    # key_b must not see workforce A's task.
    resp = client.get(f"/v1/chat/tasks/{task_a}", headers=_bearer(key_b))
    assert resp.status_code == 404, resp.text
    assert resp.json()["error"]["code"] == "task_not_found"


# ===== Error-branch coverage (regression guard for the stable error-code
# mapping in _raise_v1_for_workforce_http_error / raise_for_turn_rejection).
# These are the least-tested but most breakage-prone paths, so each asserts
# on the exact error.code SDK clients switch on. =====


def test_archived_workforce_run_returns_workforce_archived():
    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers, name="Archived WF")
    full_key = _create_workforce_key(headers, workforce_id)
    _set_workforce_status(workforce_id, "archived")

    resp = client.post(
        f"/v1/workforces/{workforce_id}/runs",
        headers=_bearer(full_key),
        json={"message": {"role": "user", "content": "go"}},
    )
    assert resp.status_code == 409, resp.text
    assert resp.json()["error"]["code"] == "workforce_archived"


def test_draft_workforce_run_returns_workforce_not_active():
    headers = _admin_headers()
    workforce_id = _create_draft_workforce(headers)
    full_key = _create_workforce_key(headers, workforce_id)

    resp = client.post(
        f"/v1/workforces/{workforce_id}/runs",
        headers=_bearer(full_key),
        json={"message": {"role": "user", "content": "go"}},
    )
    assert resp.status_code == 409, resp.text
    assert resp.json()["error"]["code"] == "workforce_not_active"


def test_invalid_execution_mode_returns_invalid_input():
    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers, name="Bad Mode WF")
    full_key = _create_workforce_key(headers, workforce_id)

    resp = client.post(
        f"/v1/workforces/{workforce_id}/runs",
        headers=_bearer(full_key),
        json={
            "message": {"role": "user", "content": "go"},
            "execution_mode": "turbo-nonsense",
        },
    )
    assert resp.status_code == 422, resp.text
    assert resp.json()["error"]["code"] == "invalid_input"


def test_bad_file_id_returns_file_not_found_not_workforce_not_found():
    """A bad message.files id must surface as file_not_found, not the
    misleading workforce_not_found (Roger review, PR #968)."""
    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers, name="Bad File WF")
    full_key = _create_workforce_key(headers, workforce_id)

    resp = client.post(
        f"/v1/workforces/{workforce_id}/runs",
        headers=_bearer(full_key),
        json={
            "message": {
                "role": "user",
                "content": "go",
                "files": ["file-does-not-exist"],
            }
        },
    )
    assert resp.status_code == 404, resp.text
    assert resp.json()["error"]["code"] == "file_not_found"


def test_idempotency_conflict_when_original_task_deleted():
    """Reusing an idempotency key whose run's task was deleted (task_id
    SET NULL) can't be replayed -> idempotency_conflict 409."""
    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers, name="Idem Conflict WF")
    full_key = _create_workforce_key(headers, workforce_id)

    first = client.post(
        f"/v1/workforces/{workforce_id}/runs",
        headers=_bearer(full_key),
        json={
            "message": {"role": "user", "content": "go"},
            "idempotency_key": "conflict-key",
        },
    )
    assert first.status_code == 202, first.text
    run_id = first.json()["workforce_run_id"]

    # Simulate the run's task being deleted (FK is ON DELETE SET NULL).
    from xagent.web.models.workforce import WorkforceRun

    db = _direct_db_session()
    try:
        db.query(WorkforceRun).filter(WorkforceRun.id == run_id).update(
            {"task_id": None}
        )
        db.commit()
    finally:
        db.close()

    resp = client.post(
        f"/v1/workforces/{workforce_id}/runs",
        headers=_bearer(full_key),
        json={
            "message": {"role": "user", "content": "go again"},
            "idempotency_key": "conflict-key",
        },
    )
    assert resp.status_code == 409, resp.text
    assert resp.json()["error"]["code"] == "idempotency_conflict"


def test_append_workforce_turn_rejection_maps_to_stable_codes(monkeypatch):
    """A non-transient workforce turn rejection on append maps to its own
    stable v1 code (not the misleading retryable task_busy)."""
    from xagent.web.services.task_orchestrator import TaskTurnError

    headers = _admin_headers()
    workforce_id = _create_active_workforce(headers, name="Append Reject WF")
    full_key = _create_workforce_key(headers, workforce_id)

    run = client.post(
        f"/v1/workforces/{workforce_id}/runs",
        headers=_bearer(full_key),
        json={"message": {"role": "user", "content": "first"}},
    ).json()
    task_id = run["task_id"]

    # Terminal status so _resolve/append reaches the turn kickoff.
    db = _direct_db_session()
    try:
        task = db.query(Task).filter(Task.id == task_id).one()
        task.status = TaskStatus.COMPLETED
        db.commit()
    finally:
        db.close()

    for reason, expected_code, expected_status in (
        ("workforce_config_changed", "workforce_config_changed", 409),
        ("workforce_archived", "workforce_archived", 409),
        ("workforce_run_not_found", "task_not_found", 404),
        ("busy", "task_busy", 409),
    ):

        def _reject(*_args, reason=reason, **_kwargs):
            raise TaskTurnError(reason)

        monkeypatch.setattr(
            task_start.TaskTurnOrchestrator,
            "claim_append_turn_no_commit",
            _reject,
        )
        resp = client.post(
            f"/v1/chat/tasks/{task_id}/messages",
            headers=_bearer(full_key),
            json={
                "workforce_id": workforce_id,
                "message": {"role": "user", "content": "next"},
            },
        )
        assert resp.status_code == expected_status, resp.text
        assert resp.json()["error"]["code"] == expected_code