Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

22.4 KB
Raw Blame
"""``Idempotency-Key`` on ``POST /v1/chat/tasks`` and the append endpoint."""

from __future__ import annotations

from datetime import datetime, timedelta, timezone
from typing import Any
from unittest.mock import MagicMock, patch

import pytest

from xagent.web.api.v1.idempotency import fingerprint_payload
from xagent.web.models.api_idempotency_key import ApiIdempotencyKey
from xagent.web.models.chat_message import TaskChatMessage
from xagent.web.models.task import Task, TaskStatus
from xagent.web.schemas.v1 import AppendMessageRequest
from xagent.web.services import api_idempotency

from ..conftest import _admin_headers, _direct_db_session, client

pytestmark = pytest.mark.usefixtures("_test_db")


@pytest.fixture(autouse=True)
def mock_schedule_bg():
    with patch(
        "xagent.web.services.task_orchestrator._schedule_bg",
        new=MagicMock(),
    ) as mocked:
        yield mocked


def _create_agent_with_key(name: str = "idempotency agent") -> tuple[int, str]:
    headers = _admin_headers()
    agent_resp = client.post(
        "/api/agents",
        headers=headers,
        json={
            "name": name,
            "description": "test",
            "instructions": "you are a test agent",
            "execution_mode": "balanced",
        },
    )
    assert agent_resp.status_code == 200, agent_resp.text
    agent_id = agent_resp.json()["id"]
    key_resp = client.post(f"/api/agents/{agent_id}/api-key", headers=headers)
    assert key_resp.status_code == 200, key_resp.text
    return agent_id, key_resp.json()["full_key"]


def _headers(full_key: str, idempotency_key: str | None = None) -> dict[str, str]:
    headers = {"Authorization": f"Bearer {full_key}"}
    if idempotency_key is not None:
        headers["Idempotency-Key"] = idempotency_key
    return headers


def _create(
    full_key: str,
    agent_id: int,
    idempotency_key: str | None = None,
    content: str = "first turn",
) -> Any:
    return client.post(
        "/v1/chat/tasks",
        headers=_headers(full_key, idempotency_key),
        json={"agent_id": agent_id, "message": {"role": "user", "content": content}},
    )


def _append(
    full_key: str,
    agent_id: int,
    task_id: int,
    idempotency_key: str | None = None,
    content: str = "next turn",
) -> Any:
    return client.post(
        f"/v1/chat/tasks/{task_id}/messages",
        headers=_headers(full_key, idempotency_key),
        json={
            "agent_id": agent_id,
            "message": {"role": "user", "content": content},
        },
    )


def _task_count() -> int:
    db = _direct_db_session()
    try:
        return db.query(Task).count()
    finally:
        db.close()


def _user_message_count(task_id: int) -> int:
    db = _direct_db_session()
    try:
        return (
            db.query(TaskChatMessage)
            .filter(TaskChatMessage.task_id == task_id, TaskChatMessage.role == "user")
            .count()
        )
    finally:
        db.close()


def _set_status(task_id: int, status: TaskStatus) -> None:
    db = _direct_db_session()
    try:
        db.query(Task).filter(Task.id == task_id).one().status = status
        db.commit()
    finally:
        db.close()


def _only_row() -> dict[str, Any]:
    db = _direct_db_session()
    try:
        row = db.query(ApiIdempotencyKey).one()
        return {
            "id": row.id,
            "state": row.state,
            "task_id": row.task_id,
            "fingerprint": row.request_fingerprint,
        }
    finally:
        db.close()


def _patch_row(**values: Any) -> None:
    db = _direct_db_session()
    try:
        row = db.query(ApiIdempotencyKey).one()
        for name, value in values.items():
            setattr(row, name, value)
        db.commit()
    finally:
        db.close()


def test_without_header_every_create_starts_a_task() -> None:
    agent_id, key = _create_agent_with_key()

    first = _create(key, agent_id)
    second = _create(key, agent_id)

    assert first.status_code == 202 and second.status_code == 202
    assert first.json()["task_id"] != second.json()["task_id"]
    assert _task_count() == 2
    db = _direct_db_session()
    try:
        assert db.query(ApiIdempotencyKey).count() == 0
    finally:
        db.close()


def test_same_key_and_body_replays_the_original_task() -> None:
    agent_id, key = _create_agent_with_key()

    first = _create(key, agent_id, "retry-1")
    replay = _create(key, agent_id, "retry-1")

    assert first.status_code == 202
    assert "idempotent-replayed" not in first.headers
    assert replay.status_code == 202
    assert replay.headers["Idempotent-Replayed"] == "true"
    assert replay.json() == first.json()
    assert _task_count() == 1
    assert _only_row()["state"] == "completed"


def test_same_key_with_different_body_is_422() -> None:
    agent_id, key = _create_agent_with_key()
    assert _create(key, agent_id, "retry-2", content="one").status_code == 202

    changed = _create(key, agent_id, "retry-2", content="two")

    assert changed.status_code == 422
    assert changed.json()["error"]["code"] == "idempotency_key_reused"
    assert _task_count() == 1


def _create_body_dump(agent_id: int, content: str) -> dict[str, Any]:
    from xagent.web.schemas.v1 import CreateTaskRequest

    return fingerprint_payload(
        CreateTaskRequest(
            agent_id=agent_id, message={"role": "user", "content": content}
        )
    )


def test_retry_while_first_request_is_in_flight_is_409() -> None:
    agent_id, key = _create_agent_with_key()
    # The first request reserved the key and has not finished.
    in_flight = api_idempotency.acquire_sync(
        scope_kind="agent",
        scope_id=agent_id,
        operation="create_task",
        key="retry-3",
        fingerprint=api_idempotency.request_fingerprint(
            _create_body_dump(agent_id, "first turn")
        ),
    )
    assert isinstance(in_flight, api_idempotency.IdempotencyClaim)

    retry = _create(key, agent_id, "retry-3")

    assert retry.status_code == 409
    assert retry.json()["error"]["code"] == "idempotency_key_in_progress"
    assert retry.headers["Retry-After"]
    assert _task_count() == 0


def test_expired_key_can_be_reused_with_a_different_body() -> None:
    agent_id, key = _create_agent_with_key()
    first = _create(key, agent_id, "retry-4", content="one")
    assert first.status_code == 202
    _patch_row(expires_at=datetime.now(timezone.utc) - timedelta(seconds=1))

    again = _create(key, agent_id, "retry-4", content="two")

    assert again.status_code == 202
    assert again.json()["task_id"] != first.json()["task_id"]
    assert "idempotent-replayed" not in again.headers
    assert _task_count() == 2


def test_stale_reservation_that_created_nothing_is_taken_over() -> None:
    agent_id, key = _create_agent_with_key()
    api_idempotency.acquire_sync(
        scope_kind="agent",
        scope_id=agent_id,
        operation="create_task",
        key="retry-5",
        fingerprint=api_idempotency.request_fingerprint(
            _create_body_dump(agent_id, "first turn")
        ),
    )
    _patch_row(lease_expires_at=datetime.now(timezone.utc) - timedelta(seconds=1))

    response = _create(key, agent_id, "retry-5")

    assert response.status_code == 202
    assert _task_count() == 1
    assert _only_row()["task_id"] == response.json()["task_id"]


def test_accepted_task_with_unrecorded_response_is_not_started_again() -> None:
    agent_id, key = _create_agent_with_key()
    first = _create(key, agent_id, "retry-6")
    task_id = first.json()["task_id"]
    # The process died after the create committed, before the response was
    # recorded.
    _patch_row(
        state="in_progress",
        response_body=None,
        response_status=None,
        lease_expires_at=datetime.now(timezone.utc) - timedelta(seconds=1),
    )

    retry = _create(key, agent_id, "retry-6")

    assert retry.status_code == 409
    error = retry.json()["error"]
    assert error["code"] == "idempotency_outcome_unknown"
    assert error["details"] == {"task_id": task_id}
    # The message must not imply the task is running: its outcome is unknown.
    assert f"GET /v1/chat/tasks/{task_id}" in error["message"]
    assert _task_count() == 1


def test_rejected_create_releases_the_key() -> None:
    agent_id, key = _create_agent_with_key()

    rejected = _create(key, agent_id + 999, "retry-7")
    assert rejected.status_code == 404
    db = _direct_db_session()
    try:
        assert db.query(ApiIdempotencyKey).count() == 0
    finally:
        db.close()

    # The same key now works for the corrected request.
    accepted = _create(key, agent_id, "retry-7")
    assert accepted.status_code == 202
    assert _task_count() == 1


def test_keys_are_scoped_to_the_credential_owner() -> None:
    agent_a, key_a = _create_agent_with_key("agent a")
    agent_b, key_b = _create_agent_with_key("agent b")

    first = _create(key_a, agent_a, "shared-key")
    second = _create(key_b, agent_b, "shared-key")

    assert first.status_code == 202 and second.status_code == 202
    assert first.json()["task_id"] != second.json()["task_id"]
    assert _task_count() == 2


def test_replay_fails_when_the_original_task_was_deleted() -> None:
    agent_id, key = _create_agent_with_key()
    first = _create(key, agent_id, "retry-8")
    task_id = first.json()["task_id"]
    _set_status(task_id, TaskStatus.COMPLETED)
    db = _direct_db_session()
    try:
        db.query(Task).filter(Task.id == task_id).delete()
        db.commit()
    finally:
        db.close()

    retry = _create(key, agent_id, "retry-8")

    assert retry.status_code == 409
    assert retry.json()["error"]["code"] == "idempotency_conflict"


@pytest.mark.parametrize("value", ["", "x" * 256, "has space", "tab\tkey"])
def test_malformed_key_is_422(value: str) -> None:
    agent_id, key = _create_agent_with_key()

    response = client.post(
        "/v1/chat/tasks",
        headers={
            "Authorization": f"Bearer {key}",
            "Idempotency-Key": value.encode("utf-8").decode("latin-1"),
        },
        json={"agent_id": agent_id, "message": {"role": "user", "content": "hi"}},
    )

    assert response.status_code == 422
    assert response.json()["error"]["code"] == "idempotency_key_invalid"
    assert _task_count() == 0


def test_append_same_key_replays_without_a_second_turn() -> None:
    agent_id, key = _create_agent_with_key()
    task_id = _create(key, agent_id).json()["task_id"]
    _set_status(task_id, TaskStatus.COMPLETED)

    first = _append(key, agent_id, task_id, "append-1")
    assert first.status_code == 202, first.text
    messages_after_first = _user_message_count(task_id)
    # Without replay protection this retry would be rejected as task_busy or,
    # once the turn finished, start another turn.
    _set_status(task_id, TaskStatus.COMPLETED)
    replay = _append(key, agent_id, task_id, "append-1")

    assert replay.status_code == 202
    assert replay.headers["Idempotent-Replayed"] == "true"
    assert replay.json() == first.json()
    assert _user_message_count(task_id) == messages_after_first


def test_append_same_key_with_different_body_is_422() -> None:
    agent_id, key = _create_agent_with_key()
    task_id = _create(key, agent_id).json()["task_id"]
    _set_status(task_id, TaskStatus.COMPLETED)
    assert _append(key, agent_id, task_id, "append-2", "one").status_code == 202
    _set_status(task_id, TaskStatus.COMPLETED)

    changed = _append(key, agent_id, task_id, "append-2", "two")

    assert changed.status_code == 422
    assert changed.json()["error"]["code"] == "idempotency_key_reused"


def test_append_key_is_scoped_to_its_task() -> None:
    agent_id, key = _create_agent_with_key()
    task_a = _create(key, agent_id).json()["task_id"]
    task_b = _create(key, agent_id).json()["task_id"]
    for task_id in (task_a, task_b):
        _set_status(task_id, TaskStatus.COMPLETED)

    on_a = _append(key, agent_id, task_a, "append-3")
    on_b = _append(key, agent_id, task_b, "append-3")

    assert on_a.status_code == 202 and on_b.status_code == 202
    assert on_a.json()["task_id"] == task_a
    assert on_b.json()["task_id"] == task_b


def test_append_without_header_is_unchanged() -> None:
    agent_id, key = _create_agent_with_key()
    task_id = _create(key, agent_id).json()["task_id"]
    _set_status(task_id, TaskStatus.COMPLETED)

    first = _append(key, agent_id, task_id)
    _set_status(task_id, TaskStatus.COMPLETED)
    second = _append(key, agent_id, task_id)

    assert first.status_code == 202 and second.status_code == 202
    assert "idempotent-replayed" not in second.headers


def test_a_slow_owner_is_fenced_out_after_its_reservation_was_taken_over() -> None:
    agent_id, key = _create_agent_with_key()
    slow = api_idempotency.acquire_sync(
        scope_kind="agent",
        scope_id=agent_id,
        operation="create_task",
        key="retry-9",
        fingerprint=api_idempotency.request_fingerprint(
            _create_body_dump(agent_id, "first turn")
        ),
    )
    assert isinstance(slow, api_idempotency.IdempotencyClaim)
    _patch_row(lease_expires_at=datetime.now(timezone.utc) - timedelta(seconds=1))

    # The retry takes the reservation over and creates the task.
    retry = _create(key, agent_id, "retry-9")
    assert retry.status_code == 202

    # The slow original finally reaches its bind: it must not create a second
    # task, and its release must not delete the row its retry owns.
    db = _direct_db_session()
    try:
        with pytest.raises(api_idempotency.IdempotencyRejected):
            api_idempotency.bind_task_no_commit(db, slow, task_id=999999)
    finally:
        db.rollback()
        db.close()
    api_idempotency.release_sync(slow)
    api_idempotency.complete_sync(slow, status=202, body={"stale": True})

    row = _only_row()
    assert row["task_id"] == retry.json()["task_id"]
    replay = _create(key, agent_id, "retry-9")
    assert replay.json() == retry.json()
    assert _task_count() == 1


def test_a_failing_release_does_not_replace_the_original_error() -> None:
    agent_id, key = _create_agent_with_key()

    with patch.object(
        api_idempotency, "release_sync", side_effect=RuntimeError("db down")
    ):
        rejected = _create(key, agent_id + 999, "retry-10")

    assert rejected.status_code == 404


def test_a_failing_purge_does_not_fail_the_request() -> None:
    agent_id, key = _create_agent_with_key()

    with patch.object(api_idempotency, "delete", side_effect=RuntimeError("locked")):
        response = _create(key, agent_id, "retry-11")

    assert response.status_code == 202


def test_append_stale_reservation_that_created_nothing_is_taken_over() -> None:
    agent_id, key = _create_agent_with_key()
    task_id = _create(key, agent_id).json()["task_id"]
    _set_status(task_id, TaskStatus.COMPLETED)
    body = fingerprint_payload(
        AppendMessageRequest(
            agent_id=agent_id, message={"role": "user", "content": "next turn"}
        )
    )
    api_idempotency.acquire_sync(
        scope_kind="agent",
        scope_id=agent_id,
        operation=f"append_message:{task_id}",
        key="append-2",
        fingerprint=api_idempotency.request_fingerprint(body),
    )
    _patch_row(lease_expires_at=datetime.now(timezone.utc) - timedelta(seconds=1))

    response = _append(key, agent_id, task_id, "append-2")

    assert response.status_code == 202, response.text
    assert _only_row()["task_id"] == task_id


def test_a_taken_over_owner_cannot_bind_or_release_before_the_new_owner_binds() -> None:
    agent_id, _key = _create_agent_with_key()
    fingerprint = api_idempotency.request_fingerprint(
        _create_body_dump(agent_id, "first turn")
    )
    scope = {"scope_kind": "agent", "scope_id": agent_id, "operation": "create_task"}
    slow = api_idempotency.acquire_sync(
        **scope, key="retry-12", fingerprint=fingerprint
    )
    _patch_row(lease_expires_at=datetime.now(timezone.utc) - timedelta(seconds=1))
    new_owner = api_idempotency.acquire_sync(
        **scope, key="retry-12", fingerprint=fingerprint
    )
    assert isinstance(slow, api_idempotency.IdempotencyClaim)
    assert isinstance(new_owner, api_idempotency.IdempotencyClaim)
    assert new_owner.token != slow.token

    # Nothing is bound yet, so only the token tells the two owners apart.
    db = _direct_db_session()
    try:
        with pytest.raises(api_idempotency.IdempotencyRejected):
            api_idempotency.bind_task_no_commit(db, slow, task_id=999999)
    finally:
        db.rollback()
        db.close()
    api_idempotency.release_sync(slow)

    assert _only_row()["state"] == "in_progress"


def test_a_bound_row_whose_task_was_deleted_is_never_taken_over() -> None:
    agent_id, key = _create_agent_with_key()
    first = _create(key, agent_id, "retry-13")
    task_id = first.json()["task_id"]
    _set_status(task_id, TaskStatus.COMPLETED)
    # The response was never recorded, then the task was deleted, then the
    # lease ran out: nothing may start a second task under this key.
    _patch_row(
        state="bound",
        response_body=None,
        response_status=None,
        lease_expires_at=datetime.now(timezone.utc) - timedelta(seconds=1),
    )
    db = _direct_db_session()
    try:
        db.query(Task).filter(Task.id == task_id).delete()
        db.commit()
    finally:
        db.close()
    assert _only_row()["task_id"] is None

    retry = _create(key, agent_id, "retry-13")

    assert retry.status_code == 409
    assert retry.json()["error"]["code"] == "idempotency_conflict"
    assert _task_count() == 0


def test_a_retry_that_only_changes_metadata_replays_the_original() -> None:
    agent_id, key = _create_agent_with_key()
    body = {"agent_id": agent_id, "message": {"role": "user", "content": "hi"}}
    first = client.post(
        "/v1/chat/tasks",
        headers=_headers(key, "retry-14"),
        json={**body, "metadata": {"trace_id": "attempt-1"}},
    )
    retry = client.post(
        "/v1/chat/tasks",
        headers=_headers(key, "retry-14"),
        json={**body, "metadata": {"trace_id": "attempt-2"}},
    )

    assert first.status_code == 202, first.text
    assert retry.status_code == 202
    assert retry.headers["Idempotent-Replayed"] == "true"
    assert retry.json() == first.json()
    assert _task_count() == 1


# Bodies Pydantic accepts but cannot write back as JSON. ``prompt_context``
# values are ``Any`` until the route checks them, so a list nested past the
# serializer's depth limit (255), or an object key holding a lone surrogate,
# reaches the fingerprint. 400 levels stay below the depth at which
# ``json.loads`` itself fails on Python 3.11, so the body is still parsed.
_UNENCODABLE_VALUES = pytest.mark.parametrize(
    "value",
    ["[" * 400 + "]" * 400, '{"\\ud800": 1}'],
    ids=["deep_nesting", "lone_surrogate_key"],
)
_ROUTE_CHECK_MESSAGE = "prompt_context values must be strings, numbers, or booleans."
_UNENCODABLE_MESSAGE = (
    "The request body cannot be encoded as JSON, so it cannot be matched "
    "against an Idempotency-Key."
)


def _post_raw(
    path: str,
    full_key: str,
    body: str,
    idempotency_key: str | None = None,
) -> Any:
    return client.post(
        path,
        headers={
            **_headers(full_key, idempotency_key),
            "Content-Type": "application/json",
        },
        content=body,
    )


def _prompt_context_body(agent_id: int, prompt_context_value: str) -> str:
    return (
        f'{{"agent_id": {agent_id}, '
        '"message": {"role": "user", "content": "x"}, '
        f'"prompt_context": {{"k": {prompt_context_value}}}}}'
    )


def _idempotency_row_count() -> int:
    db = _direct_db_session()
    try:
        return db.query(ApiIdempotencyKey).count()
    finally:
        db.close()


@_UNENCODABLE_VALUES
def test_without_header_an_unencodable_create_reaches_the_route_checks(
    value: str,
) -> None:
    agent_id, key = _create_agent_with_key()

    response = _post_raw("/v1/chat/tasks", key, _prompt_context_body(agent_id, value))

    assert response.status_code == 422, response.text
    assert response.json()["error"]["message"] == _ROUTE_CHECK_MESSAGE
    assert _task_count() == 0


@_UNENCODABLE_VALUES
def test_with_header_an_unencodable_create_is_422_and_reserves_nothing(
    value: str,
) -> None:
    agent_id, key = _create_agent_with_key()

    response = _post_raw(
        "/v1/chat/tasks", key, _prompt_context_body(agent_id, value), "retry-15"
    )

    assert response.status_code == 422, response.text
    assert response.json()["error"] == {
        "code": "invalid_input",
        "message": _UNENCODABLE_MESSAGE,
    }
    assert _task_count() == 0
    assert _idempotency_row_count() == 0
    # The same key still works for a body that can be encoded.
    assert _create(key, agent_id, "retry-15").status_code == 202


@_UNENCODABLE_VALUES
def test_without_header_an_unencodable_append_reaches_the_route_checks(
    value: str,
) -> None:
    agent_id, key = _create_agent_with_key()
    task_id = _create(key, agent_id).json()["task_id"]
    _set_status(task_id, TaskStatus.COMPLETED)

    response = _post_raw(
        f"/v1/chat/tasks/{task_id}/messages", key, _prompt_context_body(agent_id, value)
    )

    assert response.status_code == 422, response.text
    assert response.json()["error"]["message"] == _ROUTE_CHECK_MESSAGE
    assert _user_message_count(task_id) == 1


@_UNENCODABLE_VALUES
def test_with_header_an_unencodable_append_is_422_and_reserves_nothing(
    value: str,
) -> None:
    agent_id, key = _create_agent_with_key()
    task_id = _create(key, agent_id).json()["task_id"]
    _set_status(task_id, TaskStatus.COMPLETED)

    response = _post_raw(
        f"/v1/chat/tasks/{task_id}/messages",
        key,
        _prompt_context_body(agent_id, value),
        "retry-16",
    )

    assert response.status_code == 422, response.text
    assert response.json()["error"] == {
        "code": "invalid_input",
        "message": _UNENCODABLE_MESSAGE,
    }
    assert _user_message_count(task_id) == 1
    assert _idempotency_row_count() == 0
    # The same key still works for a body that can be encoded.
    _set_status(task_id, TaskStatus.COMPLETED)
    assert _append(key, agent_id, task_id, "retry-16").status_code == 202


@_UNENCODABLE_VALUES
def test_a_malformed_key_is_rejected_before_the_body_is_encoded(value: str) -> None:
    agent_id, key = _create_agent_with_key()

    response = _post_raw(
        "/v1/chat/tasks", key, _prompt_context_body(agent_id, value), "bad key"
    )

    assert response.status_code == 422, response.text
    assert response.json()["error"]["code"] == "idempotency_key_invalid"
    assert _task_count() == 0
    assert _idempotency_row_count() == 0