Enterprise agent platform for building, deploying, and operating AI teammates.
"""The auto-resume sweeper: dispatch, give-ups, budgets, housekeeping, loop.
Nothing records ``scheduled`` rows yet, so every case seeds them directly.
Runs on SQLite (scan + compare-and-swap) and, when
``XAGENT_TEST_POSTGRES_URL`` is set, on PostgreSQL (``FOR UPDATE OF tasks
SKIP LOCKED``) through the shared ``engine`` fixture; the cases that need two
real transactions are PostgreSQL only.
"""
from __future__ import annotations
import asyncio
import logging
import random
import sys
import threading
import uuid
from collections.abc import Iterator
from datetime import datetime, timedelta, timezone
from typing import Any
import pytest
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from sqlalchemy.exc import IntegrityError, OperationalError
from sqlalchemy.exc import TimeoutError as SATimeoutError
from sqlalchemy.orm import Session, sessionmaker
from tests.web.services.task_database_shared import engine as engine_fixture
from xagent.web.models import database
from xagent.web.models.agent import Agent
from xagent.web.models.database import Base
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.task_admission import TaskAdmissionBucket, TaskAdmissionTicket
from xagent.web.models.task_auto_recovery import TaskAutoRecovery, TaskRecoveryEvent
from xagent.web.models.task_command import TaskExecutionCommand
from xagent.web.models.trigger import (
AgentTrigger,
TriggerRun,
TriggerRunStatus,
TriggerType,
)
from xagent.web.models.user import User
from xagent.web.models.user_channel import UserChannel
from xagent.web.services import ops_signals, task_auto_resume, task_execution_admission
from xagent.web.services.task_auto_recovery import (
TASK_INTERRUPTION_PAUSED_TRIGGER_ERROR,
)
from xagent.web.services.task_auto_resume import (
DISPATCH_CHECK_GRACE_SECONDS,
TASK_AUTO_RESUME_DISABLED_TRIGGER_ERROR,
TASK_AUTO_RESUME_DISPATCH_FAILED_TRIGGER_ERROR,
TASK_AUTO_RESUME_EXHAUSTED_TRIGGER_ERROR,
TASK_AUTO_RESUME_EXPIRED_TRIGGER_ERROR,
TASK_AUTO_RESUME_SUPERSEDED_TRIGGER_ERROR,
auto_resume_command_id,
public_auto_resume_view,
run_auto_resume_tick,
)
from xagent.web.services.task_command_transport import (
COMMAND_ID_PATTERN,
TaskCommandOwnerStateError,
)
from xagent.web.services.task_execution_admission import (
AdmissionPolicy,
AdmissionQueueFull,
)
from xagent.web.services.task_lease_service import utc_now
engine = engine_fixture
RUN = "run-auto-resume"
PV = 5
_ENV = (
"XAGENT_TASK_AUTO_RESUME_ENABLED",
"XAGENT_TASK_AUTO_RESUME_MAX_PER_TICK",
"XAGENT_TASK_AUTO_RESUME_MAX_INFLIGHT",
"XAGENT_TASK_AUTO_RESUME_WINDOW_SECONDS",
"XAGENT_TASK_AUTO_RESUME_MAX_TOTAL_PER_RUN",
"XAGENT_TASK_AUTO_RESUME_SHORT_MAX_NO_PROGRESS",
"XAGENT_TASK_AUTO_RESUME_LLM_MAX_ELAPSED_SECONDS",
"XAGENT_SHARED_TASK_EXECUTION_ENABLED",
)
@pytest.fixture(autouse=True)
def _clean_signals():
ops_signals.clear_degradation(ops_signals.TASK_AUTO_RESUME_UNAVAILABLE)
yield
ops_signals.clear_degradation(ops_signals.TASK_AUTO_RESUME_UNAVAILABLE)
# Captured at import, before any fixture patches ``database``.
_REAL_DATABASE_ACCESSORS = {
"get_session_local": database.get_session_local,
"get_engine": database.get_engine,
}
def _stale_database_accessors() -> set[str]:
"""Loaded modules whose imported database accessors are not the real ones."""
return {
f"{name}.{attr}"
for name, module in list(sys.modules.items())
if name.startswith("xagent.") and module is not None
for attr, real in _REAL_DATABASE_ACCESSORS.items()
if getattr(module, attr, real) is not real
}
@pytest.fixture
def factory(engine, monkeypatch) -> Iterator[sessionmaker]:
Base.metadata.create_all(engine)
result = sessionmaker(engine, autoflush=False)
# Patch the globals behind the accessors, never the accessors: a module
# first imported mid-test binds ``get_session_local`` by name and would
# keep this test's database after monkeypatch restores ``database``.
monkeypatch.setattr(database, "_SessionLocal", result)
# Selects the PostgreSQL row-lock path or the SQLite CAS path.
monkeypatch.setattr(database, "_engine", engine)
for name in _ENV:
monkeypatch.delenv(name, raising=False)
# Only flag what this test leaked; other files may still leak the old way.
stale_before = _stale_database_accessors()
yield result
leaked = sorted(_stale_database_accessors() - stale_before)
assert not leaked, f"modules bound a patched database accessor: {leaked}"
@pytest.fixture
def notified(monkeypatch) -> list[int]:
calls: list[int] = []
monkeypatch.setattr(
task_auto_resume, "notify_task_command_dispatcher", lambda: calls.append(1)
)
return calls
def _aware(value: datetime | None) -> datetime | None:
if value is None:
return None
return value if value.tzinfo else value.replace(tzinfo=timezone.utc)
def _user(db: Session) -> User:
user = User(username=f"auto-resume-{uuid.uuid4().hex[:8]}", password_hash="x")
db.add(user)
db.flush()
return user
def _seed(
db: Session,
now: datetime,
*,
user: User | None = None,
state: str = "scheduled",
reason: str = "lease_expired",
due_in: float | None = -1,
interrupted_ago: float = 60,
episode_ago: float | None = None,
no_progress: int = 0,
total: int = 0,
last_command_id: str | None = None,
run_id: str = RUN,
paused_state_version: int = PV,
**task_fields: Any,
) -> int:
"""A PAUSED task at its fence and its recovery row."""
fields: dict[str, Any] = {
"status": TaskStatus.PAUSED,
"control_state": "paused",
"run_id": run_id,
"state_version": paused_state_version,
}
fields.update(task_fields)
task = Task(
user_id=(user or _user(db)).id,
title="Auto resume",
description="auto resume test",
execution_mode="balanced",
**fields,
)
db.add(task)
db.flush()
db.add(
TaskAutoRecovery(
task_id=task.id,
run_id=run_id,
reason=reason,
state=state,
paused_state_version=paused_state_version,
interrupted_at=now - timedelta(seconds=interrupted_ago),
episode_started_at=now
- timedelta(
seconds=episode_ago if episode_ago is not None else interrupted_ago
),
no_progress_resumes=no_progress,
total_resumes=total,
next_attempt_at=(
now + timedelta(seconds=due_in) if due_in is not None else None
),
last_command_id=last_command_id,
)
)
db.flush()
return int(task.id)
def _trigger(
db: Session, user: User, task_id: int, *, later_run: bool = False
) -> tuple[int, int | None]:
agent = Agent(user_id=user.id, name="auto resume agent")
db.add(agent)
db.flush()
trigger = AgentTrigger(
user_id=user.id,
agent_id=agent.id,
type=TriggerType.SCHEDULED.value,
name="tick",
config={},
)
db.add(trigger)
db.flush()
run = TriggerRun(
trigger_id=trigger.id,
task_id=task_id,
status=TriggerRunStatus.RUNNING.value,
idempotency_key=f"tick-{task_id}-1",
started_at=utc_now(),
)
db.add(run)
db.flush()
later = None
if later_run:
next_run = TriggerRun(
trigger_id=trigger.id,
status=TriggerRunStatus.RUNNING.value,
idempotency_key=f"tick-{task_id}-2",
started_at=utc_now(),
)
db.add(next_run)
db.flush()
later = int(next_run.id)
return int(run.id), later
def _trigger_task(
db: Session, now: datetime, *, later_run: bool = False, **kwargs: Any
) -> tuple[int, int]:
user = _user(db)
task_id = _seed(
db,
now,
user=user,
source="trigger",
agent_config={"trigger_type": "scheduled"},
**kwargs,
)
run_id, _later = _trigger(db, user, task_id, later_run=later_run)
return task_id, run_id
def _row(factory: sessionmaker, task_id: int) -> TaskAutoRecovery:
with factory() as db:
row = db.get(TaskAutoRecovery, task_id)
db.expunge_all()
return row
def _events(factory: sessionmaker, task_id: int) -> list[TaskRecoveryEvent]:
with factory() as db:
events = list(
db.scalars(
sa.select(TaskRecoveryEvent)
.where(TaskRecoveryEvent.task_id == task_id)
.order_by(TaskRecoveryEvent.id)
)
)
db.expunge_all()
return events
def _commands(factory: sessionmaker, task_id: int) -> list[TaskExecutionCommand]:
with factory() as db:
commands = list(
db.scalars(
sa.select(TaskExecutionCommand)
.where(TaskExecutionCommand.task_id == task_id)
.order_by(TaskExecutionCommand.id)
)
)
db.expunge_all()
return commands
def _trigger_run(factory: sessionmaker, run_id: int) -> TriggerRun:
with factory() as db:
run = db.get(TriggerRun, run_id)
db.expunge_all()
return run
def _sqlite_only(engine) -> None:
if engine.dialect.name != "sqlite":
pytest.skip("covered on SQLite; the PostgreSQL path is pinned separately")
# --------------------------------------------------------------------------
# Command id and public view
# --------------------------------------------------------------------------
def test_command_id_is_unique_per_fence_and_attempt_and_fits_the_column():
assert auto_resume_command_id(5, 1, "run-1") == "auto-resume:5:1:run-1"
long_run = "r" * 64
command_id = auto_resume_command_id(123456, 17, long_run)
assert len(command_id) == 64
assert command_id.startswith("auto-resume:123456:17:")
assert COMMAND_ID_PATTERN.fullmatch(command_id)
assert COMMAND_ID_PATTERN.fullmatch(
auto_resume_command_id(1, 1, "123e4567-e89b-12d3-a456-426614174000")
)
# A counter restart at a new fence never reuses an id.
assert auto_resume_command_id(6, 1, long_run) != auto_resume_command_id(
5, 1, long_run
)
@pytest.mark.parametrize(
("state", "detail", "expected"),
[
("scheduled", None, ("scheduled", None)),
("dispatched", None, ("resuming", None)),
("exhausted", None, ("stopped", "limit_reached")),
("expired", None, ("stopped", "expired")),
("superseded", None, ("stopped", "superseded")),
("dispatch_failed", None, ("stopped", "failed")),
("ineligible", "ineligible:preview", ("stopped", "ineligible")),
("manual", "auto_resume_disabled", ("stopped", "disabled")),
# Recorded for a channel task before channel tasks were scheduled.
("manual", "channel_hold_pending", ("stopped", "ineligible")),
("manual", None, ("stopped", "ineligible")),
("manual", "user_cancelled", ("stopped", "cancelled")),
("stale", "dispatch:run_id", None),
("disabled", None, None),
],
)
def test_public_view_exposes_only_the_client_contract(state, detail, expected):
at = datetime(2026, 10, 10, 8, 0, 12, tzinfo=timezone.utc)
view = public_auto_resume_view(
state,
state_detail=detail,
reason="lease_expired",
attempt=2,
next_attempt_at=at,
)
if expected is None:
assert view is None
return
assert (view["status"], view["stop_reason"]) == expected
assert set(view) == {
"status",
"stop_reason",
"reason",
"attempt",
"next_attempt_at",
}
assert view["next_attempt_at"] == (
"2026-10-10T08:00:12+00:00" if state == "scheduled" else None
)
def _recorded(state: str, detail: str | None = None) -> Any:
from xagent.web.services.task_settlement_report import RecordedInterruption
return RecordedInterruption(
task_id=3,
run_id=RUN,
reason="lease_expired",
state=state,
state_detail=detail,
paused_state_version=PV,
next_attempt_at=(
datetime(2026, 10, 10, 8, 0, 12, tzinfo=timezone.utc)
if state == "scheduled"
else None
),
attempt=1,
)
@pytest.mark.parametrize(
("state", "detail", "trigger_error", "notice"),
[
("scheduled", None, None, ("scheduled", None)),
(
"exhausted",
None,
TASK_AUTO_RESUME_EXHAUSTED_TRIGGER_ERROR,
("stopped", "limit_reached"),
),
(
"expired",
None,
TASK_AUTO_RESUME_EXPIRED_TRIGGER_ERROR,
("stopped", "expired"),
),
(
"superseded",
None,
TASK_AUTO_RESUME_SUPERSEDED_TRIGGER_ERROR,
("stopped", "superseded"),
),
# Recorded manual outcomes keep the caller's message and stay silent:
# the pause the client already shows is all there is.
("manual", "auto_resume_disabled", "default", None),
# Recorded for a channel task before channel tasks were scheduled.
("manual", "channel_hold_pending", "default", None),
("manual", "supersede_check_failed", "default", None),
("manual", None, "default", None),
("ineligible", "ineligible:source_a2a", "default", None),
],
)
def test_recorded_interruption_maps_to_trigger_error_and_notice(
state, detail, trigger_error, notice
):
recorded = _recorded(state, detail)
assert (
task_auto_resume.trigger_error_for_recorded(recorded, default="default")
== trigger_error
)
built = task_auto_resume.recorded_notice(recorded)
if notice is None:
assert built is None
return
assert built is not None
assert (built.auto_resume["status"], built.auto_resume["stop_reason"]) == notice
assert (built.task_id, built.run_id, built.state_version) == (3, RUN, PV)
assert built.auto_resume["next_attempt_at"] == (
"2026-10-10T08:00:12+00:00" if state == "scheduled" else None
)
def test_nothing_recorded_keeps_the_default_trigger_error():
assert task_auto_resume.trigger_error_for_recorded(None, default="x") == "x"
assert task_auto_resume.recorded_notice(None) is None
# --------------------------------------------------------------------------
# Dispatch
# --------------------------------------------------------------------------
def test_due_row_is_dispatched_as_one_fenced_resume_command(factory, notified):
now = utc_now()
with factory() as db:
task_id = _seed(db, now, no_progress=1, total=2)
db.commit()
report = run_auto_resume_tick(now=now)
assert (report.dispatched, report.staged_commands) == (1, 1)
row = _row(factory, task_id)
expected_id = f"auto-resume:{PV}:3:{RUN}"
assert row.state == "dispatched"
assert (row.no_progress_resumes, row.total_resumes) == (2, 3)
assert row.last_command_id == expected_id
assert _aware(row.next_attempt_at) == now + timedelta(
seconds=DISPATCH_CHECK_GRACE_SECONDS
)
(command,) = _commands(factory, task_id)
assert command.command_id == expected_id
assert command.kind == "resume"
assert command.status == "pending"
assert command.target_run_id == RUN
assert command.target_state_version == PV
assert command.payload == {
"type": "resume_task",
"auto_resume": {
"expected_run_id": RUN,
"expected_state_version": PV,
"reason": "lease_expired",
"attempt": 3,
},
}
(event,) = _events(factory, task_id)
assert (event.event, event.reason, event.attempt, event.run_id) == (
"auto_resumed",
"lease_expired",
3,
RUN,
)
assert event.detail == {"kind": "normal"}
assert notified == [1]
(notice,) = report.notices
assert (notice.task_id, notice.run_id, notice.state_version) == (
task_id,
RUN,
PV,
)
assert notice.auto_resume["status"] == "resuming"
assert notice.auto_resume["attempt"] == 3
message = notice.message()
assert message["type"] == "task_recovery_update"
assert "total_resumes" not in str(message)
@pytest.mark.parametrize(
("source", "previous", "expected"),
[
pytest.param("sdk", "interactive", "batch", id="sdk-ignores-previous"),
pytest.param("future_unknown", "interactive", "interactive", id="inherits"),
],
)
def test_auto_resume_command_gets_the_upstream_dispatch_class(
factory, notified, source, previous, expected
):
now = utc_now()
with factory() as db:
task_id = _seed(db, now, source=source)
if previous is not None:
db.add(
TaskExecutionCommand(
task_id=task_id,
command_id="earlier-start",
kind="start",
payload={},
status="completed",
dispatch_class=previous,
)
)
db.commit()
assert run_auto_resume_tick(now=now).staged_commands == 1
resume = [c for c in _commands(factory, task_id) if c.kind == "resume"]
assert [c.dispatch_class for c in resume] == [expected]
@pytest.mark.asyncio
async def test_auto_resume_uses_normal_xinference_priority(
factory, notified, monkeypatch
):
from xagent.core.model.chat.routing_context import xinference_interactive
from xagent.web.services import task_command_execution
from xagent.web.services.task_command_transport import (
TaskCommandKind,
claim_task_command,
)
now = utc_now()
with factory() as db:
_seed(db, now, source="internal")
db.commit()
assert run_auto_resume_tick(now=now).staged_commands == 1
with factory() as db:
command = claim_task_command(db, runner_id="worker")
assert command is not None
assert command.kind == TaskCommandKind.RESUME
assert command.command_id == auto_resume_command_id(PV, 1, RUN)
assert command.dispatch_class == "interactive"
async def observe_priority(_command):
return {"interactive": xinference_interactive.get()}
monkeypatch.setattr(
task_command_execution, "_execute_durable_task_command", observe_priority
)
assert await task_command_execution.execute_durable_task_command(command) == {
"interactive": False
}
def test_dispatch_does_not_require_an_empty_runner(factory, notified):
# A PAUSED task may hold an idle shared coordinator's owner lease.
now = utc_now()
with factory() as db:
task_id = _seed(
db,
now,
runner_id="idle-coordinator",
lease_expires_at=now + timedelta(minutes=5),
)
db.commit()
assert run_auto_resume_tick(now=now).dispatched == 1
assert _row(factory, task_id).state == "dispatched"
def test_row_not_yet_due_is_untouched(factory, notified):
now = utc_now()
with factory() as db:
task_id = _seed(db, now, due_in=5)
db.commit()
report = run_auto_resume_tick(now=now)
assert report.dispatched == 0
assert _row(factory, task_id).state == "scheduled"
assert _commands(factory, task_id) == []
assert _events(factory, task_id) == []
assert notified == []
@pytest.mark.parametrize(
("field", "task_fields"),
[
("status", {"status": TaskStatus.WAITING_FOR_USER}),
("control_state", {"control_state": "resume_requested"}),
("run_id", {"run_id": "run-other"}),
("state_version", {"state_version": PV + 1}),
],
)
def test_each_fence_column_makes_the_row_stale(factory, notified, field, task_fields):
now = utc_now()
with factory() as db:
task_id = _seed(db, now)
task = db.get(Task, task_id)
for key, value in task_fields.items():
setattr(task, key, value)
db.commit()
report = run_auto_resume_tick(now=now)
assert (report.dispatched, report.stale) == (0, 1)
row = _row(factory, task_id)
assert row.state == "stale"
assert row.state_detail == f"dispatch:{field}"
assert row.next_attempt_at is None
assert _commands(factory, task_id) == []
(event,) = _events(factory, task_id)
assert event.event == "stale"
assert event.detail["mismatch"] == field
assert event.detail["at"] == "dispatch"
assert notified == []
# A person's own action moved the task; nothing to announce.
assert report.notices == []
@pytest.mark.parametrize(
("status", "expected"),
[
(TaskStatus.RUNNING, TriggerRunStatus.RUNNING.value),
(TaskStatus.WAITING_FOR_USER, TriggerRunStatus.RUNNING.value),
(TaskStatus.COMPLETED, TriggerRunStatus.COMPLETED.value),
(TaskStatus.FAILED, TriggerRunStatus.FAILED.value),
# A user's resume that failed to start rolled the task back to
# paused on the same run, two versions on. The row goes stale and
# the run stays RUNNING until a later run settles it, as after any
# user pause (#2177); deleting the task fails it.
(TaskStatus.PAUSED, TriggerRunStatus.RUNNING.value),
],
)
def test_stale_never_fails_a_trigger_run_it_only_mirrors_an_ended_task(
factory, notified, status, expected
):
now = utc_now()
with factory() as db:
task_id, trigger_run_id = _trigger_task(db, now)
task = db.get(Task, task_id)
task.status = status
if status is TaskStatus.PAUSED:
task.state_version = int(task.state_version) + 2
db.commit()
assert run_auto_resume_tick(now=now).stale == 1
run = _trigger_run(factory, trigger_run_id)
assert run.status == expected
if status is TaskStatus.COMPLETED:
assert run.error_message is None
def _workforce_missing(db: Session, now: datetime) -> int:
return _seed(db, now, agent_config={"workforce_run_id": 999_999})
def _channel_task(db: Session, now: datetime, **kwargs: Any) -> int:
user = _user(db)
channel = UserChannel(
user_id=user.id, channel_type="slack", channel_name="ops", config={}
)
db.add(channel)
db.flush()
return _seed(db, now, user=user, channel_id=channel.id, **kwargs)
@pytest.mark.parametrize(
("case", "state", "detail", "event", "stop_reason"),
[
(
"workforce_missing",
"ineligible",
"ineligible:workforce_run_missing",
"ineligible",
"ineligible",
),
# Leaving an announced ``scheduled`` row: the client must hear it.
(
"channel_inline",
"ineligible",
"ineligible:channel_inline",
"ineligible",
"ineligible",
),
# A channel task's own (shorter) window.
("channel_expired", "expired", None, "expired", "expired"),
("expired", "expired", None, "expired", "expired"),
("no_progress", "exhausted", None, "exhausted", "limit_reached"),
("total", "exhausted", None, "exhausted", "limit_reached"),
("elapsed", "exhausted", None, "exhausted", "limit_reached"),
("no_policy", "manual", None, "ineligible", "ineligible"),
],
)
def test_dispatch_time_limits_stop_without_a_command(
factory, notified, monkeypatch, case, state, detail, event, stop_reason
):
monkeypatch.setenv(
"XAGENT_SHARED_TASK_EXECUTION_ENABLED",
"false" if case == "channel_inline" else "true",
)
now = utc_now()
with factory() as db:
if case == "workforce_missing":
task_id = _workforce_missing(db, now)
elif case == "channel_inline":
task_id = _channel_task(db, now)
elif case == "channel_expired":
task_id = _channel_task(db, now, interrupted_ago=1_800 + 60)
elif case == "expired":
task_id = _seed(db, now, interrupted_ago=86_400 + 60)
elif case == "no_progress":
task_id = _seed(db, now, no_progress=3, total=3)
elif case == "total":
task_id = _seed(db, now, total=20)
elif case == "elapsed":
task_id = _seed(
db, now, reason="llm_unavailable", episode_ago=7_200, total=5
)
else:
task_id = _seed(db, now, reason="user_pause")
db.commit()
report = run_auto_resume_tick(now=now)
assert report.dispatched == 0
row = _row(factory, task_id)
assert (row.state, row.state_detail) == (state, detail)
assert row.next_attempt_at is None
assert _commands(factory, task_id) == []
(recorded,) = _events(factory, task_id)
assert recorded.event == event
assert recorded.detail["state"] == state
assert recorded.detail["at"] == "dispatch"
assert report.gave_up == 1
assert notified == []
assert [n.auto_resume["stop_reason"] for n in report.notices] == (
[stop_reason] if stop_reason else []
)
def test_shared_channel_row_is_dispatched_like_any_other(
factory, notified, monkeypatch
):
monkeypatch.setenv("XAGENT_SHARED_TASK_EXECUTION_ENABLED", "true")
now = utc_now()
with factory() as db:
task_id = _channel_task(db, now, interrupted_ago=1_800 - 60)
db.commit()
report = run_auto_resume_tick(now=now)
assert (report.dispatched, report.gave_up) == (1, 0)
row = _row(factory, task_id)
assert (row.state, row.state_detail) == ("dispatched", None)
assert [c.command_id for c in _commands(factory, task_id)] == [row.last_command_id]
@pytest.mark.parametrize(
("case", "state", "message"),
[
("superseded", "superseded", TASK_AUTO_RESUME_SUPERSEDED_TRIGGER_ERROR),
("expired", "expired", TASK_AUTO_RESUME_EXPIRED_TRIGGER_ERROR),
("exhausted", "exhausted", TASK_AUTO_RESUME_EXHAUSTED_TRIGGER_ERROR),
("no_policy", "manual", TASK_INTERRUPTION_PAUSED_TRIGGER_ERROR),
],
)
def test_dispatch_give_up_fails_the_running_trigger_run(
factory, notified, case, state, message
):
now = utc_now()
kwargs: dict[str, Any] = {
"superseded": {"later_run": True},
"expired": {"interrupted_ago": 86_400 + 60},
"exhausted": {"total": 20},
"no_policy": {"reason": "user_pause"},
}[case]
with factory() as db:
task_id, trigger_run_id = _trigger_task(db, now, **kwargs)
db.commit()
run_auto_resume_tick(now=now)
assert _row(factory, task_id).state == state
run = _trigger_run(factory, trigger_run_id)
assert run.status == TriggerRunStatus.FAILED.value
assert run.error_message == message
assert _commands(factory, task_id) == []
def test_unsuperseded_trigger_task_is_dispatched_and_its_run_stays_running(
factory, notified
):
now = utc_now()
with factory() as db:
task_id, trigger_run_id = _trigger_task(db, now)
db.commit()
assert run_auto_resume_tick(now=now).dispatched == 1
assert _trigger_run(factory, trigger_run_id).status == "running"
assert _events(factory, task_id)[0].detail == {"kind": "trigger_scheduled"}
# --------------------------------------------------------------------------
# Budget
# --------------------------------------------------------------------------
def test_max_per_tick_bounds_dispatches(factory, notified, monkeypatch):
monkeypatch.setenv("XAGENT_TASK_AUTO_RESUME_MAX_PER_TICK", "2")
now = utc_now()
with factory() as db:
ids = [_seed(db, now, due_in=-10 + index) for index in range(3)]
db.commit()
report = run_auto_resume_tick(now=now)
assert (report.dispatched, report.budget) == (2, 2)
assert [_row(factory, task_id).state for task_id in ids] == [
"dispatched",
"dispatched",
"scheduled",
]
assert notified == [1]
def _capture_counters(monkeypatch) -> list[str]:
names: list[str] = []
real_counter = task_auto_resume.increment_counter
monkeypatch.setattr(
task_auto_resume,
"increment_counter",
lambda name, **kw: (names.append(name), real_counter(name, **kw)),
)
return names
def _inflight_running(db: Session, now: datetime) -> None:
"""A: an auto-resumed run still RUNNING (row confirmed, check cleared)."""
_seed(
db,
now,
state="dispatched",
due_in=None,
status=TaskStatus.RUNNING,
control_state="running",
run_id="run-a",
state_version=PV + 2,
paused_state_version=PV,
)
def _inflight_queued(db: Session, now: datetime) -> None:
"""B: a dispatched command still queued at an unchanged fence."""
_dispatched(db, now, command_status="pending", due_in=30, run_id="run-b")
@pytest.mark.parametrize(
("inflight", "expected"),
[((), 0), (("running",), 1), (("queued",), 1), (("running", "queued"), 2)],
)
def test_max_inflight_counts_running_and_queued_resumes(
factory, notified, monkeypatch, inflight, expected
):
monkeypatch.setenv("XAGENT_TASK_AUTO_RESUME_MAX_INFLIGHT", "3")
now = utc_now()
with factory() as db:
for kind in inflight:
(_inflight_running if kind == "running" else _inflight_queued)(db, now)
# A finished resumed run and a paused row past its fence are not in
# flight.
_seed(
db,
now,
state="dispatched",
due_in=None,
status=TaskStatus.COMPLETED,
run_id="run-done",
)
ids = [_seed(db, now, due_in=-10 + index) for index in range(4)]
db.commit()
report = run_auto_resume_tick(now=now)
assert report.inflight == expected
assert report.budget == 3 - expected
assert report.dispatched == 3 - expected
assert sum(_row(factory, i).state == "dispatched" for i in ids) == 3 - expected
def _add_completed_command(db: Session, task_id: int, command_id: str) -> None:
"""A completed command of the task's owner, flushed so ids follow call order."""
db.add(
TaskExecutionCommand(
task_id=task_id,
actor_user_id=db.get(Task, task_id).user_id,
command_id=command_id,
kind="message" if command_id.startswith("message") else "resume",
payload={},
status="completed",
)
)
db.flush()
@pytest.mark.parametrize(
("later_command", "counted"),
[
(None, True),
# The sweeper's own later dispatch is still automatic.
("auto-resume:5:2:run-a", True),
# A person resumed it by hand since: not an automatic resume any more.
("resume-by-hand", False),
("message-1", False),
# The reserved prefix is case-sensitive, as the transport checks it.
("AUTO-RESUME:5:2:run-a", False),
],
)
def test_inflight_running_excludes_runs_a_person_resumed(
factory, later_command, counted
):
now = utc_now()
dispatch_id = auto_resume_command_id(PV, 1, "run-a")
with factory() as db:
task_id = _seed(
db,
now,
state="dispatched",
due_in=None,
status=TaskStatus.RUNNING,
control_state="running",
run_id="run-a",
state_version=PV + 4,
paused_state_version=PV,
last_command_id=dispatch_id,
)
for command_id in (dispatch_id, later_command):
if command_id is not None:
_add_completed_command(db, task_id, command_id)
db.commit()
with factory() as db:
assert task_auto_resume.count_inflight(db) == (1 if counted else 0)
def test_inflight_running_checks_each_run_against_its_own_dispatch(factory):
# Two auto-resumed runs RUNNING at once. Each run's "a person acted since"
# check must compare against that run's own dispatch: on PostgreSQL a
# check that sees every dispatch fails outright once there are two.
now = utc_now()
with factory() as db:
tasks = {}
for run_id in ("run-a", "run-b"):
tasks[run_id] = _seed(
db,
now,
state="dispatched",
due_in=None,
status=TaskStatus.RUNNING,
control_state="running",
run_id=run_id,
state_version=PV + 4,
paused_state_version=PV,
last_command_id=auto_resume_command_id(PV, 1, run_id),
)
# run-b's dispatch is the oldest command, and run-a has a message a
# person sent before run-a's own dispatch: run-a is still automatic.
for run_id, command_id in (
("run-b", auto_resume_command_id(PV, 1, "run-b")),
("run-a", "message-before-dispatch"),
("run-a", auto_resume_command_id(PV, 1, "run-a")),
):
_add_completed_command(db, tasks[run_id], command_id)
db.commit()
with factory() as db:
assert task_auto_resume.count_inflight(db) == 2
def test_tick_dispatches_while_an_auto_resumed_run_is_running(factory, notified):
# The sweeper keeps filling free slots while earlier resumes still run,
# instead of waiting for every one of them to stop.
now = utc_now()
with factory() as db:
# A running resumed run whose dispatch command has completed, next to
# one whose dispatch is still queued.
_dispatched(
db,
now,
command_status="completed",
due_in=None,
status=TaskStatus.RUNNING,
control_state="running",
run_id="run-a",
state_version=PV + 2,
)
_inflight_queued(db, now)
due = _seed(db, now, due_in=-10)
db.commit()
report = run_auto_resume_tick(now=now)
assert (report.inflight, report.dispatched) == (2, 1)
assert _row(factory, due).state == "dispatched"
def test_give_ups_still_advance_when_no_dispatch_budget_is_left(
factory, notified, monkeypatch
):
monkeypatch.setenv("XAGENT_TASK_AUTO_RESUME_MAX_INFLIGHT", "1")
now = utc_now()
with factory() as db:
_inflight_queued(db, now)
expired = _seed(db, now, due_in=-20, interrupted_ago=86_400 + 60)
stale = _seed(db, now, due_in=-15, state_version=PV + 1)
waiting = _seed(db, now, due_in=-10)
after = _seed(db, now, due_in=-5, interrupted_ago=86_400 + 60)
db.commit()
report = run_auto_resume_tick(now=now)
assert (report.budget, report.dispatched) == (0, 0)
assert _row(factory, expired).state == "expired"
assert _row(factory, stale).state == "stale"
# A row that needs a dispatch is left unwritten and the scan goes on, so
# it cannot hold back the give-ups behind it.
assert _row(factory, waiting).state == "scheduled"
assert _row(factory, after).state == "expired"
assert _commands(factory, waiting) == []
assert notified == []
# --------------------------------------------------------------------------
# Races and admission
# --------------------------------------------------------------------------
def test_a_lost_compare_and_swap_writes_nothing(factory, notified, engine, monkeypatch):
_sqlite_only(engine)
now = utc_now()
with factory() as db:
task_id = _seed(db, now)
db.commit()
# A second sweeper read the row before the first one dispatched it.
with factory() as seen:
stale_task = seen.get(Task, task_id)
stale_row = seen.get(TaskAutoRecovery, task_id)
seen.expunge_all()
assert run_auto_resume_tick(now=now).dispatched == 1
monkeypatch.setattr(
task_auto_resume,
"_load_locked_no_commit",
lambda _db, _id: (stale_task, stale_row),
)
with factory() as db:
result = task_auto_resume._process_due_candidate_no_commit(
db, task_id, now=now, can_dispatch=True
)
db.rollback()
assert result.outcome is task_auto_resume._Outcome.RACED
assert len(_commands(factory, task_id)) == 1
assert [e.event for e in _events(factory, task_id)] == ["auto_resumed"]
assert _row(factory, task_id).total_resumes == 1
def _process_with_stale_view(
factory: sessionmaker, monkeypatch, task_id: int, now: datetime, change
) -> Any:
"""Process ``task_id`` from a view read before ``change`` committed."""
with factory() as seen:
stale_task = seen.get(Task, task_id)
stale_row = seen.get(TaskAutoRecovery, task_id)
seen.expunge_all()
with factory() as db:
change(db)
db.commit()
monkeypatch.setattr(
task_auto_resume,
"_load_locked_no_commit",
lambda _db, _id: (stale_task, stale_row),
)
with factory() as db:
result = task_auto_resume._process_due_candidate_no_commit(
db, task_id, now=now, can_dispatch=True
)
if result.writes:
db.commit()
else:
db.rollback()
return result
@pytest.mark.parametrize(
("column", "value"),
[
("status", TaskStatus.WAITING_FOR_USER),
("control_state", "resume_requested"),
("run_id", "run-other"),
("state_version", PV + 1),
],
)
def test_dispatch_cas_rechecks_each_fence_column_in_the_database(
factory, notified, monkeypatch, column, value
):
# SQLite has no row lock: the task moved after the sweeper read it, and
# only the CAS's EXISTS over ``tasks`` can notice.
now = utc_now()
with factory() as db:
task_id = _seed(db, now)
db.commit()
def move(db: Session) -> None:
setattr(db.get(Task, task_id), column, value)
result = _process_with_stale_view(factory, monkeypatch, task_id, now, move)
assert result.outcome is task_auto_resume._Outcome.RACED
assert _row(factory, task_id).state == "scheduled"
assert _commands(factory, task_id) == []
assert _events(factory, task_id) == []
@pytest.mark.parametrize("column", ["paused_state_version", "run_id"])
def test_stale_cas_does_not_touch_a_rerecorded_row(
factory, notified, monkeypatch, column
):
# The sweeper saw the task moved off the row's fence; meanwhile a new
# interruption re-recorded the row at a new fence. Closing it as stale
# would drop a live schedule.
now = utc_now()
with factory() as db:
task_id = _seed(db, now, state_version=PV + 1)
db.commit()
def rerecord(db: Session) -> None:
row = db.get(TaskAutoRecovery, task_id)
if column == "paused_state_version":
row.paused_state_version = PV + 1
else:
row.run_id = "run-next"
db.get(Task, task_id).run_id = "run-next"
result = _process_with_stale_view(factory, monkeypatch, task_id, now, rerecord)
assert result.outcome is task_auto_resume._Outcome.RACED
row = _row(factory, task_id)
assert row.state == "scheduled"
assert _events(factory, task_id) == []
def test_give_up_cas_rechecks_the_fence(factory, notified, monkeypatch):
now = utc_now()
with factory() as db:
task_id = _seed(db, now, interrupted_ago=86_400 + 60)
db.commit()
def resume_by_hand(db: Session) -> None:
db.get(Task, task_id).state_version = PV + 1
result = _process_with_stale_view(
factory, monkeypatch, task_id, now, resume_by_hand
)
assert result.outcome is task_auto_resume._Outcome.RACED
assert _row(factory, task_id).state == "scheduled"
@pytest.mark.parametrize("existing", ["pending", "completed", "foreign"])
def test_an_existing_command_with_the_same_id_commits_only_if_it_will_run(
factory, notified, caplog, existing
):
now = utc_now()
with factory() as db:
task_id = _seed(db, now)
task = db.get(Task, task_id)
if existing == "foreign":
db.add(
TaskExecutionCommand(
task_id=task_id,
actor_user_id=task.user_id,
command_id=auto_resume_command_id(PV, 1, RUN),
kind="resume",
payload={"type": "resume_task"},
status="pending",
)
)
db.commit()
if existing != "foreign":
# An earlier dispatch of this exact (fence, attempt) whose CAS a
# second sweeper then saw undone: the row reads scheduled again.
assert run_auto_resume_tick(now=now).dispatched == 1
with factory() as db:
row = db.get(TaskAutoRecovery, task_id)
row.state, row.total_resumes, row.no_progress_resumes = "scheduled", 0, 0
row.next_attempt_at = now - timedelta(seconds=1)
command = db.scalars(
sa.select(TaskExecutionCommand).where(
TaskExecutionCommand.task_id == task_id
)
).one()
command.status = existing
db.commit()
notified.clear()
events_before = len(_events(factory, task_id))
with caplog.at_level(logging.WARNING, logger=task_auto_resume.__name__):
report = run_auto_resume_tick(now=now)
assert "already existed" in caplog.text
assert len(_commands(factory, task_id)) == 1
assert report.staged_commands == 0
assert notified == []
row = _row(factory, task_id)
if existing == "pending":
# That command is this dispatch and will still run.
assert report.dispatched == 1
assert row.state == "dispatched"
assert len(_events(factory, task_id)) == events_before + 1
else:
# A finished command or someone else's payload: nothing commits.
assert (report.dispatched, report.raced) == (0, 1)
assert (row.state, row.total_resumes) == ("scheduled", 0)
assert len(_events(factory, task_id)) == events_before
def test_full_admission_queue_postpones_only_that_row(factory, notified, monkeypatch):
full_for: set[int] = set()
real = task_execution_admission.stage_task_admission
def admission(db, command, policy):
if command.task_id in full_for:
raise AdmissionQueueFull("Execution queue is full")
real(db, command, policy)
monkeypatch.setattr(
"xagent.web.services.task_execution_admission.stage_task_admission", admission
)
now = utc_now()
with factory() as db:
first = _seed(db, now, due_in=-10)
second = _seed(db, now, due_in=-5)
db.commit()
full_for.add(first)
report = run_auto_resume_tick(now=now)
assert report.admission_full
assert report.dispatched == 1
row = _row(factory, first)
assert row.state == "scheduled"
assert (row.no_progress_resumes, row.total_resumes) == (0, 0)
assert row.last_command_id is None
assert _aware(row.next_attempt_at) == now + timedelta(seconds=30)
assert _commands(factory, first) == []
assert _events(factory, first) == []
# The scan went on: the next candidate was dispatched in the same tick.
assert _row(factory, second).state == "dispatched"
assert len(_commands(factory, second)) == 1
assert notified == [1]
def _team_admission(
factory: sessionmaker, monkeypatch, request, *, full_for: User, install: str
) -> None:
"""Admission buckets per user (a stand-in for a team), the user's bucket
already holding its one pending slot."""
monkeypatch.setenv("XAGENT_SHARED_TASK_EXECUTION_ENABLED", "true")
def classify(db, command):
task = db.get(Task, command.task_id)
if task.source != "sdk":
return None
return AdmissionPolicy(bucket=f"team:{task.user_id}", capacity=1, max_pending=1)
if install == "legacy":
task_execution_admission.set_task_admission_hook(classify)
request.addfinalizer(
lambda: task_execution_admission.set_task_admission_hook(None)
)
else:
task_execution_admission.set_command_classifier(
lambda db, command, view: task_execution_admission.CommandClassification(
admission=classify(db, command)
),
admission_enabled=True,
)
request.addfinalizer(task_execution_admission.reset_command_classifier)
now = utc_now()
with factory() as db:
holder = _seed(db, now, user=full_for, state="manual", due_in=None)
command = TaskExecutionCommand(
task_id=holder,
actor_user_id=full_for.id,
command_id=f"holder-{holder}",
kind="resume",
payload={"type": "resume_task"},
status="pending",
)
db.add(command)
db.add(
TaskAdmissionBucket(key=f"team:{full_for.id}", capacity=1, max_pending=1)
)
db.flush()
db.add(
TaskAdmissionTicket(
command_id=command.id,
task_id=holder,
bucket_key=f"team:{full_for.id}",
)
)
db.commit()
@pytest.mark.parametrize("install", ["legacy", "classifier"])
@pytest.mark.parametrize("team_a_rows", [1, 200])
def test_admission_full_in_one_team_does_not_block_another(
factory, notified, monkeypatch, request, team_a_rows, install
):
now = utc_now()
with factory() as db:
team_a, team_b = _user(db), _user(db)
db.commit()
team_a_id, team_b_id = team_a.id, team_b.id
with factory() as db:
team_a, team_b = db.get(User, team_a_id), db.get(User, team_b_id)
_team_admission(factory, monkeypatch, request, full_for=team_a, install=install)
# Team A's rows are due first; team B's row and an internal row the
# host's hook does not classify come after them.
a_rows = [
_seed(db, now, user=team_a, source="sdk", due_in=-1000 + index)
for index in range(team_a_rows)
]
b_row = _seed(db, now, user=team_b, source="sdk", due_in=-1)
unhooked = _seed(db, now, user=team_b, due_in=-1)
db.commit()
# Each scan handles ``4 * max_per_tick`` rows, so a team ahead of everyone
# else delays another team by at most ceil(rows / 80) scans, plus the one
# that dispatches it.
ticks_bound = -(-team_a_rows // (4 * 20)) + 1
for tick in range(ticks_bound):
report = run_auto_resume_tick(now=now + timedelta(seconds=5 * tick))
if tick == 0:
assert report.admission_full
if _row(factory, b_row).state == "dispatched":
break
assert _row(factory, b_row).state == "dispatched"
assert _row(factory, unhooked).state == "dispatched"
assert len(_commands(factory, b_row)) == 1
if team_a_rows == 1:
assert tick == 0
row = _row(factory, a_rows[0])
assert (row.state, _aware(row.next_attempt_at)) == (
"scheduled",
now + timedelta(seconds=30),
)
# Team A never got through its full bucket.
assert all(_commands(factory, task_id) == [] for task_id in a_rows[:5])
def test_candidate_failure_is_isolated(factory, notified, monkeypatch):
now = utc_now()
with factory() as db:
broken = _seed(db, now, due_in=-10)
healthy = _seed(db, now, due_in=-5)
db.commit()
original = task_auto_resume.auto_recovery_eligibility
def eligibility(db, task):
if task.id == broken:
raise RuntimeError("boom")
return original(db, task)
monkeypatch.setattr(task_auto_resume, "auto_recovery_eligibility", eligibility)
report = run_auto_resume_tick(now=now)
assert (report.failed, report.dispatched) == (1, 1)
assert _row(factory, broken).state == "scheduled"
assert _row(factory, healthy).state == "dispatched"
def test_postgresql_candidate_query_locks_tasks_with_skip_locked():
captured: list[Any] = []
class _Db:
def execute(self, statement):
captured.append(statement)
class _Result:
def first(self):
return None
return _Result()
task_auto_resume.select_next_candidate_for_update(
_Db(),
task_auto_resume.TaskAutoRecoveryState.SCHEDULED,
now=utc_now(),
after=(utc_now(), 3),
)
sql = " ".join(str(captured[0].compile(dialect=postgresql.dialect())).split())
assert "FOR UPDATE OF tasks SKIP LOCKED" in sql
assert "task_auto_recovery.next_attempt_at <=" in sql
assert (
"ORDER BY task_auto_recovery.next_attempt_at, task_auto_recovery.task_id" in sql
)
assert "LIMIT" in sql
# --------------------------------------------------------------------------
# PostgreSQL: partitioning and lock order
# --------------------------------------------------------------------------
def _postgresql_only(engine) -> None:
if engine.dialect.name != "postgresql":
pytest.skip("needs two real PostgreSQL transactions")
def test_postgresql_workers_partition_candidates(factory, engine):
_postgresql_only(engine)
now = utc_now()
with factory() as db:
first = _seed(db, now, due_in=-10)
second = _seed(db, now, due_in=-5)
db.commit()
state = task_auto_resume.TaskAutoRecoveryState.SCHEDULED
with factory() as one, factory() as two:
picked_one = task_auto_resume.select_next_candidate_for_update(
one, state, now=now
)
# A worker that waited on the first one's lock would fail here, not hang.
two.execute(sa.text("SET LOCAL lock_timeout = '1s'"))
picked_two = task_auto_resume.select_next_candidate_for_update(
two, state, now=now
)
assert picked_one is not None and picked_two is not None
assert (picked_one[1], picked_two[1]) == (first, second)
one.rollback()
two.rollback()
def test_postgresql_sweeper_skips_a_task_a_settlement_holds_without_deadlock(
factory, engine, notified
):
"""A settlement holds ``tasks`` and then writes the recovery row.
Locking the recovery row first (``FOR UPDATE OF task_auto_recovery``)
would let the sweeper take it, then block on the key-share lock its
command and event inserts need on ``tasks`` while the settlement blocks
on the recovery row: a deadlock. Locking ``tasks`` with SKIP LOCKED
leaves the row to a later tick. Every sweeper statement runs under a
short ``lock_timeout`` so a regression fails here instead of hanging.
"""
_postgresql_only(engine)
@sa.event.listens_for(engine, "connect")
def short_lock_timeout(connection, _record):
# Outside a transaction, so the pool's reset-on-return keeps it.
connection.autocommit = True
with connection.cursor() as cursor:
cursor.execute("SET lock_timeout = '1500ms'")
connection.autocommit = False
try:
_sweeper_skips_a_held_task(factory, engine)
finally:
sa.event.remove(engine, "connect", short_lock_timeout)
def _sweeper_skips_a_held_task(factory: sessionmaker, engine) -> None:
engine.dispose()
now = utc_now()
with factory() as db:
task_id = _seed(db, now)
db.commit()
holder = factory()
try:
# The settlement: lock the task row first, as every settling path does.
holder.execute(
sa.select(Task.id).where(Task.id == task_id).with_for_update()
).one()
outcome: dict[str, Any] = {}
def sweep() -> None:
try:
outcome["report"] = run_auto_resume_tick(now=now)
except BaseException as exc: # pragma: no cover - reported below
outcome["error"] = exc
sweeper = threading.Thread(target=sweep)
sweeper.start()
sweeper.join(timeout=10)
assert not sweeper.is_alive()
assert "error" not in outcome, outcome.get("error")
report = outcome["report"]
assert (report.dispatched, report.failed) == (0, 0)
# The settlement now writes the recovery row and commits unhindered.
holder.execute(sa.text("SET LOCAL lock_timeout = '1500ms'"))
holder.execute(
sa.update(TaskAutoRecovery)
.where(TaskAutoRecovery.task_id == task_id)
.values(last_error="settled")
)
holder.commit()
finally:
holder.close()
assert _row(factory, task_id).state == "scheduled"
# Released, the next tick dispatches it.
assert run_auto_resume_tick(now=now).dispatched == 1
assert _row(factory, task_id).state == "dispatched"
# --------------------------------------------------------------------------
# Housekeeping
# --------------------------------------------------------------------------
def _dispatched(
db: Session, now: datetime, *, command_status: str | None, **kwargs: Any
) -> int:
command_id = auto_resume_command_id(PV, 1, RUN)
task_id = _seed(
db,
now,
state="dispatched",
last_command_id=command_id,
no_progress=kwargs.pop("no_progress", 1),
total=kwargs.pop("total", 1),
**kwargs,
)
if command_status is not None:
db.add(
TaskExecutionCommand(
task_id=task_id,
actor_user_id=db.get(Task, task_id).user_id,
command_id=command_id,
kind="resume",
payload={"type": "resume_task"},
status=command_status,
result=(
{"rejection_reason": "stale_run"}
if command_status == "failed"
else {"resume_outcome": "already_in_progress"}
if command_status == "completed"
else None
),
)
)
db.flush()
return task_id
# --------------------------------------------------------------------------
# Drain with the switch off
# --------------------------------------------------------------------------
# --------------------------------------------------------------------------
# Housekeeping
# --------------------------------------------------------------------------
@pytest.mark.parametrize("command_status", ["failed", None])
def test_housekeeping_marks_a_failed_or_lost_dispatch(
factory, notified, command_status
):
now = utc_now()
with factory() as db:
task_id = _dispatched(db, now, command_status=command_status)
user = db.get(User, db.get(Task, task_id).user_id)
trigger_run_id, _later = _trigger(db, user, task_id)
db.commit()
report = run_auto_resume_tick(now=now)
row = _row(factory, task_id)
assert row.state == "dispatch_failed"
assert row.next_attempt_at is None
(event,) = _events(factory, task_id)
assert event.event == "dispatch_failed"
assert event.detail["at"] == "housekeeping"
assert event.detail["rejection_reason"] == (
"stale_run" if command_status == "failed" else None
)
run = _trigger_run(factory, trigger_run_id)
assert run.status == TriggerRunStatus.FAILED.value
assert run.error_message == TASK_AUTO_RESUME_DISPATCH_FAILED_TRIGGER_ERROR
assert report.gave_up == 1
assert [n.auto_resume["stop_reason"] for n in report.notices] == ["failed"]
def test_housekeeping_reschedules_a_completed_resume_that_did_not_happen(
factory, notified, monkeypatch
):
monkeypatch.setattr(task_auto_resume, "_AUTO_RESUME_RNG", random.Random(7))
now = utc_now()
with factory() as db:
task_id = _dispatched(db, now, command_status="completed")
db.commit()
report = run_auto_resume_tick(now=now)
row = _row(factory, task_id)
assert row.state == "scheduled"
# The dispatch happened; its counters stand.
assert (row.no_progress_resumes, row.total_resumes) == (1, 1)
# Second short backoff (20s) plus at most 10s of jitter.
assert now + timedelta(seconds=20) <= _aware(row.next_attempt_at)
assert _aware(row.next_attempt_at) <= now + timedelta(seconds=30)
(event,) = _events(factory, task_id)
assert (event.event, event.attempt) == ("scheduled", 2)
assert event.detail == {"at": "housekeeping", "after": "already_in_progress"}
assert report.rescheduled == 1
(notice,) = report.notices
assert notice.auto_resume["status"] == "scheduled"
assert notice.auto_resume["attempt"] == 2
assert notice.auto_resume["next_attempt_at"] is not None
def test_an_announced_resume_that_ends_in_manual_is_announced_stopped(
factory, notified, monkeypatch
):
"""A housekeeping reschedule announced ``scheduled``; when the next
dispatch finds the task out of reach (a policy-less reason here), the
client must hear that the resume stopped, at the same fence."""
now = utc_now()
with factory() as db:
task_id = _dispatched(db, now, command_status="completed")
db.commit()
first = run_auto_resume_tick(now=now)
(scheduled,) = first.notices
assert scheduled.auto_resume["status"] == "scheduled"
with factory() as db:
db.get(TaskAutoRecovery, task_id).reason = "user_pause"
db.commit()
later = run_auto_resume_tick(now=now + timedelta(minutes=5))
assert _row(factory, task_id).state == "manual"
(stopped,) = later.notices
assert (stopped.auto_resume["status"], stopped.auto_resume["stop_reason"]) == (
"stopped",
"ineligible",
)
assert stopped.state_version == scheduled.state_version
def test_housekeeping_stops_a_completed_resume_at_its_limit(factory, notified):
now = utc_now()
with factory() as db:
task_id = _dispatched(
db, now, command_status="completed", no_progress=3, total=3
)
db.commit()
run_auto_resume_tick(now=now)
assert _row(factory, task_id).state == "exhausted"
(event,) = _events(factory, task_id)
assert event.event == "exhausted"
assert event.detail["after"] == "already_in_progress"
def test_housekeeping_cas_misses_a_row_redispatched_since_it_was_read(
factory, notified, monkeypatch
):
now = utc_now()
with factory() as db:
task_id = _dispatched(db, now, command_status="failed")
db.commit()
with factory() as seen:
stale_task = seen.get(Task, task_id)
stale_row = seen.get(TaskAutoRecovery, task_id)
seen.expunge_all()
redispatched = auto_resume_command_id(PV, 2, RUN)
with factory() as db:
db.get(TaskAutoRecovery, task_id).last_command_id = redispatched
db.commit()
monkeypatch.setattr(
task_auto_resume,
"_load_locked_no_commit",
lambda _db, _id: (stale_task, stale_row),
)
with factory() as db:
result = task_auto_resume._housekeep_candidate_no_commit(db, task_id, now=now)
db.rollback()
assert result.outcome is task_auto_resume._Outcome.RACED
row = _row(factory, task_id)
assert (row.state, row.last_command_id) == ("dispatched", redispatched)
def test_housekeeping_reschedule_rechecks_the_fence(factory, notified, monkeypatch):
# Read at the fence with its command completed, the task was resumed by
# hand before the CAS: scheduling another resume would race the person.
now = utc_now()
with factory() as db:
task_id = _dispatched(db, now, command_status="completed")
db.commit()
with factory() as seen:
stale_task = seen.get(Task, task_id)
stale_row = seen.get(TaskAutoRecovery, task_id)
seen.expunge_all()
with factory() as db:
db.get(Task, task_id).state_version = PV + 1
db.commit()
monkeypatch.setattr(
task_auto_resume,
"_load_locked_no_commit",
lambda _db, _id: (stale_task, stale_row),
)
with factory() as db:
result = task_auto_resume._housekeep_candidate_no_commit(db, task_id, now=now)
db.rollback()
assert result.outcome is task_auto_resume._Outcome.RACED
assert _row(factory, task_id).state == "dispatched"
assert _events(factory, task_id) == []
def test_housekeeping_rearm_rechecks_the_fence(factory, notified, monkeypatch):
# Read while still at the fence, the task resumed before the CAS: the
# rearm must miss, so the next check confirms instead.
now = utc_now()
with factory() as db:
task_id = _dispatched(db, now, command_status="pending")
db.commit()
with factory() as seen:
stale_task = seen.get(Task, task_id)
stale_row = seen.get(TaskAutoRecovery, task_id)
seen.expunge_all()
with factory() as db:
db.get(Task, task_id).state_version = PV + 1
db.commit()
monkeypatch.setattr(
task_auto_resume,
"_load_locked_no_commit",
lambda _db, _id: (stale_task, stale_row),
)
with factory() as db:
result = task_auto_resume._housekeep_candidate_no_commit(db, task_id, now=now)
db.rollback()
assert result.outcome is task_auto_resume._Outcome.RACED
assert _aware(_row(factory, task_id).next_attempt_at) == now - timedelta(seconds=1)
@pytest.mark.parametrize("command_status", ["completed", "failed", "pending", None])
def test_housekeeping_confirms_once_the_fence_moved(
factory, notified, monkeypatch, command_status
):
now = utc_now()
with factory() as db:
task_id = _dispatched(db, now, command_status=command_status)
task = db.get(Task, task_id)
task.status = TaskStatus.RUNNING
task.control_state = "running"
task.state_version = PV + 2
db.commit()
counters = _capture_counters(monkeypatch)
report = run_auto_resume_tick(now=now)
assert report.confirmed == 1
assert "xagent.task.auto_resume.confirmed" in counters
row = _row(factory, task_id)
assert (row.state, row.next_attempt_at) == ("dispatched", None)
assert _events(factory, task_id) == []
assert report.notices == []
def test_housekeeping_fails_a_resume_whose_last_attempt_rolled_back(factory, notified):
"""The resume wrote ``resume_requested``, failed to start and rolled the
task back to ``paused`` two versions on, on its command's last attempt:
the fence moved, but nothing resumed the task, so it is a failed
dispatch rather than a confirmed one."""
now = utc_now()
with factory() as db:
task_id = _dispatched(db, now, command_status="failed")
user = db.get(User, db.get(Task, task_id).user_id)
trigger_run_id, _later = _trigger(db, user, task_id)
db.get(Task, task_id).state_version = PV + 2
db.commit()
report = run_auto_resume_tick(now=now)
assert (report.confirmed, report.gave_up) == (0, 1)
row = _row(factory, task_id)
assert (row.state, row.next_attempt_at) == ("dispatch_failed", None)
(event,) = _events(factory, task_id)
assert event.event == "dispatch_failed"
run = _trigger_run(factory, trigger_run_id)
assert run.status == TriggerRunStatus.FAILED.value
assert run.error_message == TASK_AUTO_RESUME_DISPATCH_FAILED_TRIGGER_ERROR
# The notice names the pause the client shows (the rolled-back version),
# not the row's old fence, or a viewer at that version drops it.
(notice,) = report.notices
assert (notice.state_version, notice.auto_resume["status"]) == (
PV + 2,
"stopped",
)
@pytest.mark.parametrize("command_status", ["pending", "processing"])
def test_housekeeping_rechecks_a_queued_command_later(
factory, notified, monkeypatch, command_status
):
now = utc_now()
with factory() as db:
task_id = _dispatched(db, now, command_status=command_status)
db.commit()
counters = _capture_counters(monkeypatch)
report = run_auto_resume_tick(now=now)
assert report.rearmed == 1
assert "xagent.task.auto_resume.rearmed" in counters
row = _row(factory, task_id)
assert row.state == "dispatched"
assert _aware(row.next_attempt_at) == now + timedelta(
seconds=DISPATCH_CHECK_GRACE_SECONDS
)
assert _events(factory, task_id) == []
def test_housekeeping_ignores_dispatches_not_yet_due(factory, notified):
now = utc_now()
with factory() as db:
task_id = _dispatched(db, now, command_status=None, due_in=10)
db.commit()
run_auto_resume_tick(now=now)
assert _row(factory, task_id).state == "dispatched"
# --------------------------------------------------------------------------
# Drain with the switch off
# --------------------------------------------------------------------------
def test_switch_off_drains_scheduled_rows_to_manual(factory, notified, monkeypatch):
monkeypatch.setenv("XAGENT_TASK_AUTO_RESUME_ENABLED", "false")
monkeypatch.setattr(task_auto_resume, "_last_inflight", 7)
now = utc_now()
with factory() as db:
task_id, trigger_run_id = _trigger_task(db, now, due_in=3_600)
moved = _seed(db, now, state_version=PV + 1)
db.commit()
report = run_auto_resume_tick(now=now)
assert report.inflight is None
assert task_auto_resume._last_inflight == 0
row = _row(factory, task_id)
assert (row.state, row.state_detail) == ("manual", "auto_resume_disabled")
assert row.next_attempt_at is None
(event,) = _events(factory, task_id)
assert event.event == "skipped_disabled"
assert event.detail["at"] == "drain"
run = _trigger_run(factory, trigger_run_id)
assert run.status == TriggerRunStatus.FAILED.value
assert run.error_message == TASK_AUTO_RESUME_DISABLED_TRIGGER_ERROR
assert _row(factory, moved).state == "stale"
assert _commands(factory, task_id) == []
assert report.gave_up == 1
assert notified == []
# The stale row is not announced.
assert [n.auto_resume["stop_reason"] for n in report.notices] == ["disabled"]
@pytest.mark.parametrize("command_status", ["completed", "failed", None])
def test_switch_off_closes_a_dispatch_whose_command_ended(
factory, notified, monkeypatch, command_status
):
monkeypatch.setenv("XAGENT_TASK_AUTO_RESUME_ENABLED", "false")
now = utc_now()
with factory() as db:
task_id = _dispatched(db, now, command_status=command_status)
user = db.get(User, db.get(Task, task_id).user_id)
trigger_run_id, _later = _trigger(db, user, task_id)
db.commit()
report = run_auto_resume_tick(now=now)
row = _row(factory, task_id)
assert (row.state, row.state_detail) == ("manual", "auto_resume_disabled")
assert [e.event for e in _events(factory, task_id)] == ["skipped_disabled"]
run = _trigger_run(factory, trigger_run_id)
assert (run.status, run.error_message) == (
TriggerRunStatus.FAILED.value,
TASK_AUTO_RESUME_DISABLED_TRIGGER_ERROR,
)
assert report.gave_up == 1
assert [n.auto_resume["stop_reason"] for n in report.notices] == ["disabled"]
@pytest.mark.parametrize("command_status", ["pending", "processing"])
def test_housekeeping_rearms_a_retry_queued_after_a_rollback(
factory, notified, monkeypatch, command_status
):
"""A rolled-back resume moved the version by two while its retry waits
out its backoff: nothing resumed the task, so it is re-checked, not
confirmed."""
now = utc_now()
with factory() as db:
task_id = _dispatched(db, now, command_status=command_status)
db.get(Task, task_id).state_version = PV + 2
db.commit()
counters = _capture_counters(monkeypatch)
report = run_auto_resume_tick(now=now)
assert (report.confirmed, report.rearmed) == (0, 1)
assert "xagent.task.auto_resume.confirmed" not in counters
row = _row(factory, task_id)
assert row.state == "dispatched"
assert row.next_attempt_at is not None
assert _events(factory, task_id) == []
def test_switch_off_closes_a_resume_whose_last_attempt_rolled_back(
factory, notified, monkeypatch
):
"""The fence moved only because the failed resume rolled back: with the
switch off it is closed like any other ended dispatch, not left racing."""
monkeypatch.setenv("XAGENT_TASK_AUTO_RESUME_ENABLED", "false")
now = utc_now()
with factory() as db:
task_id = _dispatched(db, now, command_status="failed")
user = db.get(User, db.get(Task, task_id).user_id)
trigger_run_id, _later = _trigger(db, user, task_id)
db.get(Task, task_id).state_version = PV + 2
db.commit()
report = run_auto_resume_tick(now=now)
row = _row(factory, task_id)
assert (row.state, row.state_detail) == ("manual", "auto_resume_disabled")
assert _trigger_run(factory, trigger_run_id).status == (
TriggerRunStatus.FAILED.value
)
assert report.gave_up == 1
(notice,) = report.notices
assert (notice.state_version, notice.auto_resume["stop_reason"]) == (
PV + 2,
"disabled",
)
def test_switch_off_leaves_a_queued_dispatch_to_the_claim_guard(
factory, notified, monkeypatch
):
monkeypatch.setenv("XAGENT_TASK_AUTO_RESUME_ENABLED", "false")
now = utc_now()
with factory() as db:
queued = _dispatched(db, now, command_status="pending")
moved = _dispatched(db, now, command_status="completed", state_version=PV + 2)
db.commit()
report = run_auto_resume_tick(now=now)
assert (report.rearmed, report.confirmed) == (1, 1)
row = _row(factory, queued)
assert row.state == "dispatched"
assert _aware(row.next_attempt_at) == now + timedelta(
seconds=DISPATCH_CHECK_GRACE_SECONDS
)
assert (_row(factory, moved).state, _row(factory, moved).next_attempt_at) == (
"dispatched",
None,
)
assert _events(factory, queued) == []
# --------------------------------------------------------------------------
# Loop
# --------------------------------------------------------------------------
class _Stop(Exception):
pass
async def _run_loop(monkeypatch, outcomes: list[Any], *, clock: list[float]):
"""Drive the loop through ``outcomes`` (exceptions or reports)."""
sleeps: list[float] = []
states: list[bool] = []
async def run_inline(operation):
return operation()
def tick(*, now):
outcome = outcomes.pop(0)
if isinstance(outcome, BaseException):
raise outcome
return outcome
async def sleep(delay):
sleeps.append(delay)
states.append(
ops_signals.TASK_AUTO_RESUME_UNAVAILABLE
in ops_signals.active_degradations()
)
clock[0] += delay
if not outcomes:
raise asyncio.CancelledError
monkeypatch.setattr(task_auto_resume, "run_db_io_cancellation_safe", run_inline)
monkeypatch.setattr(task_auto_resume, "run_auto_resume_tick", tick)
monkeypatch.setattr(task_auto_resume.asyncio, "sleep", sleep)
monkeypatch.setattr(task_auto_resume, "_clock", lambda: clock[0])
with pytest.raises(asyncio.CancelledError):
await task_auto_resume.run_task_auto_resume_loop(poll_interval_seconds=5)
return sleeps, states
@pytest.mark.asyncio
async def test_loop_backs_off_raises_the_signal_and_recovers(monkeypatch, caplog):
ok = task_auto_resume.AutoResumeTickReport()
outcomes: list[Any] = [RuntimeError("db down")] * 5 + [ok, ok]
with caplog.at_level(logging.WARNING, logger=task_auto_resume.__name__):
sleeps, signal = await _run_loop(monkeypatch, outcomes, clock=[0.0])
assert sleeps == [10, 20, 40, 60, 60, 5, 5]
assert signal == [False, False, True, True, True, False, False]
# Logged on the first failure, then at most once a minute.
failures = [r for r in caplog.records if "tick failed" in r.getMessage()]
assert [r.getMessage() for r in failures] == [
"Auto-resume tick failed (failures=1)",
"Auto-resume tick failed (failures=4)",
"Auto-resume tick failed (failures=5)",
]
@pytest.mark.asyncio
async def test_loop_publishes_each_ticks_notices(monkeypatch):
published: list[Any] = []
async def publish(notices):
published.extend(notices)
monkeypatch.setattr(task_auto_resume, "publish_recovery_notices", publish)
notice = task_auto_resume.RecoveryNotice(
task_id=1, run_id="r", state_version=2, auto_resume={"status": "resuming"}
)
report = task_auto_resume.AutoResumeTickReport(notices=(notice,))
await _run_loop(monkeypatch, [report], clock=[0.0])
assert published == [notice]
@pytest.mark.asyncio
async def test_loop_lets_cancellation_through(monkeypatch):
async def run_inline(operation):
raise asyncio.CancelledError
monkeypatch.setattr(task_auto_resume, "run_db_io_cancellation_safe", run_inline)
with pytest.raises(asyncio.CancelledError):
await task_auto_resume.run_task_auto_resume_loop(poll_interval_seconds=5)
@pytest.mark.asyncio
async def test_publish_is_best_effort(monkeypatch):
sent: list[tuple[dict[str, Any], int]] = []
async def sink(message, task_id):
sent.append((message, task_id))
if len(sent) == 1:
raise RuntimeError("socket gone")
monkeypatch.setattr("xagent.web.services.task_events._task_event_sink", sink)
notices = [
task_auto_resume.RecoveryNotice(
task_id=task_id, run_id="r", state_version=2, auto_resume={}
)
for task_id in (1, 2)
]
await task_auto_resume.publish_recovery_notices(notices)
assert [task_id for _message, task_id in sent] == [1, 2]
assert sent[0][0]["type"] == "task_recovery_update"
# --------------------------------------------------------------------------
# Command order
# --------------------------------------------------------------------------
def test_user_commands_queue_behind_a_pending_auto_resume(factory, notified):
from xagent.web.services.task_command_transport import (
TaskCommandKind,
claim_task_command,
finish_task_command,
stage_task_command,
)
now = utc_now()
with factory() as db:
task_id = _seed(db, now)
db.commit()
assert run_auto_resume_tick(now=now).dispatched == 1
with factory() as db:
user_id = int(db.get(Task, task_id).user_id)
for command_id, kind in (
("message-1", TaskCommandKind.MESSAGE),
("pause-1", TaskCommandKind.PAUSE),
):
stage_task_command(
db,
task_id=task_id,
actor_user_id=user_id,
command_id=command_id,
kind=kind,
payload={"type": kind.value},
)
db.commit()
claimed: list[str] = []
with factory() as db:
first = claim_task_command(db, runner_id="worker")
assert first is not None
claimed.append(first.command_id)
# Per-task order: nothing else is claimable while it is unfinished.
assert claim_task_command(db, runner_id="worker") is None
assert finish_task_command(first.id, "worker", result={})
with factory() as db:
second = claim_task_command(db, runner_id="worker")
assert second is not None
claimed.append(second.command_id)
assert claimed == [auto_resume_command_id(PV, 1, RUN), "message-1"]
# --------------------------------------------------------------------------
# Tick robustness
# --------------------------------------------------------------------------
def test_nothing_due_costs_one_exists_and_no_counts(factory, notified, monkeypatch):
now = utc_now()
with factory() as db:
_seed(db, now, due_in=60)
_seed(db, now, state="dispatched", due_in=60, run_id="run-b")
_seed(db, now, state="manual", due_in=None, run_id="run-c")
db.commit()
def unexpected(*_args, **_kwargs):
raise AssertionError("the inert tick must stop at the EXISTS")
monkeypatch.setattr(task_auto_resume, "count_inflight", unexpected)
monkeypatch.setattr(task_auto_resume, "_candidates", unexpected)
report = run_auto_resume_tick(now=now)
assert (report.dispatched, report.inflight) == (0, None)
@pytest.mark.parametrize("enabled", ["true", "false"])
def test_due_dispatched_rows_alone_make_the_tick_run(
factory, notified, monkeypatch, enabled
):
monkeypatch.setenv("XAGENT_TASK_AUTO_RESUME_ENABLED", enabled)
now = utc_now()
with factory() as db:
task_id = _dispatched(db, now, command_status="pending")
db.commit()
assert run_auto_resume_tick(now=now).rearmed == 1
assert _aware(_row(factory, task_id).next_attempt_at) > now
def test_dispatch_cas_requires_the_row_to_still_be_scheduled(
factory, notified, monkeypatch
):
# Another sweeper closed the row (here: drained it with the switch off)
# without touching the task or the counters.
now = utc_now()
with factory() as db:
task_id = _seed(db, now)
db.commit()
def drained(db: Session) -> None:
db.get(TaskAutoRecovery, task_id).state = "manual"
result = _process_with_stale_view(factory, monkeypatch, task_id, now, drained)
assert result.outcome is task_auto_resume._Outcome.RACED
assert _commands(factory, task_id) == []
assert _row(factory, task_id).state == "manual"
def test_dispatch_cas_pins_the_attempt_it_read(factory, notified, monkeypatch):
now = utc_now()
with factory() as db:
task_id = _seed(db, now)
db.commit()
def dispatched_meanwhile(db: Session) -> None:
db.get(TaskAutoRecovery, task_id).total_resumes = 1
result = _process_with_stale_view(
factory, monkeypatch, task_id, now, dispatched_meanwhile
)
assert result.outcome is task_auto_resume._Outcome.RACED
assert _commands(factory, task_id) == []
assert _row(factory, task_id).total_resumes == 1
def test_naive_now_is_read_as_utc(factory, notified):
now = utc_now()
with factory() as db:
due = _seed(db, now, due_in=-1)
later = _seed(db, now, due_in=30, run_id="run-later")
db.commit()
report = run_auto_resume_tick(now=now.replace(tzinfo=None))
assert report.dispatched == 1
assert _row(factory, due).state == "dispatched"
assert _row(factory, later).state == "scheduled"
assert _aware(_row(factory, due).next_attempt_at) == now + timedelta(
seconds=DISPATCH_CHECK_GRACE_SECONDS
)
def test_pool_timeout_aborts_the_tick_and_is_not_a_candidate_failure(
factory, notified, monkeypatch
):
from sqlalchemy.exc import TimeoutError as PoolTimeout
now = utc_now()
with factory() as db:
first = _seed(db, now, due_in=-10)
second = _seed(db, now, due_in=-5)
db.commit()
failures: list[str] = []
real_counter = task_auto_resume.increment_counter
monkeypatch.setattr(
task_auto_resume,
"increment_counter",
lambda name, **kw: (failures.append(name), real_counter(name, **kw)),
)
def exhausted(db, task):
raise PoolTimeout("QueuePool limit reached")
monkeypatch.setattr(task_auto_resume, "auto_recovery_eligibility", exhausted)
with pytest.raises(PoolTimeout):
run_auto_resume_tick(now=now)
assert "xagent.task.auto_resume.candidate_failed" not in failures
assert _row(factory, first).state == "scheduled"
assert _row(factory, second).state == "scheduled"
def test_a_tick_that_fails_keeps_what_it_committed_for_announcing(
factory, notified, monkeypatch
):
from sqlalchemy.exc import TimeoutError as PoolTimeout
now = utc_now()
with factory() as db:
first = _seed(db, now, due_in=-10)
second = _seed(db, now, due_in=-5)
db.commit()
real = task_auto_resume.auto_recovery_eligibility
calls: list[int] = []
def second_times_out(db, task):
calls.append(int(task.id))
if int(task.id) == second:
raise PoolTimeout("QueuePool limit reached")
return real(db, task)
monkeypatch.setattr(task_auto_resume, "auto_recovery_eligibility", second_times_out)
with pytest.raises(PoolTimeout) as raised:
run_auto_resume_tick(now=now)
assert _row(factory, first).state == "dispatched"
partial = task_auto_resume.committed_tick_report(raised.value)
assert partial is not None
assert [(n.task_id, n.auto_resume["status"]) for n in partial.notices] == [
(first, "resuming")
]
@pytest.mark.asyncio
async def test_loop_announces_what_a_failed_tick_committed(monkeypatch):
published: list[Any] = []
async def publish(notices):
published.extend(notices)
monkeypatch.setattr(task_auto_resume, "publish_recovery_notices", publish)
notice = task_auto_resume.RecoveryNotice(
task_id=1, run_id="r", state_version=2, auto_resume={"status": "resuming"}
)
failure = RuntimeError("pool timeout")
failure.__dict__["auto_resume_partial_report"] = (
task_auto_resume.AutoResumeTickReport(notices=[notice])
)
await _run_loop(monkeypatch, [failure], clock=[0.0])
assert published == [notice]
def test_integrity_error_while_staging_is_a_race_and_writes_nothing(
factory, notified, monkeypatch
):
now = utc_now()
with factory() as db:
task_id = _seed(db, now)
db.commit()
def conflict(*_args, **_kwargs):
raise sa.exc.IntegrityError("INSERT", {}, Exception("duplicate"))
monkeypatch.setattr(task_auto_resume, "stage_task_command", conflict)
report = run_auto_resume_tick(now=now)
assert (report.dispatched, report.raced, report.failed) == (0, 1, 0)
row = _row(factory, task_id)
assert (row.state, row.total_resumes) == ("scheduled", 0)
assert _events(factory, task_id) == []
assert notified == []
def test_notify_fires_even_when_the_tick_fails_after_a_dispatch(
factory, notified, monkeypatch
):
from sqlalchemy.exc import TimeoutError as PoolTimeout
now = utc_now()
with factory() as db:
first = _seed(db, now, due_in=-10)
_seed(db, now, due_in=-5, run_id="run-second")
db.commit()
real = task_auto_resume._process_due_candidate_no_commit
calls = 0
def second_times_out(db, task_id, **kwargs):
nonlocal calls
calls += 1
if calls == 2:
raise PoolTimeout("QueuePool limit reached")
return real(db, task_id, **kwargs)
monkeypatch.setattr(
task_auto_resume, "_process_due_candidate_no_commit", second_times_out
)
with pytest.raises(PoolTimeout):
run_auto_resume_tick(now=now)
assert _row(factory, first).state == "dispatched"
assert notified == [1]
def test_scan_and_housekeeping_limits_bound_each_tick(factory, notified, monkeypatch):
monkeypatch.setenv("XAGENT_TASK_AUTO_RESUME_MAX_PER_TICK", "1")
now = utc_now()
with factory() as db:
# Housekeeping takes at most MAX_PER_TICK rows.
queued = [
_dispatched(db, now, command_status="pending", due_in=-10 + i)
for i in range(2)
]
# The scan takes at most 4 x MAX_PER_TICK rows, give-ups included.
expired = [
_seed(
db,
now,
due_in=-20 + i,
interrupted_ago=86_400 + 60,
run_id=f"run-{i}",
)
for i in range(5)
]
db.commit()
report = run_auto_resume_tick(now=now)
assert report.rearmed == 1
assert [_aware(_row(factory, t).next_attempt_at) > now for t in queued] == [
True,
False,
]
assert report.gave_up == 4
assert [_row(factory, t).state for t in expired] == ["expired"] * 4 + ["scheduled"]
@pytest.mark.parametrize("command_status", ["completed", "failed", None])
def test_unconfirmed_dispatches_whose_command_ended_are_not_in_flight(
factory, notified, monkeypatch, command_status
):
# Until housekeeping closes it, a row whose command has ended must not
# hold an in-flight slot.
monkeypatch.setenv("XAGENT_TASK_AUTO_RESUME_MAX_INFLIGHT", "1")
now = utc_now()
with factory() as db:
# Not due a check yet, so housekeeping leaves it alone this tick.
ended = _dispatched(
db, now, command_status=command_status, due_in=30, run_id="run-ended"
)
due = _seed(db, now)
db.commit()
report = run_auto_resume_tick(now=now)
assert (report.inflight, report.dispatched) == (0, 1)
assert _row(factory, due).state == "dispatched"
row = _row(factory, ended)
assert row.state == "dispatched"
assert _aware(row.next_attempt_at) == now + timedelta(seconds=30)
assert _events(factory, ended) == []
@pytest.mark.parametrize(
"postpone_error",
[
RuntimeError("postpone failed"),
OperationalError("UPDATE", {}, Exception("connection reset")),
SATimeoutError("QueuePool limit reached"),
],
ids=["other_error", "db_unavailable", "pool_timeout"],
)
def test_failure_while_postponing(
factory, notified, monkeypatch, caplog, postpone_error
):
"""A postpone that fails on a database outage or a pool timeout ends the
tick; any other failure is logged and the scan goes on."""
real = task_execution_admission.stage_task_admission
full_for: set[int] = set()
def admission(db, command, policy):
if command.task_id in full_for:
raise AdmissionQueueFull("Execution queue is full")
real(db, command, policy)
def broken(*_args, **_kwargs):
raise postpone_error
monkeypatch.setattr(
"xagent.web.services.task_execution_admission.stage_task_admission", admission
)
monkeypatch.setattr(task_auto_resume, "_postpone_candidate", broken)
now = utc_now()
with factory() as db:
refused = _seed(db, now, due_in=-10)
healthy = _seed(db, now, due_in=-5)
db.commit()
full_for.add(refused)
with caplog.at_level(logging.WARNING, logger=task_auto_resume.__name__):
if isinstance(postpone_error, RuntimeError):
report = run_auto_resume_tick(now=now)
assert "postpone" in caplog.text
assert report.dispatched == 1
assert _row(factory, healthy).state == "dispatched"
else:
with pytest.raises(type(postpone_error)):
run_auto_resume_tick(now=now)
# Nothing behind the refused row ran.
assert _row(factory, healthy).state == "scheduled"
# Nothing committed for the refused row: the next tick retries it.
row = _row(factory, refused)
assert (row.state, row.total_resumes) == ("scheduled", 0)
assert _aware(row.next_attempt_at) == now - timedelta(seconds=10)
class _HostHookRefusal(Exception):
"""A host admission hook's own error type: directly an ``Exception``."""
def _raising_stage(exc: BaseException, task_id_box: list[int]):
"""``stage_task_command`` that raises ``exc`` for the first seeded task."""
real = task_auto_resume.stage_task_command
def stage(db, *, task_id, **kwargs):
if task_id == task_id_box[0]:
raise exc
return real(db, task_id=task_id, **kwargs)
return stage
@pytest.mark.parametrize(
"exc",
[
RuntimeError("Execution admission requires shared task execution"),
TaskCommandOwnerStateError("pending writes failed to flush"),
ValueError("Drain the admission bucket before changing its policy"),
AttributeError("x"),
TypeError("y"),
_HostHookRefusal("team membership required"),
IntegrityError("INSERT", {}, Exception("duplicate")),
OperationalError("SELECT", {}, Exception("connection reset")),
SATimeoutError("QueuePool limit reached"),
],
ids=[
"hook_runtime_error",
"owner_state_error",
"value_error",
"attribute_error",
"type_error",
"host_hook_error",
"integrity_error",
"db_unavailable",
"pool_timeout",
],
)
def test_staging_refusal_postpones(factory, notified, monkeypatch, caplog, exc):
"""Every refused row is postponed. ``ValueError`` and ``RuntimeError``
count as refused (WARNING); any other class counts as a failed candidate
(ERROR)."""
counters: list[tuple[str, dict[str, Any] | None]] = []
real_counter = task_auto_resume.increment_counter
monkeypatch.setattr(
task_auto_resume,
"increment_counter",
lambda name, attributes=None, **kw: (
counters.append((name, attributes)),
real_counter(name, attributes=attributes, **kw),
),
)
now = utc_now()
with factory() as db:
refused = _seed(db, now, due_in=-10)
healthy = _seed(db, now, due_in=-5)
db.commit()
monkeypatch.setattr(
task_auto_resume, "stage_task_command", _raising_stage(exc, [refused])
)
with caplog.at_level(logging.WARNING, logger=task_auto_resume.__name__):
if isinstance(exc, SATimeoutError):
with pytest.raises(SATimeoutError):
run_auto_resume_tick(now=now)
# Unchanged: a pool timeout ends the tick, nothing behind it ran.
assert _row(factory, healthy).state == "scheduled"
assert _aware(_row(factory, refused).next_attempt_at) == now - timedelta(
seconds=10
)
return
report = run_auto_resume_tick(now=now)
row = _row(factory, refused)
assert (row.state, row.total_resumes, row.last_command_id) == ("scheduled", 0, None)
assert _events(factory, refused) == []
assert _commands(factory, refused) == []
# The scan continued past the refused row in every case.
assert _row(factory, healthy).state == "dispatched"
assert report.dispatched == 1
problems = [r for r in caplog.records if r.levelno >= logging.WARNING]
refused_count = counters.count(
("xagent.task.auto_resume.skipped", {"outcome": "staging_refused"})
)
failed_count = counters.count(("xagent.task.auto_resume.candidate_failed", None))
if isinstance(exc, IntegrityError):
assert report.raced == 1
assert _aware(row.next_attempt_at) == now - timedelta(seconds=10)
assert problems == []
elif isinstance(exc, OperationalError):
assert report.failed == 1
assert _aware(row.next_attempt_at) == now - timedelta(seconds=10)
elif isinstance(exc, (ValueError, RuntimeError)):
assert (report.failed, refused_count, failed_count) == (0, 1, 0)
assert _aware(row.next_attempt_at) == now + timedelta(seconds=60)
assert len(problems) == 1 and problems[0].levelno == logging.WARNING
assert problems[0].exc_info is not None and problems[0].exc_info[1] is exc
else:
assert (report.failed, refused_count, failed_count) == (1, 0, 1)
assert _aware(row.next_attempt_at) == now + timedelta(seconds=60)
assert len(problems) == 1 and problems[0].levelno == logging.ERROR
assert problems[0].exc_info is not None and problems[0].exc_info[1] is exc
def test_dispatch_delay_is_observed(factory, notified, monkeypatch):
observed: list[tuple[str, float, str]] = []
monkeypatch.setattr(
task_auto_resume,
"observe_value",
lambda name, value, *, unit, attributes=None: observed.append(
(name, value, unit)
),
)
now = utc_now()
with factory() as db:
_seed(db, now, due_in=-12)
db.commit()
run_auto_resume_tick(now=now)
assert observed == [("xagent.task.auto_resume.dispatch_delay_seconds", 12.0, "s")]