Enterprise agent platform for building, deploying, and operating AI teammates.
"""Durable-file ownership regressions for task execution finalization."""
from __future__ import annotations
import asyncio
import threading
from pathlib import Path
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from tests.shared.async_waits import DB_PROGRESS_TIMEOUT
from xagent.core.file_storage.factory import get_unscoped_file_storage
from xagent.core.file_storage.storage import FsspecFileStorage
from xagent.web.api import websocket as websocket_api
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.uploaded_file import UploadedFile
from xagent.web.models.user import User
from xagent.web.services import task_execution as task_execution_service
from xagent.web.services.task_lease_service import (
TaskLease,
TaskLeaseHeartbeatOutcome,
)
from xagent.web.services.task_settlement_report import SettlementReport
from xagent.web.services.uploaded_file_store import (
UploadedFileStore,
UploadedFileVersionConflict,
)
from .conftest import (
_direct_db_session,
_install_one_slot_queue_pool,
)
pytestmark = pytest.mark.usefixtures("_test_db")
def _seed_running_task(*, runner_id: str, run_id: str) -> tuple[int, int]:
db = _direct_db_session()
try:
user = User(
username=f"output-owner-{run_id}",
password_hash="hash",
is_admin=False,
)
db.add(user)
db.flush()
task = Task(
user_id=int(user.id),
title="Durable output",
status=TaskStatus.RUNNING,
lease_attempt_id="test-attempt",
runner_id=runner_id,
run_id=run_id,
)
db.add(task)
db.commit()
return int(task.id), int(user.id)
finally:
db.close()
@pytest.mark.parametrize("state", ["detached", "compensating"])
def test_output_prepare_excludes_unavailable_metadata(state) -> None:
task_id, user_id = _seed_running_task(
runner_id="compensating-runner",
run_id="compensating-run",
)
db = _direct_db_session()
try:
db.add(
UploadedFile(
file_id="compensating-output",
user_id=user_id,
task_id=None if state == "detached" else task_id,
filename="compensating.txt",
storage_path="/tmp/compensating.txt",
storage_key=(
f"users/{user_id}/tasks/{task_id}/outputs/"
"compensating-output/compensating.txt"
),
storage_status="compensating"
if state == "compensating"
else "available",
detached_reason="task_deleted" if state == "detached" else None,
mime_type="text/plain",
file_size=12,
)
)
db.commit()
finally:
db.close()
prepared = task_execution_service._prepare_task_file_outputs_isolated(
task_id=task_id,
task_user_id=user_id,
file_outputs=[{"file_id": "compensating-output"}],
resolved_scope_segments=(),
)
assert prepared.normalized_outputs == ()
assert prepared.mutations == ()
@pytest.mark.asyncio
async def test_output_path_resolution_releases_pool_between_multiple_outputs(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
task_id, user_id = _seed_running_task(
runner_id="path-runner",
run_id="path-run",
)
uploads_dir = tmp_path / "uploads"
output_dir = uploads_dir / f"user_{user_id}" / f"web_task_{task_id}" / "output"
output_dir.mkdir(parents=True)
output_paths = [output_dir / "first.txt", output_dir / "second.txt"]
for path in output_paths:
path.write_text(path.name, encoding="utf-8")
monkeypatch.setenv("XAGENT_UPLOADS_DIR", str(uploads_dir))
monkeypatch.setenv("XAGENT_FILE_STORAGE_URI", (tmp_path / "objects").as_uri())
get_unscoped_file_storage.cache_clear()
engine = _install_one_slot_queue_pool(monkeypatch, pool_timeout=0.5)
second_resolution_started = threading.Event()
allow_second_resolution = threading.Event()
original_resolve = task_execution_service._resolve_output_storage_path
resolution_count = 0
def gated_resolve(raw_path: str):
nonlocal resolution_count
resolution_count += 1
resolved = original_resolve(raw_path)
if resolution_count == 2:
second_resolution_started.set()
assert allow_second_resolution.wait(timeout=3)
return resolved
monkeypatch.setattr(
task_execution_service,
"_resolve_output_storage_path",
gated_resolve,
)
prepared: task_execution_service._PreparedTaskFileOutputs | None = None
try:
worker = asyncio.create_task(
asyncio.to_thread(
task_execution_service._prepare_task_file_outputs_isolated,
task_id=task_id,
task_user_id=user_id,
file_outputs=[
{"path": str(path), "filename": path.name} for path in output_paths
],
resolved_scope_segments=(),
)
)
assert await asyncio.to_thread(
second_resolution_started.wait, DB_PROGRESS_TIMEOUT
)
assert engine.pool.checkedout() == 0
allow_second_resolution.set()
prepared = await worker
finally:
allow_second_resolution.set()
if prepared is not None:
task_execution_service._settle_prepared_task_file_outputs(
prepared,
metadata_committed=False,
)
engine.dispose()
get_unscoped_file_storage.cache_clear()
@pytest.mark.asyncio
async def test_output_path_resolution_releases_pool_after_owner_lookup(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
task_id, user_id = _seed_running_task(
runner_id="owner-runner",
run_id="owner-run",
)
uploads_dir = tmp_path / "uploads"
output_path = (
uploads_dir / f"user_{user_id}" / f"web_task_{task_id}" / "output" / "owner.txt"
)
output_path.parent.mkdir(parents=True)
output_path.write_text("owner", encoding="utf-8")
monkeypatch.setenv("XAGENT_UPLOADS_DIR", str(uploads_dir))
monkeypatch.setenv("XAGENT_FILE_STORAGE_URI", (tmp_path / "objects").as_uri())
get_unscoped_file_storage.cache_clear()
engine = _install_one_slot_queue_pool(monkeypatch, pool_timeout=0.5)
resolution_started = threading.Event()
allow_resolution = threading.Event()
original_resolve = task_execution_service._resolve_output_storage_path
def gated_resolve(raw_path: str):
resolved = original_resolve(raw_path)
resolution_started.set()
assert allow_resolution.wait(timeout=3)
return resolved
monkeypatch.setattr(
task_execution_service,
"_resolve_output_storage_path",
gated_resolve,
)
prepared: task_execution_service._PreparedTaskFileOutputs | None = None
try:
worker = asyncio.create_task(
asyncio.to_thread(
task_execution_service._prepare_task_file_outputs_isolated,
task_id=task_id,
task_user_id=None,
file_outputs=[{"path": str(output_path), "filename": output_path.name}],
resolved_scope_segments=(),
)
)
assert await asyncio.to_thread(resolution_started.wait, DB_PROGRESS_TIMEOUT)
assert engine.pool.checkedout() == 0
allow_resolution.set()
prepared = await worker
finally:
allow_resolution.set()
if prepared is not None:
task_execution_service._settle_prepared_task_file_outputs(
prepared,
metadata_committed=False,
)
engine.dispose()
get_unscoped_file_storage.cache_clear()
@pytest.mark.asyncio
async def test_output_staging_holds_no_pool_slot_or_task_lock(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
task_id, user_id = _seed_running_task(runner_id="runner-a", run_id="run-a")
uploads_dir = tmp_path / "uploads"
output_path = (
uploads_dir / f"user_{user_id}" / f"web_task_{task_id}" / "output" / "a.txt"
)
output_path.parent.mkdir(parents=True)
output_path.write_text("durable output", encoding="utf-8")
monkeypatch.setenv("XAGENT_UPLOADS_DIR", str(uploads_dir))
monkeypatch.setenv("XAGENT_FILE_STORAGE_URI", (tmp_path / "objects").as_uri())
get_unscoped_file_storage.cache_clear()
engine = _install_one_slot_queue_pool(monkeypatch, pool_timeout=0.5)
put_started = threading.Event()
allow_put = threading.Event()
original_put_file = FsspecFileStorage.put_file
def blocking_put_file(
self: FsspecFileStorage,
source: Path,
key: str,
content_type: str | None = None,
):
assert engine.pool.checkedout() == 0
put_started.set()
assert allow_put.wait(timeout=DB_PROGRESS_TIMEOUT)
return original_put_file(self, source, key, content_type)
monkeypatch.setattr(FsspecFileStorage, "put_file", blocking_put_file)
try:
staging = asyncio.create_task(
asyncio.to_thread(
task_execution_service._prepare_task_file_outputs_isolated,
task_id=task_id,
task_user_id=user_id,
file_outputs=[{"path": str(output_path), "filename": "a.txt"}],
resolved_scope_segments=(),
)
)
assert await asyncio.to_thread(put_started.wait, DB_PROGRESS_TIMEOUT)
assert engine.pool.checkedout() == 0
def read_and_lock_task() -> tuple[str, str]:
db = _direct_db_session()
try:
row = db.query(Task).filter(Task.id == task_id).with_for_update().one()
return str(row.runner_id), str(row.run_id)
finally:
db.close()
assert await asyncio.to_thread(read_and_lock_task) == ("runner-a", "run-a")
allow_put.set()
prepared = await staging
assert "/_versions/" in prepared.staged_files[0].storage_key
finalized = task_execution_service._finalize_task_execution_result_isolated(
task_id=task_id,
task_user_id=user_id,
pre_run_status=TaskStatus.RUNNING,
result={"success": True, "output": "done", "file_outputs": []},
expected_run_id="run-a",
task_lease=TaskLease(
attempt_id="test-attempt",
task_id=task_id,
runner_id="runner-a",
run_id="run-a",
),
resolved_scope_segments=(),
prepared_outputs=prepared,
)
assert not finalized.late_result
check_db = _direct_db_session()
try:
assert check_db.get(Task, task_id).output == "done"
assert (
check_db.query(UploadedFile)
.filter(UploadedFile.task_id == task_id)
.count()
== 1
)
finally:
check_db.close()
finally:
allow_put.set()
engine.dispose()
get_unscoped_file_storage.cache_clear()
@pytest.mark.asyncio
async def test_takeover_during_output_upload_cannot_commit_old_run_metadata(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
task_id, user_id = _seed_running_task(runner_id="runner-old", run_id="run-old")
uploads_dir = tmp_path / "uploads"
output_path = (
uploads_dir / f"user_{user_id}" / f"web_task_{task_id}" / "output" / "stale.txt"
)
output_path.parent.mkdir(parents=True)
output_path.write_text("stale output", encoding="utf-8")
object_root = tmp_path / "objects"
monkeypatch.setenv("XAGENT_UPLOADS_DIR", str(uploads_dir))
monkeypatch.setenv("XAGENT_FILE_STORAGE_URI", object_root.as_uri())
get_unscoped_file_storage.cache_clear()
engine = _install_one_slot_queue_pool(monkeypatch, pool_timeout=0.5)
put_started = threading.Event()
allow_put = threading.Event()
original_put_file = FsspecFileStorage.put_file
def blocking_put_file(
self: FsspecFileStorage,
source: Path,
key: str,
content_type: str | None = None,
):
assert engine.pool.checkedout() == 0
put_started.set()
assert allow_put.wait(timeout=DB_PROGRESS_TIMEOUT)
return original_put_file(self, source, key, content_type)
monkeypatch.setattr(FsspecFileStorage, "put_file", blocking_put_file)
try:
staging = asyncio.create_task(
asyncio.to_thread(
task_execution_service._prepare_task_file_outputs_isolated,
task_id=task_id,
task_user_id=user_id,
file_outputs=[{"path": str(output_path), "filename": "stale.txt"}],
resolved_scope_segments=(),
)
)
assert await asyncio.to_thread(put_started.wait, DB_PROGRESS_TIMEOUT)
takeover_db = _direct_db_session()
try:
task = takeover_db.query(Task).filter(Task.id == task_id).one()
task.runner_id = "runner-new"
task.run_id = "run-new"
takeover_db.commit()
finally:
takeover_db.close()
allow_put.set()
prepared = await staging
finalized = task_execution_service._finalize_task_execution_result_isolated(
task_id=task_id,
task_user_id=user_id,
pre_run_status=TaskStatus.RUNNING,
result={"success": True, "output": "stale", "file_outputs": []},
expected_run_id="run-old",
task_lease=TaskLease(
attempt_id="test-attempt",
task_id=task_id,
runner_id="runner-old",
run_id="run-old",
),
resolved_scope_segments=(),
prepared_outputs=prepared,
)
assert finalized.late_result
check_db = _direct_db_session()
try:
assert check_db.query(UploadedFile).count() == 0
current = check_db.query(Task).filter(Task.id == task_id).one()
assert (current.runner_id, current.run_id) == ("runner-new", "run-new")
finally:
check_db.close()
assert not any(path.is_file() for path in object_root.rglob("*"))
finally:
allow_put.set()
engine.dispose()
get_unscoped_file_storage.cache_clear()
@pytest.mark.parametrize(
("terminal_state", "result_status", "expect_committed"),
[
("a2a_canceled", "completed", False),
("unknown_failed", "completed", False),
("running", "interrupted", True),
],
)
def test_late_result_files_commit_only_with_a_pause_transition(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
terminal_state: str,
result_status: str,
expect_committed: bool,
) -> None:
"""A result the finalizer ignores must not persist its file outputs."""
run_id = f"late-{terminal_state}"
task_id, user_id = _seed_running_task(runner_id="late-runner", run_id=run_id)
uploads_dir = tmp_path / "uploads"
output_path = (
uploads_dir / f"user_{user_id}" / f"web_task_{task_id}" / "output" / "late.txt"
)
output_path.parent.mkdir(parents=True)
output_path.write_text("late output", encoding="utf-8")
object_root = tmp_path / "objects"
monkeypatch.setenv("XAGENT_UPLOADS_DIR", str(uploads_dir))
monkeypatch.setenv("XAGENT_FILE_STORAGE_URI", object_root.as_uri())
get_unscoped_file_storage.cache_clear()
try:
prepared = task_execution_service._prepare_task_file_outputs_isolated(
task_id=task_id,
task_user_id=user_id,
file_outputs=[{"path": str(output_path), "filename": "late.txt"}],
resolved_scope_segments=(),
)
assert prepared.staged_files
assert any(path.is_file() for path in object_root.rglob("*"))
if terminal_state != "running":
db = _direct_db_session()
try:
task = db.query(Task).filter(Task.id == task_id).one()
task.status = TaskStatus.FAILED
task.error_message = "Task canceled."
if terminal_state == "a2a_canceled":
task.agent_config = {"a2a_state": "TASK_STATE_CANCELED"}
db.commit()
finally:
db.close()
result: dict[str, Any] = {
"success": result_status == "completed",
"status": result_status,
"output": "late result",
"file_outputs": [],
}
if terminal_state == "unknown_failed":
result["injection_outcome_unknown"] = True
finalized = task_execution_service._finalize_task_execution_result_isolated(
task_id=task_id,
task_user_id=user_id,
pre_run_status=TaskStatus.RUNNING,
result=result,
expected_run_id=run_id,
task_lease=TaskLease(
attempt_id="test-attempt",
task_id=task_id,
runner_id="late-runner",
run_id=run_id,
),
resolved_scope_segments=(),
prepared_outputs=prepared,
)
assert finalized.waiting_for_control
check_db = _direct_db_session()
try:
file_count = (
check_db.query(UploadedFile)
.filter(UploadedFile.task_id == task_id)
.count()
)
task = check_db.get(Task, task_id)
if expect_committed:
assert file_count == 1
assert task.status == TaskStatus.PAUSED
else:
assert file_count == 0
assert task.status == TaskStatus.FAILED
assert task.error_message == "Task canceled."
assert task.output is None
finally:
check_db.close()
assert (
any(path.is_file() for path in object_root.rglob("*")) is expect_committed
)
finally:
get_unscoped_file_storage.cache_clear()
@pytest.mark.parametrize(
"variant", ["unknown_success", "unknown_waiting", "interrupted"]
)
def test_first_run_keeps_a_pending_resume_for_input_accepted_earlier(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
variant: str,
) -> None:
"""A resume requested for earlier input survives later unknown input.
Message A was accepted and its handoff is waiting on this run; message B
then became unknown. The finalizer keeps ``resume_requested`` and A's
handoff still acquires the lease, since acquisition never reads the
control state.
"""
run_id = f"first-run-{variant}"
task_id, user_id = _seed_running_task(runner_id="first-runner", run_id=run_id)
db = _direct_db_session()
try:
task = db.query(Task).filter(Task.id == task_id).one()
task.control_state = "resume_requested"
db.commit()
finally:
db.close()
uploads_dir = tmp_path / "uploads"
output_path = (
uploads_dir / f"user_{user_id}" / f"web_task_{task_id}" / "output" / "kept.txt"
)
output_path.parent.mkdir(parents=True)
output_path.write_text("kept output", encoding="utf-8")
monkeypatch.setenv("XAGENT_UPLOADS_DIR", str(uploads_dir))
monkeypatch.setenv("XAGENT_FILE_STORAGE_URI", (tmp_path / "objects").as_uri())
get_unscoped_file_storage.cache_clear()
try:
prepared = task_execution_service._prepare_task_file_outputs_isolated(
task_id=task_id,
task_user_id=user_id,
file_outputs=[{"path": str(output_path), "filename": "kept.txt"}],
resolved_scope_segments=(),
)
result: dict[str, Any] = {
"success": variant == "unknown_success",
"status": {
"unknown_success": "completed",
"unknown_waiting": "waiting_for_user",
"interrupted": "interrupted",
}[variant],
"output": "answer produced before the unknown input",
"file_outputs": [],
}
if variant != "interrupted":
result["injection_outcome_unknown"] = True
task_execution_service._finalize_task_execution_result_isolated(
task_id=task_id,
task_user_id=user_id,
pre_run_status=TaskStatus.RUNNING,
result=result,
expected_run_id=run_id,
task_lease=TaskLease(
attempt_id="test-attempt",
task_id=task_id,
runner_id="first-runner",
run_id=run_id,
),
resolved_scope_segments=(),
prepared_outputs=prepared,
)
check_db = _direct_db_session()
try:
task = check_db.get(Task, task_id)
assert task.status == TaskStatus.PAUSED
assert task.control_state == "resume_requested"
# A pause transition owns its files, including an unknown success
# whose answer is preserved in history.
assert (
check_db.query(UploadedFile)
.filter(UploadedFile.task_id == task_id)
.count()
== 1
)
if variant == "unknown_success":
assert task.output == "answer produced before the unknown input"
finally:
check_db.close()
handoff = task_execution_service._acquire_resume_task_lease(
task_id, user_id, run_id
)
assert handoff is not None
check_db = _direct_db_session()
try:
assert check_db.get(Task, task_id).status == TaskStatus.RUNNING
finally:
check_db.close()
finally:
get_unscoped_file_storage.cache_clear()
def test_metadata_version_change_rolls_back_entire_task_finalization(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
task_id, user_id = _seed_running_task(
runner_id="cas-runner",
run_id="cas-run",
)
uploads_dir = tmp_path / "uploads"
output_path = (
uploads_dir / f"user_{user_id}" / f"web_task_{task_id}" / "output" / "cas.txt"
)
output_path.parent.mkdir(parents=True)
output_path.write_text("old bytes", encoding="utf-8")
monkeypatch.setenv("XAGENT_UPLOADS_DIR", str(uploads_dir))
monkeypatch.setenv("XAGENT_FILE_STORAGE_URI", (tmp_path / "objects").as_uri())
get_unscoped_file_storage.cache_clear()
storage = get_unscoped_file_storage()
try:
db = _direct_db_session()
try:
existing = UploadedFileStore(db).create_from_local_path(
local_path=output_path,
user_id=user_id,
task_id=task_id,
file_id="cas-output",
filename="cas.txt",
storage_key=(
f"users/{user_id}/tasks/{task_id}/outputs/cas-output/"
"_versions/11111111111111111111111111111111/output/cas.txt"
),
workspace_relative_path="output/cas.txt",
workspace_category="output",
)
original_key = str(existing.storage_key)
db.commit()
finally:
db.close()
output_path.write_text("stale finalizer bytes", encoding="utf-8")
prepared = task_execution_service._prepare_task_file_outputs_isolated(
task_id=task_id,
task_user_id=user_id,
file_outputs=[{"path": str(output_path), "filename": "cas.txt"}],
resolved_scope_segments=(),
)
staged_key = prepared.staged_files[0].storage_key
assert storage.exists(staged_key)
external_key = (
f"users/{user_id}/tasks/{task_id}/outputs/cas-output/"
"_versions/22222222222222222222222222222222/output/cas.txt"
)
concurrent_db = _direct_db_session()
try:
concurrent_record = (
concurrent_db.query(UploadedFile)
.filter(UploadedFile.file_id == "cas-output")
.one()
)
concurrent_record.storage_key = external_key
concurrent_record.checksum = "external-checksum"
concurrent_record.etag = "external-etag"
concurrent_db.commit()
finally:
concurrent_db.close()
with pytest.raises(UploadedFileVersionConflict):
task_execution_service._finalize_task_execution_result_isolated(
task_id=task_id,
task_user_id=user_id,
pre_run_status=TaskStatus.RUNNING,
result={
"success": True,
"output": "must not commit",
"file_outputs": [],
},
expected_run_id="cas-run",
task_lease=TaskLease(
attempt_id="test-attempt",
task_id=task_id,
runner_id="cas-runner",
run_id="cas-run",
),
resolved_scope_segments=(),
prepared_outputs=prepared,
)
check_db = _direct_db_session()
try:
current = (
check_db.query(UploadedFile)
.filter(UploadedFile.file_id == "cas-output")
.one()
)
task = check_db.query(Task).filter(Task.id == task_id).one()
assert current.storage_key == external_key
assert current.checksum == "external-checksum"
assert task.status == TaskStatus.RUNNING
assert task.output is None
finally:
check_db.close()
assert not storage.exists(staged_key)
assert storage.exists(original_key)
finally:
get_unscoped_file_storage.cache_clear()
def test_resumed_finalizer_uses_same_prepared_output_transaction(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
task_id, user_id = _seed_running_task(
runner_id="resume-runner",
run_id="resume-run",
)
uploads_dir = tmp_path / "uploads"
output_path = (
uploads_dir
/ f"user_{user_id}"
/ f"web_task_{task_id}"
/ "output"
/ "resume.txt"
)
output_path.parent.mkdir(parents=True)
output_path.write_text("resume output", encoding="utf-8")
monkeypatch.setenv("XAGENT_UPLOADS_DIR", str(uploads_dir))
monkeypatch.setenv("XAGENT_FILE_STORAGE_URI", (tmp_path / "objects").as_uri())
get_unscoped_file_storage.cache_clear()
try:
prepared = task_execution_service._prepare_task_file_outputs_isolated(
task_id=task_id,
task_user_id=user_id,
file_outputs=[{"path": str(output_path), "filename": "resume.txt"}],
resolved_scope_segments=(),
)
finalized = task_execution_service._finalize_resumed_task(
task_id,
status="completed",
success=True,
output="resume done",
task_owner_user_id=user_id,
result={"success": True, "output": "resume done", "file_outputs": []},
task_lease=TaskLease(
attempt_id="test-attempt",
task_id=task_id,
runner_id="resume-runner",
run_id="resume-run",
),
prepared_outputs=prepared,
)
assert not finalized["late_result"]
assert finalized["final_status"] == TaskStatus.COMPLETED.value
db = _direct_db_session()
try:
output = (
db.query(UploadedFile).filter(UploadedFile.task_id == task_id).one()
)
assert output.filename == "resume.txt"
task = db.query(Task).filter(Task.id == task_id).one()
assert task.status == TaskStatus.COMPLETED
finally:
db.close()
finally:
get_unscoped_file_storage.cache_clear()
def test_lost_resume_owner_compensates_new_version_without_deleting_committed_object(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
task_id, user_id = _seed_running_task(
runner_id="version-runner-old",
run_id="version-run-old",
)
uploads_dir = tmp_path / "uploads"
output_path = (
uploads_dir
/ f"user_{user_id}"
/ f"web_task_{task_id}"
/ "output"
/ "versioned.txt"
)
output_path.parent.mkdir(parents=True)
output_path.write_text("committed bytes", encoding="utf-8")
monkeypatch.setenv("XAGENT_UPLOADS_DIR", str(uploads_dir))
monkeypatch.setenv("XAGENT_FILE_STORAGE_URI", (tmp_path / "objects").as_uri())
get_unscoped_file_storage.cache_clear()
try:
db = _direct_db_session()
try:
existing = UploadedFileStore(db).create_from_local_path(
local_path=output_path,
user_id=user_id,
task_id=task_id,
file_id="committed-output",
filename="versioned.txt",
storage_key=(
f"users/{user_id}/tasks/{task_id}/outputs/"
"committed-output/output/versioned.txt"
),
workspace_relative_path="output/versioned.txt",
workspace_category="output",
)
committed_key = str(existing.storage_key)
db.commit()
finally:
db.close()
output_path.write_text("stale replacement", encoding="utf-8")
prepared = task_execution_service._prepare_task_file_outputs_isolated(
task_id=task_id,
task_user_id=user_id,
file_outputs=[{"path": str(output_path), "filename": "versioned.txt"}],
resolved_scope_segments=(),
)
assert prepared.staged_files[0].storage_key != committed_key
takeover_db = _direct_db_session()
try:
task = takeover_db.query(Task).filter(Task.id == task_id).one()
task.runner_id = "version-runner-new"
task.run_id = "version-run-new"
takeover_db.commit()
finally:
takeover_db.close()
finalized = task_execution_service._finalize_resumed_task(
task_id,
status="completed",
success=True,
output="stale",
task_owner_user_id=user_id,
result={"success": True, "output": "stale", "file_outputs": []},
task_lease=TaskLease(
attempt_id="test-attempt",
task_id=task_id,
runner_id="version-runner-old",
run_id="version-run-old",
),
prepared_outputs=prepared,
)
assert finalized["late_result"]
check_db = _direct_db_session()
try:
record = (
check_db.query(UploadedFile)
.filter(UploadedFile.file_id == "committed-output")
.one()
)
assert record.storage_key == committed_key
finally:
check_db.close()
with get_unscoped_file_storage().open_read(committed_key) as handle:
assert handle.read() == b"committed bytes"
assert not get_unscoped_file_storage().exists(
prepared.staged_files[0].storage_key
)
finally:
get_unscoped_file_storage.cache_clear()
@pytest.mark.parametrize("release_succeeds", [True, False])
def test_resumed_unknown_failed_result_commits_only_a_confirmed_release(
monkeypatch: pytest.MonkeyPatch,
release_succeeds: bool,
) -> None:
task_id, user_id = _seed_running_task(
runner_id="failed-runner",
run_id="failed-run",
)
db = _direct_db_session()
try:
task = db.query(Task).filter(Task.id == task_id).one()
task.status = TaskStatus.FAILED
db.commit()
finally:
db.close()
real_release = task_execution_service.release_task_lease_no_commit
def release(db: Any, lease: TaskLease, **kwargs: Any) -> bool:
db.query(Task).filter(Task.id == lease.task_id).update(
{Task.error_message: "written before release"},
synchronize_session=False,
)
if release_succeeds:
return real_release(db, lease, **kwargs)
return False
monkeypatch.setattr(task_execution_service, "release_task_lease_no_commit", release)
prepared = task_execution_service._prepare_task_file_outputs_isolated(
task_id=task_id,
task_user_id=user_id,
file_outputs=[],
resolved_scope_segments=(),
)
finalized = task_execution_service._finalize_resumed_task(
task_id,
status="interrupted",
success=False,
output=None,
task_owner_user_id=user_id,
result={"success": False, "injection_outcome_unknown": True},
task_lease=TaskLease(
attempt_id="test-attempt",
task_id=task_id,
runner_id="failed-runner",
run_id="failed-run",
),
prepared_outputs=prepared,
)
assert finalized["late_result"]
check_db = _direct_db_session()
try:
task = check_db.query(Task).filter(Task.id == task_id).one()
assert task.status == TaskStatus.FAILED
if release_succeeds:
assert task.runner_id is None
assert task.error_message == "written before release"
else:
# A failed release must not commit anything flushed before it.
assert task.runner_id == "failed-runner"
assert task.error_message != "written before release"
finally:
check_db.close()
@pytest.mark.parametrize(
("status", "success"),
[
("waiting_for_user", False),
("interrupted", False),
("completed", True),
],
)
def test_resumed_unknown_input_pauses_whatever_the_run_reported(
status: str, success: bool
) -> None:
run_id = f"resumed-unknown-{status}"
task_id, user_id = _seed_running_task(runner_id="resumed-runner", run_id=run_id)
prepared = task_execution_service._prepare_task_file_outputs_isolated(
task_id=task_id,
task_user_id=user_id,
file_outputs=[],
resolved_scope_segments=(),
)
finalized = task_execution_service._finalize_resumed_task(
task_id,
status=status,
success=success,
output="answer before the unknown input" if success else None,
task_owner_user_id=user_id,
result={"success": success, "injection_outcome_unknown": True},
task_lease=TaskLease(
attempt_id="test-attempt",
task_id=task_id,
runner_id="resumed-runner",
run_id=run_id,
),
prepared_outputs=prepared,
)
assert not finalized.get("late_result")
check_db = _direct_db_session()
try:
task = check_db.query(Task).filter(Task.id == task_id).one()
assert task.status == TaskStatus.PAUSED
assert task.control_state == "paused"
finally:
check_db.close()
def test_superseded_output_is_deleted_only_after_exact_metadata_commit(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
task_id, user_id = _seed_running_task(
runner_id="commit-runner",
run_id="commit-run",
)
uploads_dir = tmp_path / "uploads"
output_path = (
uploads_dir
/ f"user_{user_id}"
/ f"web_task_{task_id}"
/ "output"
/ "committed.txt"
)
output_path.parent.mkdir(parents=True)
output_path.write_text("old bytes", encoding="utf-8")
monkeypatch.setenv("XAGENT_UPLOADS_DIR", str(uploads_dir))
monkeypatch.setenv("XAGENT_FILE_STORAGE_URI", (tmp_path / "objects").as_uri())
get_unscoped_file_storage.cache_clear()
try:
db = _direct_db_session()
try:
existing = UploadedFileStore(db).create_from_local_path(
local_path=output_path,
user_id=user_id,
task_id=task_id,
file_id="replace-after-commit",
filename="committed.txt",
storage_key=(
f"users/{user_id}/tasks/{task_id}/outputs/"
"replace-after-commit/_versions/"
"0123456789abcdef0123456789abcdef/output/committed.txt"
),
workspace_relative_path="output/committed.txt",
workspace_category="output",
)
old_key = str(existing.storage_key)
db.commit()
finally:
db.close()
output_path.write_text("new bytes", encoding="utf-8")
prepared = task_execution_service._prepare_task_file_outputs_isolated(
task_id=task_id,
task_user_id=user_id,
file_outputs=[{"path": str(output_path), "filename": "committed.txt"}],
resolved_scope_segments=(),
)
new_key = prepared.staged_files[0].storage_key
storage = get_unscoped_file_storage()
assert storage.exists(old_key)
assert storage.exists(new_key)
original_cleanup = (
task_execution_service.cleanup_superseded_uploaded_file_objects
)
cleanup_observations: list[tuple[str, TaskStatus]] = []
def observe_committed_metadata(claims): # type: ignore[no-untyped-def]
check_db = _direct_db_session()
try:
record = (
check_db.query(UploadedFile)
.filter(UploadedFile.file_id == "replace-after-commit")
.one()
)
task = check_db.query(Task).filter(Task.id == task_id).one()
cleanup_observations.append(
(str(record.storage_key), TaskStatus(task.status))
)
finally:
check_db.close()
return original_cleanup(claims)
monkeypatch.setattr(
task_execution_service,
"cleanup_superseded_uploaded_file_objects",
observe_committed_metadata,
)
finalized = task_execution_service._finalize_task_execution_result_isolated(
task_id=task_id,
task_user_id=user_id,
pre_run_status=TaskStatus.RUNNING,
result={"success": True, "output": "done", "file_outputs": []},
expected_run_id="commit-run",
task_lease=TaskLease(
attempt_id="test-attempt",
task_id=task_id,
runner_id="commit-runner",
run_id="commit-run",
),
resolved_scope_segments=(),
prepared_outputs=prepared,
)
assert not finalized.late_result
assert cleanup_observations == [(new_key, TaskStatus.COMPLETED)]
assert not storage.exists(old_key)
assert storage.exists(new_key)
finally:
get_unscoped_file_storage.cache_clear()
def _assert_cleanup_failure_does_not_reclassify_committed_finalization(
*,
resume: bool,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
run_kind = "resume" if resume else "background"
runner_id = f"{run_kind}-cleanup-runner"
run_id = f"{run_kind}-cleanup-run"
task_id, user_id = _seed_running_task(runner_id=runner_id, run_id=run_id)
uploads_dir = tmp_path / "uploads"
output_path = (
uploads_dir
/ f"user_{user_id}"
/ f"web_task_{task_id}"
/ "output"
/ f"{run_kind}.txt"
)
output_path.parent.mkdir(parents=True)
output_path.write_text("old bytes", encoding="utf-8")
monkeypatch.setenv("XAGENT_UPLOADS_DIR", str(uploads_dir))
monkeypatch.setenv("XAGENT_FILE_STORAGE_URI", (tmp_path / "objects").as_uri())
get_unscoped_file_storage.cache_clear()
storage = get_unscoped_file_storage()
try:
db = _direct_db_session()
try:
existing = UploadedFileStore(db).create_from_local_path(
local_path=output_path,
user_id=user_id,
task_id=task_id,
file_id=f"{run_kind}-cleanup-output",
filename=output_path.name,
storage_key=(
f"users/{user_id}/tasks/{task_id}/outputs/"
f"{run_kind}-cleanup-output/_versions/"
f"11111111111111111111111111111111/output/{output_path.name}"
),
workspace_relative_path=f"output/{output_path.name}",
workspace_category="output",
)
old_key = str(existing.storage_key)
db.commit()
finally:
db.close()
output_path.write_text("new bytes", encoding="utf-8")
prepared = task_execution_service._prepare_task_file_outputs_isolated(
task_id=task_id,
task_user_id=user_id,
file_outputs=[{"path": str(output_path), "filename": output_path.name}],
resolved_scope_segments=(),
)
new_key = prepared.staged_files[0].storage_key
assert storage.exists(old_key)
assert storage.exists(new_key)
cleanup_attempts = 0
def fail_cleanup_after_commit(claims): # type: ignore[no-untyped-def]
nonlocal cleanup_attempts
cleanup_attempts += 1
assert claims
check_db = _direct_db_session()
try:
record = (
check_db.query(UploadedFile)
.filter(UploadedFile.file_id == f"{run_kind}-cleanup-output")
.one()
)
task = check_db.query(Task).filter(Task.id == task_id).one()
assert record.storage_key == new_key
assert task.status == TaskStatus.COMPLETED
finally:
check_db.close()
assert storage.exists(new_key)
raise RuntimeError("cleanup reference query unavailable")
monkeypatch.setattr(
task_execution_service,
"cleanup_superseded_uploaded_file_objects",
fail_cleanup_after_commit,
)
lease = TaskLease(
attempt_id="test-attempt",
task_id=task_id,
runner_id=runner_id,
run_id=run_id,
)
if resume:
resumed_finalization = task_execution_service._finalize_resumed_task(
task_id,
status="completed",
success=True,
output="done",
task_owner_user_id=user_id,
result={"success": True, "output": "done", "file_outputs": []},
task_lease=lease,
prepared_outputs=prepared,
)
assert resumed_finalization["late_result"] is False
assert resumed_finalization["final_status"] == TaskStatus.COMPLETED.value
else:
background_finalization = (
task_execution_service._finalize_task_execution_result_isolated(
task_id=task_id,
task_user_id=user_id,
pre_run_status=TaskStatus.RUNNING,
result={"success": True, "output": "done", "file_outputs": []},
expected_run_id=run_id,
task_lease=lease,
resolved_scope_segments=(),
prepared_outputs=prepared,
)
)
assert background_finalization.late_result is False
assert (
background_finalization.final_task_status == TaskStatus.COMPLETED.value
)
assert cleanup_attempts == 1
check_db = _direct_db_session()
try:
record = (
check_db.query(UploadedFile)
.filter(UploadedFile.file_id == f"{run_kind}-cleanup-output")
.one()
)
task = check_db.query(Task).filter(Task.id == task_id).one()
assert record.storage_key == new_key
assert task.status == TaskStatus.COMPLETED
finally:
check_db.close()
assert storage.exists(new_key)
assert storage.exists(old_key)
finally:
get_unscoped_file_storage.cache_clear()
def test_background_cleanup_failure_does_not_reclassify_committed_finalization(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_assert_cleanup_failure_does_not_reclassify_committed_finalization(
resume=False,
monkeypatch=monkeypatch,
tmp_path=tmp_path,
)
def test_resume_cleanup_failure_does_not_reclassify_committed_finalization(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_assert_cleanup_failure_does_not_reclassify_committed_finalization(
resume=True,
monkeypatch=monkeypatch,
tmp_path=tmp_path,
)
@pytest.mark.asyncio
async def test_legacy_preview_staging_releases_request_pool_slot(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
task_id, user_id = _seed_running_task(
runner_id="legacy-preview-runner",
run_id="legacy-preview-run",
)
uploads_dir = tmp_path / "uploads"
relative_path = f"user_{user_id}/web_task_{task_id}/output/legacy-preview.txt"
local_path = uploads_dir / relative_path
local_path.parent.mkdir(parents=True)
local_path.write_text("legacy preview", encoding="utf-8")
monkeypatch.setenv("XAGENT_UPLOADS_DIR", str(uploads_dir))
monkeypatch.setenv("XAGENT_FILE_STORAGE_URI", (tmp_path / "objects").as_uri())
get_unscoped_file_storage.cache_clear()
engine = _install_one_slot_queue_pool(monkeypatch, pool_timeout=0.5)
request_db = _direct_db_session()
put_pool_counts: list[int] = []
original_put_file = FsspecFileStorage.put_file
def observe_put_file(
self: FsspecFileStorage,
source: Path,
key: str,
content_type: str | None = None,
) -> Any:
put_pool_counts.append(getattr(engine.pool, "checkedout")())
return original_put_file(self, source, key, content_type)
monkeypatch.setattr(FsspecFileStorage, "put_file", observe_put_file)
try:
request_db.query(User).filter(User.id == user_id).one()
assert getattr(engine.pool, "checkedout")() == 1
response = await websocket_api.redirect_legacy_preview(
relative_path,
request_db,
User(id=user_id),
)
assert response.status_code == 307
assert response.headers["location"].startswith("/api/files/public/preview/")
assert put_pool_counts == [0]
finally:
request_db.close()
engine.dispose()
get_unscoped_file_storage.cache_clear()
@pytest.mark.asyncio
async def test_legacy_preview_metadata_failure_compensates_staged_object(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
task_id, user_id = _seed_running_task(
runner_id="legacy-failure-runner",
run_id="legacy-failure-run",
)
uploads_dir = tmp_path / "uploads"
relative_path = f"user_{user_id}/web_task_{task_id}/output/legacy-failure.txt"
local_path = uploads_dir / relative_path
local_path.parent.mkdir(parents=True)
local_path.write_text("must be compensated", encoding="utf-8")
object_root = tmp_path / "objects"
monkeypatch.setenv("XAGENT_UPLOADS_DIR", str(uploads_dir))
monkeypatch.setenv("XAGENT_FILE_STORAGE_URI", object_root.as_uri())
get_unscoped_file_storage.cache_clear()
request_db = _direct_db_session()
def fail_metadata(*_args: Any, **_kwargs: Any) -> Any:
raise RuntimeError("metadata insert failed")
monkeypatch.setattr(
UploadedFileStore,
"upsert_already_durable",
fail_metadata,
)
try:
with pytest.raises(RuntimeError, match="metadata insert failed"):
await websocket_api.redirect_legacy_preview(
relative_path,
request_db,
User(id=user_id),
)
check_db = _direct_db_session()
try:
assert (
check_db.query(UploadedFile)
.filter(UploadedFile.storage_path == str(local_path.resolve()))
.count()
== 0
)
finally:
check_db.close()
assert not any(path.is_file() for path in object_root.rglob("*"))
finally:
request_db.close()
get_unscoped_file_storage.cache_clear()
@pytest.mark.asyncio
async def test_cancelled_legacy_preview_drains_failed_registration_compensation(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
task_id, user_id = _seed_running_task(
runner_id="legacy-cancel-runner",
run_id="legacy-cancel-run",
)
uploads_dir = tmp_path / "uploads"
relative_path = f"user_{user_id}/web_task_{task_id}/output/legacy-cancel.txt"
local_path = uploads_dir / relative_path
local_path.parent.mkdir(parents=True)
local_path.write_text("cancelled preview", encoding="utf-8")
object_root = tmp_path / "objects"
monkeypatch.setenv("XAGENT_UPLOADS_DIR", str(uploads_dir))
monkeypatch.setenv("XAGENT_FILE_STORAGE_URI", object_root.as_uri())
get_unscoped_file_storage.cache_clear()
request_db = _direct_db_session()
metadata_started = threading.Event()
allow_metadata_failure = threading.Event()
def fail_metadata_after_cancel(*_args: Any, **_kwargs: Any) -> Any:
metadata_started.set()
assert allow_metadata_failure.wait(timeout=3)
raise RuntimeError("metadata failed after caller cancellation")
monkeypatch.setattr(
UploadedFileStore,
"upsert_already_durable",
fail_metadata_after_cancel,
)
registration = asyncio.create_task(
websocket_api.redirect_legacy_preview(
relative_path, request_db, User(id=user_id)
)
)
try:
assert await asyncio.to_thread(metadata_started.wait, DB_PROGRESS_TIMEOUT)
registration.cancel()
await asyncio.sleep(0)
assert not registration.done()
allow_metadata_failure.set()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(registration, timeout=DB_PROGRESS_TIMEOUT)
check_db = _direct_db_session()
try:
assert (
check_db.query(UploadedFile)
.filter(UploadedFile.storage_path == str(local_path.resolve()))
.count()
== 0
)
finally:
check_db.close()
assert not any(path.is_file() for path in object_root.rglob("*"))
finally:
allow_metadata_failure.set()
request_db.close()
get_unscoped_file_storage.cache_clear()
@pytest.mark.asyncio
async def test_cancelled_output_staging_compensates_late_result_before_return(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Caller cancellation must not orphan objects produced by the worker."""
stage_started = threading.Event()
allow_stage = threading.Event()
compensation_finished = threading.Event()
prepared = task_execution_service._PreparedTaskFileOutputs((), (), ())
def gated_stage(
**_kwargs: Any,
) -> task_execution_service._PreparedTaskFileOutputs:
stage_started.set()
assert allow_stage.wait(timeout=3)
return prepared
def record_compensation(
actual: task_execution_service._PreparedTaskFileOutputs,
*,
metadata_committed: bool,
) -> None:
assert actual is prepared
assert metadata_committed is False
compensation_finished.set()
monkeypatch.setattr(
task_execution_service,
"_prepare_task_file_outputs_isolated",
gated_stage,
)
monkeypatch.setattr(
task_execution_service,
"_settle_prepared_task_file_outputs",
record_compensation,
)
staging = asyncio.create_task(
task_execution_service._prepare_task_file_outputs_cancellation_safe(
task_id=42,
task_user_id=1,
file_outputs=[{"path": "/slow/output.txt"}],
resolved_scope_segments=(),
)
)
assert await asyncio.to_thread(stage_started.wait, 2)
staging.cancel()
await asyncio.sleep(0)
assert not staging.done()
allow_stage.set()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(staging, timeout=3)
assert compensation_finished.is_set()
@pytest.mark.asyncio
async def test_resume_output_staging_finishes_while_heartbeat_is_still_active() -> None:
lease = TaskLease(
attempt_id="test-attempt",
task_id=42,
runner_id="resume-runner",
run_id="resume-run",
)
heartbeat_stop = asyncio.Event()
stage_started = threading.Event()
allow_stage = threading.Event()
prepared = task_execution_service._PreparedTaskFileOutputs((), (), ())
async def heartbeat() -> TaskLeaseHeartbeatOutcome:
await heartbeat_stop.wait()
return TaskLeaseHeartbeatOutcome()
heartbeat_task = asyncio.create_task(heartbeat())
class FakeTracker:
quota_interrupt_reason = None
def __init__(self, *, task_id: int, **kwargs: Any) -> None:
assert task_id == 42
assert kwargs == {
"expected_run_id": "resume-run",
"expected_runner_id": "resume-runner",
}
async def start_tracking(self) -> None:
return None
async def complete_tracking(self) -> None:
return None
async def interrupt_reason_for_quota(self) -> None:
return None
def gated_stage(
**_kwargs: Any,
) -> task_execution_service._PreparedTaskFileOutputs:
assert not heartbeat_stop.is_set()
assert not heartbeat_task.done()
stage_started.set()
assert allow_stage.wait(timeout=3)
assert not heartbeat_stop.is_set()
assert not heartbeat_task.done()
return prepared
def finalize(*_args: Any, **kwargs: Any) -> dict[str, Any]:
assert kwargs["prepared_outputs"] is prepared
assert heartbeat_stop.is_set()
assert heartbeat_task.done()
return {
"task_title": "resume",
"task_description": "",
"task_execution_mode": "flash",
"task_agent_id": None,
"agent_name": None,
"agent_logo_url": None,
"final_status": TaskStatus.COMPLETED.value,
"lease_released": True,
"report": SettlementReport(),
"normalized_outputs": [],
"output": "done",
"late_result": False,
}
agent_service = MagicMock()
agent_service.resume_execution_by_id = AsyncMock(
return_value={
"status": "completed",
"success": True,
"output": "done",
"file_outputs": [{"path": "/slow/output.txt"}],
}
)
with (
patch(
"xagent.web.services.task_execution._prepare_task_file_outputs_isolated",
side_effect=gated_stage,
),
patch(
"xagent.web.services.task_execution._finalize_resumed_task",
side_effect=finalize,
),
patch(
"xagent.web.api.websocket.manager",
MagicMock(broadcast_to_task=AsyncMock()),
),
patch(
"xagent.web.services.task_execution.background_task_manager.promote_resume_task"
),
patch(
"xagent.web.services.task_execution.background_task_manager.cleanup_task"
),
patch("xagent.web.tracking.task_tracker.TaskTracker", FakeTracker),
):
resume_task = asyncio.create_task(
task_execution_service.execute_resume_background(
task_id=42,
agent_service=agent_service,
task_owner_user_id=1,
expected_run_id="resume-run",
resolved_execution_scope=None,
preacquired_lease=lease,
preacquired_heartbeat_stop=heartbeat_stop,
preacquired_heartbeat_task=heartbeat_task,
)
)
assert await asyncio.to_thread(stage_started.wait, 2)
assert not heartbeat_stop.is_set()
assert not heartbeat_task.done()
allow_stage.set()
await asyncio.wait_for(resume_task, timeout=3)