Enterprise agent platform for building, deploying, and operating AI teammates.
"""The production collector/recovery lifecycle retains every cleanup obligation."""
from __future__ import annotations
import copy
import hashlib
import os
from concurrent.futures import ThreadPoolExecutor
from contextlib import ExitStack
from datetime import UTC, datetime, timedelta
from pathlib import Path
from threading import Event
from uuid import uuid4
import pytest
import sqlalchemy as sa
from sqlalchemy.orm import sessionmaker
from xagent.core.file_storage.factory import get_unscoped_file_storage
from xagent.core.tools.core.RAG_tools.storage.file_reference import file_cleanup_lock
from xagent.web.models import database
from xagent.web.models.database import Base, configure_db, get_engine, get_session_local
from xagent.web.models.uploaded_file import UploadedFile
from xagent.web.models.uploaded_file_cleanup_fence import UploadedFileCleanupFence
from xagent.web.models.user import User
from xagent.web.services import uploaded_file_cleanup as cleanup
from xagent.web.services import uploaded_file_cleanup_resources as resources
from xagent.web.services import uploaded_file_store as store
from xagent.web.services.kb_reference_protection import install_file_reference_validator
from xagent.web.services.managed_file_ref import (
ManagedFileRef,
)
from xagent.web.services.orphan_upload_gc import (
TASKLESS_SHARE_UPLOAD_SOURCE,
_claim_orphan,
_orphan_candidates,
cleanup_orphaned_taskless_uploads,
)
from xagent.web.services.uploaded_file_cleanup_publication import (
FilePublicationUnavailable,
)
from xagent.web.services.uploaded_file_recovery import (
recover_stale_uploaded_file_compensations_batch_isolated,
)
PAYLOAD = b"owned upload bytes"
DIGEST = hashlib.sha256(PAYLOAD).hexdigest()
@pytest.fixture(
params=["sqlite", pytest.param("postgresql", marks=pytest.mark.postgresql)]
)
def lifecycle(request, tmp_path, monkeypatch):
from tests.shared.postgres_disposable import disposable_database_factory
previous = database._SessionLocal, database._engine
for variable, name in (
("XAGENT_UPLOADS_DIR", "uploads"),
("XAGENT_FILE_MATERIALIZE_DIR", "materialized"),
("XAGENT_STORAGE_ROOT", "storage"),
("LANCEDB_DIR", "lance"),
):
monkeypatch.setenv(variable, str(tmp_path / name))
monkeypatch.setenv("XAGENT_FILE_STORAGE_URI", (tmp_path / "objects").as_uri())
get_unscoped_file_storage.cache_clear()
stack = ExitStack()
if request.param == "sqlite":
configure_db(f"sqlite:///{tmp_path / 'metadata.db'}")
engine = get_engine()
else:
make = stack.enter_context(disposable_database_factory("complete_cleanup"))
engine = make("boundary")
database._engine = engine
database._SessionLocal = sessionmaker(bind=engine, autoflush=False)
install_file_reference_validator()
Base.metadata.create_all(engine)
sessions = get_session_local()
file_id = str(uuid4())
source = tmp_path / "uploads" / "user_1" / "source.txt"
source.parent.mkdir(parents=True)
source.write_bytes(PAYLOAD)
key = f"users/1/uploads/{file_id}/source.txt"
durable = get_unscoped_file_storage().put_file(source, key)
with sessions.begin() as db:
db.add(User(id=1, username="owner", password_hash="unused"))
db.flush()
db.add(
UploadedFile(
file_id=file_id,
user_id=1,
filename="source.txt",
storage_path=str(source),
storage_key=key,
storage_backend=durable.backend,
storage_uri=durable.uri,
storage_status="available",
checksum=DIGEST,
file_size=len(PAYLOAD),
upload_source=TASKLESS_SHARE_UPLOAD_SOURCE,
created_at=datetime.now(UTC) - timedelta(days=10),
)
)
materialized = get_unscoped_file_storage().materialize(key, "source.txt")
previews = []
for directory, filename in (
("pptx_pdf_cache", f"{file_id}.preview.pdf"),
("svg_png_cache", f"{file_id}.asset.preview.png"),
):
path = tmp_path / "storage" / directory / filename
path.parent.mkdir(parents=True)
path.write_bytes(b"derived bytes")
previews.append(path)
yield sessions, source, materialized, previews, key, file_id
engine.dispose()
database._SessionLocal, database._engine = previous
get_unscoped_file_storage.cache_clear()
stack.close()
def collect(lifecycle):
sessions, *_ = lifecycle
with sessions() as db:
return cleanup_orphaned_taskless_uploads(db, older_than_seconds=1)
def claim(lifecycle):
sessions, *_ = lifecycle
with sessions() as db:
candidate = _orphan_candidates(
db, cutoff=datetime.now(UTC), limit=1, after=None
)[0]
token = _claim_orphan(db, candidate)
assert token is not None
return candidate, token
def recover(lifecycle):
return recover_stale_uploaded_file_compensations_batch_isolated(
session_factory=lifecycle[0],
cutoff=datetime.now(UTC) + timedelta(days=1),
batch_size=10,
)
def retained(lifecycle, done):
sessions, _, _, _, key, file_id = lifecycle
with sessions() as db:
row = db.query(UploadedFile).one()
assert row.storage_status == "compensating"
assert row.storage_key == key
assert row.cleanup_manifest["done"] == done
assert row.cleanup_manifest["file_id"] == file_id
return copy.deepcopy(row.cleanup_manifest)
def completed(lifecycle):
sessions, source, materialized, previews, key, file_id = lifecycle
with sessions() as db:
assert db.query(UploadedFile).count() == 0
assert db.get(UploadedFileCleanupFence, file_id) is not None
assert not get_unscoped_file_storage().exists(key)
assert not source.exists()
assert not materialized.exists()
assert not any(path.exists() for path in previews)
def test_actual_collector_finishes_every_resource(lifecycle):
assert collect(lifecycle).deleted == 1
completed(lifecycle)
@pytest.mark.parametrize("phase", ["durable", "local", "previews", "settlement"])
def test_each_failure_retains_metadata_and_converges_in_fresh_session(
lifecycle, monkeypatch, phase
):
sessions, source, materialized, previews, key, _ = lifecycle
with monkeypatch.context() as failing:
if phase == "durable":
def fail_delete(_self, _key):
raise OSError("transient durable delete")
failing.setattr(type(get_unscoped_file_storage()), "delete", fail_delete)
elif phase in {"local", "previews"}:
original = resources.os.unlink
preview_directories = {
(info.st_dev, info.st_ino)
for preview in previews
for info in [preview.parent.stat()]
}
def fail_unlink(path, *args, **kwargs):
parent = os.fstat(kwargs["dir_fd"])
is_preview = (parent.st_dev, parent.st_ino) in preview_directories
if is_preview == (phase == "previews"):
raise OSError("transient unlink")
return original(path, *args, **kwargs)
failing.setattr(resources.os, "unlink", fail_unlink)
else:
def fail_settlement(*_args, **_kwargs):
raise OSError("transient settlement")
failing.setattr(
store, "settle_uploaded_file_compensation_no_commit", fail_settlement
)
assert collect(lifecycle).deleted == 0
expected = {
"durable": [],
"local": ["durable"],
"previews": ["durable", "local"],
"settlement": ["durable", "local", "previews"],
}[phase]
manifest = retained(lifecycle, expected)
assert manifest["local"][0]["path"] == str(source)
assert manifest["local"][1]["path"] == str(materialized)
assert get_unscoped_file_storage().exists(key) is (phase == "durable")
assert recover(lifecycle).deleted == 1
completed(lifecycle)
@pytest.mark.parametrize(
"done", [[], ["durable"], ["durable", "local"], ["durable", "local", "previews"]]
)
def test_interruption_after_committed_phase_restarts_without_losing_handle(
lifecycle, monkeypatch, done
):
original = cleanup._save_manifest
class Interrupted(BaseException):
pass
if not done:
claim(lifecycle)
else:
def interrupt(*args, **kwargs):
result = original(*args, **kwargs)
manifest = args[-1]
if manifest["done"] == done:
raise Interrupted()
return result
with monkeypatch.context() as failing:
failing.setattr(cleanup, "_save_manifest", interrupt)
with pytest.raises(Interrupted):
collect(lifecycle)
retained(lifecycle, done)
assert recover(lifecycle).deleted == 1
completed(lifecycle)
def test_missing_objects_and_copies_are_success(lifecycle):
_, source, materialized, previews, key, _ = lifecycle
get_unscoped_file_storage().delete(key)
for path in (source, materialized, *previews):
path.unlink()
assert collect(lifecycle).deleted == 1
completed(lifecycle)
@pytest.mark.parametrize("presence", ["exists", "unknown"])
def test_non_absent_presence_keeps_all_obligations(lifecycle, presence):
candidate, token = claim(lifecycle)
outcome = cleanup.run_uploaded_file_cleanup(
session_factory=lifecycle[0],
row_id=candidate.row_id,
user_id=1,
file_id=candidate.file_id,
task_id=None,
storage_key=candidate.storage_key,
expected_updated_at=token,
compensation_delete=lambda **_kwargs: presence,
)
assert outcome == presence
retained(lifecycle, [])
assert lifecycle[1].exists() and lifecycle[2].exists()
assert recover(lifecycle).deleted == 1
completed(lifecycle)
def test_claim_commit_precedes_all_destructive_io_and_returns_connections(
lifecycle, monkeypatch
):
sessions, source, *_ = lifecycle
engine = sessions.kw["bind"]
active = set()
def checked_out(connection, _record, _proxy):
active.add(id(connection))
def returned(connection, _record):
active.discard(id(connection))
sa.event.listen(engine, "checkout", checked_out)
sa.event.listen(engine, "checkin", returned) # codespell:ignore checkin
original_delete = get_unscoped_file_storage().delete
original_unlink = resources.os.unlink
def verify():
assert not active
with sessions() as db:
row = db.query(UploadedFile).one()
assert row.storage_status == "compensating"
assert row.cleanup_manifest is not None
def delete(key):
verify()
# The reference lock is available even while slow storage is active.
from xagent.core.tools.core.RAG_tools.storage.file_reference import (
file_reference_lock,
)
with file_reference_lock([lifecycle[-1]]):
pass
return original_delete(key)
def unlink(path, *args, **kwargs):
verify()
return original_unlink(path, *args, **kwargs)
monkeypatch.setattr(get_unscoped_file_storage(), "delete", delete)
monkeypatch.setattr(resources.os, "unlink", unlink)
try:
assert collect(lifecycle).deleted == 1
finally:
sa.event.remove(engine, "checkout", checked_out)
sa.event.remove(engine, "checkin", returned) # codespell:ignore checkin
completed(lifecycle)
def test_failed_claim_commit_leaves_local_and_durable_bytes(lifecycle, monkeypatch):
sessions, source, materialized, previews, key, _ = lifecycle
with sessions() as db:
candidate = _orphan_candidates(
db, cutoff=datetime.now(UTC), limit=1, after=None
)[0]
monkeypatch.setattr(
db, "commit", lambda: (_ for _ in ()).throw(OSError("commit failed"))
)
with pytest.raises(OSError):
_claim_orphan(db, candidate)
db.rollback()
assert (
source.exists()
and materialized.exists()
and all(path.exists() for path in previews)
)
assert get_unscoped_file_storage().exists(key)
with sessions() as db:
assert db.query(UploadedFile).one().storage_status == "available"
@pytest.mark.parametrize(
"kind", ["external", "shared", "escaping", "replacement", "checksum", "root"]
)
def test_source_boundaries_preserve_unowned_and_uncertain_resources(
lifecycle, tmp_path, monkeypatch, kind
):
sessions, source, _, _, _, _ = lifecycle
retained_path = source
if kind == "external":
external = tmp_path / "external.txt"
external.write_bytes(PAYLOAD)
with sessions.begin() as db:
db.query(UploadedFile).update({UploadedFile.storage_path: str(external)})
source.unlink()
retained_path = external
elif kind == "shared":
retained_path = tmp_path / "shared.txt"
os.link(source, retained_path)
elif kind == "escaping":
outside = tmp_path / "outside.txt"
outside.write_bytes(PAYLOAD)
source.unlink()
source.symlink_to(outside)
retained_path = outside
candidate, token = claim(lifecycle)
if kind == "replacement":
original = source.with_suffix(".original")
source.rename(original)
source.write_bytes(b"new publication")
elif kind == "checksum":
source.write_bytes(b"modified managed content")
elif kind == "root":
monkeypatch.setenv("XAGENT_UPLOADS_DIR", str(tmp_path / "another-root"))
outcome = cleanup.run_uploaded_file_cleanup(
session_factory=sessions,
row_id=candidate.row_id,
user_id=1,
file_id=candidate.file_id,
task_id=None,
storage_key=candidate.storage_key,
expected_updated_at=token,
compensation_delete=store.delete_uploaded_file_compensation_object,
)
if kind in {"external", "shared"}:
assert outcome == "deleted"
assert retained_path.read_bytes() == PAYLOAD
else:
assert outcome == "pending"
with sessions() as db:
assert db.query(UploadedFile).one().cleanup_manifest is not None
assert retained_path.exists()
if kind == "replacement":
assert source.read_bytes() == b"new publication"
elif kind == "root":
assert get_unscoped_file_storage().exists(lifecycle[-2])
def test_preview_symlink_boundary_preserves_target_and_handle(lifecycle, tmp_path):
sessions, _, _, previews, _, _ = lifecycle
outside = tmp_path / "outside-preview"
outside.mkdir()
target = outside / previews[0].name
target.write_bytes(b"external content")
directory = previews[0].parent
previews[0].unlink()
directory.rmdir()
directory.symlink_to(outside, target_is_directory=True)
assert collect(lifecycle).deleted == 0
retained(lifecycle, ["durable", "local"])
assert target.read_bytes() == b"external content"
def test_cleanup_claim_cannot_settle_on_durable_absence_alone(lifecycle):
candidate, token = claim(lifecycle)
with lifecycle[0]() as db:
assert (
store.settle_uploaded_file_compensation_no_commit(
db,
row_id=candidate.row_id,
user_id=1,
file_id=candidate.file_id,
task_id=None,
storage_key=candidate.storage_key,
expected_updated_at=token,
presence="absent",
)
is None
)
db.commit()
retained(lifecycle, [])
def test_two_independent_workers_wait_for_inflight_delete_and_only_one_settles(
lifecycle, monkeypatch
):
from filelock import FileLock, Timeout
candidate, token = claim(lifecycle)
sessions, *_ = lifecycle
entered, proceed, blocked = Event(), Event(), Event()
original_delete = store.delete_uploaded_file_compensation_object
original_acquire = FileLock.acquire
calls = []
def delete(**kwargs):
calls.append(kwargs)
entered.set()
assert proceed.wait(10)
return original_delete(**kwargs)
def acquire(lock, *args, **kwargs):
try:
return original_acquire(lock, *args, **{**kwargs, "timeout": 0})
except Timeout:
blocked.set()
return original_acquire(lock, *args, **kwargs)
monkeypatch.setattr(FileLock, "acquire", acquire)
def run():
return cleanup.run_uploaded_file_cleanup(
session_factory=sessions,
row_id=candidate.row_id,
user_id=1,
file_id=candidate.file_id,
task_id=None,
storage_key=candidate.storage_key,
expected_updated_at=token,
compensation_delete=delete,
take_over=True,
)
with ThreadPoolExecutor(max_workers=2) as pool:
winner = pool.submit(run)
try:
assert entered.wait(10)
loser = pool.submit(run)
assert blocked.wait(10)
with sessions() as db:
assert db.query(UploadedFile).one().storage_status == "compensating"
finally:
proceed.set()
assert winner.result(timeout=10) == "deleted"
assert loser.result(timeout=10) == "stale"
assert len(calls) == 1
completed(lifecycle)
def test_stale_settlement_and_worker_cannot_touch_newer_claim(lifecycle):
candidate, old = claim(lifecycle)
sessions, _, _, _, key, file_id = lifecycle
with file_cleanup_lock(file_id), sessions() as db:
newer = store.take_over_uploaded_file_compensation_no_commit(
db,
row_id=candidate.row_id,
user_id=1,
file_id=file_id,
task_id=None,
storage_key=key,
expected_updated_at=old,
)
assert newer != old
db.commit()
calls = []
assert (
cleanup.run_uploaded_file_cleanup(
session_factory=sessions,
row_id=candidate.row_id,
user_id=1,
file_id=file_id,
task_id=None,
storage_key=key,
expected_updated_at=old,
compensation_delete=lambda **kwargs: calls.append(kwargs) or "absent",
)
== "stale"
)
assert calls == []
with sessions() as db:
assert (
store.settle_uploaded_file_compensation_no_commit(
db,
row_id=candidate.row_id,
user_id=1,
file_id=file_id,
task_id=None,
storage_key=key,
expected_updated_at=old,
presence="absent",
)
is None
)
assert recover(lifecycle).deleted == 1
completed(lifecycle)
def test_managed_copy_loaded_before_claim_cannot_publish_after_cleanup(lifecycle):
sessions, source, materialized, _, _, file_id = lifecycle
with sessions() as db:
ref = ManagedFileRef(db.query(UploadedFile).one())
# Keep the caller's snapshot alive across cleanup on another connection.
db.expunge_all()
assert collect(lifecycle).deleted == 1
with pytest.raises(FilePublicationUnavailable):
ref.ensure_local()
with pytest.raises(FilePublicationUnavailable):
ref.materialize(allow_existing_local=False)
assert not source.exists() and not materialized.exists()
def test_existing_copy_publication_blocks_claim_until_it_finishes(
lifecycle, monkeypatch
):
from filelock import FileLock, Timeout
sessions, source, materialized, _, key, file_id = lifecycle
source.unlink()
with sessions() as db:
row = db.query(UploadedFile).one()
ref = ManagedFileRef(row)
db.expunge_all()
entered, proceed, blocked = Event(), Event(), Event()
original = get_unscoped_file_storage().copy_to_path
original_acquire = FileLock.acquire
def copy_to_path(*args, **kwargs):
entered.set()
assert proceed.wait(10)
return original(*args, **kwargs)
def acquire(lock, *args, **kwargs):
try:
return original_acquire(lock, *args, **{**kwargs, "timeout": 0})
except Timeout:
blocked.set()
return original_acquire(lock, *args, **kwargs)
monkeypatch.setattr(get_unscoped_file_storage(), "copy_to_path", copy_to_path)
monkeypatch.setattr(FileLock, "acquire", acquire)
with ThreadPoolExecutor(max_workers=2) as pool:
publication = pool.submit(ref.ensure_local)
try:
assert entered.wait(10)
collection = pool.submit(collect, lifecycle)
assert blocked.wait(10)
finally:
proceed.set()
assert publication.result(timeout=10) == source
assert collection.result(timeout=10).deleted == 1
completed(lifecycle)
def test_retired_worker_cannot_delete_a_fresh_publication(lifecycle):
sessions, source, _, _, key, file_id = lifecycle
candidate, token = claim(lifecycle)
assert recover(lifecycle).deleted == 1
source.write_bytes(b"new published bytes")
new_key = f"users/1/uploads/{file_id}/_versions/{uuid4()}/source.txt"
staged = store.stage_uploaded_file_from_local_path(
local_path=source,
user_id=1,
file_id=file_id,
storage_key=new_key,
)
with sessions.begin() as db:
store.UploadedFileStore(db).add_already_durable(staged.to_record())
assert (
cleanup.run_uploaded_file_cleanup(
session_factory=sessions,
row_id=candidate.row_id,
user_id=1,
file_id=file_id,
task_id=None,
storage_key=key,
expected_updated_at=token,
compensation_delete=store.delete_uploaded_file_compensation_object,
)
== "stale"
)
assert source.read_bytes() == b"new published bytes"
assert get_unscoped_file_storage().exists(new_key)
def test_production_compensation_local_failure_preserves_original_contract(
lifecycle, monkeypatch
):
sessions, source, _, _, key, file_id = lifecycle
with monkeypatch.context() as failing:
failing.setattr(
resources.os,
"unlink",
lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("unlink failed")),
)
with pytest.raises(store.DurableStorageOperationError):
store.compensate_registered_uploads_sync(
[store.RegisteredUploadCompensationClaim(1, file_id, None, key)]
)
retained(lifecycle, ["durable"])
assert recover(lifecycle).deleted == 1
completed(lifecycle)
@pytest.mark.parametrize(
"phase", ["claim", "durable", "local", "previews", "quarantine"]
)
def test_process_exit_and_new_worker_converge(lifecycle, monkeypatch, phase):
import subprocess
import sys
sessions, *_ = lifecycle
engine = sessions.kw["bind"]
if engine.dialect.name == "postgresql":
with engine.connect() as connection:
name = connection.execute(sa.text("SELECT current_database()")).scalar()
url = sa.engine.make_url(os.environ["XAGENT_TEST_POSTGRES_URL"]).set(
database=name
)
else:
url = engine.url
monkeypatch.setenv(
"CLEANUP_TEST_DATABASE", url.render_as_string(hide_password=False)
)
script = """
import os, sys
from datetime import UTC, datetime
from xagent.web.models.database import configure_db, get_session_local
from xagent.web.services import uploaded_file_cleanup as cleanup
from xagent.web.services import uploaded_file_cleanup_resources as resources
from xagent.web.services.orphan_upload_gc import _claim_orphan, _orphan_candidates, cleanup_orphaned_taskless_uploads
configure_db(os.environ["CLEANUP_TEST_DATABASE"])
phase = sys.argv[1]
if phase == "claim":
with get_session_local()() as db:
candidate = _orphan_candidates(db, cutoff=datetime.now(UTC), limit=1, after=None)[0]
_claim_orphan(db, candidate)
os._exit(37)
if phase == "quarantine":
original_unlink = resources.os.unlink
def stop(path, *args, **kwargs):
if ".cleanup-" in str(path):
os._exit(37)
return original_unlink(path, *args, **kwargs)
resources.os.unlink = stop
else:
original = cleanup._save_manifest
def stop(*args, **kwargs):
result = original(*args, **kwargs)
if phase in args[-1]["done"]:
os._exit(37)
return result
cleanup._save_manifest = stop
with get_session_local()() as db:
cleanup_orphaned_taskless_uploads(db, older_than_seconds=1)
raise AssertionError("Did not interrupt the requested phase")
"""
result = subprocess.run(
[sys.executable, "-c", script, phase],
capture_output=True,
text=True,
timeout=45,
)
assert result.returncode == 37, result.stderr
expected = {
"claim": [],
"durable": ["durable"],
"local": ["durable", "local"],
"previews": ["durable", "local", "previews"],
"quarantine": ["durable"],
}[phase]
manifest = retained(lifecycle, expected)
if phase == "quarantine":
resource = manifest["local"][0]
assert (Path(resource["path"]).parent / resource["quarantine"]).exists()
assert recover(lifecycle).deleted == 1
completed(lifecycle)
def test_populated_upgrade_fresh_schema_retry_and_rollback_constraint(lifecycle):
from alembic.migration import MigrationContext
from alembic.operations import Operations
from tests.shared.postgres_disposable import load_migration_module
sessions, *_ = lifecycle
engine = sessions.kw["bind"]
migration = load_migration_module(
Path(
"src/xagent/migrations/versions/20261005_uploaded_file_cleanup_manifest.py"
)
)
fresh = sa.inspect(engine).get_columns("uploaded_files")
with sessions.begin() as db:
db.query(UploadedFile).one().cleanup_manifest = {}
with sessions.begin() as db:
db.query(UploadedFile).one().cleanup_manifest = None
with (
engine.begin() as connection,
Operations.context(MigrationContext.configure(connection)),
):
assert connection.execute(
sa.text("SELECT cleanup_manifest IS NULL FROM uploaded_files")
).scalar()
migration.downgrade()
assert "cleanup_manifest" not in {
column["name"] for column in sa.inspect(engine).get_columns("uploaded_files")
}
with pytest.raises(OSError), engine.begin() as connection:
if engine.dialect.name == "sqlite":
connection.exec_driver_sql("BEGIN IMMEDIATE")
with Operations.context(MigrationContext.configure(connection)):
migration.upgrade()
raise OSError("interrupted upgrade")
assert "cleanup_manifest" not in {
column["name"] for column in sa.inspect(engine).get_columns("uploaded_files")
}
for _ in range(2):
with (
engine.begin() as connection,
Operations.context(MigrationContext.configure(connection)),
):
migration.upgrade()
upgraded = sa.inspect(engine).get_columns("uploaded_files")
assert sorted(
(c["name"], str(c["type"]), c["nullable"]) for c in upgraded
) == sorted([(c["name"], str(c["type"]), c["nullable"]) for c in fresh])
with sessions() as db:
row = db.query(UploadedFile).one()
assert row.storage_status == "available" and row.checksum == DIGEST
claim(lifecycle)
with (
engine.begin() as connection,
Operations.context(MigrationContext.configure(connection)),
):
with pytest.raises(RuntimeError, match="Finish pending upload cleanup"):
migration.downgrade()
assert recover(lifecycle).deleted == 1
with (
engine.begin() as connection,
Operations.context(MigrationContext.configure(connection)),
):
migration.downgrade()
with engine.connect() as connection:
assert (
connection.execute(
sa.text("SELECT count(*) FROM uploaded_file_cleanup_fences")
).scalar()
== 1
)
def test_migration_refuses_offline_ddl():
from alembic.migration import MigrationContext
from alembic.operations import Operations
from tests.shared.postgres_disposable import load_migration_module
migration = load_migration_module(
Path(
"src/xagent/migrations/versions/20261005_uploaded_file_cleanup_manifest.py"
)
)
with Operations.context(
MigrationContext.configure(dialect_name="postgresql", opts={"as_sql": True})
):
for operation in (migration.upgrade, migration.downgrade):
with pytest.raises(RuntimeError, match="online migration"):
operation()
def test_abandoned_managed_copy_and_converter_temporaries_are_owned(lifecycle):
_, source, materialized, previews, _, file_id = lifecycle
source_prefix = hashlib.sha256(file_id.encode()).hexdigest()[:24]
temporary = [
source.with_name(f".{source_prefix}.{source.name}.interrupted.tmp"),
materialized.with_name(f".{materialized.name}.interrupted.tmp"),
previews[0].with_name(f"{previews[0].name}.interrupted.tmp"),
]
converter = previews[0].parent / f".{file_id}.preview-interrupted"
converter.mkdir()
(converter / "partial.pdf").write_bytes(b"partial conversion")
for path in temporary:
path.write_bytes(b"partial copy")
unrelated = source.with_name(f".{source.name}.legacy-unknown.tmp")
unrelated.write_bytes(b"unproven ownership")
assert collect(lifecycle).deleted == 1
completed(lifecycle)
assert not any(path.exists() for path in temporary)
assert not converter.exists()
assert unrelated.read_bytes() == b"unproven ownership"
def test_preview_directory_partial_failure_can_be_retried(lifecycle, monkeypatch):
_, _, _, previews, _, file_id = lifecycle
converter = previews[0].parent / f".{file_id}.preview-interrupted"
converter.mkdir()
for name in ("first.pdf", "second.pdf"):
(converter / name).write_bytes(b"partial")
unlink = resources.os.unlink
deleted = Event()
def fail_after_first(path, *args, **kwargs):
if str(path).endswith(".pdf") and str(path) in {"first.pdf", "second.pdf"}:
if deleted.is_set():
raise OSError("directory disposal interrupted")
deleted.set()
return unlink(path, *args, **kwargs)
with monkeypatch.context() as failing:
failing.setattr(resources.os, "unlink", fail_after_first)
assert collect(lifecycle).deleted == 0
retained(lifecycle, ["durable", "local"])
assert deleted.is_set()
assert recover(lifecycle).deleted == 1
completed(lifecycle)
assert not converter.exists()
@pytest.mark.asyncio
@pytest.mark.parametrize("protection", ["task", "target", "uncertain_delete"])
async def test_cancelled_production_upload_preserves_skipped_or_pending_claim(
lifecycle, monkeypatch, protection
):
import asyncio
import io
from fastapi.datastructures import UploadFile
from xagent.web.api import files
from xagent.web.models.kb_ingest_target import KBIngestTarget
from xagent.web.models.task import Task
from xagent.web.services.file_turn import bind_turn_files_no_commit
sessions = lifecycle[0]
registered, release = Event(), Event()
identifiers = []
original = files.register_local_uploads_sync
def register_then_pause(registrations):
result = original(registrations)
identifiers.extend(item.file_id for item in result)
with sessions.begin() as db:
if protection == "task":
task = Task(user_id=1, title="owns cancelled upload")
db.add(task)
db.flush()
assert not bind_turn_files_no_commit(
file_ids=identifiers, task_id=int(task.id), owner_user_id=1, db=db
)
elif protection == "target":
db.add(
KBIngestTarget(
user_id=1,
collection="protected",
target_path="cancelled.txt",
file_id=identifiers[0],
latest_file_sha256=DIGEST,
)
)
registered.set()
assert release.wait(10)
return result
monkeypatch.setattr(files, "register_local_uploads_sync", register_then_pause)
if protection == "uncertain_delete":
def fail_delete(*_args):
raise OSError("delete unavailable")
monkeypatch.setattr(type(get_unscoped_file_storage()), "delete", fail_delete)
worker = asyncio.create_task(
files.store_uploaded_files(
upload_items=[
UploadFile(filename="cancelled.txt", file=io.BytesIO(PAYLOAD))
],
task_type="general",
task_id=None,
folder=None,
user_id=1,
single_file_mode=True,
)
)
assert await asyncio.to_thread(registered.wait, 10)
worker.cancel()
release.set()
with pytest.raises(asyncio.CancelledError):
await worker
with sessions() as db:
row = db.query(UploadedFile).filter_by(file_id=identifiers[0]).one()
assert Path(row.storage_path).read_bytes() == PAYLOAD
assert get_unscoped_file_storage().exists(row.storage_key)
if protection == "uncertain_delete":
assert row.storage_status == "compensating"
assert row.cleanup_manifest["done"] == []
else:
assert row.storage_status == "available"
assert row.cleanup_manifest is None
@pytest.mark.asyncio
async def test_request_finalizer_preserves_replacement_after_compensation(
lifecycle, monkeypatch
):
import io
from fastapi.datastructures import UploadFile
from xagent.web.api import files
source = []
original = files.compensate_registered_uploads_sync
def fail_register(registrations):
source.extend(item.local_path for item in registrations)
raise RuntimeError("failed registration")
def replace_after_compensation(claims):
original(claims)
source[0].rename(source[0].with_suffix(".retired"))
source[0].write_bytes(b"new publication")
monkeypatch.setattr(files, "register_local_uploads_sync", fail_register)
monkeypatch.setattr(
files, "compensate_registered_uploads_sync", replace_after_compensation
)
with pytest.raises(RuntimeError, match="failed registration"):
await files.store_uploaded_files(
upload_items=[
UploadFile(filename="replacement.txt", file=io.BytesIO(PAYLOAD))
],
task_type="general",
task_id=None,
folder=None,
user_id=1,
single_file_mode=True,
)
assert source[0].read_bytes() == b"new publication"
@pytest.mark.parametrize("first", ["bind", "claim"])
def test_independent_task_binder_and_claim_have_one_winner(lifecycle, first):
from threading import local
from xagent.web.models.task import Task
from xagent.web.services.file_turn import bind_turn_files_no_commit
sessions, source, _, _, _, file_id = lifecycle
with sessions.begin() as db:
task = Task(user_id=1, title="racing attachment")
db.add(task)
db.flush()
task_id = int(task.id)
with sessions() as db:
candidate = _orphan_candidates(
db, cutoff=datetime.now(UTC), limit=1, after=None
)[0]
entered, proceed, attempted = Event(), Event(), Event()
worker = local()
engine = sessions.kw["bind"]
def before(_conn, _cursor, statement, *_args):
if getattr(worker, "role", None) == "loser" and statement.startswith("UPDATE"):
attempted.set()
def after(_conn, _cursor, statement, *_args):
if getattr(worker, "role", None) == "winner" and statement.startswith(
"UPDATE uploaded_files"
):
entered.set()
assert proceed.wait(10)
sa.event.listen(engine, "before_cursor_execute", before)
sa.event.listen(engine, "after_cursor_execute", after)
def execute(action, role):
worker.role = role
with sessions() as db:
if action == "bind":
missing = bind_turn_files_no_commit(
file_ids=[file_id], task_id=task_id, owner_user_id=1, db=db
)
db.commit()
return missing
return _claim_orphan(db, candidate)
try:
with ThreadPoolExecutor(max_workers=2) as pool:
winner = pool.submit(execute, first, "winner")
try:
assert entered.wait(10)
loser = pool.submit(
execute, "claim" if first == "bind" else "bind", "loser"
)
assert attempted.wait(10)
assert not loser.done()
finally:
proceed.set()
winning, losing = winner.result(timeout=10), loser.result(timeout=10)
if first == "bind":
assert winning == [] and losing is None
assert source.read_bytes() == PAYLOAD
with sessions() as db:
row = db.query(UploadedFile).one()
assert row.task_id == task_id and row.storage_status == "available"
else:
assert winning is not None and losing == [file_id]
assert recover(lifecycle).deleted == 1
completed(lifecycle)
finally:
sa.event.remove(engine, "before_cursor_execute", before)
sa.event.remove(engine, "after_cursor_execute", after)
@pytest.mark.parametrize("target", ["source", "materialized", "preview"])
def test_file_replaced_by_directory_before_claim_is_preserved(lifecycle, target):
_, source, materialized, previews, _, _ = lifecycle
path = {"source": source, "materialized": materialized, "preview": previews[0]}[
target
]
path.unlink()
path.mkdir()
child = path / "unrelated.txt"
child.write_bytes(b"replacement tree")
assert collect(lifecycle).deleted == 0
assert child.read_bytes() == b"replacement tree"
retained(lifecycle, ["durable", "local"] if target == "preview" else [])
def test_missing_converter_child_does_not_settle_surviving_quarantine(
lifecycle, monkeypatch
):
_, _, _, previews, _, file_id = lifecycle
directory = previews[0].parent / f".{file_id}.preview-interrupted"
directory.mkdir()
for name in ("first.pdf", "second.pdf"):
(directory / name).write_bytes(b"partial preview")
unlink = resources.os.unlink
raced = Event()
def disappear_before_unlink(path, *args, **kwargs):
if str(path) in {"first.pdf", "second.pdf"} and not raced.is_set():
raced.set()
unlink(path, *args, **kwargs)
return unlink(path, *args, **kwargs)
with monkeypatch.context() as failing:
failing.setattr(resources.os, "unlink", disappear_before_unlink)
assert collect(lifecycle).deleted == 0
assert raced.is_set()
manifest = retained(lifecycle, ["durable", "local"])
resource = next(r for r in manifest["previews"] if r["path"] == str(directory))
quarantine = directory.parent / resource["quarantine"]
assert len(list(quarantine.glob("*.pdf"))) == 1
assert recover(lifecycle).deleted == 1
completed(lifecycle)
assert not quarantine.exists()
@pytest.mark.parametrize(
"basename",
["source.txt", "r" * 206 + ".txt", "r" * 246 + ".txt", "还" * 80 + ".txt"],
)
def test_process_death_inside_real_restoration_cleans_both_temporaries(
lifecycle, monkeypatch, basename
):
import subprocess
import sys
sessions, source, materialized, previews, key, file_id = lifecycle
if basename != source.name:
renamed_source = source.with_name(basename)
source.rename(renamed_source)
renamed_cache = materialized.with_name(basename)
materialized.rename(renamed_cache)
with sessions.begin() as db:
row = db.query(UploadedFile).one()
row.storage_path, row.filename = str(renamed_source), basename
source = renamed_source
lifecycle = sessions, source, renamed_cache, previews, key, file_id
engine = sessions.kw["bind"]
if engine.dialect.name == "postgresql":
with engine.connect() as connection:
name = connection.execute(sa.text("SELECT current_database()")).scalar()
url = sa.engine.make_url(os.environ["XAGENT_TEST_POSTGRES_URL"]).set(
database=name
)
else:
url = engine.url
monkeypatch.setenv(
"CLEANUP_TEST_DATABASE", url.render_as_string(hide_password=False)
)
source.unlink()
script = """
import os
from xagent.core.file_storage import storage
from xagent.web.models.database import configure_db, get_session_local
from xagent.web.models.uploaded_file import UploadedFile
from xagent.web.services.managed_file_ref import ManagedFileRef
configure_db(os.environ["CLEANUP_TEST_DATABASE"])
def stop(source, destination, *args):
destination.write(source.read(2))
destination.flush()
os._exit(37)
storage.shutil.copyfileobj = stop
with get_session_local()() as db:
ManagedFileRef(db.query(UploadedFile).one()).ensure_local()
raise AssertionError("The actual restoration producer was not interrupted")
"""
result = subprocess.run(
[sys.executable, "-c", script], capture_output=True, text=True, timeout=45
)
assert result.returncode == 37, result.stderr
prefix = hashlib.sha256(file_id.encode()).hexdigest()[:24]
copies = list(source.parent.iterdir())
assert len(copies) == 2
assert any(path.name.startswith(f".{prefix}.") for path in copies)
assert any(path.name.startswith(f"..{prefix}.") for path in copies)
assert collect(lifecycle).deleted == 1
completed(lifecycle)
assert not any(path.exists() for path in copies)
def test_migration_without_upload_table_defers_to_fresh_model_creation(lifecycle):
from alembic.migration import MigrationContext
from alembic.operations import Operations
from tests.shared.postgres_disposable import load_migration_module
engine = lifecycle[0].kw["bind"]
migration = load_migration_module(
Path(
"src/xagent/migrations/versions/20261005_uploaded_file_cleanup_manifest.py"
)
)
with engine.begin() as connection:
Base.metadata.drop_all(connection)
with Operations.context(MigrationContext.configure(connection)):
migration.upgrade()
migration.upgrade()
migration.downgrade()
assert not sa.inspect(connection).has_table("uploaded_files")
Base.metadata.create_all(connection)
column = next(
c
for c in sa.inspect(connection).get_columns("uploaded_files")
if c["name"] == "cleanup_manifest"
)
assert column["nullable"]
assert isinstance(column["type"], sa.JSON)
@pytest.mark.asyncio
@pytest.mark.parametrize("operation", ["restore", "materialize", "svg", "pptx"])
async def test_publication_contention_preserves_caller_error_contract(
lifecycle, monkeypatch, operation
):
from types import SimpleNamespace
from fastapi import HTTPException
from filelock import FileLock, Timeout
from xagent.web.api import files
from xagent.web.services.managed_file_ref import DurableStorageOperationError
sessions, source, materialized, _, key, file_id = lifecycle
entered, release = Event(), Event()
checked_out = [0]
@sa.event.listens_for(sessions.kw["bind"], "checkout")
def checkout(*_args):
checked_out[0] += 1
@sa.event.listens_for(sessions.kw["bind"], "checkin") # codespell:ignore checkin
def returned(*_args):
checked_out[0] -= 1
def slow_publication():
with file_cleanup_lock(file_id):
entered.set()
assert release.wait(10)
original_acquire = FileLock.acquire
conversions = []
async def unexpected_conversion(*args, **kwargs):
conversions.append(args)
raise AssertionError("Conversion must not start while publication is blocked")
def bounded_acquire(lock, *args, **kwargs):
if str(lock.lock_file).endswith(".cleanup.lock"):
kwargs["timeout"] = 0.02
return original_acquire(lock, *args, **kwargs)
with ThreadPoolExecutor(max_workers=1) as pool:
owner = pool.submit(slow_publication)
assert entered.wait(5)
monkeypatch.setattr(FileLock, "acquire", bounded_acquire)
try:
with sessions() as db:
if operation in {"restore", "materialize"}:
source.unlink()
materialized.unlink()
ref = ManagedFileRef(db.query(UploadedFile).one())
with pytest.raises(DurableStorageOperationError) as fault:
ref.ensure_local() if operation == "restore" else ref.materialize()
assert fault.value.storage_key == key
assert isinstance(fault.value.__cause__, Timeout)
elif operation == "svg":
with pytest.raises(HTTPException) as fault:
await files._inline_preview_response(
source,
filename="source.svg",
media_type="image/svg+xml",
file_id=file_id,
)
assert fault.value.status_code == 503
assert isinstance(
fault.value.__cause__, DurableStorageOperationError
)
else:
pptx = source.with_suffix(".pptx")
row = db.query(UploadedFile).one()
row.filename = pptx.name
row.storage_path = str(pptx)
db.commit()
source.unlink()
monkeypatch.setattr(
files.asyncio, "create_subprocess_exec", unexpected_conversion
)
with pytest.raises(HTTPException) as fault:
await files.preview_pptx_as_pdf(
file_id, user=SimpleNamespace(id=1), db=db
)
assert fault.value.status_code == 503
assert isinstance(
fault.value.__cause__, DurableStorageOperationError
)
assert isinstance(fault.value.__cause__.__cause__, Timeout)
assert not conversions
assert checked_out[0] == 0
finally:
release.set()
owner.result(timeout=5)
with sessions() as db:
assert db.query(UploadedFile).one().storage_status == "available"
if operation in {"restore", "materialize", "pptx"}:
assert not source.exists()
else:
assert source.read_bytes() == PAYLOAD