Enterprise agent platform for building, deploying, and operating AI teammates.
from hashlib import sha256
from pathlib import Path
import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import QueuePool
from xagent.core.file_storage.types import StoredObject
from xagent.web.models.database import Base
from xagent.web.models.uploaded_file import UploadedFile
from xagent.web.models.user import User
from xagent.web.services.startup_file_storage_sync import (
_acquire_file_lock,
_release_file_lock,
sync_registered_files_to_durable_storage,
)
class FakeStorage:
def __init__(
self,
existing_keys: set[str] | None = None,
backend: str = "s3",
object_sizes: dict[str, int] | None = None,
object_checksums: dict[str, str] | None = None,
):
self.backend = backend
self.existing_keys = set(existing_keys or set())
self.object_sizes = object_sizes or {}
self.object_checksums = object_checksums or {}
self.list_calls: list[str] = []
self.put_calls: list[tuple[Path, str]] = []
def list(self, prefix: str) -> list[StoredObject]:
self.list_calls.append(prefix)
normalized = prefix.rstrip("/")
return [
StoredObject(
backend=self.backend,
key=key,
uri=f"s3://bucket/{key}",
size=self.object_sizes.get(key, 0),
)
for key in sorted(self.existing_keys)
if key.startswith(normalized + "/") or key == normalized
]
def put_file(
self, source: Path, key: str, content_type: str | None = None
) -> StoredObject:
del content_type
self.put_calls.append((source, key))
self.existing_keys.add(key)
return StoredObject(
backend=self.backend,
key=key,
uri=f"s3://bucket/{key}",
size=source.stat().st_size,
checksum="checksum",
etag="etag",
)
def stat(self, key: str) -> StoredObject:
return StoredObject(
backend=self.backend,
key=key,
uri=f"s3://bucket/{key}",
size=self.object_sizes.get(key, 0),
checksum=self.object_checksums.get(key, f"sha256:{key}"),
etag="etag",
)
class HashAwareFakeStorage(FakeStorage):
def __init__(
self,
existing_keys: set[str] | None = None,
*,
missing_hash_keys: set[str] | None = None,
object_checksums: dict[str, str] | None = None,
):
super().__init__(existing_keys, object_checksums=object_checksums)
self.missing_hash_keys = set(missing_hash_keys or set())
self.stat_calls: list[str] = []
self.content_hash_calls: list[str] = []
def stat(self, key: str) -> StoredObject:
self.stat_calls.append(key)
if key in self.missing_hash_keys:
return StoredObject(
backend=self.backend,
key=key,
uri=f"s3://bucket/{key}",
size=self.object_sizes.get(key, 0),
)
return StoredObject(
backend=self.backend,
key=key,
uri=f"s3://bucket/{key}",
size=self.object_sizes.get(key, 0),
checksum=self.object_checksums.get(key, f"sha256:{key}"),
etag="etag",
)
def content_hash(self, key: str) -> str:
self.content_hash_calls.append(key)
if key in self.missing_hash_keys:
raise RuntimeError("missing content hash")
return self.object_checksums.get(key, f"sha256:{key}")
class FailingOnceStorage(FakeStorage):
def __init__(self, failed_key: str):
super().__init__()
self.failed_key = failed_key
def put_file(
self, source: Path, key: str, content_type: str | None = None
) -> StoredObject:
if key == self.failed_key:
self.put_calls.append((source, key))
raise RuntimeError("transient upload failure")
return super().put_file(source, key, content_type)
def _session():
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
SessionLocal = sessionmaker(bind=engine)
return SessionLocal()
def _user(db, username="sync-user"):
user = User(username=username, password_hash="hash", is_admin=False)
db.add(user)
db.commit()
db.refresh(user)
return user
def _record(db, *, user, local_path: Path, file_id: str, **overrides) -> UploadedFile:
values = {
"file_id": file_id,
"user_id": int(user.id),
"filename": local_path.name,
"storage_path": str(local_path),
"mime_type": "text/plain",
"file_size": local_path.stat().st_size if local_path.exists() else 0,
"storage_status": "legacy",
}
values.update(overrides)
record = UploadedFile(**values)
db.add(record)
db.flush()
return record
def test_sync_uploads_registered_local_file_missing_remote_object(tmp_path):
db = _session()
user = _user(db)
local_path = tmp_path / "uploads" / "input.txt"
local_path.parent.mkdir()
local_path.write_text("content", encoding="utf-8")
record = _record(db, user=user, local_path=local_path, file_id="file-123")
db.commit()
storage = FakeStorage()
result = sync_registered_files_to_durable_storage(db, storage=storage)
db.refresh(record)
expected_key = f"users/{int(user.id)}/uploads/file-123/input.txt"
assert result.scanned == 1
assert result.uploaded == 1
assert result.already_present == 0
assert result.skipped_missing_local == 0
assert storage.list_calls == [f"users/{int(user.id)}"]
assert storage.put_calls == [(local_path, expected_key)]
assert record.storage_backend == "s3"
assert record.storage_key == expected_key
assert record.storage_uri == f"s3://bucket/{expected_key}"
assert record.checksum == "checksum"
assert record.etag == "etag"
assert record.storage_status == "available"
def test_sync_does_not_upload_when_remote_key_is_present(tmp_path):
db = _session()
user = _user(db)
local_path = tmp_path / "uploads" / "input.txt"
local_path.parent.mkdir()
local_path.write_text("content", encoding="utf-8")
key = f"users/{int(user.id)}/uploads/file-123/input.txt"
record = _record(
db,
user=user,
local_path=local_path,
file_id="file-123",
storage_backend="s3",
storage_key=key,
storage_uri=f"s3://bucket/{key}",
storage_status="available",
checksum="checksum",
)
db.commit()
storage = FakeStorage({key})
result = sync_registered_files_to_durable_storage(db, storage=storage)
db.refresh(record)
assert result.scanned == 1
assert result.already_present == 1
assert result.uploaded == 0
assert storage.put_calls == []
assert record.storage_status == "available"
def test_sync_skips_compensating_rows_instead_of_reviving_them(tmp_path):
db = _session()
user = _user(db)
local_path = tmp_path / "uploads" / "cancelled.txt"
local_path.parent.mkdir()
local_path.write_text("content", encoding="utf-8")
key = f"users/{int(user.id)}/uploads/file-cancelled/cancelled.txt"
record = _record(
db,
user=user,
local_path=local_path,
file_id="file-cancelled",
storage_backend="s3",
storage_key=key,
storage_uri=f"s3://bucket/{key}",
storage_status="compensating",
checksum=sha256(local_path.read_bytes()).hexdigest(),
)
db.commit()
storage = FakeStorage(
{key},
object_sizes={key: local_path.stat().st_size},
object_checksums={key: sha256(local_path.read_bytes()).hexdigest()},
)
result = sync_registered_files_to_durable_storage(db, storage=storage)
db.refresh(record)
assert result.scanned == 0
assert storage.list_calls == []
assert storage.put_calls == []
assert record.storage_status == "compensating"
def test_sync_releases_pool_before_storage_io(tmp_path):
engine = create_engine(
f"sqlite:///{tmp_path / 'startup-sync.db'}",
poolclass=QueuePool,
pool_size=1,
max_overflow=0,
pool_timeout=0.1,
connect_args={"check_same_thread": False},
)
Base.metadata.create_all(engine)
SessionLocal = sessionmaker(bind=engine)
db = SessionLocal()
user = _user(db)
local_path = tmp_path / "uploads" / "pool.txt"
local_path.parent.mkdir()
local_path.write_text("content", encoding="utf-8")
_record(db, user=user, local_path=local_path, file_id="file-pool")
db.commit()
assert db.query(User).filter(User.id == user.id).one() is not None
assert engine.pool.checkedout() == 1
class PoolAwareStorage(FakeStorage):
def list(self, prefix: str) -> list[StoredObject]:
assert engine.pool.checkedout() == 0
return super().list(prefix)
def put_file(
self,
source: Path,
key: str,
content_type: str | None = None,
) -> StoredObject:
assert engine.pool.checkedout() == 0
return super().put_file(source, key, content_type)
try:
result = sync_registered_files_to_durable_storage(
db,
storage=PoolAwareStorage(),
batch_size=1,
)
assert result.uploaded == 1
finally:
db.close()
engine.dispose()
def test_sync_rejects_dirty_compatibility_session_before_storage_io(tmp_path):
engine = create_engine(
f"sqlite:///{tmp_path / 'startup-sync-dirty.db'}",
poolclass=QueuePool,
pool_size=1,
max_overflow=0,
pool_timeout=0.1,
connect_args={"check_same_thread": False},
)
Base.metadata.create_all(engine)
SessionLocal = sessionmaker(bind=engine)
db = SessionLocal()
db.add(User(username="dirty-sync-user", password_hash="hash", is_admin=False))
storage = FakeStorage()
try:
with pytest.raises(RuntimeError, match="pending writes"):
sync_registered_files_to_durable_storage(db, storage=storage)
assert storage.list_calls == []
assert storage.put_calls == []
finally:
db.rollback()
db.close()
engine.dispose()
def test_sync_advances_detached_cursor_across_multiple_batches(tmp_path):
db = _session()
user = _user(db)
local_paths = []
for index in range(3):
local_path = tmp_path / "uploads" / f"batch-{index}.txt"
local_path.parent.mkdir(exist_ok=True)
local_path.write_text(f"content-{index}", encoding="utf-8")
_record(
db,
user=user,
local_path=local_path,
file_id=f"file-batch-{index}",
)
local_paths.append(local_path)
db.commit()
storage = FakeStorage()
result = sync_registered_files_to_durable_storage(
db,
storage=storage,
batch_size=1,
)
assert result.scanned == 3
assert result.uploaded == 3
assert [source for source, _key in storage.put_calls] == local_paths
assert storage.list_calls == [f"users/{int(user.id)}"]
def test_sync_revalidates_available_row_without_checksum(tmp_path):
db = _session()
user = _user(db)
local_path = tmp_path / "uploads" / "input.txt"
local_path.parent.mkdir()
local_path.write_text("content", encoding="utf-8")
key = f"users/{int(user.id)}/uploads/file-123/input.txt"
record = _record(
db,
user=user,
local_path=local_path,
file_id="file-123",
storage_backend="s3",
storage_key=key,
storage_uri=f"s3://bucket/{key}",
storage_status="available",
checksum=None,
)
db.commit()
local_checksum = sha256(local_path.read_bytes()).hexdigest()
storage = HashAwareFakeStorage({key}, object_checksums={key: local_checksum})
storage.object_sizes[key] = local_path.stat().st_size
result = sync_registered_files_to_durable_storage(db, storage=storage)
db.refresh(record)
assert result.scanned == 1
assert result.already_present == 1
assert result.uploaded == 0
assert storage.stat_calls == [key]
assert record.checksum == local_checksum
assert record.storage_status == "available"
def test_sync_refreshes_metadata_when_remote_key_present_but_record_incomplete(
tmp_path,
):
db = _session()
user = _user(db)
local_path = tmp_path / "uploads" / "input.txt"
local_path.parent.mkdir()
local_path.write_text("content", encoding="utf-8")
key = f"users/{int(user.id)}/uploads/file-123/input.txt"
record = _record(
db,
user=user,
local_path=local_path,
file_id="file-123",
storage_key=key,
storage_status="legacy",
)
db.commit()
storage = FakeStorage(
{key},
object_sizes={key: local_path.stat().st_size},
object_checksums={key: sha256(local_path.read_bytes()).hexdigest()},
)
result = sync_registered_files_to_durable_storage(db, storage=storage)
db.refresh(record)
assert result.scanned == 1
assert result.already_present == 1
assert result.uploaded == 0
assert storage.put_calls == []
assert record.storage_backend == "s3"
assert record.storage_key == key
assert record.storage_uri == f"s3://bucket/{key}"
assert record.storage_status == "available"
def test_sync_repairs_hashless_remote_object_from_local_file(tmp_path):
db = _session()
user = _user(db)
local_path = tmp_path / "uploads" / "input.txt"
local_path.parent.mkdir()
local_path.write_text("content", encoding="utf-8")
key = f"users/{int(user.id)}/uploads/file-123/input.txt"
record = _record(
db,
user=user,
local_path=local_path,
file_id="file-123",
storage_key=key,
storage_status="legacy",
)
db.commit()
storage = HashAwareFakeStorage({key}, missing_hash_keys={key})
result = sync_registered_files_to_durable_storage(db, storage=storage)
db.refresh(record)
assert result.already_present == 1
assert result.uploaded == 1
assert result.failed == 0
assert storage.stat_calls == [key]
assert storage.content_hash_calls == []
assert storage.put_calls == [(local_path, key)]
assert record.storage_status == "available"
assert record.checksum == "checksum"
def test_sync_refreshes_when_remote_size_is_zero_but_local_file_is_not(tmp_path):
db = _session()
user = _user(db)
local_path = tmp_path / "uploads" / "input.txt"
local_path.parent.mkdir()
local_path.write_text("content", encoding="utf-8")
key = f"users/{int(user.id)}/uploads/file-123/input.txt"
record = _record(
db,
user=user,
local_path=local_path,
file_id="file-123",
storage_key=key,
storage_status="legacy",
)
db.commit()
storage = FakeStorage({key}, object_sizes={key: 0})
result = sync_registered_files_to_durable_storage(db, storage=storage)
db.refresh(record)
assert result.already_present == 1
assert result.uploaded == 1
assert result.failed == 0
assert storage.put_calls == [(local_path, key)]
assert record.storage_status == "available"
def test_sync_reports_remote_metadata_lookup_failure_when_local_missing(
tmp_path,
):
db = _session()
user = _user(db)
local_path = tmp_path / "uploads" / "missing.txt"
key = f"users/{int(user.id)}/uploads/file-123/missing.txt"
record = _record(
db,
user=user,
local_path=local_path,
file_id="file-123",
storage_key=key,
storage_status="legacy",
)
db.commit()
storage = HashAwareFakeStorage({key}, missing_hash_keys={key})
result = sync_registered_files_to_durable_storage(db, storage=storage)
db.refresh(record)
assert result.already_present == 0
assert result.uploaded == 0
assert result.skipped_missing_local == 0
assert result.failed == 1
assert storage.put_calls == []
assert record.storage_status == "legacy"
assert record.checksum is None
def test_sync_continues_when_remote_metadata_lookup_fails(tmp_path):
db = _session()
user = _user(db)
first_path = tmp_path / "uploads" / "first.txt"
second_path = tmp_path / "uploads" / "second.txt"
first_path.parent.mkdir()
first_path.write_text("first", encoding="utf-8")
second_path.write_text("second", encoding="utf-8")
first_key = f"users/{int(user.id)}/uploads/file-first/first.txt"
second_key = f"users/{int(user.id)}/uploads/file-second/second.txt"
first_record = _record(
db,
user=user,
local_path=first_path,
file_id="file-first",
storage_key=first_key,
storage_status="legacy",
)
second_record = _record(
db,
user=user,
local_path=second_path,
file_id="file-second",
storage_key=second_key,
storage_status="legacy",
)
db.commit()
second_checksum = sha256(second_path.read_bytes()).hexdigest()
storage = HashAwareFakeStorage(
{first_key, second_key},
missing_hash_keys={first_key},
object_checksums={second_key: second_checksum},
)
storage.object_sizes[second_key] = second_path.stat().st_size
result = sync_registered_files_to_durable_storage(db, storage=storage)
db.refresh(first_record)
db.refresh(second_record)
assert result.scanned == 2
assert result.failed == 0
assert result.already_present == 2
assert result.uploaded == 1
assert storage.put_calls == [(first_path, first_key)]
assert first_record.storage_status == "available"
assert first_record.checksum == "checksum"
assert second_record.storage_status == "available"
assert second_record.checksum == second_checksum
def test_sync_reuploads_available_row_when_remote_key_is_missing(tmp_path):
db = _session()
user = _user(db)
local_path = tmp_path / "uploads" / "input.txt"
local_path.parent.mkdir()
local_path.write_text("content", encoding="utf-8")
key = f"users/{int(user.id)}/uploads/file-123/input.txt"
record = _record(
db,
user=user,
local_path=local_path,
file_id="file-123",
storage_backend="s3",
storage_key=key,
storage_uri=f"s3://bucket/{key}",
storage_status="available",
checksum="checksum",
)
db.commit()
storage = FakeStorage()
result = sync_registered_files_to_durable_storage(db, storage=storage)
db.refresh(record)
assert result.uploaded == 1
assert storage.put_calls == [(local_path, key)]
assert record.storage_status == "available"
assert record.etag == "etag"
def test_sync_skips_missing_local_file(tmp_path):
db = _session()
user = _user(db)
local_path = tmp_path / "uploads" / "missing.txt"
record = _record(db, user=user, local_path=local_path, file_id="file-missing")
db.commit()
storage = FakeStorage()
result = sync_registered_files_to_durable_storage(db, storage=storage)
db.refresh(record)
assert result.scanned == 1
assert result.uploaded == 0
assert result.skipped_missing_local == 1
assert storage.put_calls == []
assert record.storage_status == "legacy"
def test_sync_lists_once_per_user_for_many_files(tmp_path):
db = _session()
user = _user(db)
for index in range(3):
local_path = tmp_path / "uploads" / f"input-{index}.txt"
local_path.parent.mkdir(exist_ok=True)
local_path.write_text(f"content {index}", encoding="utf-8")
_record(db, user=user, local_path=local_path, file_id=f"file-{index}")
db.commit()
storage = FakeStorage()
result = sync_registered_files_to_durable_storage(db, storage=storage)
assert result.scanned == 3
assert result.uploaded == 3
assert storage.list_calls == [f"users/{int(user.id)}"]
def test_sync_skips_non_s3_backend(tmp_path):
db = _session()
user = _user(db)
local_path = tmp_path / "uploads" / "input.txt"
local_path.parent.mkdir()
local_path.write_text("content", encoding="utf-8")
_record(db, user=user, local_path=local_path, file_id="file-123")
db.commit()
storage = FakeStorage(backend="file")
result = sync_registered_files_to_durable_storage(db, storage=storage)
assert result.scanned == 0
assert result.uploaded == 0
assert result.skipped_backend == 1
assert storage.list_calls == []
assert storage.put_calls == []
def test_sync_continues_after_single_file_upload_failure(tmp_path):
db = _session()
user = _user(db)
first_path = tmp_path / "uploads" / "first.txt"
second_path = tmp_path / "uploads" / "second.txt"
first_path.parent.mkdir()
first_path.write_text("first", encoding="utf-8")
second_path.write_text("second", encoding="utf-8")
first_record = _record(db, user=user, local_path=first_path, file_id="file-first")
second_record = _record(
db, user=user, local_path=second_path, file_id="file-second"
)
db.commit()
failed_key = f"users/{int(user.id)}/uploads/file-first/first.txt"
second_key = f"users/{int(user.id)}/uploads/file-second/second.txt"
storage = FailingOnceStorage(failed_key)
result = sync_registered_files_to_durable_storage(db, storage=storage)
db.refresh(first_record)
db.refresh(second_record)
assert result.scanned == 2
assert result.failed == 1
assert result.uploaded == 1
assert storage.put_calls == [(first_path, failed_key), (second_path, second_key)]
assert first_record.storage_status == "legacy"
assert second_record.storage_status == "available"
assert second_record.storage_key == second_key
def test_startup_sync_file_lock_prevents_second_holder(monkeypatch, tmp_path):
monkeypatch.setenv(
"XAGENT_FILE_STORAGE_STARTUP_SYNC_LOCK_FILE", str(tmp_path / "sync.lock")
)
first_lock = _acquire_file_lock()
assert first_lock is not None
try:
assert _acquire_file_lock() is None
finally:
_release_file_lock(first_lock)
def test_startup_sync_file_lock_can_be_reacquired(monkeypatch, tmp_path):
monkeypatch.setenv(
"XAGENT_FILE_STORAGE_STARTUP_SYNC_LOCK_FILE", str(tmp_path / "sync.lock")
)
first_lock = _acquire_file_lock()
assert first_lock is not None
_release_file_lock(first_lock)
second_lock = _acquire_file_lock()
assert second_lock is not None
_release_file_lock(second_lock)
def test_sync_waits_for_file_lock_then_runs_idempotent_pass(monkeypatch, tmp_path):
db = _session()
user = _user(db)
local_path = tmp_path / "uploads" / "input.txt"
local_path.parent.mkdir()
local_path.write_text("content", encoding="utf-8")
record = _record(db, user=user, local_path=local_path, file_id="file-123")
db.commit()
storage = FakeStorage()
import xagent.web.services.startup_file_storage_sync as sync_module
lock = object()
attempts = 0
waits: list[None] = []
def acquire_after_contention():
nonlocal attempts
attempts += 1
if attempts == 1:
return None
return lock
monkeypatch.setattr(sync_module, "_acquire_file_lock", acquire_after_contention)
monkeypatch.setattr(sync_module, "_release_file_lock", lambda acquired: None)
monkeypatch.setattr(
sync_module,
"_wait_for_lock_holder",
lambda: waits.append(None),
raising=False,
)
result = sync_registered_files_to_durable_storage(db, storage=storage)
db.refresh(record)
expected_key = f"users/{int(user.id)}/uploads/file-123/input.txt"
assert attempts == 2
assert waits == [None]
assert result.locked is False
assert result.scanned == 1
assert result.uploaded == 1
assert storage.put_calls == [(local_path, expected_key)]
assert record.storage_status == "available"
def test_sync_reports_the_provider_fault_behind_an_adoption_failure(tmp_path, caplog):
"""An adoption failure must arrive classified, not just as a traceback.
``adopt_existing_object``'s stat and content-hash calls are two of the raise
sites #1467 asked to cover, and this sync is their only caller. The bare
``except Exception`` here already produced a traceback via
``logger.exception``, so the cause was never lost -- what was missing is the
classified provider fields, which is what makes a burst of these
aggregatable by cause rather than readable one at a time.
"""
import logging
db = _session()
user = _user(db)
local_path = tmp_path / "uploads" / "input.txt"
local_path.parent.mkdir()
local_path.write_text("content", encoding="utf-8")
key = f"users/{int(user.id)}/uploads/file-123/input.txt"
_record(
db,
user=user,
local_path=local_path,
file_id="file-123",
storage_key=key,
storage_status="legacy",
)
db.commit()
class ThrottledStorage(FakeStorage):
def stat(self, stat_key):
# Shaped like the real chain: botocore ClientError under s3fs's
# translation under the wrap ManagedFileRef applies.
class ClientError(Exception):
def __init__(self) -> None:
super().__init__("An error occurred (SlowDown)")
self.response = {
"Error": {"Code": "SlowDown"},
"ResponseMetadata": {"HTTPStatusCode": 503},
}
raise OSError(16, "SlowDown occurred") from ClientError()
storage = ThrottledStorage({key})
logger_name = "xagent.web.services.startup_file_storage_sync"
with caplog.at_level(logging.WARNING, logger=logger_name):
result = sync_registered_files_to_durable_storage(db, storage=storage)
assert result.failed == 1
records = [
logging.Formatter("%(message)s").format(entry)
for entry in caplog.records
if entry.name == logger_name and entry.levelno == logging.WARNING
]
assert len(records) == 1, caplog.records
rendered = records[0]
assert "during startup durable adoption" in rendered
assert "file_id=file-123" in rendered
assert f"storage_key={key}" in rendered
# The code recovered from below the errno, not the EBUSY that masked it.
assert "provider_code=SlowDown" in rendered
assert "provider_http_status=503" in rendered