Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

26.2 KB
Raw Blame
"""The startup file-storage sync gate must never hold a host's control paths.

Covers the 2026-11 incident: a prewarmed standby's activation started the
sync, the gate held the host's lifecycle probe behind it, and a sync that stood
still left the host blind. Also covers the read-only standby preflight that
keeps a fully-synced standby from gating at all, and the sync's own bounds.
"""

import asyncio
import threading
import time
from importlib import import_module
from pathlib import Path
from unittest.mock import Mock

import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

from xagent.core.file_storage.types import StoredObject
from xagent.web.models.database import Base
from xagent.web.models.uploaded_file import UploadedFile
from xagent.web.models.user import User
from xagent.web.services import startup_file_storage_sync as sync_module
from xagent.web.services.startup_file_storage_sync import (
    StartupFileStoragePreflight,
    StartupSyncCancelled,
    StartupSyncControl,
    preflight_registered_files,
)

app_module = import_module("xagent.web.app")


@pytest.fixture(autouse=True)
def _isolated_prefixes(monkeypatch):
    monkeypatch.setattr(app_module, "_file_storage_startup_sync_exempt_prefixes", set())
    monkeypatch.setattr(
        app_module, "get_file_storage_startup_sync_enabled", lambda: True
    )


def _app_with_stuck_sync(*, finish_after: float | None = None) -> FastAPI:
    test_app = FastAPI()
    test_app.add_middleware(app_module.FileStorageStartupSyncGateMiddleware)

    @test_app.on_event("startup")
    async def _startup() -> None:
        async def _sync() -> None:
            await asyncio.sleep(finish_after if finish_after is not None else 3600)

        test_app.state.file_storage_startup_sync_task = asyncio.create_task(_sync())

    @test_app.get("/internal/thin-gate/lifecycle")
    async def _lifecycle() -> dict[str, str]:
        return {"status": "ok"}

    @test_app.get("/api/client")
    async def _client() -> dict[str, str]:
        return {"status": "served"}

    return test_app


# --- M1: host-registered exemptions ------------------------------------------


def test_registered_prefix_answers_while_sync_is_stuck():
    app_module.register_file_storage_startup_sync_exempt_prefix("/internal/thin-gate/")
    with TestClient(_app_with_stuck_sync()) as client:
        started = time.monotonic()
        response = client.get("/internal/thin-gate/lifecycle")
    assert response.status_code == 200
    assert time.monotonic() - started < 1.0


def test_registered_prefix_answers_after_sync_failed():
    app_module.register_file_storage_startup_sync_exempt_prefix("/internal/thin-gate/")
    test_app = _app_with_stuck_sync()
    with TestClient(test_app) as client:
        test_app.state.file_storage_startup_sync_error = RuntimeError("s3 down")
        assert client.get("/internal/thin-gate/lifecycle").status_code == 200
        assert client.get("/api/client").status_code == 503


def test_unregistered_path_is_still_held_until_sync_completes():
    app_module.register_file_storage_startup_sync_exempt_prefix("/internal/thin-gate/")
    with TestClient(_app_with_stuck_sync(finish_after=0.3)) as client:
        started = time.monotonic()
        response = client.get("/api/client")
    assert response.status_code == 200
    assert time.monotonic() - started >= 0.25


def test_without_registration_lifecycle_is_held():
    """Pins the incident: the default gate holds every non-exempt path."""
    with TestClient(_app_with_stuck_sync(finish_after=0.3)) as client:
        started = time.monotonic()
        assert client.get("/internal/thin-gate/lifecycle").status_code == 200
        assert time.monotonic() - started >= 0.25


def test_prefix_must_be_absolute():
    with pytest.raises(ValueError):
        app_module.register_file_storage_startup_sync_exempt_prefix("internal/")


def test_prefix_match_is_a_prefix_not_a_substring():
    app_module.register_file_storage_startup_sync_exempt_prefix("/internal/thin-gate/")
    assert app_module._is_file_storage_startup_sync_exempt("/internal/thin-gate/x")
    assert not app_module._is_file_storage_startup_sync_exempt(
        "/api/internal/thin-gate/x"
    )
    assert not app_module._is_file_storage_startup_sync_exempt("/internal/thin-gates")


# --- status accessor ----------------------------------------------------------


@pytest.mark.asyncio
async def test_status_reports_running_failed_done_with_elapsed(monkeypatch):
    test_app = FastAPI()
    assert (
        app_module.get_file_storage_startup_sync_status(test_app)["state"] == "pending"
    )

    release = asyncio.Event()

    async def _attempt(*_args) -> None:
        await release.wait()

    monkeypatch.setattr(app_module, "_run_bounded_file_storage_sync_attempt", _attempt)
    task = app_module.start_file_storage_startup_sync_task(test_app)
    await asyncio.sleep(0.05)
    running = app_module.get_file_storage_startup_sync_status(test_app)
    assert running["state"] == "running"
    assert running["gating"] is True
    assert running["elapsed_seconds"] >= 0.04

    test_app.state.file_storage_startup_sync_error = RuntimeError("boom")
    assert (
        app_module.get_file_storage_startup_sync_status(test_app)["state"] == "failed"
    )
    test_app.state.file_storage_startup_sync_error = None

    release.set()
    await task
    done = app_module.get_file_storage_startup_sync_status(test_app)
    assert done["state"] == "done"
    assert done["elapsed_seconds"] is not None


def test_status_disabled(monkeypatch):
    monkeypatch.setattr(
        app_module, "get_file_storage_startup_sync_enabled", lambda: False
    )
    assert (
        app_module.get_file_storage_startup_sync_status(FastAPI())["state"]
        == "disabled"
    )


# --- S2: preflight and non-gating background sync ------------------------------


class _Storage:
    backend = "s3"

    def __init__(self, keys=()):
        self.keys = set(keys)
        self.put_calls = []

    def list(self, prefix):
        return [
            StoredObject(backend="s3", key=k, uri=f"s3://b/{k}", size=1)
            for k in sorted(self.keys)
            if k.startswith(prefix.rstrip("/") + "/")
        ]

    def put_file(self, *a, **k):  # pragma: no cover - must never run
        self.put_calls.append(a)
        raise AssertionError("preflight must not write storage")


def _db(tmp_path):
    engine = create_engine("sqlite:///:memory:")
    Base.metadata.create_all(engine)
    factory = sessionmaker(bind=engine)
    with factory() as db:
        user = User(username="u", password_hash="h", is_admin=False)
        db.add(user)
        db.commit()
        return factory, int(user.id)


def _row(factory, user_id, path: Path, file_id, **kw):
    with factory() as db:
        db.add(
            UploadedFile(
                file_id=file_id,
                user_id=user_id,
                filename=path.name,
                storage_path=str(path),
                mime_type="text/plain",
                file_size=0,
                storage_status=kw.pop("storage_status", "legacy"),
                **kw,
            )
        )
        db.commit()


def _durable_row(factory, user_id, path, file_id):
    key = f"users/{user_id}/uploads/{file_id}/{path.name}"
    _row(
        factory,
        user_id,
        path,
        file_id,
        storage_backend="s3",
        storage_key=key,
        storage_status="available",
        checksum="c",
    )
    return key


def test_preflight_clean_when_every_file_is_already_durable(tmp_path):
    factory, uid = _db(tmp_path)
    local = tmp_path / "a.txt"
    local.write_text("x")
    key = _durable_row(factory, uid, local, "f1")
    gone = tmp_path / "gone.txt"  # missing locally and remotely: nothing to write
    _row(factory, uid, gone, "f2")
    storage = _Storage({key})

    result = preflight_registered_files(storage=storage, session_factory=factory)

    assert result.clean
    assert result.scanned == 2
    assert result.skipped_missing_local == 1
    assert storage.put_calls == []


def test_preflight_reports_file_missing_remote_but_present_locally(tmp_path):
    factory, uid = _db(tmp_path)
    local = tmp_path / "a.txt"
    local.write_text("x")
    _row(factory, uid, local, "f1")
    result = preflight_registered_files(storage=_Storage(), session_factory=factory)
    assert not result.clean and result.needs_write == 1


def test_preflight_reports_remote_present_but_row_incomplete(tmp_path):
    factory, uid = _db(tmp_path)
    local = tmp_path / "a.txt"
    local.write_text("x")
    _row(factory, uid, local, "f1")
    key = f"users/{uid}/uploads/f1/a.txt"
    result = preflight_registered_files(
        storage=_Storage({key}), session_factory=factory
    )
    assert result.needs_write == 1


def test_preflight_writes_nothing_to_the_database(tmp_path):
    factory, uid = _db(tmp_path)
    local = tmp_path / "a.txt"
    local.write_text("x")
    _row(factory, uid, local, "f1")
    key = f"users/{uid}/uploads/f1/a.txt"
    preflight_registered_files(storage=_Storage({key}), session_factory=factory)
    with factory() as db:
        row = db.query(UploadedFile).one()
        assert row.storage_status == "legacy" and row.storage_key is None


def test_preflight_honours_cancel(tmp_path):
    factory, uid = _db(tmp_path)
    local = tmp_path / "a.txt"
    local.write_text("x")
    _row(factory, uid, local, "f1")
    control = StartupSyncControl()
    control.cancel.set()
    with pytest.raises(StartupSyncCancelled):
        preflight_registered_files(
            storage=_Storage(), session_factory=factory, control=control
        )


@pytest.mark.parametrize(
    ("task", "result", "expected_gate"),
    [
        (None, None, True),  # normal start: never ran a preflight
        (object(), None, True),  # preflight unfinished or failed
        (object(), StartupFileStoragePreflight(scanned=3, needs_write=1), True),
        (object(), StartupFileStoragePreflight(scanned=3, needs_write=0), False),
    ],
)
def test_should_gate_only_when_preflight_proved_nothing_to_write(
    task, result, expected_gate
):
    test_app = FastAPI()
    test_app.state.file_storage_preflight_task = task
    test_app.state.file_storage_preflight_result = result
    assert app_module._file_storage_sync_should_gate(test_app) is expected_gate


@pytest.mark.asyncio
async def test_clean_standby_activation_never_gates_requests(monkeypatch):
    """Activation with a clean preflight: sync runs in background, gate open."""
    test_app = FastAPI()
    test_app.state.file_storage_preflight_task = object()
    test_app.state.file_storage_preflight_result = StartupFileStoragePreflight(
        scanned=9
    )
    release = asyncio.Event()

    async def _attempt(*_args) -> None:
        await release.wait()

    monkeypatch.setattr(app_module, "_run_bounded_file_storage_sync_attempt", _attempt)
    monkeypatch.setattr(app_module, "get_shared_task_execution_enabled", lambda: True)
    for name in (
        "start_uploaded_file_recovery_task",
        "start_orphan_upload_gc_task",
        "start_retention_purge_task",
        "start_task_cleanup_retry_task",
    ):
        monkeypatch.setattr(app_module, name, Mock())

    await app_module._start_local_runtime_loops(test_app)
    try:
        assert test_app.state.file_storage_startup_sync_gating is False
        # Sync still running, yet the wait returns at once and /ready is not 503.
        await asyncio.wait_for(
            app_module.wait_for_file_storage_startup_sync(test_app), timeout=0.5
        )
        assert not test_app.state.file_storage_startup_sync_task.done()
    finally:
        release.set()
        await test_app.state.file_storage_startup_sync_task


@pytest.mark.asyncio
async def test_standby_with_files_to_write_still_gates(monkeypatch):
    test_app = FastAPI()
    test_app.state.file_storage_preflight_task = object()
    test_app.state.file_storage_preflight_result = StartupFileStoragePreflight(
        scanned=9, needs_write=2
    )
    release = asyncio.Event()

    async def _attempt(*_args) -> None:
        await release.wait()

    monkeypatch.setattr(app_module, "_run_bounded_file_storage_sync_attempt", _attempt)
    monkeypatch.setattr(app_module, "get_shared_task_execution_enabled", lambda: True)
    for name in (
        "start_uploaded_file_recovery_task",
        "start_orphan_upload_gc_task",
        "start_retention_purge_task",
        "start_task_cleanup_retry_task",
    ):
        monkeypatch.setattr(app_module, name, Mock())

    await app_module._start_local_runtime_loops(test_app)
    try:
        assert test_app.state.file_storage_startup_sync_gating is True
        with pytest.raises(asyncio.TimeoutError):
            await asyncio.wait_for(
                app_module.wait_for_file_storage_startup_sync(test_app), timeout=0.4
            )
    finally:
        release.set()
        await test_app.state.file_storage_startup_sync_task


def test_non_gating_sync_does_not_fail_requests_when_it_errors():
    test_app = FastAPI()
    test_app.state.file_storage_startup_sync_gating = False
    test_app.state.file_storage_startup_sync_error = RuntimeError("s3 down")
    asyncio.run(app_module.wait_for_file_storage_startup_sync(test_app))


# --- S1: bounded sync -----------------------------------------------------------


@pytest.mark.asyncio
async def test_attempt_logs_progress_dumps_stacks_and_cancels_when_stalled(monkeypatch):
    monkeypatch.setattr(
        app_module, "FILE_STORAGE_STARTUP_SYNC_PROGRESS_INTERVAL_SECONDS", 0.05
    )
    monkeypatch.setattr(app_module, "FILE_STORAGE_STARTUP_SYNC_DUMP_AFTER_SECONDS", 0.1)
    monkeypatch.setattr(app_module, "FILE_STORAGE_STARTUP_SYNC_STALL_SECONDS", 0.3)
    monkeypatch.setattr(
        app_module, "FILE_STORAGE_STARTUP_SYNC_CANCEL_GRACE_SECONDS", 2.0
    )
    dump = Mock()
    monkeypatch.setattr(app_module, "_dump_all_thread_stacks", dump)
    saw_cancel = threading.Event()

    def _stuck_sync() -> None:
        control = app_module._file_storage_sync_control.get()
        while True:
            try:
                control.checkpoint("stuck", progress=False)
            except StartupSyncCancelled:
                saw_cancel.set()
                raise
            time.sleep(0.01)

    monkeypatch.setattr(app_module, "run_startup_file_storage_sync", _stuck_sync)

    with pytest.raises(TimeoutError, match="no progress"):
        await app_module._run_bounded_file_storage_sync_attempt()

    dump.assert_called_once()
    assert saw_cancel.wait(1.0)


@pytest.mark.asyncio
async def test_attempt_runs_on_dedicated_executor_not_the_default_one(monkeypatch):
    names: list[str] = []
    monkeypatch.setattr(
        app_module,
        "run_startup_file_storage_sync",
        lambda: names.append(threading.current_thread().name),
    )
    await app_module._run_bounded_file_storage_sync_attempt()
    assert names and names[0].startswith("startup-file-sync")


@pytest.mark.asyncio
async def test_saturated_default_executor_does_not_stall_the_sync(monkeypatch):
    loop = asyncio.get_running_loop()
    gate = threading.Event()
    from concurrent.futures import ThreadPoolExecutor

    small = ThreadPoolExecutor(max_workers=1)
    loop.set_default_executor(small)
    blocker = loop.run_in_executor(None, gate.wait)  # default pool now full
    ran = threading.Event()
    monkeypatch.setattr(app_module, "run_startup_file_storage_sync", ran.set)
    try:
        await asyncio.wait_for(app_module._run_bounded_file_storage_sync_attempt(), 2.0)
        assert ran.is_set()
    finally:
        gate.set()
        await blocker
        small.shutdown(wait=True)


def test_locked_result_from_an_earlier_attempt_is_not_success(monkeypatch):
    monkeypatch.setattr(
        sync_module,
        "sync_registered_files_to_durable_storage",
        Mock(return_value=sync_module.StartupFileStorageSyncResult(locked=True)),
    )
    token = app_module._file_storage_sync_control.set(StartupSyncControl())
    try:
        with pytest.raises(RuntimeError, match="previous attempt"):
            app_module.run_startup_file_storage_sync()
    finally:
        app_module._file_storage_sync_control.reset(token)


def test_stuck_sync_does_not_stop_lifecycle_from_answering(monkeypatch):
    """End to end through the real middleware with the real retry loop."""
    app_module.register_file_storage_startup_sync_exempt_prefix("/internal/thin-gate/")
    test_app = FastAPI()
    test_app.add_middleware(app_module.FileStorageStartupSyncGateMiddleware)
    release = threading.Event()
    monkeypatch.setattr(
        app_module, "run_startup_file_storage_sync", lambda: release.wait(30)
    )

    @test_app.on_event("startup")
    async def _startup() -> None:
        app_module.start_file_storage_startup_sync_task(test_app)

    @test_app.get("/internal/thin-gate/lifecycle")
    async def _lifecycle() -> dict:
        return app_module.get_file_storage_startup_sync_status(test_app)

    @test_app.get("/api/client")
    async def _client() -> dict:
        return {}

    try:
        with TestClient(test_app) as client:
            started = time.monotonic()
            body = client.get("/internal/thin-gate/lifecycle").json()
            assert time.monotonic() - started < 1.0
            assert body["state"] == "running"
            release.set()
            assert client.get("/api/client").status_code == 200
    finally:
        release.set()


@pytest.mark.asyncio
async def test_ready_is_not_held_by_a_non_gating_sync(monkeypatch):
    import xagent.web.models.database as database_module
    import xagent.web.services.interaction_rollout as ir

    monkeypatch.setattr(
        ir,
        "_policy",
        ir.InteractionRolloutPolicy(
            mode="legacy",
            native_sources=frozenset(),
            native_protocol_version=1,
            shared_task_execution=True,
        ),
    )
    monkeypatch.setattr(database_module, "_SessionLocal", None)
    state = app_module.app.state
    task = asyncio.create_task(asyncio.sleep(30))
    monkeypatch.setattr(state, "file_storage_startup_sync_task", task, raising=False)
    monkeypatch.setattr(state, "file_storage_startup_sync_error", None, raising=False)
    try:
        monkeypatch.setattr(
            state, "file_storage_startup_sync_gating", True, raising=False
        )
        assert (await app_module.readiness_check()).status_code == 503
        monkeypatch.setattr(
            state, "file_storage_startup_sync_gating", False, raising=False
        )
        assert (await app_module.readiness_check()).status_code == 200
    finally:
        task.cancel()


# --- review: stall (not total) timeout, resume, cancel before storage I/O --------


class _SlowListStorage(_Storage):
    """Every per-user listing succeeds but takes a while (listings cache off)."""

    def __init__(self, keys=(), delay=0.1, on_list=None):
        super().__init__(keys)
        self.delay = delay
        self.list_prefixes: list[str] = []
        self.on_list = on_list

    def list(self, prefix):
        self.list_prefixes.append(prefix)
        if self.on_list is not None:
            self.on_list(len(self.list_prefixes))
        time.sleep(self.delay)
        return super().list(prefix)


def _many_user_db(tmp_path, users=8):
    # A file database: the sync runs on executor threads.
    engine = create_engine(f"sqlite:///{tmp_path}/sync.sqlite")
    Base.metadata.create_all(engine)
    factory = sessionmaker(bind=engine)
    keys = set()
    ids = []
    with factory() as db:
        for n in range(users):
            user = User(username=f"u{n}", password_hash="h", is_admin=False)
            db.add(user)
            db.commit()
            ids.append(int(user.id))
    for uid in ids:
        path = tmp_path / f"f{uid}.txt"
        path.write_text("x")
        keys.add(_durable_row(factory, uid, path, f"file{uid}"))
    return factory, keys


@pytest.mark.asyncio
async def test_steadily_progressing_scan_longer_than_the_stall_window_completes(
    monkeypatch, tmp_path
):
    """Regression: a total deadline restarted a healthy long scan forever."""
    factory, keys = _many_user_db(tmp_path, users=8)
    storage = _SlowListStorage(keys, delay=0.1)  # ~0.8 s in total
    monkeypatch.setattr(
        app_module, "FILE_STORAGE_STARTUP_SYNC_PROGRESS_INTERVAL_SECONDS", 0.05
    )
    monkeypatch.setattr(app_module, "FILE_STORAGE_STARTUP_SYNC_STALL_SECONDS", 0.3)
    monkeypatch.setattr(app_module, "FILE_STORAGE_STARTUP_SYNC_DUMP_AFTER_SECONDS", 60)
    monkeypatch.setenv(
        "XAGENT_FILE_STORAGE_STARTUP_SYNC_LOCK_FILE", str(tmp_path / "l.lock")
    )
    real = sync_module.sync_registered_files_to_durable_storage
    monkeypatch.setattr(
        sync_module,
        "sync_registered_files_to_durable_storage",
        lambda **kw: real(storage=storage, session_factory=factory, **kw),
    )
    test_app = FastAPI()
    task = app_module.start_file_storage_startup_sync_task(
        test_app, retry_interval_seconds=0
    )
    await asyncio.wait_for(task, 10)

    assert test_app.state.file_storage_startup_sync_completed is True
    assert test_app.state.file_storage_startup_sync_error is None
    # one pass: no restarts, each user listed exactly once
    assert len(storage.list_prefixes) == 8
    await asyncio.wait_for(app_module.wait_for_file_storage_startup_sync(test_app), 1)


@pytest.mark.asyncio
async def test_retry_after_a_stall_resumes_instead_of_restarting(monkeypatch, tmp_path):
    factory, keys = _many_user_db(tmp_path, users=6)
    stall_once = {"done": False}

    def on_list(call_number):
        if call_number == 4 and not stall_once["done"]:
            stall_once["done"] = True
            time.sleep(0.6)  # a real stall inside the 4th listing

    storage = _SlowListStorage(keys, delay=0.01, on_list=on_list)
    monkeypatch.setattr(
        app_module, "FILE_STORAGE_STARTUP_SYNC_PROGRESS_INTERVAL_SECONDS", 0.05
    )
    monkeypatch.setattr(app_module, "FILE_STORAGE_STARTUP_SYNC_STALL_SECONDS", 0.3)
    monkeypatch.setattr(
        app_module, "FILE_STORAGE_STARTUP_SYNC_CANCEL_GRACE_SECONDS", 0.05
    )
    monkeypatch.setattr(app_module, "_dump_all_thread_stacks", Mock())
    monkeypatch.setenv(
        "XAGENT_FILE_STORAGE_STARTUP_SYNC_LOCK_FILE", str(tmp_path / "l.lock")
    )
    real = sync_module.sync_registered_files_to_durable_storage
    monkeypatch.setattr(
        sync_module,
        "sync_registered_files_to_durable_storage",
        lambda **kw: real(storage=storage, session_factory=factory, **kw),
    )
    test_app = FastAPI()
    task = app_module.start_file_storage_startup_sync_task(
        test_app, retry_interval_seconds=0.7
    )
    await asyncio.wait_for(task, 15)

    assert test_app.state.file_storage_startup_sync_completed is True
    # The stalled attempt finished user 4 before it saw the cancel, and the
    # retry continued after it: every user is listed exactly once (a restart
    # from the beginning would list users 1-3 again).
    assert sorted(storage.list_prefixes) == sorted(set(storage.list_prefixes))
    assert len(storage.list_prefixes) == 6
    # The cancelled attempt's thread is still counted until it returns.
    assert await app_module.drain_workers(10)


@pytest.mark.asyncio
async def test_a_real_stall_still_trips_the_timeout_and_dumps_stacks(monkeypatch):
    monkeypatch.setattr(
        app_module, "FILE_STORAGE_STARTUP_SYNC_PROGRESS_INTERVAL_SECONDS", 0.05
    )
    monkeypatch.setattr(app_module, "FILE_STORAGE_STARTUP_SYNC_DUMP_AFTER_SECONDS", 0.1)
    monkeypatch.setattr(app_module, "FILE_STORAGE_STARTUP_SYNC_STALL_SECONDS", 0.3)
    monkeypatch.setattr(
        app_module, "FILE_STORAGE_STARTUP_SYNC_CANCEL_GRACE_SECONDS", 0.05
    )
    dump = Mock()
    monkeypatch.setattr(app_module, "_dump_all_thread_stacks", dump)
    release = threading.Event()
    monkeypatch.setattr(
        app_module, "run_startup_file_storage_sync", lambda: release.wait(10)
    )
    try:
        with pytest.raises(TimeoutError, match="no progress"):
            await app_module._run_bounded_file_storage_sync_attempt()
        dump.assert_called_once()
    finally:
        release.set()
        # The stalled worker keeps its count until the thread returns; let it
        # finish so the fixture's leak check does not race its last instructions.
        assert await app_module.drain_workers(10)


def test_cancel_during_the_first_listing_stops_before_the_next_users_listing(tmp_path):
    factory, keys = _many_user_db(tmp_path, users=3)
    control = StartupSyncControl()
    storage = _SlowListStorage(keys, delay=0, on_list=lambda n: control.cancel.set())
    with pytest.raises(StartupSyncCancelled):
        preflight_registered_files(
            storage=storage, session_factory=factory, control=control
        )
    assert len(storage.list_prefixes) == 1


def test_cancel_during_the_first_listing_stops_the_sync_too(tmp_path, monkeypatch):
    factory, keys = _many_user_db(tmp_path, users=3)
    control = StartupSyncControl()
    storage = _SlowListStorage(keys, delay=0, on_list=lambda n: control.cancel.set())
    monkeypatch.setenv(
        "XAGENT_FILE_STORAGE_STARTUP_SYNC_LOCK_FILE", str(tmp_path / "l.lock")
    )
    with pytest.raises(StartupSyncCancelled):
        sync_module.sync_registered_files_to_durable_storage(
            storage=storage, session_factory=factory, control=control
        )
    assert len(storage.list_prefixes) == 1


def test_resume_cursor_never_moves_past_a_failed_file(tmp_path, monkeypatch):
    factory, keys = _many_user_db(tmp_path, users=3)
    with factory() as db:  # user 1's row is not durable and its upload fails
        row = db.query(UploadedFile).order_by(UploadedFile.id).first()
        row.storage_status = "legacy"
        row.storage_key = None
        row.checksum = None
        row.storage_backend = None
        db.commit()
        # Read the identity while the session is open: the row detaches on exit.
        first_user_key = (
            f"users/{row.user_id}/uploads/{row.file_id}/{Path(row.storage_path).name}"
        )
    storage = _SlowListStorage(keys - {first_user_key}, delay=0)

    def failing_put(*args, **kwargs):
        raise RuntimeError("s3 down")

    storage.put_file = failing_put
    monkeypatch.setenv(
        "XAGENT_FILE_STORAGE_STARTUP_SYNC_LOCK_FILE", str(tmp_path / "l.lock")
    )
    control = StartupSyncControl()

    result = sync_module.sync_registered_files_to_durable_storage(
        storage=storage, session_factory=factory, control=control
    )

    assert result.failed == 1
    assert control.resume.cursor is None  # a retry starts at the failed file