Skip to content

Automate browser based workflows with AI

61.3 KB
Raw Blame
"""Phase 2 (SKY-15288) incremental same-key S3 compose — protocol & lifecycle tests.

A fake async S3 client models the exact MPU surface the compose path drives (create with Metadata /
upload_part_copy with CopySourceIfMatch fencing / upload_part / complete / abort / head with Metadata /
put / upload_fileobj), so these assert the product lifecycle logic — atomic same-key visibility, stale-ETag
conflict preservation, generation-token reconciliation, single-slot fallback ordering, and fd ownership —
without real S3. The real-S3 protocol probe (temp bucket) validates the wire protocol separately.
"""

from __future__ import annotations

import asyncio
import contextlib
import gc
import os
import tempfile
from datetime import UTC, datetime
from typing import Any
from unittest.mock import AsyncMock, MagicMock

import pytest
from botocore.exceptions import ClientError

from skyvern.config import settings
from skyvern.forge.sdk.api.aws import (
    _MAX_COMPOSE_PART_BYTES,
    AsyncAWSClient,
    _ComposeState,
    _ComposeStateCache,
    _recording_content_type,
)
from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType
from skyvern.forge.sdk.artifact.storage.s3 import S3Storage

MIB = 1024 * 1024
KEY = "v1/prod/org/task/rec.webm"
URI = f"s3://bucket/{KEY}"
KEY_MP4 = "v1/prod/org/task/rec.mp4"
URI_MP4 = f"s3://bucket/{KEY_MP4}"
KEY_XYZ = "v1/prod/org/task/rec.skyvernunknown15466"  # project-unique ext, guess_type -> None on all platforms
URI_XYZ = f"s3://bucket/{KEY_XYZ}"
GEN = "skyvern-compose-gen"


def _client_error(code: str, status: int = 400) -> ClientError:
    return ClientError({"Error": {"Code": code}, "ResponseMetadata": {"HTTPStatusCode": status}}, "op")


class FakeS3:
    def __init__(self) -> None:
        # key -> (bytes, etag, metadata)
        self.objects: dict[str, tuple[bytes, str, dict[str, str]]] = {}
        self.mpus: dict[str, dict[str, Any]] = {}
        self.aborted: list[str] = []
        self.abort_attempts: list[str] = []  # every abort attempt (incl. failed), in order
        self.abort_transient_failures = 0  # first N abort calls raise a transient (non-expired) error, then work
        self.ops: list[str] = []  # ordered op log (cleanup-before-replacement ordering assertions)
        self.completed: list[str] = []
        self.content_types: dict[str, str] = {}  # key -> ContentType seen on the write that authored it
        self._n = 0
        self.fail: dict[str, Exception] = {}  # op -> exception to raise once (ClientError OR transport error)
        self.on_complete: Any = None  # optional hook(key, uploadid) to simulate ambiguity/foreign writes
        self.on_before_complete: Any = None  # hook(key, uploadid) firing AFTER copy, BEFORE the IfMatch check
        self.complete_omit_etag = False  # model a successful Complete whose response omits ETag
        self.head_malformed = False  # model a HEAD 200 whose body omits ContentLength/ETag

    def _fail(self, op: str) -> None:
        err = self.fail.pop(op, None)
        if err is not None:
            raise err

    async def put_object(self, *, Bucket: str, Key: str, Body: Any, **kw: Any) -> dict[str, Any]:
        self.ops.append("put_object")
        self._fail("put_object")
        data = Body if isinstance(Body, bytes) else Body.read()
        self._n += 1
        etag = f'"put-{self._n}"'
        self.objects[Key] = (data, etag, {})
        if kw.get("ContentType"):
            self.content_types[Key] = kw["ContentType"]
        return {"ETag": etag}

    async def upload_fileobj(
        self, Fileobj: Any, Bucket: str, Key: str, ExtraArgs: dict | None = None, Config: Any = None
    ) -> None:
        self.ops.append("upload_fileobj")
        self._fail("upload_fileobj")
        chunks = []
        while True:
            c = Fileobj.read(1 << 20)
            if not c:
                break
            chunks.append(c)
        self._n += 1
        meta = dict((ExtraArgs or {}).get("Metadata") or {})  # real S3 preserves user metadata on put/multipart
        self.objects[Key] = (b"".join(chunks), f'"stream-{self._n}"', meta)
        if (ExtraArgs or {}).get("ContentType"):
            self.content_types[Key] = ExtraArgs["ContentType"]

    async def create_multipart_upload(
        self, *, Bucket: str, Key: str, Metadata: dict | None = None, **kw: Any
    ) -> dict[str, Any]:
        self.ops.append("create")
        self._fail("create")
        self._n += 1
        uid = f"mpu-{self._n}"
        self.mpus[uid] = {"key": Key, "parts": {}, "meta": dict(Metadata or {})}
        if kw.get("ContentType"):  # S3 carries the MPU ContentType onto the completed object
            self.content_types[Key] = kw["ContentType"]
        return {"UploadId": uid}

    async def upload_part_copy(
        self,
        *,
        Bucket: str,
        Key: str,
        UploadId: str,
        PartNumber: int,
        CopySource: dict[str, str],
        CopySourceIfMatch: str,
        CopySourceRange: str,
    ) -> dict[str, Any]:
        self.ops.append("copy")
        self._fail("copy")
        src = self.objects.get(CopySource["Key"])
        if src is None:
            raise _client_error("NoSuchKey", 404)
        data, etag, _ = src
        if CopySourceIfMatch.strip('"') != etag.strip('"'):
            raise _client_error("PreconditionFailed", 412)
        a, b = CopySourceRange.split("=", 1)[1].split("-")
        self.mpus[UploadId]["parts"][PartNumber] = data[int(a) : int(b) + 1]
        return {"CopyPartResult": {"ETag": f'"copy-{PartNumber}"'}}

    async def upload_part(self, *, Bucket: str, Key: str, UploadId: str, PartNumber: int, Body: Any) -> dict[str, Any]:
        self.ops.append("uploadpart")
        self._fail("uploadpart")
        body = Body if isinstance(Body, bytes) else Body.read()
        assert len(body) > 0, "final tail part must never be empty"
        self.mpus[UploadId]["parts"][PartNumber] = body
        return {"ETag": f'"part-{PartNumber}"'}

    async def complete_multipart_upload(
        self, *, Bucket: str, Key: str, UploadId: str, MultipartUpload: dict[str, Any], IfMatch: str | None = None
    ) -> dict[str, Any]:
        if self.on_before_complete is not None:
            # A foreign generation lands AFTER our UploadPartCopy but BEFORE Complete.
            self.on_before_complete(Key, UploadId)
        if IfMatch is not None:
            cur = self.objects.get(Key)
            cur_etag = cur[1] if cur is not None else None
            if cur_etag is None or cur_etag.strip('"') != IfMatch.strip('"'):
                # The destination changed under us: the conditional Complete is rejected (never overwrites).
                raise _client_error("PreconditionFailed", 412)
        m = self.mpus[UploadId]
        data = b"".join(
            m["parts"][p["PartNumber"]] for p in sorted(MultipartUpload["Parts"], key=lambda x: x["PartNumber"])
        )
        etag = f'"complete-{UploadId}"'
        self.objects[Key] = (data, etag, dict(m["meta"]))
        self.completed.append(UploadId)
        if self.on_complete is not None:
            self.on_complete(Key, UploadId)  # may mutate to simulate ambiguity/foreign writes
        self._fail("complete")
        return {} if self.complete_omit_etag else {"ETag": etag}

    async def abort_multipart_upload(self, *, Bucket: str, Key: str, UploadId: str) -> dict[str, Any]:
        self.ops.append("abort")
        self.abort_attempts.append(UploadId)
        if self.abort_transient_failures > 0:
            self.abort_transient_failures -= 1
            raise _client_error("RequestTimeout", 400)  # transient (non-expired) abort failure
        self.aborted.append(UploadId)
        self.mpus.pop(UploadId, None)
        return {}

    async def head_object(self, *, Bucket: str, Key: str) -> dict[str, Any]:
        self._fail("head")
        obj = self.objects.get(Key)
        if obj is None:
            raise _client_error("NotFound", 404)
        data, etag, meta = obj
        if self.head_malformed:
            return {"Metadata": dict(meta)}  # a HEAD success that omits ContentLength/ETag
        return {"ETag": etag, "ContentLength": len(data), "Metadata": dict(meta)}


def _make_client(fake: FakeS3) -> AsyncAWSClient:
    c = AsyncAWSClient()

    @contextlib.asynccontextmanager
    async def _cm() -> Any:
        yield fake

    c._s3_client = _cm  # type: ignore[method-assign]
    return c


def _write(path: str, n: int, fill: bytes = b"\xab") -> None:
    with open(path, "wb") as f:
        f.write(fill * n)


@pytest.fixture
def tmpfile() -> Any:
    fd, path = tempfile.mkstemp(suffix=".webm")
    os.close(fd)
    yield path
    with contextlib.suppress(FileNotFoundError):
        os.unlink(path)


async def _seed(fake: FakeS3, n: int, fill: bytes = b"\xab") -> None:
    fake.objects[KEY] = (fill * n, f'"seed-{n}"', {})


def _seed_base(c: AsyncAWSClient, fake: FakeS3, n: int, fill: bytes = b"\xab") -> None:
    """Seed a process-AUTHORED base: the remote object AND the local compose state that a prior verified
    full replacement would have recorded. Compose only ever runs off such an owned base."""
    etag = f'"seed-{n}"'  # S3 etags are quoted on the wire
    fake.objects[KEY] = (fill * n, etag, {GEN: "prior-token"})
    c._compose_state.set(URI, _ComposeState(n, etag.strip('"')))  # _ComposeState holds the stripped etag


def _obj(fake: FakeS3) -> bytes:
    return fake.objects[KEY][0]


# --- unsupported / fallback (remote unchanged) -------------------------------------------------


@pytest.mark.asyncio
async def test_initial_step_full_replacement_when_no_object(tmpfile: str) -> None:
    fake = FakeS3()
    c = _make_client(fake)
    _write(tmpfile, 10 * MIB)
    assert await c.store_recording_prefix(URI, tmpfile, 10 * MIB, serialize_key=URI) == URI
    assert _obj(fake) == b"\xab" * (10 * MIB)  # full-prefix fallback seeded the object
    assert not fake.mpus and not fake.completed  # no compose attempted


@pytest.mark.asyncio
async def test_small_base_falls_back_to_full_replacement(tmpfile: str) -> None:
    fake = FakeS3()
    await _seed(fake, 2 * MIB)  # < 5 MiB copy floor
    c = _make_client(fake)
    _write(tmpfile, 4 * MIB)
    await c.store_recording_prefix(URI, tmpfile, 4 * MIB, serialize_key=URI)
    assert _obj(fake) == b"\xab" * (4 * MIB)
    assert not fake.completed


@pytest.mark.asyncio
async def test_over_5gib_base_guard_falls_back_without_mpu(tmpfile: str) -> None:
    fake = FakeS3()
    c = _make_client(fake)
    await _seed(fake, 8 * MIB)
    base_len = _MAX_COMPOSE_PART_BYTES + 1  # confirmed base over the single-copy-part limit
    c._compose_state.set(URI, _ComposeState(base_len, '"e"'))
    _write(tmpfile, 8 * MIB)
    # length MUST exceed base_len (a growing snapshot) so the truncate/no-op guards do NOT fire first and the
    # >5GiB base guard is actually reached. The file itself stays 8 MiB — the guard rejects before any tail
    # read, and the same-slot full replacement then uploads whatever the file holds.
    length = base_len + 2
    await c.store_recording_prefix(URI, tmpfile, length, serialize_key=URI)
    assert not fake.mpus  # >5GiB base guard fired before any create_multipart_upload
    assert not fake.completed
    assert _obj(fake) == b"\xab" * (8 * MIB)  # full-prefix fallback wrote the file bytes, no server-side copy


@pytest.mark.asyncio
async def test_custom_endpoint_refuses_compose_and_full_replaces(tmpfile: str, monkeypatch: Any) -> None:
    """A customer-supplied S3-compatible endpoint may ignore the AWS-specific IfMatch / CopySourceIfMatch
    fence and silently degrade compose to an unconditional overwrite. Compose must refuse when a custom
    endpoint is configured and fall back to the verified full-prefix replacement — even with an owned base
    that would otherwise compose, and even when the feature flag is enabled."""
    monkeypatch.setattr(settings, "RECORDING_INCREMENTAL_COMPOSE_ENABLED", True)
    fake = FakeS3()
    c = AsyncAWSClient(endpoint_url="https://minio.example.internal")

    @contextlib.asynccontextmanager
    async def _cm() -> Any:
        yield fake

    c._s3_client = _cm  # type: ignore[method-assign]
    _seed_base(c, fake, 8 * MIB)  # an owned base that WOULD compose against real AWS S3
    _write(tmpfile, 11 * MIB)
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    assert not fake.mpus and not fake.completed  # no create_multipart_upload / upload_part_copy
    assert _obj(fake) == b"\xab" * (11 * MIB)  # full-prefix replacement wrote the file bytes


# --- successful compose ------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_compose_success_copy_tail_complete(tmpfile: str) -> None:
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 11 * MIB)
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    assert _obj(fake) == b"\xab" * (11 * MIB)
    assert fake.completed and not fake.aborted
    assert c._compose_state.get(URI).length == 11 * MIB


@pytest.mark.asyncio
async def test_compose_chains_and_uses_cached_state(tmpfile: str) -> None:
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 11 * MIB)
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    fake.fail["head"] = _client_error("ServiceUnavailable", 503)  # cached state means no head needed
    _write(tmpfile, 13 * MIB)
    await c.store_recording_prefix(URI, tmpfile, 13 * MIB, serialize_key=URI)
    assert len(_obj(fake)) == 13 * MIB


# SKY-15466: generic S3 seams never infer ContentType from a (user-named) key; RECORDING opts in via allowlist.
@pytest.mark.parametrize("uri,key", [("s3://b/x.html", "x.html"), ("s3://b/x.svg", "x.svg"), (URI_MP4, KEY_MP4)])
@pytest.mark.asyncio
async def test_generic_seams_never_infer_recording_opts_in(tmpfile: str, uri: str, key: str) -> None:
    fake = FakeS3()
    c = _make_client(fake)
    _write(tmpfile, 4096)
    await c.upload_file(uri, b"x")  # generic put: no content_type
    with open(tmpfile, "rb") as fh:
        await c.upload_file_stream(uri, fh)  # save_legacy_file path: no content_type
    assert key not in fake.content_types  # never inferred from any (untrusted) name; both seams
    assert _recording_content_type("a.webm") == "video/webm" and _recording_content_type("x.html") is None


@pytest.mark.asyncio
async def test_recording_prefix_reseed_then_compose_advance_mp4_content_type(tmpfile: str) -> None:
    fake = FakeS3()
    c = _make_client(fake)
    # first prefix reseeds via full replacement (_full_replace_and_seed -> upload_fileobj)
    _write(tmpfile, 11 * MIB)
    await c.store_recording_prefix(URI_MP4, tmpfile, 11 * MIB, serialize_key=URI_MP4)
    assert fake.content_types.get(KEY_MP4) == "video/mp4"
    # second, larger prefix triggers a real compose advance (create_multipart_upload must carry ContentType)
    _write(tmpfile, 13 * MIB)
    await c.store_recording_prefix(URI_MP4, tmpfile, 13 * MIB, serialize_key=URI_MP4)
    assert fake.completed  # a compose advance actually ran
    assert fake.content_types.get(KEY_MP4) == "video/mp4"


# --- conflict preservation (blocker 2) ---------------------------------------------------------


@pytest.mark.asyncio
async def test_stale_etag_conflict_preserves_remote(tmpfile: str) -> None:
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 11 * MIB)
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    # A foreign writer replaces the object with a NEWER, longer generation.
    foreign = b"\x01" * (20 * MIB)
    fake.objects[KEY] = (foreign, '"foreign"', {})
    _write(tmpfile, 13 * MIB)
    await c.store_recording_prefix(URI, tmpfile, 13 * MIB, serialize_key=URI)
    # 412 on copy => conflict => remote preserved, NOT overwritten with our 13 MiB prefix.
    assert _obj(fake) == foreign
    assert fake.aborted  # our MPU aborted
    assert URI not in c._compose_state  # base reseeds next step


# --- review 3920309801: fence Complete against a destination change after copy -----------------


@pytest.mark.asyncio
async def test_foreign_generation_after_copy_before_complete_is_fenced_out(tmpfile: str) -> None:
    """A foreign producer can replace the destination AFTER our UploadPartCopy but BEFORE our Complete.
    The Complete is fenced with IfMatch on the verified base ETag, so it is rejected (412) instead of
    unconditionally overwriting the newer generation: preserve the foreign bytes, abort our MPU, clear
    state, never full-replace."""
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 11 * MIB)
    foreign = b"\x0a" * (14 * MIB)

    def foreign_lands_before_complete(key: str, uid: str) -> None:
        fake.objects[key] = (foreign, '"foreign-newgen"', {GEN: "not-our-token"})

    fake.on_before_complete = foreign_lands_before_complete
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    assert _obj(fake) == foreign  # newer foreign generation preserved, NOT clobbered by our Complete
    assert fake.aborted and not fake.completed  # our MPU aborted; our Complete never wrote
    assert URI not in c._compose_state  # conflict path cleared local state


@pytest.mark.asyncio
async def test_fenced_complete_succeeds_when_destination_unchanged(tmpfile: str) -> None:
    """Normal fenced Complete: the destination still holds the verified base at Complete time, so the
    IfMatch fence passes and the compose lands."""
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 11 * MIB)
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    assert _obj(fake) == b"\xab" * (11 * MIB)  # compose landed under the fence
    assert fake.completed and not fake.aborted
    assert c._compose_state.get(URI).length == 11 * MIB


# --- ambiguous complete (blocker 3) ------------------------------------------------------------


@pytest.mark.asyncio
async def test_ambiguous_complete_reconciled_by_token(tmpfile: str) -> None:
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 11 * MIB)
    # Complete writes the object (with our token) then the response is "lost".
    fake.fail["complete"] = _client_error("RequestTimeout", 400)
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    assert len(_obj(fake)) == 11 * MIB
    assert not fake.aborted  # reconciled as durable via length + token; never abort a completed object


@pytest.mark.asyncio
async def test_ambiguous_complete_same_length_foreign_gen_not_accepted(tmpfile: str) -> None:
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 11 * MIB)

    def foreign_same_length(key: str, uid: str) -> None:
        # Simulate a DIFFERENT generation of the SAME length (foreign token) landing at the key.
        fake.objects[key] = (b"\x09" * (11 * MIB), '"foreign-same-len"', {GEN: "not-our-token"})

    fake.on_complete = foreign_same_length
    fake.fail["complete"] = _client_error("RequestTimeout", 400)
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    # Same length but foreign token => conflict => preserve foreign bytes, do NOT overwrite.
    assert _obj(fake) == b"\x09" * (11 * MIB)
    assert fake.aborted
    assert URI not in c._compose_state


@pytest.mark.asyncio
async def test_ambiguous_complete_old_base_intact_falls_back(tmpfile: str) -> None:
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 11 * MIB)

    def revert_to_old_base(key: str, uid: str) -> None:
        fake.objects[key] = (b"\xab" * (8 * MIB), '"seed-8388608"', {})  # complete "didn't land"

    fake.on_complete = revert_to_old_base
    fake.fail["complete"] = _client_error("RequestTimeout", 400)
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    # Old base intact => unsupported => safe full replacement to our 11 MiB prefix.
    assert _obj(fake) == b"\xab" * (11 * MIB)
    assert fake.aborted


# --- review 3917237493: unreadable HEAD after ambiguous complete must PRESERVE, never clobber ---


@pytest.mark.asyncio
async def test_ambiguous_complete_head_transport_failure_preserves_remote(tmpfile: str) -> None:
    """A NON-ClientError HEAD failure (endpoint/read timeout) during ambiguous-complete reconciliation must
    resolve to conflict and preserve the remote — never fall through to an unconditional full replacement
    that could clobber a foreign generation written during the same network blip."""
    from botocore.exceptions import EndpointConnectionError

    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 11 * MIB)

    def foreign_lands(key: str, uid: str) -> None:
        fake.objects[key] = (b"\x07" * (11 * MIB), '"foreign"', {GEN: "not-our-token"})

    fake.on_complete = foreign_lands
    fake.fail["complete"] = _client_error("RequestTimeout", 400)  # ambiguous complete
    fake.fail["head"] = EndpointConnectionError(endpoint_url="https://s3")  # transport failure, NOT a ClientError
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    assert _obj(fake) == b"\x07" * (11 * MIB)  # foreign generation preserved, NOT overwritten with our prefix
    assert fake.aborted  # our MPU aborted
    assert URI not in c._compose_state  # base reseeds next step


@pytest.mark.asyncio
async def test_reconcile_complete_reraises_expired_token_for_retry() -> None:
    """An expired token on the reconciliation HEAD is recoverable, NOT a conflict: it must be re-raised so
    _s3_with_retry refreshes the session and retries the transaction, rather than being swallowed as
    "conflict" (which would skip the step on a recoverable expiry)."""
    from skyvern.forge.sdk.api.aws import S3Uri

    fake = FakeS3()
    c = _make_client(fake)
    fake.objects[KEY] = (b"\xab" * (8 * MIB), '"seed-8"', {GEN: "prior"})
    base = _ComposeState(8 * MIB, "seed-8")
    fake.fail["head"] = _client_error("ExpiredTokenException", 400)
    with pytest.raises(ClientError) as ei:
        await c._reconcile_complete(fake, S3Uri(URI), 11 * MIB, base, "token")
    assert ei.value.response["Error"]["Code"] == "ExpiredTokenException"


@pytest.mark.asyncio
async def test_ambiguous_complete_malformed_head_preserves_remote(tmpfile: str) -> None:
    """A HEAD that returns 200 but omits ContentLength/ETag is just as unreadable as a failed HEAD: the parse
    must not escape reconciliation and trigger a clobbering full replacement. Resolve to conflict, preserve."""
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 11 * MIB)

    def foreign_lands(key: str, uid: str) -> None:
        fake.objects[key] = (b"\x06" * (11 * MIB), '"foreign"', {GEN: "not-our-token"})

    fake.on_complete = foreign_lands
    fake.fail["complete"] = _client_error("RequestTimeout", 400)
    fake.head_malformed = True  # HEAD 200 without ContentLength/ETag
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    assert _obj(fake) == b"\x06" * (11 * MIB)  # preserved, not clobbered by a full replacement
    assert fake.aborted
    assert URI not in c._compose_state


@pytest.mark.asyncio
async def test_full_replace_malformed_head_pops_stale_compose_state(tmpfile: str) -> None:
    """A malformed HEAD (200 without ContentLength/ETag) after the full-replacement PUT must guard its read
    and pop the previous compose state rather than raise; a raise would strand the stale base cached and cost
    a snapshot self-healing as a 412 next step."""
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 2 * MIB)  # base below the 5-MiB copy floor -> ComposeUnsupportedError -> full replace
    _write(tmpfile, 4 * MIB)
    fake.head_malformed = True  # the seed-and-verify HEAD omits ContentLength/ETag
    assert await c.store_recording_prefix(URI, tmpfile, 4 * MIB, serialize_key=URI) == URI
    assert URI not in c._compose_state  # stale base popped, not left cached
    assert _obj(fake) == b"\xab" * (4 * MIB)  # replacement still wrote the file
    assert not fake.mpus and not fake.completed  # full-replace path, no compose MPU


# --- review 3917237500: Complete without ETag must validate length + token before caching -------


@pytest.mark.asyncio
async def test_complete_without_etag_foreign_gen_not_cached(tmpfile: str) -> None:
    """A successful Complete whose response omits ETag must reconcile by HEAD and validate our exact
    length + generation token; a foreign same-length generation that raced in must NOT be cached (else the
    next snapshot's conditional copy splices the foreign prefix with our tail)."""
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 11 * MIB)
    fake.complete_omit_etag = True

    def foreign_same_len(key: str, uid: str) -> None:
        fake.objects[key] = (b"\x05" * (11 * MIB), '"foreign-noetag"', {GEN: "not-our-token"})

    fake.on_complete = foreign_same_len
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    assert URI not in c._compose_state  # foreign generation NOT cached as our base
    assert _obj(fake) == b"\x05" * (11 * MIB)  # foreign preserved, not clobbered


@pytest.mark.asyncio
async def test_complete_without_etag_own_gen_cached(tmpfile: str) -> None:
    """Regression guard: a successful Complete omitting ETag, with OUR generation at the key, still caches
    the base (validated by exact length + token) so the compose chain continues."""
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 11 * MIB)
    fake.complete_omit_etag = True
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    assert len(_obj(fake)) == 11 * MIB
    assert c._compose_state.get(URI) is not None and c._compose_state.get(URI).length == 11 * MIB


# --- pre-complete failure aborts + preserves old object (remote unchanged => fallback) ----------


@pytest.mark.asyncio
@pytest.mark.parametrize("failop", ["create", "copy", "uploadpart"])
async def test_precomplete_failure_aborts_then_full_replacement(tmpfile: str, failop: str) -> None:
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 11 * MIB)
    fake.fail[failop] = _client_error("InternalError", 500)
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    # Remote was unchanged by the failed compose, so the same-slot full replacement lands our prefix.
    assert _obj(fake) == b"\xab" * (11 * MIB)
    if failop != "create":
        assert fake.aborted


# --- single-slot ordering (blocker 1) ----------------------------------------------------------


@pytest.mark.asyncio
async def test_failed_compose_fallback_and_next_step_stay_ordered() -> None:
    """Step A's compose fails and must complete its full-prefix fallback before step B (queued) writes."""
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    order: list[str] = []
    gate = asyncio.Event()
    orig_fileobj = fake.upload_fileobj

    async def gated_fileobj(
        Fileobj: Any, Bucket: str, Key: str, ExtraArgs: dict | None = None, Config: Any = None
    ) -> None:
        await gate.wait()  # hold A inside its fallback
        await orig_fileobj(Fileobj, Bucket, Key, ExtraArgs, Config)
        order.append("A_fallback")

    fake.upload_fileobj = gated_fileobj  # type: ignore[method-assign]
    fake.fail["copy"] = _client_error("InternalError", 500)  # force A to fall back

    fa = tempfile.mkstemp(suffix=".webm")[1]
    fb = tempfile.mkstemp(suffix=".webm")[1]
    _write(fa, 11 * MIB)
    _write(fb, 13 * MIB)
    try:
        a = asyncio.create_task(c.store_recording_prefix(URI, fa, 11 * MIB, serialize_key=URI))
        await asyncio.sleep(0.05)  # let A reserve its slot and enter the gated fallback
        orig_complete = fake.complete_multipart_upload

        async def b_complete(**kw: Any) -> dict[str, Any]:
            order.append("B_complete")
            return await orig_complete(**kw)

        fake.complete_multipart_upload = b_complete  # type: ignore[method-assign]
        b = asyncio.create_task(c.store_recording_prefix(URI, fb, 13 * MIB, serialize_key=URI))
        await asyncio.sleep(0.05)
        assert "B_complete" not in order  # B must not overtake A's unfinished fallback
        gate.set()
        await asyncio.gather(a, b)
        assert order == ["A_fallback", "B_complete"]  # strict ordering held across compose+fallback
        assert len(_obj(fake)) == 13 * MIB
    finally:
        for f in (fa, fb):
            with contextlib.suppress(FileNotFoundError):
                os.unlink(f)


# --- fd ownership before detach (blocker 4) ----------------------------------------------------


@pytest.mark.asyncio
async def test_fd_owned_before_detach_survives_unlink(tmpfile: str) -> None:
    fake = FakeS3()
    await _seed(fake, 8 * MIB)
    c = _make_client(fake)
    _write(tmpfile, 11 * MIB)
    # Block the step at its queued wait, unlink the path, then release: the pre-opened fd must survive.
    _prev, prev_done = c._begin_serialized_write(URI)
    task = asyncio.create_task(c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI))
    await asyncio.sleep(0.05)
    os.unlink(tmpfile)  # caller removes the file after the transfer was queued
    prev_done.set_result(None)
    assert await task == URI
    assert len(_obj(fake)) == 11 * MIB  # tail read from the owned fd despite the unlink


# --- storage gating / feature-off exact Phase 1 ------------------------------------------------


def _recording_artifact(uri: str, atype: ArtifactType = ArtifactType.RECORDING) -> Artifact:
    return Artifact(
        artifact_id="art_1",
        artifact_type=atype,
        uri=uri,
        bundle_key=None,
        organization_id="o_1",
        created_at=datetime.now(UTC),
        modified_at=datetime.now(UTC),
    )


def _mock_storage() -> S3Storage:
    s = S3Storage()
    s.async_client = AsyncMock()
    s.async_client.forget_compose_state = MagicMock()
    return s


@pytest.mark.asyncio
async def test_storage_gating_off_uses_exact_phase1_stream(tmpfile: str, monkeypatch: Any) -> None:
    monkeypatch.setattr(settings, "RECORDING_INCREMENTAL_COMPOSE_ENABLED", False)
    s = _mock_storage()
    _write(tmpfile, 10 * MIB)
    await s.store_artifact_prefix_from_path(_recording_artifact(URI), tmpfile, 10 * MIB)
    s.async_client.store_recording_prefix.assert_not_called()
    s.async_client.upload_file_stream.assert_awaited_once()


@pytest.mark.asyncio
async def test_storage_gating_on_recording_uses_compose_path(tmpfile: str, monkeypatch: Any) -> None:
    monkeypatch.setattr(settings, "RECORDING_INCREMENTAL_COMPOSE_ENABLED", True)
    s = _mock_storage()
    _write(tmpfile, 10 * MIB)
    await s.store_artifact_prefix_from_path(_recording_artifact(URI), tmpfile, 10 * MIB)
    s.async_client.store_recording_prefix.assert_awaited_once()
    s.async_client.upload_file_stream.assert_not_called()


@pytest.mark.asyncio
async def test_storage_non_recording_uses_phase1_even_when_enabled(tmpfile: str, monkeypatch: Any) -> None:
    monkeypatch.setattr(settings, "RECORDING_INCREMENTAL_COMPOSE_ENABLED", True)
    s = _mock_storage()
    _write(tmpfile, 10 * MIB)
    await s.store_artifact_prefix_from_path(_recording_artifact(URI, ArtifactType.SKYVERN_LOG), tmpfile, 10 * MIB)
    s.async_client.store_recording_prefix.assert_not_called()
    s.async_client.upload_file_stream.assert_awaited_once()


@pytest.mark.asyncio
async def test_terminal_forgets_compose_state(monkeypatch: Any) -> None:
    monkeypatch.setattr(settings, "RECORDING_INCREMENTAL_COMPOSE_ENABLED", True)
    s = _mock_storage()
    art = _recording_artifact(URI)
    await s.store_artifact(art, b"final-bytes", supersede_queued_prefixes=True)
    s.async_client.forget_compose_state.assert_called_once_with(art.uri)
    s.async_client.upload_file.assert_awaited_once()


# --- never regress/truncate a newer remote generation (addendum 1) -----------------------------


@pytest.mark.asyncio
async def test_larger_remote_generation_preserved_not_truncated(tmpfile: str) -> None:
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 20 * MIB)  # process-authored base LONGER than this step's snapshot
    _write(tmpfile, 13 * MIB)
    await c.store_recording_prefix(URI, tmpfile, 13 * MIB, serialize_key=URI)
    assert len(_obj(fake)) == 20 * MIB  # preserved; never overwritten with the shorter 13 MiB prefix
    assert not fake.completed  # no compose, and crucially no full-PUT truncation
    assert URI not in c._compose_state


@pytest.mark.asyncio
async def test_same_length_remote_is_noop(tmpfile: str) -> None:
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 8 * MIB)
    before = _obj(fake)
    await c.store_recording_prefix(URI, tmpfile, 8 * MIB, serialize_key=URI)
    assert _obj(fake) == before  # no-op: no MPU, no full-PUT
    assert not fake.completed and not fake.mpus


# --- newest pre-terminal fallback contract (addendum 2) ----------------------------------------


async def _park_newest(c: AsyncAWSClient, fake: FakeS3, path: str, length: int) -> asyncio.Task:
    """Reserve a queued prefix, seal a terminal behind it so the queued prefix is the fallback target,
    and drive the queued prefix so it parks. Returns the terminal task (already reserved, awaiting)."""
    blk_prev, blk_done = c._begin_serialized_write(URI)  # blocker so the queued prefix waits
    q = asyncio.create_task(c.store_recording_prefix(URI, path, length, serialize_key=URI))
    await asyncio.sleep(0.03)  # queued prefix reserves its slot, then awaits blk_done
    return blk_done, q


@pytest.mark.asyncio
async def test_superseded_newest_parks_and_terminal_failure_uploads_it(tmpfile: str) -> None:
    fake = FakeS3()
    await _seed(fake, 8 * MIB)
    c = _make_client(fake)
    _write(tmpfile, 11 * MIB)
    blk_done, q = await _park_newest(c, fake, tmpfile, 11 * MIB)
    fake.fail["put_object"] = _client_error("InternalError", 500)  # terminal finalize fails -> use fallback
    t = asyncio.create_task(c.upload_file(URI, b"terminal-final", serialize_key=URI, supersede_queued=True))
    await asyncio.sleep(0.03)  # terminal reserves the seal (its prev = the queued prefix's done)
    blk_done.set_result(None)  # release: queued prefix parks; terminal runs, put fails, uploads fallback
    await asyncio.gather(q, t)
    assert _obj(fake) == b"\xab" * (11 * MIB)  # newest pre-terminal snapshot uploaded on terminal failure
    assert URI not in c._object_write_fallback  # consumed + cleaned up


@pytest.mark.asyncio
async def test_superseded_newest_parked_then_terminal_success_discards_it(tmpfile: str) -> None:
    fake = FakeS3()
    await _seed(fake, 8 * MIB)
    c = _make_client(fake)
    _write(tmpfile, 11 * MIB)
    blk_done, q = await _park_newest(c, fake, tmpfile, 11 * MIB)
    t = asyncio.create_task(c.upload_file(URI, b"terminal-final", serialize_key=URI, supersede_queued=True))
    await asyncio.sleep(0.03)
    blk_done.set_result(None)  # queued prefix parks; terminal put succeeds -> fallback discarded, not uploaded
    await asyncio.gather(q, t)
    assert _obj(fake) == b"terminal-final"  # terminal full replacement wins
    assert URI not in c._object_write_fallback


@pytest.mark.asyncio
async def test_superseded_older_prefix_skips_without_parking(tmpfile: str) -> None:
    fake = FakeS3()
    await _seed(fake, 8 * MIB)
    c = _make_client(fake)
    _write(tmpfile, 11 * MIB)
    # Seal the key with a DIFFERENT (sentinel) fallback target so this queued prefix is NOT the newest.
    sentinel: asyncio.Future = asyncio.get_running_loop().create_future()
    c._object_write_sealed.add(URI)
    c._object_write_fallback_target[URI] = sentinel
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    assert URI not in c._object_write_fallback  # older superseded prefix skips, does not park
    assert not fake.completed


# --- MUST_FIX 1: a GC'd abandoned reader must NOT close the fd the same-slot fallback still reads ------


@pytest.mark.asyncio
async def test_gc_during_fallback_does_not_close_shared_fd(tmpfile: str) -> None:
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 11 * MIB)
    # Compose creates a tail reader over the shared fd, then fails; the compose falls back to a full
    # replacement in the same slot. A gc.collect() before the fallback reads would, if any reader owned the
    # shared fd, run its __del__ -> close() and the fallback would die with "read of closed file".
    fake.fail["uploadpart"] = _client_error("InternalError", 500)
    orig_fileobj = fake.upload_fileobj

    async def gc_then_upload(
        Fileobj: Any, Bucket: str, Key: str, ExtraArgs: dict | None = None, Config: Any = None
    ) -> None:
        gc.collect()
        await orig_fileobj(Fileobj, Bucket, Key, ExtraArgs, Config)

    fake.upload_fileobj = gc_then_upload  # type: ignore[method-assign]
    assert await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI) == URI
    assert _obj(fake) == b"\xab" * (11 * MIB)  # fallback read the shared fd successfully after GC


# --- MUST_FIX 2: expired-token refresh+retry parity for compose AND the inline full-replace -----------


@pytest.mark.asyncio
async def test_compose_refreshes_and_retries_on_expired_token(tmpfile: str) -> None:
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 11 * MIB)
    refreshed: list[bool] = []
    c.refresh_session = lambda: refreshed.append(True)  # type: ignore[method-assign]
    fake.fail["copy"] = _client_error("ExpiredTokenException", 400)  # first copy expires; popped -> retry ok
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    assert refreshed  # session was refreshed instead of silently degrading
    assert len(_obj(fake)) == 11 * MIB and fake.completed  # compose landed on the retry
    assert c._compose_state.get(URI).length == 11 * MIB


@pytest.mark.asyncio
async def test_full_replace_refreshes_and_retries_on_expired_token(tmpfile: str) -> None:
    fake = FakeS3()
    c = _make_client(fake)  # no owned base -> full-replace path
    _write(tmpfile, 10 * MIB)
    refreshed: list[bool] = []
    c.refresh_session = lambda: refreshed.append(True)  # type: ignore[method-assign]
    fake.fail["upload_fileobj"] = _client_error("ExpiredTokenException", 400)
    await c.store_recording_prefix(URI, tmpfile, 10 * MIB, serialize_key=URI)
    assert refreshed
    assert _obj(fake) == b"\xab" * (10 * MIB)
    assert c._compose_state.get(URI) is not None  # verified generation seeded after the retry


# --- Aron review finding 2: expired-token orphan MPU must be aborted on the REFRESHED client ----


class _GenBackend:
    """Shared S3 state across client 'generations'. Generation 1 is a stale/expired client: its complete
    AND abort both raise ExpiredTokenException (one expired session token invalidates the whole client, so
    aborting the orphaned MPU on that same client is futile). Generation 2 (post-refresh) works. Records
    which upload ids were aborted by which generation so a test can assert the refreshed client aborts the
    exact orphan before creating a replacement MPU."""

    def __init__(self) -> None:
        self.objects: dict[str, tuple[bytes, str, dict[str, str]]] = {}
        self.mpus: dict[str, dict[str, Any]] = {}
        self.completed: list[str] = []
        self.completed_ids: set[str] = set()  # upload ids whose Complete actually landed
        self.aborted_by_gen: dict[int, list[str]] = {}  # successful aborts, per generation
        self.abort_attempts_by_gen: dict[int, list[str]] = {}  # every abort attempt (incl. failed), per gen
        self.fail_abort_gens: set[int] = set()  # generations whose abort raises (e.g. a transient failure)
        self.expired_code = "ExpiredTokenException"  # exact S3 error code to raise (S3 also uses "ExpiredToken")
        self.complete_fail_gens: set[int] = {1}  # generations whose Complete raises expired-token (orphan MPU)
        self.complete_omit_etag = False  # Complete succeeds but omits ETag
        self.head_expired_gens: set[int] = set()  # generations whose head_object raises expired-token
        self.nosuchupload_on_completed_abort = True  # abort of an already-completed id raises NoSuchUpload
        self._n = 0
        self.gen = 0

    def next_client(self) -> _GenClient:
        self.gen += 1
        return _GenClient(self, self.gen)


class _GenClient:
    def __init__(self, be: _GenBackend, gen: int) -> None:
        self.be = be
        self.gen = gen

    async def create_multipart_upload(self, *, Bucket: str, Key: str, Metadata: dict | None = None, **kw: Any) -> Any:
        self.be._n += 1
        uid = f"mpu-{self.be._n}"
        self.be.mpus[uid] = {"key": Key, "parts": {}, "meta": dict(Metadata or {})}
        return {"UploadId": uid}

    async def upload_part_copy(
        self,
        *,
        Bucket: str,
        Key: str,
        UploadId: str,
        PartNumber: int,
        CopySource: dict,
        CopySourceIfMatch: str,
        CopySourceRange: str,
    ) -> Any:
        data, etag, _ = self.be.objects[CopySource["Key"]]
        if CopySourceIfMatch.strip('"') != etag.strip('"'):
            raise _client_error("PreconditionFailed", 412)
        a, b = CopySourceRange.split("=", 1)[1].split("-")
        self.be.mpus[UploadId]["parts"][PartNumber] = data[int(a) : int(b) + 1]
        return {"CopyPartResult": {"ETag": f'"copy-{PartNumber}"'}}

    async def upload_part(self, *, Bucket: str, Key: str, UploadId: str, PartNumber: int, Body: Any) -> Any:
        body = Body if isinstance(Body, bytes) else Body.read()
        self.be.mpus[UploadId]["parts"][PartNumber] = body
        return {"ETag": f'"part-{PartNumber}"'}

    async def complete_multipart_upload(
        self, *, Bucket: str, Key: str, UploadId: str, MultipartUpload: dict, IfMatch: str | None = None
    ) -> Any:
        if self.gen in self.be.complete_fail_gens:
            raise _client_error(self.be.expired_code, 400)  # token expires at complete -> MPU orphaned
        if IfMatch is not None:
            cur = self.be.objects.get(Key)
            if cur is None or cur[1].strip('"') != IfMatch.strip('"'):
                raise _client_error("PreconditionFailed", 412)
        m = self.be.mpus[UploadId]
        data = b"".join(
            m["parts"][p["PartNumber"]] for p in sorted(MultipartUpload["Parts"], key=lambda x: x["PartNumber"])
        )
        etag = f'"complete-{UploadId}"'
        self.be.objects[Key] = (data, etag, dict(m["meta"]))
        self.be.completed.append(UploadId)
        self.be.completed_ids.add(UploadId)
        return {} if self.be.complete_omit_etag else {"ETag": etag}

    async def abort_multipart_upload(self, *, Bucket: str, Key: str, UploadId: str) -> Any:
        self.be.abort_attempts_by_gen.setdefault(self.gen, []).append(UploadId)  # record EVERY attempt
        if UploadId in self.be.completed_ids and self.be.nosuchupload_on_completed_abort:
            # An already-completed upload can no longer be aborted: real S3 returns NoSuchUpload.
            raise _client_error("NoSuchUpload", 404)
        if self.gen in self.be.complete_fail_gens or self.gen in self.be.fail_abort_gens:
            raise _client_error(self.be.expired_code, 400)  # stale client or injected transient abort failure
        self.be.aborted_by_gen.setdefault(self.gen, []).append(UploadId)
        self.be.mpus.pop(UploadId, None)
        return {}

    async def head_object(self, *, Bucket: str, Key: str) -> Any:
        if self.gen in self.be.head_expired_gens:
            raise _client_error(self.be.expired_code, 400)
        obj = self.be.objects.get(Key)
        if obj is None:
            raise _client_error("NotFound", 404)
        data, etag, meta = obj
        return {"ETag": etag, "ContentLength": len(data), "Metadata": dict(meta)}


def _make_client_gen(be: _GenBackend) -> AsyncAWSClient:
    c = AsyncAWSClient()

    @contextlib.asynccontextmanager
    async def _cm() -> Any:
        yield be.next_client()

    c._s3_client = _cm  # type: ignore[method-assign]
    return c


# S3 reports an expired STS/web-identity token under EITHER exact code; both must drive the same refresh+retry
# path (the compose retry, not just the helper). "ExpiredToken" is the bare S3 code the review flagged.
@pytest.mark.asyncio
@pytest.mark.parametrize("expired_code", ["ExpiredTokenException", "ExpiredToken"])
async def test_expired_token_orphan_mpu_aborted_on_refreshed_client(tmpfile: str, expired_code: str) -> None:
    be = _GenBackend()
    be.expired_code = expired_code
    be.objects[KEY] = (b"\xab" * (8 * MIB), '"seed-8"', {GEN: "prior"})  # process-owned base
    c = _make_client_gen(be)
    c._compose_state.set(URI, _ComposeState(8 * MIB, "seed-8"))
    c.refresh_session = lambda: None  # type: ignore[method-assign]  # no real STS session in the test
    _write(tmpfile, 11 * MIB)
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    # gen1 created mpu-1, then expired at complete; aborting mpu-1 on that stale gen1 client also failed.
    # The refreshed gen2 client MUST abort the exact orphaned mpu-1 before completing the replacement mpu-2.
    assert be.aborted_by_gen.get(2) == ["mpu-1"]  # refreshed client aborted the exact old upload id
    assert 1 not in be.aborted_by_gen  # the stale client did NOT successfully abort it
    assert be.completed == ["mpu-2"]  # the replacement compose completed on the retry
    assert "mpu-1" not in be.mpus  # the orphan MPU is gone (no lingering incomplete upload)
    assert len(be.objects[KEY][0]) == 11 * MIB  # composed recording landed
    assert c._compose_state.get(URI).length == 11 * MIB


@pytest.mark.asyncio
async def test_expired_token_orphan_abort_failure_fails_closed_no_replacement(tmpfile: str) -> None:
    """If the REFRESHED client's abort of the carried orphan ALSO fails, fail closed: do NOT create a
    replacement compose MPU on top of an un-aborted orphan. Preserve the existing remote object, drop stale
    compose state (so the next step authors a fresh verified generation), and skip — the bucket
    AbortIncompleteMultipartUpload lifecycle rule is the documented backstop for the leaked MPU."""
    be = _GenBackend()
    base_bytes = b"\xab" * (8 * MIB)
    be.objects[KEY] = (base_bytes, '"seed-8"', {GEN: "prior"})
    be.fail_abort_gens = {2}  # the refreshed (gen2) client's abort of the orphan ALSO fails
    c = _make_client_gen(be)
    c._compose_state.set(URI, _ComposeState(8 * MIB, "seed-8"))
    c.refresh_session = lambda: None  # type: ignore[method-assign]
    _write(tmpfile, 11 * MIB)
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    assert be.abort_attempts_by_gen.get(2) == ["mpu-1"]  # the orphan abort WAS attempted on the refreshed client
    assert be.aborted_by_gen.get(2) is None  # ...and it failed
    assert "mpu-2" not in be.mpus and be.completed == []  # fail-closed: NO replacement MPU created/completed
    assert be.objects[KEY][0] == base_bytes  # existing remote object preserved, not overwritten
    assert URI not in c._compose_state  # stale compose state dropped -> next step authors a fresh generation


@pytest.mark.asyncio
async def test_abort_compose_treats_nosuchupload_as_success() -> None:
    """A NoSuchUpload from abort means the upload no longer exists (e.g. an ambiguous Complete actually
    landed) — that IS the cleanup goal, so _abort_compose reports success (True), not an unabortable orphan.
    A genuine failure (AccessDenied) or any other error still reports False so it is not masked."""
    from skyvern.forge.sdk.api.aws import S3Uri

    c = _make_client(FakeS3())
    u = S3Uri(URI)

    async def _abort_raises(code: str) -> bool:
        client = MagicMock()
        client.abort_multipart_upload = AsyncMock(
            side_effect=_client_error(code, 404 if code == "NoSuchUpload" else 403)
        )
        return await c._abort_compose(client, u, "mpu-x")

    assert await _abort_raises("NoSuchUpload") is True  # already gone -> cleanup success
    assert await _abort_raises("AccessDenied") is False  # genuine failure not masked
    assert await _abort_raises("InternalError") is False  # any other error still reported as failure


@pytest.mark.asyncio
async def test_generic_failure_inline_abort_transient_then_fresh_cleanup_before_full_replace(tmpfile: str) -> None:
    """A non-expired pre-Complete failure leaves the remote unchanged (safe to full-replace), but the inline
    abort of our MPU fails transiently. The transaction MUST clean up on a freshly-created client BEFORE the
    full replacement — so no incomplete MPU leaks — and then land exactly one safe replacement, in that order
    (cleanup precedes replacement)."""
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 11 * MIB)
    fake.fail["uploadpart"] = _client_error("InternalError", 500)  # non-expired pre-Complete failure
    fake.abort_transient_failures = 1  # inline abort fails once; the fresh-client abort then succeeds
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    assert fake.abort_attempts == ["mpu-1", "mpu-1"]  # inline attempt, then the fresh-client retry (same id)
    assert fake.aborted == ["mpu-1"] and "mpu-1" not in fake.mpus  # orphan cleaned up on the fresh client
    assert not fake.completed  # no compose Complete
    assert _obj(fake) == b"\xab" * (11 * MIB)  # the safe full replacement landed our prefix
    assert fake.ops.count("upload_fileobj") == 1  # exactly one replacement, no duplicate PUT
    last_abort = max(i for i, o in enumerate(fake.ops) if o == "abort")
    assert fake.ops.index("upload_fileobj") > last_abort  # cleanup precedes the replacement
    assert c._compose_state.get(URI) is not None and c._compose_state.get(URI).length == 11 * MIB


@pytest.mark.asyncio
async def test_fresh_cleanup_refreshes_session_before_building_client(tmpfile: str) -> None:
    """The fresh-client cleanup may be reached because the token expired between the primary op and the inline
    abort. _s3_client reuses the cached session (recreated only on the 45-min TTL), so the cleanup must refresh
    the session BEFORE constructing the fresh client — otherwise it carries the same expired credentials and
    fails identically, and since this path returns normally _s3_with_retry never sees the token error."""
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 11 * MIB)
    fake.fail["uploadpart"] = _client_error("InternalError", 500)  # non-expired pre-Complete failure
    fake.abort_transient_failures = 1  # inline abort fails once -> the fresh-client cleanup path is taken
    calls: list[str] = []
    real_refresh = c.refresh_session

    def _spy() -> None:
        calls.append("refresh")
        real_refresh()

    c.refresh_session = _spy  # type: ignore[method-assign]
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    assert calls == ["refresh"]  # session refreshed once, on the fresh-cleanup path
    assert fake.aborted == ["mpu-1"]  # cleanup then succeeded on the refreshed fresh client


@pytest.mark.asyncio
async def test_conflict_inline_abort_transient_then_fresh_cleanup_preserves_remote(tmpfile: str) -> None:
    """A copy 412 conflict against a moved remote, where the inline abort fails transiently: the MPU must be
    cleaned up on a fresh client, the remote generation preserved (never overwritten), and NO replacement
    written."""
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 11 * MIB)
    foreign = b"\x01" * (20 * MIB)
    fake.objects[KEY] = (foreign, '"foreign"', {})  # base moved under us -> copy 412 conflict
    fake.abort_transient_failures = 1  # inline abort fails; fresh-client abort succeeds
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    assert fake.abort_attempts == ["mpu-1", "mpu-1"]  # inline attempt + fresh-client retry
    assert fake.aborted == ["mpu-1"] and "mpu-1" not in fake.mpus  # orphan cleaned up on the fresh client
    assert _obj(fake) == foreign  # remote preserved, never overwritten
    assert not fake.completed and fake.ops.count("upload_fileobj") == 0  # conflict path never full-replaces
    assert URI not in c._compose_state  # stale compose state cleared


@pytest.mark.asyncio
async def test_fenced_complete_conflict_inline_abort_transient_then_fresh_cleanup_preserves_remote(
    tmpfile: str,
) -> None:
    """A foreign generation lands AFTER our copy but BEFORE our fenced Complete (IfMatch 412), and the inline
    abort of our MPU then fails transiently. The fenced-Complete conflict site must converge exactly like the
    copy conflict site: clean up the MPU on a fresh client, preserve the foreign remote, and write NO
    replacement."""
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    _write(tmpfile, 11 * MIB)
    foreign = b"\x0a" * (14 * MIB)

    def foreign_lands_before_complete(key: str, uid: str) -> None:
        fake.objects[key] = (foreign, '"foreign-newgen"', {GEN: "not-our-token"})

    fake.on_before_complete = foreign_lands_before_complete
    fake.abort_transient_failures = 1  # inline abort fails; the fresh-client abort then succeeds
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    assert fake.abort_attempts == ["mpu-1", "mpu-1"]  # inline attempt + fresh-client retry (same id)
    assert fake.aborted == ["mpu-1"] and "mpu-1" not in fake.mpus  # orphan cleaned up on the fresh client
    assert _obj(fake) == foreign  # newer foreign generation preserved, never overwritten by a Complete/replace
    assert not fake.completed and fake.ops.count("upload_fileobj") == 0  # fenced-conflict path never full-replaces
    assert URI not in c._compose_state  # stale compose state cleared


@pytest.mark.asyncio
async def test_generic_failure_fresh_cleanup_also_fails_fails_closed_no_replacement(tmpfile: str) -> None:
    """If the inline abort AND the fresh-client abort both fail, fail closed: preserve the remote, clear stale
    compose state, and create NO replacement MPU / full PUT on top of the un-aborted orphan. The bucket
    AbortIncompleteMultipartUpload lifecycle rule is the only remaining backstop for the leaked MPU."""
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    base = _obj(fake)
    _write(tmpfile, 11 * MIB)
    fake.fail["uploadpart"] = _client_error("InternalError", 500)  # non-expired pre-Complete failure
    fake.abort_transient_failures = 99  # both the inline AND the fresh-client abort fail
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    assert fake.abort_attempts == ["mpu-1", "mpu-1"]  # tried inline, then a fresh client
    assert fake.aborted == [] and "mpu-1" in fake.mpus  # neither succeeded -> orphan remains (lifecycle backstop)
    assert _obj(fake) == base  # remote preserved: NO full replacement over the un-aborted orphan
    assert not fake.completed and fake.ops.count("upload_fileobj") == 0  # no replacement / new PUT authored
    assert URI not in c._compose_state  # stale compose state cleared; next step authors a fresh generation


@pytest.mark.asyncio
async def test_completed_no_etag_then_expired_reconcile_then_nosuchupload_abort_is_safe(tmpfile: str) -> None:
    """Full interleaving: Complete succeeds but omits ETag; the reconciliation HEAD then returns an expired
    token, so the transaction refreshes and retries; the refreshed abort of the (already-completed) orphan
    returns NoSuchUpload. That NoSuchUpload must count as cleanup success — no fail-closed skip, no false
    unabortable orphan — leaving a safe remote outcome and no lingering incomplete MPU."""
    be = _GenBackend()
    be.objects[KEY] = (b"\xab" * (8 * MIB), '"seed-8"', {GEN: "prior"})
    be.complete_fail_gens = set()  # gen1 Complete SUCCEEDS (does not raise)
    be.complete_omit_etag = True  # ...but omits ETag -> reconciliation HEAD is consulted
    be.head_expired_gens = {1}  # ...and that HEAD returns an expired token on the stale gen1 client
    c = _make_client_gen(be)
    c._compose_state.set(URI, _ComposeState(8 * MIB, "seed-8"))
    c.refresh_session = lambda: None  # type: ignore[method-assign]
    _write(tmpfile, 11 * MIB)
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    # gen1 Complete landed mpu-1 (our 11 MiB generation), returned no ETag; the reconcile HEAD expired ->
    # refresh + retry. gen2 aborts the orphan mpu-1 -> NoSuchUpload (already completed) -> treated as success.
    assert "mpu-1" in be.abort_attempts_by_gen.get(2, [])  # the orphan abort was attempted on the refreshed client
    assert be.completed == ["mpu-1"]  # the original Complete had actually landed our generation
    assert be.objects[KEY][0] == b"\xab" * (11 * MIB)  # safe remote: our completed generation is preserved
    assert be._n == 2  # gen2 was NOT fail-closed: it proceeded to create a replacement (mpu-2)...
    assert "mpu-2" not in be.mpus  # ...which then conflicted on the moved destination and was aborted (no leak)
    assert URI not in c._compose_state  # conflict path cleared local state; next step reseeds


# --- MUST_FIX 3: compose-state cache is bounded (LRU + TTL) and cleared after the terminal drain -------


def test_compose_state_cache_bounds_by_lru_and_ttl() -> None:
    now = [0.0]
    cache = _ComposeStateCache(max_size=2, ttl_seconds=10, clock=lambda: now[0])
    cache.set("a", _ComposeState(1, "a"))
    cache.set("b", _ComposeState(2, "b"))
    assert cache.get("a").length == 1  # touch a -> a is now most-recently-used
    cache.set("c", _ComposeState(3, "c"))  # over cap -> evict LRU, which is b (not the touched a)
    assert cache.get("b") is None
    assert cache.get("a").length == 1 and cache.get("c").length == 3
    now[0] = 100  # advance past TTL
    assert cache.get("a") is None and cache.get("c") is None  # expired entries are dropped
    assert len(cache) == 0


@pytest.mark.asyncio
async def test_terminal_clears_compose_state_inside_serialized_slot(tmpfile: str) -> None:
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)
    assert URI in c._compose_state
    # The terminal finalize clears state AFTER draining prior writes (race-safe), not only pre-emptively.
    await c.upload_file(URI, b"final-bytes", serialize_key=URI, supersede_queued=True)
    assert URI not in c._compose_state


# --- review 3920309806: any serialized full recording write must invalidate compose state ------


@pytest.mark.asyncio
async def test_nonterminal_full_write_invalidates_compose_state_so_next_snapshot_reseeds(tmpfile: str) -> None:
    """A NON-terminal serialized full byte write (supersede_queued=False — e.g. the mid-step byte fallback in
    _sync_video_artifact_after_step) replaces the object/metadata. It must clear the cached compose base too,
    or the next streamed snapshot's conditional copy 412s against the now-stale ETag and is skipped. Prove
    the state is invalidated and the next snapshot full-reseeds and lands."""
    fake = FakeS3()
    c = _make_client(fake)
    _seed_base(c, fake, 8 * MIB)  # compose base cached at etag "seed-8388608"
    assert URI in c._compose_state
    # A serialized, NON-terminal full write replaces the object with a new ETag ("put-N").
    await c.upload_file(URI, b"\x03" * (9 * MIB), serialize_key=URI, supersede_queued=False)
    assert URI not in c._compose_state  # stale compose base invalidated inside the serialized slot
    # The next streamed snapshot must full-reseed (base is gone) and LAND, not 412-skip on the stale ETag.
    _write(tmpfile, 11 * MIB, fill=b"\xcd")
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    assert _obj(fake) == b"\xcd" * (11 * MIB)  # reseeded full replacement landed
    assert not fake.completed  # no compose (no owned base) -> no UploadPartCopy against a stale ETag
    assert c._compose_state.get(URI) is not None  # fresh verified generation seeded for the next step


# --- MUST_FIX 4: never splice an unowned/foreign remote generation --------------------------------------


@pytest.mark.asyncio
async def test_foreign_remote_object_is_not_spliced(tmpfile: str) -> None:
    fake = FakeS3()
    c = _make_client(fake)
    # A duplicate/zombie producer full-PUT a foreign >=5 MiB generation; this process has NO owned state.
    foreign = b"\x07" * (8 * MIB)
    fake.objects[KEY] = (foreign, '"foreign-gen"', {GEN: "foreign-token"})
    _write(tmpfile, 11 * MIB, fill=b"\xcd")
    await c.store_recording_prefix(URI, tmpfile, 11 * MIB, serialize_key=URI)
    # Must be a clean full replacement of OUR bytes — never foreign-prefix copy + our-tail splice.
    assert _obj(fake) == b"\xcd" * (11 * MIB)
    assert not fake.mpus and not fake.completed  # no UploadPartCopy of the foreign prefix
    # And local state is now seeded from OUR authored+verified generation for the next step to compose off.
    assert c._compose_state.get(URI) is not None


@pytest.mark.asyncio
async def test_full_replace_does_not_seed_when_foreign_writer_wins_the_race(tmpfile: str) -> None:
    fake = FakeS3()
    c = _make_client(fake)
    _write(tmpfile, 10 * MIB)
    orig_head = fake.head_object

    async def head_shows_foreign(*, Bucket: str, Key: str) -> dict[str, Any]:
        # A foreign writer overwrote our object between our PUT and this readback (token mismatch).
        fake.objects[Key] = (b"\x02" * (10 * MIB), '"foreign"', {GEN: "not-ours"})
        return await orig_head(Bucket=Bucket, Key=Key)

    fake.head_object = head_shows_foreign  # type: ignore[method-assign]
    await c.store_recording_prefix(URI, tmpfile, 10 * MIB, serialize_key=URI)
    # Readback token/length did not match ours -> do NOT seed; the next step safely full-replaces again.
    assert URI not in c._compose_state