Skip to content

Automate browser based workflows with AI

6.4 KB
Raw Blame
import asyncio
import io
import zipfile
from datetime import datetime
from types import SimpleNamespace
from typing import Any

import pytest

from skyvern.forge.sdk.artifact.manager import ArtifactManager
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.models import Step, StepStatus


def _step(step_id: str, order: int, task_id: str = "tsk_block") -> Step:
    now = datetime.utcnow()
    return Step(
        task_id=task_id,
        step_id=step_id,
        order=order,
        retry_index=0,
        status=StepStatus.running,
        is_last=False,
        organization_id="o_archive",
        created_at=now,
        modified_at=now,
    )


@pytest.mark.asyncio
async def test_a_tasks_step_archives_flush_as_one_zip_with_each_file_on_its_own_step(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    # A Task V3 block's turn steps share its ZIP (one storage PUT per block); each file's row stays on its turn.
    manager = ArtifactManager()
    block, turn_1, turn_2 = _step("stp_block", 0), _step("stp_turn_1", 1), _step("stp_turn_2", 2)
    other = _step("stp_other", 1, task_id="tsk_other")
    for step in (turn_1, turn_2, block, other):
        manager.accumulate_llm_call_to_archive(
            step=step,
            workflow_run_id="wr_archive",
            request=f"{step.step_id} request".encode(),
            response=f"{step.step_id} response".encode(),
        )

    stored: list[tuple[Any, bytes]] = []
    rows: list[Any] = []

    async def _store(artifact: Any, data: bytes) -> None:
        stored.append((artifact, data))

    async def _bulk_create(models: list[Any]) -> list[Any]:
        rows.extend(models)
        return models

    monkeypatch.setattr("skyvern.forge.sdk.artifact.manager._bundling_enabled", lambda: True)
    monkeypatch.setattr(
        "skyvern.forge.sdk.artifact.manager.app.STORAGE",
        SimpleNamespace(build_uri=lambda **kwargs: f"s3://artifacts/{kwargs['artifact_id']}", store_artifact=_store),
    )
    monkeypatch.setattr("skyvern.forge.sdk.artifact.manager.app.DATABASE.artifacts.bulk_create_artifacts", _bulk_create)

    await manager.flush_task_step_archives_as_one(block)

    [(archive, zip_bytes)] = stored
    parent, *members = rows
    assert (parent.artifact_type, parent.step_id, parent.uri) == (ArtifactType.STEP_ARCHIVE, "stp_block", archive.uri)
    with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:
        assert all(member.uri == archive.uri for member in members)
        assert sorted((member.step_id, zf.read(member.bundle_key).decode()) for member in members) == sorted(
            (step_id, f"{step_id} {kind}")
            for step_id in ("stp_block", "stp_turn_1", "stp_turn_2")
            for kind in ("request", "response")
        )
    # Another task's archive waits for its own flush, and a step flushed alone keeps its own file names.
    assert list(manager._step_archives) == ["stp_other"]
    rows.clear()
    await manager.flush_step_archive("stp_other")
    assert sorted(row.bundle_key for row in rows[1:]) == ["llm_request_0.json", "llm_response_0.json"]


@pytest.mark.asyncio
async def test_a_background_turn_flush_survives_the_upload_barrier_cancelling_it(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    # A turn flushed on its own is one object per turn; a barrier that gives up and cancels the tracked task must not
    # lose an archive the flush already popped, and the block-end flush then finds nothing of those turns left.
    manager = ArtifactManager()
    block, turn_1, turn_2 = _step("stp_block", 0), _step("stp_turn_1", 1), _step("stp_turn_2", 2)
    for step in (block, turn_1, turn_2):
        manager.accumulate_llm_call_to_archive(step=step, request=f"{step.step_id} request".encode())
    manager.accumulate_html_scrape_to_archive(turn_1, b"<p>turn 1</p>")

    release = asyncio.Event()
    stored: list[tuple[str, bytes]] = []

    async def _store(artifact: Any, data: bytes) -> None:
        await release.wait()
        stored.append((artifact.step_id, data))

    async def _bulk_create(models: list[Any]) -> list[Any]:
        return models

    monkeypatch.setattr("skyvern.forge.sdk.artifact.manager._bundling_enabled", lambda: True)
    monkeypatch.setattr(
        "skyvern.forge.sdk.artifact.manager.app.STORAGE",
        SimpleNamespace(build_uri=lambda **kwargs: f"s3://artifacts/{kwargs['artifact_id']}", store_artifact=_store),
    )
    monkeypatch.setattr("skyvern.forge.sdk.artifact.manager.app.DATABASE.artifacts.bulk_create_artifacts", _bulk_create)

    manager.flush_step_archive_in_background(turn_1)
    manager.flush_step_archive_in_background(turn_2)
    await asyncio.sleep(0)
    assert set(manager._step_archives) == {"stp_block"}
    for tracked in list(manager.upload_aiotasks_map["tsk_block"]):
        tracked.cancel()
    await asyncio.sleep(0)
    release.set()
    await asyncio.gather(*manager._background_flushes)

    turn_zips = dict(stored)
    assert sorted(turn_zips) == ["stp_turn_1", "stp_turn_2"]
    with zipfile.ZipFile(io.BytesIO(turn_zips["stp_turn_1"])) as zf:
        assert (sorted(zf.namelist()), zf.read("scrape.html")) == (
            ["llm_request_0.json", "scrape.html"],
            b"<p>turn 1</p>",
        )
    stored.clear()
    await manager.flush_task_step_archives_as_one(block)
    [(owner, zip_bytes)] = stored
    with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:
        assert (owner, zf.namelist()) == ("stp_block", ["llm_request_0.json"])


@pytest.mark.asyncio
async def test_a_background_turn_flush_against_a_stalled_store_ends_at_its_deadline(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    manager = ArtifactManager()
    turn = _step("stp_turn_1", 1)
    manager.accumulate_llm_call_to_archive(step=turn, request=b"request")

    async def _never_returns(artifact: Any, data: bytes) -> None:
        await asyncio.Event().wait()

    monkeypatch.setattr("skyvern.forge.sdk.artifact.manager.BACKGROUND_FLUSH_TIMEOUT_SECONDS", 0.05)
    monkeypatch.setattr("skyvern.forge.sdk.artifact.manager._bundling_enabled", lambda: True)
    monkeypatch.setattr(
        "skyvern.forge.sdk.artifact.manager.app.STORAGE",
        SimpleNamespace(
            build_uri=lambda **kwargs: f"s3://artifacts/{kwargs['artifact_id']}", store_artifact=_never_returns
        ),
    )

    manager.flush_step_archive_in_background(turn)
    # The outer bound only keeps a regression from hanging the suite.
    await asyncio.wait_for(asyncio.gather(*manager._background_flushes), timeout=5)

    assert manager._background_flushes == set()