Skip to content

armgit / BYOC

Public Imported from GitHub
0

One storage API for Google Drive, Nextcloud, and S3-compatible clouds, in TypeScript and Python.

5.6 KB
Raw Blame
"""AsyncBYOC driven against a real S3 server.

The unit tests exercise the client against an in-memory provider; this proves
the same code paths work when the provider is doing real HTTP, real signing, and
real error mapping.
"""

from __future__ import annotations

import uuid
from collections.abc import AsyncIterator
from pathlib import Path

import pytest
import pytest_asyncio

from byoc import AsyncBYOC, ConflictStrategy
from byoc.errors import CapabilityUnsupportedError, ObjectNotFoundError
from byoc.providers.s3 import S3CompatibleProvider

from .test_s3_minio import (
    ACCESS_KEY,
    BUCKET,
    ENDPOINT,
    REGION,
    SECRET_KEY,
    _ensure_bucket,
    requires_minio,
)

pytestmark = pytest.mark.integration


def _provider(provider_prefix: str) -> S3CompatibleProvider:
    return S3CompatibleProvider(
        endpoint=ENDPOINT,
        bucket=BUCKET,
        region=REGION,
        access_key_id=ACCESS_KEY,
        secret_access_key=SECRET_KEY,
        root_prefix=provider_prefix,
        force_path_style=True,
    )


@pytest_asyncio.fixture
async def storage() -> AsyncIterator[AsyncBYOC]:
    await _ensure_bucket()
    run = uuid.uuid4().hex[:8]
    client = AsyncBYOC(provider=_provider(f"client-{run}"), max_file_size_bytes=1_000_000)
    await client.connect()
    try:
        yield client
    finally:
        await client.disconnect()


@requires_minio
async def test_write_and_read_text(storage: AsyncBYOC) -> None:
    await storage.write_text("docs/note.md", "# Live from MinIO")
    assert await storage.read_text("docs/note.md") == "# Live from MinIO"


@requires_minio
async def test_mime_type_detection_survives_the_round_trip(storage: AsyncBYOC) -> None:
    await storage.write_bytes("images/logo.png", b"\x89PNG\r\n\x1a\n")
    meta = await storage.metadata("images/logo.png")
    assert meta.mime_type == "image/png"


@requires_minio
async def test_exists_delete_and_list(storage: AsyncBYOC) -> None:
    await storage.write_text("docs/a.txt", "a")
    await storage.write_text("docs/b.txt", "bb")

    assert await storage.exists("docs/a.txt") is True
    listed = await storage.list("docs")
    assert sorted(o.name for o in listed) == ["a.txt", "b.txt"]

    await storage.delete("docs/a.txt")
    assert await storage.exists("docs/a.txt") is False


@requires_minio
async def test_missing_object_raises_not_found(storage: AsyncBYOC) -> None:
    with pytest.raises(ObjectNotFoundError):
        await storage.read_text("docs/never-written.md")


@requires_minio
async def test_unsupported_capabilities_raise(storage: AsyncBYOC) -> None:
    """S3 has no real folders and reports no quota; both must say so honestly."""
    with pytest.raises(CapabilityUnsupportedError):
        await storage.create_folder("docs")
    with pytest.raises(CapabilityUnsupportedError):
        await storage.get_quota()


@requires_minio
async def test_backup_helper_writes_to_the_server(storage: AsyncBYOC) -> None:
    obj = await storage.backup(b'{"rows": 1}')
    assert obj.path.startswith("Backups/backup-")
    assert await storage.exists(obj.path) is True


@requires_minio
async def test_migrate_between_two_live_prefixes() -> None:
    """Two S3 prefixes stand in for two clouds: the engine only sees providers."""
    await _ensure_bucket()
    run = uuid.uuid4().hex[:8]
    source = _provider(f"mig-src-{run}")
    target = _provider(f"mig-dst-{run}")

    client = AsyncBYOC(providers=[source, target])
    # Both adapters share the same manifest id, so register them distinctly by
    # driving the engine through the source/target adapters directly.
    from byoc.migration import migrate as run_migration

    await source.connect()
    await target.connect()
    try:
        await source.upload("reports/q3.pdf", b"PDF BYTES", None)
        await source.upload("reports/q4.pdf", b"MORE BYTES", None)

        report = await run_migration(
            source=source,
            target=target,
            paths=["reports/q3.pdf", "reports/q4.pdf"],
            conflict_strategy=ConflictStrategy.OVERWRITE,
        )

        assert report.files_migrated == 2
        assert report.files_failed == 0
        assert report.bytes_transferred == 19
        assert await target.exists("reports/q3.pdf") is True
        assert await (await target.download("reports/q4.pdf")).read() == b"MORE BYTES"
    finally:
        await source.disconnect()
        await target.disconnect()
        await client.disconnect()


@requires_minio
async def test_migration_streams_instead_of_buffering(tmp_path: Path) -> None:
    """A migration must be bounded by the chunk size, not the largest file.

    Before this, the engine read each object fully before writing it, so
    migrating a file larger than memory was impossible.
    """
    from byoc import LocalFileSystemProvider

    await _ensure_bucket()
    local = LocalFileSystemProvider(tmp_path / "src")
    remote = _provider(f"mig-{uuid.uuid4().hex[:8]}")
    client = AsyncBYOC(providers=[local, remote], default_provider_id="local")
    await client.connect_all()

    total = 24 * 1024 * 1024

    async def chunks() -> AsyncIterator[bytes]:
        for _ in range(total // (1024 * 1024)):
            yield b"m" * (1024 * 1024)

    try:
        await client.use_provider("local").upload("big.bin", chunks())

        report = await client.migrate(
            source="local", target="s3-compatible", paths=["big.bin"]
        )

        assert report.files_migrated == 1, report.results
        assert report.files_failed == 0

        client.use_provider("s3-compatible")
        assert (await client.metadata("big.bin")).size == total
        await client.delete("big.bin")
    finally:
        await client.disconnect_all()