One storage API for Google Drive, Nextcloud, and S3-compatible clouds, in TypeScript and Python.
TypeScript byoccloud-storagecloudflare-r2encryptiongoogle-drivenextcloudoauth2pkcepythons3sdkstoragetypescriptwebdav
"""AsyncBYOC client and migration engine, driven by an in-memory provider."""
from __future__ import annotations
from collections.abc import AsyncIterator
import pytest
from byoc import AsyncBYOC, ConflictStrategy
from byoc.errors import CapabilityUnsupportedError, InvalidInputError
from byoc.types import (
ProviderCapabilities,
ProviderManifest,
StorageInput,
StorageObject,
StorageOutput,
StorageQuota,
UploadOptions,
)
class MemoryProvider:
"""Minimal in-memory BYOCProvider for exercising client behaviour."""
def __init__(self, provider_id: str = "memory", *, folders: bool = True, quota: bool = True):
self.id = provider_id
self.store: dict[str, tuple[bytes, str | None]] = {}
self.connected = False
self.fail_delete = False
self._folders = folders
self._quota = quota
def manifest(self) -> ProviderManifest:
return ProviderManifest(
id=self.id,
name=f"Memory ({self.id})",
category="self-hosted",
authentication="basic",
supports_user_owned_storage=True,
adapter_version="0.2.0",
)
def capabilities(self) -> ProviderCapabilities:
return ProviderCapabilities(folders=self._folders, quota=self._quota)
async def connect(self) -> None:
self.connected = True
async def disconnect(self) -> None:
self.connected = False
async def upload(
self, path: str, data: StorageInput, options: UploadOptions | None = None
) -> StorageObject:
# Mirror a real adapter: StorageInput includes async iterators, and
# every shipped provider now consumes them.
if isinstance(data, str):
payload = data.encode("utf-8")
elif isinstance(data, bytes | bytearray | memoryview):
payload = bytes(data)
else:
collected = bytearray()
async for chunk in data:
collected.extend(chunk)
payload = bytes(collected)
mime = options.mime_type if options else None
self.store[path] = (payload, mime)
return StorageObject(
id=f"{self.id}_{path}",
path=path,
name=path.rsplit("/", 1)[-1],
provider=self.id,
provider_id=path,
size=len(payload),
mime_type=mime,
)
async def download(self, path: str) -> StorageOutput:
if path not in self.store:
from byoc.errors import ObjectNotFoundError
raise ObjectNotFoundError(f"Not found: {path}", provider=self.id)
payload, mime = self.store[path]
async def stream() -> AsyncIterator[bytes]:
yield payload
async def read() -> bytes:
return payload
return StorageOutput(
metadata=StorageObject(
id=f"{self.id}_{path}",
path=path,
name=path.rsplit("/", 1)[-1],
provider=self.id,
provider_id=path,
size=len(payload),
mime_type=mime,
),
stream=stream,
read=read,
)
async def delete(self, path: str) -> None:
if self.fail_delete:
from byoc.errors import PermissionDeniedError
raise PermissionDeniedError("read-only source", provider=self.id)
self.store.pop(path, None)
async def list(self, path: str | None = None) -> list[StorageObject]:
prefix = f"{path}/" if path else ""
return [
StorageObject(
id=f"{self.id}_{key}",
path=key,
name=key.rsplit("/", 1)[-1],
provider=self.id,
provider_id=key,
)
for key in self.store
if key.startswith(prefix)
]
async def exists(self, path: str) -> bool:
return path in self.store
async def metadata(self, path: str) -> StorageObject:
return (await self.download(path)).metadata
async def quota(self) -> StorageQuota:
used = sum(len(v[0]) for v in self.store.values())
return StorageQuota(used=used, total=1000, available=1000 - used)
def test_requires_at_least_one_provider() -> None:
with pytest.raises(InvalidInputError):
AsyncBYOC()
async def test_write_and_read_text_roundtrip() -> None:
storage = AsyncBYOC(provider=MemoryProvider())
await storage.connect()
await storage.write_text("docs/note.md", "# Hello")
assert await storage.read_text("docs/note.md") == "# Hello"
async def test_paths_are_normalized_before_reaching_the_provider() -> None:
provider = MemoryProvider()
storage = AsyncBYOC(provider=provider)
await storage.write_text("///docs//../..//".replace("..", "x"), "v")
await storage.write_text("/docs//note.md/", "value")
assert "docs/note.md" in provider.store
async def test_upload_rejects_traversal() -> None:
storage = AsyncBYOC(provider=MemoryProvider())
with pytest.raises(InvalidInputError):
await storage.write_text("../escape.txt", "nope")
async def test_mime_type_is_detected_from_the_path() -> None:
provider = MemoryProvider()
storage = AsyncBYOC(provider=provider)
obj = await storage.upload("images/logo.png", b"PNG")
assert obj.mime_type == "image/png"
async def test_max_file_size_is_enforced() -> None:
storage = AsyncBYOC(provider=MemoryProvider(), max_file_size_bytes=4)
with pytest.raises(InvalidInputError):
await storage.write_bytes("big.bin", b"12345")
async def test_exists_and_delete() -> None:
storage = AsyncBYOC(provider=MemoryProvider())
await storage.write_text("a.txt", "x")
assert await storage.exists("a.txt") is True
await storage.delete("a.txt")
assert await storage.exists("a.txt") is False
# An empty path is the root, never a file.
assert await storage.exists("") is False
async def test_provider_switching() -> None:
drive = MemoryProvider("google-drive")
s3 = MemoryProvider("s3-compatible")
storage = AsyncBYOC(providers=[drive, s3], default_provider_id="s3-compatible")
assert storage.manifest().id == "s3-compatible"
await storage.write_text("only-on-s3.txt", "v")
storage.use_provider("google-drive")
assert storage.manifest().id == "google-drive"
assert await storage.exists("only-on-s3.txt") is False
assert sorted(m.id for m in storage.get_providers()) == ["google-drive", "s3-compatible"]
async def test_unknown_provider_lists_the_available_ones() -> None:
storage = AsyncBYOC(provider=MemoryProvider("memory"))
with pytest.raises(InvalidInputError) as excinfo:
storage.use_provider("dropbox")
assert "memory" in str(excinfo.value)
async def test_capability_gated_operations_raise_when_unsupported() -> None:
storage = AsyncBYOC(provider=MemoryProvider(folders=False, quota=False))
with pytest.raises(CapabilityUnsupportedError):
await storage.create_folder("docs")
with pytest.raises(CapabilityUnsupportedError):
await storage.get_quota()
# move() is absent from MemoryProvider entirely.
with pytest.raises(CapabilityUnsupportedError):
await storage.move("a.txt", "b.txt")
async def test_quota_is_returned_when_supported() -> None:
storage = AsyncBYOC(provider=MemoryProvider())
await storage.write_bytes("a.bin", b"1234")
quota = await storage.get_quota()
assert quota.used == 4
assert quota.available == 996
async def test_backup_writes_a_timestamped_file() -> None:
provider = MemoryProvider()
storage = AsyncBYOC(provider=provider)
obj = await storage.backup(b'{"rows":1}')
assert obj.path.startswith("Backups/backup-")
assert obj.path.endswith(".json")
async def test_async_context_manager_connects_and_disconnects() -> None:
provider = MemoryProvider()
async with AsyncBYOC(provider=provider) as storage:
assert provider.connected is True
await storage.write_text("x.txt", "y")
assert provider.connected is False
# -- migration ---------------------------------------------------------------
async def test_migrate_between_providers() -> None:
source = MemoryProvider("google-drive")
target = MemoryProvider("webdav")
storage = AsyncBYOC(providers=[source, target])
await source.upload("docs/a.txt", b"aaa")
await source.upload("docs/b.txt", b"bb")
report = await storage.migrate(
source="google-drive", target="webdav", paths=["docs/a.txt", "docs/b.txt"]
)
assert report.files_total == 2
assert report.files_migrated == 2
assert report.files_failed == 0
assert report.bytes_transferred == 5
assert await target.exists("docs/a.txt")
async def test_migrate_reports_partial_when_source_delete_fails() -> None:
"""The copy landed; only cleanup failed. Retrying would re-upload for nothing."""
source = MemoryProvider("src")
target = MemoryProvider("dst")
source.fail_delete = True
await source.upload("vault/key.pem", b"SECRET")
storage = AsyncBYOC(providers=[source, target])
report = await storage.migrate(
source="src", target="dst", paths=["vault/key.pem"], delete_source_after_migrate=True
)
assert report.files_migrated == 0
assert report.files_failed == 0
assert report.files_partial == 1
# Exactly one record for one input file: no double-counting.
assert len(report.results) == 1
assert report.results[0].status == "partial"
# The point of 'partial': the target copy really exists...
assert await target.exists("vault/key.pem")
# ...and the source is still there, awaiting cleanup.
assert await source.exists("vault/key.pem")
async def test_migrate_reports_failed_when_the_upload_fails() -> None:
source = MemoryProvider("src")
target = MemoryProvider("dst")
await source.upload("a.txt", b"x")
async def boom(*args: object, **kwargs: object) -> StorageObject:
raise RuntimeError("quota exceeded on target")
target.upload = boom # type: ignore[assignment]
storage = AsyncBYOC(providers=[source, target])
report = await storage.migrate(
source="src", target="dst", paths=["a.txt"], delete_source_after_migrate=True
)
assert report.files_failed == 1
assert report.files_partial == 0
# A real transfer failure must never delete the source.
assert await source.exists("a.txt")
async def test_migrate_skip_strategy() -> None:
source = MemoryProvider("src")
target = MemoryProvider("dst")
await source.upload("a.txt", b"new")
await target.upload("a.txt", b"old")
storage = AsyncBYOC(providers=[source, target])
report = await storage.migrate(
source="src", target="dst", paths=["a.txt"], conflict_strategy=ConflictStrategy.SKIP
)
assert report.files_skipped == 1
assert (await (await target.download("a.txt")).read()) == b"old"
async def test_migrate_error_strategy_records_failure() -> None:
source = MemoryProvider("src")
target = MemoryProvider("dst")
await source.upload("a.txt", b"new")
await target.upload("a.txt", b"old")
storage = AsyncBYOC(providers=[source, target])
report = await storage.migrate(
source="src", target="dst", paths=["a.txt"], conflict_strategy=ConflictStrategy.ERROR
)
assert report.files_failed == 1
async def test_migrate_progress_reaches_one_hundred_percent() -> None:
source = MemoryProvider("src")
target = MemoryProvider("dst")
for i in range(4):
await source.upload(f"f{i}.txt", b"x")
events: list[int] = []
storage = AsyncBYOC(providers=[source, target])
await storage.migrate(
source="src",
target="dst",
paths=[f"f{i}.txt" for i in range(4)],
on_progress=lambda p: events.append(p.percentage),
)
assert len(events) == 4
assert events[-1] == 100
async def test_migrate_unknown_provider_raises() -> None:
storage = AsyncBYOC(provider=MemoryProvider("only"))
with pytest.raises(InvalidInputError):
await storage.migrate(source="nope", target="only", paths=["a.txt"])
async def test_migrate_with_no_paths_returns_an_empty_report() -> None:
storage = AsyncBYOC(providers=[MemoryProvider("a"), MemoryProvider("b")])
report = await storage.migrate(source="a", target="b", paths=[])
assert report.files_total == 0
assert report.results == []
async def test_migration_pipes_the_stream_and_never_buffers_the_object() -> None:
"""Migration must consume ``stream()``, not ``read()``.
Buffering is not incorrect, only wasteful, so a round-trip assertion
cannot catch a regression here -- the bytes still arrive. What
distinguishes the two is which accessor gets used, so that is what this
pins. Without it, a change back to ``await output.read()`` would pass
every other test while making migration impossible for a file larger
than memory.
"""
source = MemoryProvider("source")
target = MemoryProvider("target")
await source.upload("big.bin", b"x" * 4096)
read_calls = 0
stream_calls = 0
original_download = source.download
async def counting_download(path: str) -> StorageOutput:
nonlocal read_calls, stream_calls
output = await original_download(path)
async def counted_stream() -> AsyncIterator[bytes]:
nonlocal stream_calls
stream_calls += 1
async for chunk in output.stream():
yield chunk
async def counted_read() -> bytes:
nonlocal read_calls
read_calls += 1
return await output.read()
return StorageOutput(
metadata=output.metadata, stream=counted_stream, read=counted_read
)
source.download = counting_download # type: ignore[method-assign]
client = AsyncBYOC(providers=[source, target], default_provider_id="source")
report = await client.migrate(source="source", target="target", paths=["big.bin"])
assert report.files_migrated == 1, report.results
assert stream_calls == 1, "migration should consume the source stream"
assert read_calls == 0, "migration must not buffer the whole object"
assert target.store["big.bin"][0] == b"x" * 4096