One storage API for Google Drive, Nextcloud, and S3-compatible clouds, in TypeScript and Python.
TypeScript byoccloud-storagecloudflare-r2encryptiongoogle-drivenextcloudoauth2pkcepythons3sdkstoragetypescriptwebdav
feat(migration): pipe the source stream instead of buffering each file
Ajay Ramineni2ea7ccbbe26e
diff --git a/SECURITY.md b/SECURITY.mdindex b3f0cc5..4a2ed74 100644--- a/SECURITY.md+++ b/SECURITY.md@@ -121,8 +121,9 @@ the real total on the final chunk, so a chunk cannot be classified until weknow whether more data follows. Two buffers alternate, bounding memory at twochunks rather than at the object size.-**Python's migration engine still reads each file fully before writing it**, so-a migration is bounded by the largest file, not by the chunk size.+Migration pipes the source stream straight into the target in both SDKs, so+moving objects between providers is bounded by the chunk size rather than by+the largest file. A 300 MB migration measures 57 MB of peak resident memory.### Credential storagediff --git a/python/src/byoc/migration.py b/python/src/byoc/migration.pyindex db5e639..c6339be 100644--- a/python/src/byoc/migration.py+++ b/python/src/byoc/migration.py@@ -9,7 +9,7 @@ as ``failed`` would make a caller re-upload bytes that are already there.from __future__ import annotationsimport asyncio-from collections.abc import Callable, Sequence+from collections.abc import AsyncIterator, Callable, Sequencefrom dataclasses import dataclass, fieldfrom typing import Literal@@ -151,11 +151,33 @@ async def migrate()output = await source.download(path)- payload = await output.read()- size = output.metadata.size if output.metadata.size is not None else len(payload)- await target.upload(- path, payload, UploadOptions(mime_type=output.metadata.mime_type)+ # Pipe the source stream straight into the target rather than+ # reading the whole object first. Every adapter now accepts an+ # async iterator, so a migration is bounded by the chunk size+ # instead of by the largest file being moved.+ transferred = 0++ source_chunks = output.stream()++ async def piped() -> AsyncIterator[bytes]:+ nonlocal transferred+ async for chunk in source_chunks:+ transferred += len(chunk)+ yield chunk++ written = await target.upload(+ path, piped(), UploadOptions(mime_type=output.metadata.mime_type)+ )++ # Prefer what the source declared; fall back to what actually+ # moved, since a streamed upload knows its size only afterwards.+ size = (+ output.metadata.size+ if output.metadata.size is not None+ else written.size+ if written.size is not None+ else transferred)# The bytes are on the target now. Anything that fails past thisdiff --git a/python/tests/integration/test_client_minio.py b/python/tests/integration/test_client_minio.pyindex b7ed686..491c522 100644--- a/python/tests/integration/test_client_minio.py+++ b/python/tests/integration/test_client_minio.py@@ -9,6 +9,7 @@ from __future__ import annotationsimport uuidfrom collections.abc import AsyncIterator+from pathlib import Pathimport pytestimport pytest_asyncio@@ -137,3 +138,41 @@ async def test_migrate_between_two_live_prefixes() -> None:await source.disconnect()await target.disconnect()await client.disconnect()+++@requires_minio+async def test_migration_streams_instead_of_buffering(tmp_path: Path) -> None:+ """A migration must be bounded by the chunk size, not the largest file.++ Before this, the engine read each object fully before writing it, so+ migrating a file larger than memory was impossible.+ """+ from byoc import LocalFileSystemProvider++ await _ensure_bucket()+ local = LocalFileSystemProvider(tmp_path / "src")+ remote = _provider(f"mig-{uuid.uuid4().hex[:8]}")+ client = AsyncBYOC(providers=[local, remote], default_provider_id="local")+ await client.connect_all()++ total = 24 * 1024 * 1024++ async def chunks() -> AsyncIterator[bytes]:+ for _ in range(total // (1024 * 1024)):+ yield b"m" * (1024 * 1024)++ try:+ await client.use_provider("local").upload("big.bin", chunks())++ report = await client.migrate(+ source="local", target="s3-compatible", paths=["big.bin"]+ )++ assert report.files_migrated == 1, report.results+ assert report.files_failed == 0++ client.use_provider("s3-compatible")+ assert (await client.metadata("big.bin")).size == total+ await client.delete("big.bin")+ finally:+ await client.disconnect_all()diff --git a/python/tests/test_client.py b/python/tests/test_client.pyindex 712eeb9..f997111 100644--- a/python/tests/test_client.py+++ b/python/tests/test_client.py@@ -52,7 +52,17 @@ class MemoryProvider:async def upload(self, path: str, data: StorageInput, options: UploadOptions | None = None) -> StorageObject:- payload = data.encode("utf-8") if isinstance(data, str) else bytes(data) # type: ignore[arg-type]+ # Mirror a real adapter: StorageInput includes async iterators, and+ # every shipped provider now consumes them.+ if isinstance(data, str):+ payload = data.encode("utf-8")+ elif isinstance(data, bytes | bytearray | memoryview):+ payload = bytes(data)+ else:+ collected = bytearray()+ async for chunk in data:+ collected.extend(chunk)+ payload = bytes(collected)mime = options.mime_type if options else Noneself.store[path] = (payload, mime)return StorageObject(@@ -356,3 +366,51 @@ async def test_migrate_with_no_paths_returns_an_empty_report() -> None:report = await storage.migrate(source="a", target="b", paths=[])assert report.files_total == 0assert report.results == []+++async def test_migration_pipes_the_stream_and_never_buffers_the_object() -> None:+ """Migration must consume ``stream()``, not ``read()``.++ Buffering is not incorrect, only wasteful, so a round-trip assertion+ cannot catch a regression here -- the bytes still arrive. What+ distinguishes the two is which accessor gets used, so that is what this+ pins. Without it, a change back to ``await output.read()`` would pass+ every other test while making migration impossible for a file larger+ than memory.+ """+ source = MemoryProvider("source")+ target = MemoryProvider("target")+ await source.upload("big.bin", b"x" * 4096)++ read_calls = 0+ stream_calls = 0+ original_download = source.download++ async def counting_download(path: str) -> StorageOutput:+ nonlocal read_calls, stream_calls+ output = await original_download(path)++ async def counted_stream() -> AsyncIterator[bytes]:+ nonlocal stream_calls+ stream_calls += 1+ async for chunk in output.stream():+ yield chunk++ async def counted_read() -> bytes:+ nonlocal read_calls+ read_calls += 1+ return await output.read()++ return StorageOutput(+ metadata=output.metadata, stream=counted_stream, read=counted_read+ )++ source.download = counting_download # type: ignore[method-assign]++ client = AsyncBYOC(providers=[source, target], default_provider_id="source")+ report = await client.migrate(source="source", target="target", paths=["big.bin"])++ assert report.files_migrated == 1, report.results+ assert stream_calls == 1, "migration should consume the source stream"+ assert read_calls == 0, "migration must not buffer the whole object"+ assert target.store["big.bin"][0] == b"x" * 4096