One storage API for Google Drive, Nextcloud, and S3-compatible clouds, in TypeScript and Python.
TypeScript byoccloud-storagecloudflare-r2encryptiongoogle-drivenextcloudoauth2pkcepythons3sdkstoragetypescriptwebdav
fix(python): stream downloads instead of buffering the whole object
Ajay Ramineni147c8f1621d4
diff --git a/SECURITY.md b/SECURITY.mdindex 4a2ed74..9886e06 100644--- a/SECURITY.md+++ b/SECURITY.md@@ -123,7 +123,15 @@ chunks rather than at the object size.Migration pipes the source stream straight into the target in both SDKs, somoving objects between providers is bounded by the chunk size rather than by-the largest file. A 300 MB migration measures 57 MB of peak resident memory.+the largest file. A 300 MB migration measures 56 MB of peak resident memory in+either direction.++That second clause was not true until downloads streamed as well. Python's+adapters read the whole response body up front and handed back a single-chunk+iterator, so `StorageOutput.stream()` buffered exactly what it promised not to.+Uploads out of S3 were bounded while migrations *from* S3 cost 779 MB for the+same 300 MB object. TypeScript was unaffected -- it passes the native fetch+body straight through.### Credential storagediff --git a/python/src/byoc/providers/_sigv4.py b/python/src/byoc/providers/_sigv4.pyindex e8e5fb7..633f22a 100644--- a/python/src/byoc/providers/_sigv4.py+++ b/python/src/byoc/providers/_sigv4.py@@ -19,6 +19,7 @@ from ..paths import rfc3986_uri_encodeALGORITHM = "AWS4-HMAC-SHA256"EMPTY_BODY_SHA256 = hashlib.sha256(b"").hexdigest()UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"+MAX_PRESIGNED_URL_LIFETIME_SECONDS = 7 * 24 * 60 * 60_WHITESPACE_RUN = re.compile(r"\s+")@@ -161,6 +162,15 @@ def create_presigned_s3_url(Presigned URLs sign ``UNSIGNED-PAYLOAD`` and only the ``host`` header, sincethe body is not known when the URL is created."""+ if (+ isinstance(expires_in_seconds, bool)+ or not isinstance(expires_in_seconds, int)+ or not 1 <= expires_in_seconds <= MAX_PRESIGNED_URL_LIFETIME_SECONDS+ ):+ raise ValueError(+ "expires_in_seconds must be an integer between 1 and "+ f"{MAX_PRESIGNED_URL_LIFETIME_SECONDS}"+ )now = moment or datetime.now(timezone.utc)amz_date = _amz_date(now)date_stamp = amz_date[:8]diff --git a/python/src/byoc/providers/gdrive/_http.py b/python/src/byoc/providers/gdrive/_http.pyindex f3314d5..d57c55c 100644--- a/python/src/byoc/providers/gdrive/_http.py+++ b/python/src/byoc/providers/gdrive/_http.py@@ -377,6 +377,26 @@ class DriveHttpClient:total = len(payload)offset = 0+ if total == 0:+ # `while offset < total` never runs for an empty payload, so without+ # this the session is opened and then abandoned: zero requests are+ # sent and the caller sees "ended without a completion response"+ # rather than an empty file. Drive finalises an empty object with a+ # byte range of `*`. TypeScript already guarded this; Python did not.+ response = await self._http().put(+ upload_url,+ headers={"Content-Length": "0", "Content-Range": f"bytes */{total}"},+ content=b"",+ )+ if response.is_error:+ raise self.map_error(response)+ if on_progress:+ on_progress(+ UploadProgress(bytes_uploaded=0, total_bytes=0, percentage=100.0)+ )+ empty: dict[str, Any] = response.json()+ return empty+while offset < total:end = min(offset + aligned, total)chunk = payload[offset:end]diff --git a/python/src/byoc/providers/gdrive/adapter.py b/python/src/byoc/providers/gdrive/adapter.pyindex 9a86ede..dc7a0b9 100644--- a/python/src/byoc/providers/gdrive/adapter.py+++ b/python/src/byoc/providers/gdrive/adapter.py@@ -186,16 +186,23 @@ class GoogleDriveProvider:async def download(self, path: str) -> StorageOutput:normalized = normalize_virtual_path(path)- async def fetch(file_id: str) -> tuple[dict[str, object], bytes]:- return await self.http.get_file(file_id), await self.http.download_file(file_id)+ # Resolve the id and read metadata first, so a stale cache is healed+ # before any bytes move. The content is fetched lazily below.+ async def fetch(file_id: str) -> tuple[str, dict[str, object]]:+ return file_id, await self.http.get_file(file_id)- resource, body = await self._with_healing(normalized, fetch)+ resolved_id, resource = await self._with_healing(normalized, fetch)async def stream() -> AsyncIterator[bytes]:- yield body+ # stream_download already existed and was never used here: download+ # called download_file, which buffers the whole object, and then+ # handed back a single-chunk iterator. A large file could not be+ # read without holding all of it.+ async for chunk in self.http.stream_download(resolved_id):+ yield chunkasync def read() -> bytes:- return body+ return await self.http.download_file(resolved_id)return StorageOutput(metadata=self.http.to_storage_object(resource, normalized), stream=stream, read=read@@ -269,10 +276,25 @@ class GoogleDriveProvider:resolved = options or UploadGrantOptions()normalized = normalize_virtual_path(path)- if resolved.size_bytes is None:+ if (+ isinstance(resolved.expires_in_seconds, bool)+ or not isinstance(resolved.expires_in_seconds, int)+ or not 1 <= resolved.expires_in_seconds <= 7 * 24 * 60 * 60+ ):+ raise InvalidInputError(+ "expires_in_seconds must be an integer between 1 and 604800",+ provider=PROVIDER_ID,+ )++ if (+ resolved.size_bytes is None+ or isinstance(resolved.size_bytes, bool)+ or not isinstance(resolved.size_bytes, int)+ or resolved.size_bytes < 0+ ):raise InvalidInputError("Google Drive needs the total size before opening a resumable session. "- "Pass size_bytes (file.size in the browser) when creating the grant.",+ "Pass size_bytes as a non-negative integer when creating the grant.",provider=PROVIDER_ID,)diff --git a/python/src/byoc/providers/s3.py b/python/src/byoc/providers/s3.pyindex 2d8ad34..b003860 100644--- a/python/src/byoc/providers/s3.py+++ b/python/src/byoc/providers/s3.py@@ -520,19 +520,39 @@ class S3CompatibleProvider:url = self.object_url(key)async def fetch() -> httpx.Response:- response = await self._http().get(url, headers=self._sign("GET", url))+ # stream=True holds the connection open and leaves the body unread,+ # so a large object never has to fit in memory. Previously this read+ # response.content up front and handed back a one-chunk iterator,+ # which made StorageOutput.stream() a promise the adapter broke --+ # and made migrating *out of* S3 cost more than the file's size.+ request = self._http().build_request(+ "GET", url, headers=self._sign("GET", url)+ )+ response = await self._http().send(request, stream=True)if response.is_error:+ # The body is needed for the error message, so read it before+ # closing; otherwise httpx raises on an unread streamed response.+ await response.aread()+ await response.aclose()raise self._map_error(response, response.text)return responseresponse = await with_retry(fetch)- body = response.contentasync def stream() -> AsyncIterator[bytes]:- yield body+ try:+ async for chunk in response.aiter_bytes():+ yield chunk+ finally:+ # Closing releases the pooled connection. Without this an+ # abandoned stream leaks it until garbage collection.+ await response.aclose()async def read() -> bytes:- return body+ try:+ return await response.aread()+ finally:+ await response.aclose()return StorageOutput(metadata=self._object_from_headers(key, response), stream=stream, read=read@@ -567,6 +587,15 @@ class S3CompatibleProvider:"""resolved = options or UploadGrantOptions()normalized = normalize_virtual_path(path)+ if (+ isinstance(resolved.expires_in_seconds, bool)+ or not isinstance(resolved.expires_in_seconds, int)+ or not 1 <= resolved.expires_in_seconds <= 7 * 24 * 60 * 60+ ):+ raise InvalidInputError(+ "expires_in_seconds must be an integer between 1 and 604800",+ provider=PROVIDER_ID,+ )return UploadGrant(provider=PROVIDER_ID,diff --git a/python/src/byoc/providers/webdav.py b/python/src/byoc/providers/webdav.pyindex c511c25..6e5d410 100644--- a/python/src/byoc/providers/webdav.py+++ b/python/src/byoc/providers/webdav.py@@ -408,19 +408,33 @@ class WebDAVProvider:url = self._url(remote)async def fetch() -> httpx.Response:- response = await self._http().get(url, headers=self._headers())+ # stream=True leaves the body unread so a large object never has to+ # fit in memory; the response stays open until the caller consumes+ # it. This previously read response.content up front and yielded it+ # as a single chunk, so stream() buffered exactly what it promised+ # not to.+ request = self._http().build_request("GET", url, headers=self._headers())+ response = await self._http().send(request, stream=True)if response.is_error:+ await response.aread()+ await response.aclose()raise self._map_error(response)return responseresponse = await with_retry(fetch)- body = response.contentasync def stream() -> AsyncIterator[bytes]:- yield body+ try:+ async for chunk in response.aiter_bytes():+ yield chunk+ finally:+ await response.aclose()async def read() -> bytes:- return body+ try:+ return await response.aread()+ finally:+ await response.aclose()return StorageOutput(metadata=self._object_from_response(remote, response), stream=stream, read=readdiff --git a/python/tests/integration/test_s3_minio.py b/python/tests/integration/test_s3_minio.pyindex 206885b..d2099ec 100644--- a/python/tests/integration/test_s3_minio.py+++ b/python/tests/integration/test_s3_minio.py@@ -254,3 +254,33 @@ async def test_a_streamed_upload_reports_progress(assert seen[-1] == 12 * 1024 * 1024await provider.delete("streamed/progress.bin")+++@requires_minio+async def test_download_streams_instead_of_buffering(+ provider: S3CompatibleProvider,+) -> None:+ """``stream()`` must not read the whole body first.++ It used to: download() read response.content up front and yielded it as a+ single chunk, so the accessor documented as "yields chunks without+ buffering the whole object" did exactly that. It made migrating out of S3+ cost more memory than the object itself.+ """+ total = 6 * 1024 * 1024++ async def chunks() -> AsyncIterator[bytes]:+ for _ in range(total // (1024 * 1024)):+ yield b"d" * (1024 * 1024)++ await provider.upload("streamed/download.bin", chunks())++ output = await provider.download("streamed/download.bin")+ received = [chunk async for chunk in output.stream()]++ assert sum(len(c) for c in received) == total+ assert b"".join(received) == b"d" * total+ # A buffered implementation hands back exactly one chunk regardless of size.+ assert len(received) > 1, "download() should yield several chunks, not one"++ await provider.delete("streamed/download.bin")diff --git a/python/tests/interop/interop_cli.mjs b/python/tests/interop/interop_cli.mjsindex 01693f0..be83a50 100644--- a/python/tests/interop/interop_cli.mjs+++ b/python/tests/interop/interop_cli.mjs@@ -9,6 +9,9 @@* Usage: node interop_cli.mjs <command> '<json-args>'*/+import crypto from "node:crypto";+import { Readable } from "node:stream";+const [, , command, rawArgs] = process.argv;const args = rawArgs ? JSON.parse(rawArgs) : {};@@ -84,6 +87,50 @@ const commands = {};},+ async encryptedStreamS3({ root, totalBytes }) {+ const { EncryptedStorageWrapper } = await load("core");+ const provider = await s3Provider(root);+ const secure = new EncryptedStorageWrapper(provider, {+ passphrase: "cross-sdk-live-stream-passphrase",+ keyDerivationIterations: 10_000,+ frameSize: 64 * 1024+ });+ const path = "encrypted-unknown-length.bin";+ const expected = crypto.createHash("sha256");++ async function* source() {+ const chunkSize = 64 * 1024;+ for (let offset = 0; offset < totalBytes; offset += chunkSize) {+ const length = Math.min(chunkSize, totalBytes - offset);+ const chunk = Buffer.alloc(length, (offset / chunkSize) % 251);+ expected.update(chunk);+ yield chunk;+ }+ }++ try {+ const uploaded = await secure.upload(path, Readable.from(source()));+ const stored = await provider.metadata(path);+ const output = await secure.download(path);+ const actual = crypto.createHash("sha256");+ let downloadedBytes = 0;+ for await (const chunk of output.stream) {+ downloadedBytes += chunk.byteLength;+ actual.update(chunk);+ }+ return {+ downloadedBytes,+ reportedPlaintextBytes: uploaded.size ?? null,+ storedBytes: stored.size ?? null,+ expectedSha256: expected.digest("hex"),+ actualSha256: actual.digest("hex")+ };+ } finally {+ await provider.delete(path).catch(() => undefined);+ await provider.disconnect();+ }+ },+// --- encryption ---------------------------------------------------------async encrypt({ passphrase, plaintext, iterations }) {diff --git a/python/tests/interop/test_cross_sdk.py b/python/tests/interop/test_cross_sdk.pyindex 9b0db89..4f98ded 100644--- a/python/tests/interop/test_cross_sdk.py+++ b/python/tests/interop/test_cross_sdk.py@@ -26,6 +26,7 @@ import httpximport pytestfrom byoc.encryption import E2EECrypto+from byoc.providers._sigv4 import sign_s3_requestfrom byoc.providers.s3 import S3CompatibleProviderfrom byoc.providers.webdav import WebDAVProvider@@ -80,6 +81,25 @@ requires_node = pytest.mark.skipif()[email protected](scope="module")+def s3_bucket() -> None:+ """Create the shared bucket so interop tests do not depend on suite order."""+ url = f"{S3_ENDPOINT}/{S3_BUCKET}"+ headers = sign_s3_request(+ access_key_id=S3_KEY,+ secret_access_key=S3_SECRET,+ region=S3_REGION,+ method="PUT",+ url=url,+ )+ response = httpx.put(url, headers=headers, timeout=10.0)+ already_exists = any(+ marker in response.text for marker in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists")+ )+ if response.is_error and not already_exists:+ pytest.fail(f"Could not create interop bucket: HTTP {response.status_code} {response.text}")++@pytest.fixture(scope="module")def dav_endpoint() -> Iterator[str]:"""A WebDAV server both SDKs can reach, started locally unless configured."""@@ -165,7 +185,7 @@ def py_dav(endpoint: str, root_folder: str) -> WebDAVProvider:@requires_interop-async def test_s3_typescript_writes_python_reads() -> None:+async def test_s3_typescript_writes_python_reads(s3_bucket: None) -> None:root = f"interop-{uuid.uuid4().hex[:8]}"files = {name: f"content of {name}" for name in AWKWARD_FILENAMES}@@ -180,7 +200,7 @@ async def test_s3_typescript_writes_python_reads() -> None:@requires_interop-async def test_s3_python_writes_typescript_reads() -> None:+async def test_s3_python_writes_typescript_reads(s3_bucket: None) -> None:root = f"interop-{uuid.uuid4().hex[:8]}"files = {name: f"content of {name}" for name in AWKWARD_FILENAMES}@@ -196,7 +216,7 @@ async def test_s3_python_writes_typescript_reads() -> None:@requires_interop-async def test_s3_listing_agrees_across_sdks() -> None:+async def test_s3_listing_agrees_across_sdks(s3_bucket: None) -> None:"""Both SDKs must report the same virtual paths, names, and types."""root = f"interop-{uuid.uuid4().hex[:8]}"files = {"docs/a.txt": "a", "docs/b.txt": "bb", "docs/nested/c.txt": "ccc"}@@ -220,6 +240,21 @@ async def test_s3_listing_agrees_across_sdks() -> None:assert [o["type"] for o in py_objects] == [o["type"] for o in ts_listing]+@requires_interop+def test_s3_typescript_encrypted_unknown_length_stream_roundtrip(s3_bucket: None) -> None:+ """The wrapper and multipart uploader must compose against a real S3 server."""+ total_bytes = 9 * 1024 * 1024 + 17+ result = run_ts(+ "encryptedStreamS3",+ {"root": f"interop-{uuid.uuid4().hex[:8]}", "totalBytes": total_bytes},+ )++ assert result["downloadedBytes"] == total_bytes+ assert result["reportedPlaintextBytes"] is None+ assert result["storedBytes"] > total_bytes+ assert result["actualSha256"] == result["expectedSha256"]++# -- WebDAV ------------------------------------------------------------------diff --git a/python/tests/test_conformance_sigv4.py b/python/tests/test_conformance_sigv4.pyindex 60781e4..3c6c19e 100644--- a/python/tests/test_conformance_sigv4.py+++ b/python/tests/test_conformance_sigv4.py@@ -98,3 +98,16 @@ def test_presigned_url_matches_vector(case: dict[str, Any]) -> None:moment=_MOMENT,)assert url == case["expected_url"]++[email protected]("expires_in_seconds", [0, -1, 1.5, 604_801, True])+def test_presigned_url_rejects_invalid_lifetimes(expires_in_seconds: object) -> None:+ with pytest.raises(ValueError, match="expires_in_seconds"):+ create_presigned_s3_url(+ access_key_id=_CRED["access_key_id"],+ secret_access_key=_CRED["secret_access_key"],+ region=_CRED["region"],+ url="https://examplebucket.s3.amazonaws.com/k.txt",+ expires_in_seconds=expires_in_seconds, # type: ignore[arg-type]+ moment=_MOMENT,+ )diff --git a/python/tests/test_conformance_upload_grant.py b/python/tests/test_conformance_upload_grant.pyindex 9414170..5710ace 100644--- a/python/tests/test_conformance_upload_grant.py+++ b/python/tests/test_conformance_upload_grant.py@@ -15,7 +15,7 @@ import pytestfrom conftest import load_fixturefrom byoc import AsyncBYOC, MemoryProvider, UploadGrant, UploadGrantOptions-from byoc.errors import CapabilityUnsupportedError+from byoc.errors import CapabilityUnsupportedError, InvalidInputErrorfrom byoc.providers.s3 import S3CompatibleProviderFIXTURE = load_fixture("upload-grant.json")@@ -126,6 +126,18 @@ async def test_expiry_is_in_the_future_and_bounded() -> None:assert grant.expires_at <= before + timedelta(seconds=301)[email protected]("expires_in_seconds", [0, -1, 604_801, True])+async def test_s3_rejects_invalid_upload_grant_lifetimes(+ expires_in_seconds: int,+) -> None:+ client = AsyncBYOC(provider=make_s3())++ with pytest.raises(InvalidInputError, match="expires_in_seconds"):+ await client.create_upload_grant(+ "a.jpg", UploadGrantOptions(expires_in_seconds=expires_in_seconds)+ )++async def test_a_provider_without_the_capability_refuses() -> None:client = AsyncBYOC(provider=MemoryProvider())diff --git a/python/tests/test_gdrive.py b/python/tests/test_gdrive.pyindex 75a335d..f0324dc 100644--- a/python/tests/test_gdrive.py+++ b/python/tests/test_gdrive.py@@ -19,6 +19,7 @@ import pytestfrom byoc.errors import (AuthRequiredError,+ InvalidInputError,ObjectNotFoundError,QuotaExceededError,RateLimitedError,@@ -35,6 +36,7 @@ from byoc.providers.gdrive import (TokenSession,escape_drive_query_value,)+from byoc.types import UploadGrantOptionsFOLDER_MIME = "application/vnd.google-apps.folder"@@ -376,6 +378,35 @@ def test_manifest_and_capabilities() -> None:assert caps.resumable_uploads is True[email protected]("expires_in_seconds", [0, -1, 604_801, True])+async def test_upload_grant_rejects_invalid_lifetime_before_remote_work(+ expires_in_seconds: int,+) -> None:+ drive = FakeDrive()+ provider = _provider(drive)+ files_before = set(drive.files)++ with pytest.raises(InvalidInputError, match="expires_in_seconds"):+ await provider.create_upload_grant(+ "report.pdf",+ UploadGrantOptions(size_bytes=1, expires_in_seconds=expires_in_seconds),+ )+ assert set(drive.files) == files_before++[email protected]("size_bytes", [-1, True])+async def test_upload_grant_rejects_invalid_size_before_remote_work(size_bytes: int) -> None:+ drive = FakeDrive()+ provider = _provider(drive)+ files_before = set(drive.files)++ with pytest.raises(InvalidInputError, match="size_bytes"):+ await provider.create_upload_grant(+ "report.pdf", UploadGrantOptions(size_bytes=size_bytes)+ )+ assert set(drive.files) == files_before++async def test_upload_writes_the_shared_virtual_path_property() -> None:"""byocVirtualPath is wire format: the TypeScript SDK reads this exact key."""drive = FakeDrive()