One storage API for Google Drive, Nextcloud, and S3-compatible clouds, in TypeScript and Python.
TypeScript byoccloud-storagecloudflare-r2encryptiongoogle-drivenextcloudoauth2pkcepythons3sdkstoragetypescriptwebdav
feat(gdrive): stream resumable uploads without buffering the object
Ajay Raminenid97a3f153489
diff --git a/SECURITY.md b/SECURITY.mdindex 8275359..b3f0cc5 100644--- a/SECURITY.md+++ b/SECURITY.md@@ -112,11 +112,17 @@ Where streaming stands, per adapter:| :--- | :--- | :--- || S3-compatible | streams (multipart) | streams (multipart) || WebDAV | streams (chunked) | streams (chunked) |-| Google Drive | **buffers** | **buffers** |+| Google Drive | streams (resumable) | streams (resumable) || Local, in-memory | streams | streams |-**Do not claim end-to-end bounded memory for the adapters marked buffers.**-Python's migration engine also still reads each file fully before writing it.+Google Drive needs a one-chunk lookahead. Its resumable protocol accepts+`Content-Range: bytes {start}-{end}/*` while the total is unknown but demands+the real total on the final chunk, so a chunk cannot be classified until we+know whether more data follows. Two buffers alternate, bounding memory at two+chunks rather than at the object size.++**Python's migration engine still reads each file fully before writing it**, so+a migration is bounded by the largest file, not by the chunk size.### Credential storagediff --git a/packages/google-drive/src/adapter.ts b/packages/google-drive/src/adapter.tsindex d014da4..a89f7cc 100644--- a/packages/google-drive/src/adapter.ts+++ b/packages/google-drive/src/adapter.ts@@ -154,8 +154,23 @@ export class GoogleDriveProvider implements BYOCProvider {const isLarge = byteLength !== undefined && byteLength >= 5 * 1024 * 1024; // 5 MB thresholdconst shouldUseResumable = options?.resumable || options?.onProgress || isLarge;+ // A stream has no length to compare against a threshold, and chunked+ // transfer is the only way to send one, so it always goes resumable.+ const isStream =+ byteLength === undefined &&+ typeof data !== "string" &&+ typeof (data as AsyncIterable<Uint8Array>)?.[Symbol.asyncIterator] === "function";+let resource;- if (shouldUseResumable) {+ if (isStream) {+ resource = await this.uploader.uploadStream(+ metadata,+ data as AsyncIterable<Uint8Array>,+ options?.mimeType ?? "application/octet-stream",+ options?.chunkSize,+ options?.onProgress+ );+ } else if (shouldUseResumable) {resource = await this.uploader.upload(metadata, data, {mimeType: options?.mimeType,chunkSize: options?.chunkSize,diff --git a/packages/google-drive/src/api/resumable.ts b/packages/google-drive/src/api/resumable.tsindex 7acb32b..d7df3f0 100644--- a/packages/google-drive/src/api/resumable.ts+++ b/packages/google-drive/src/api/resumable.ts@@ -47,6 +47,128 @@ export class ResumableUploader {return this.transmitChunks(sessionUri, bytes, totalBytes, chunkSize, mimeType, options.onProgress);}+ /**+ * Streams an async iterator into a resumable session without buffering it.+ *+ * Drive accepts `Content-Range: bytes {start}-{end}/*` while the total is+ * still unknown, and requires the real total on the final chunk. That forces+ * a one-chunk lookahead: a chunk cannot be sent as non-final until we know+ * more data follows, and cannot be sent as final until we know none does.+ * Two buffers alternate so the reader always has somewhere to write while a+ * full chunk waits to be classified.+ *+ * Memory is therefore bounded at two chunks, not at the object size.+ */+ public async uploadStream(+ metadata: Record<string, unknown>,+ source: AsyncIterable<Uint8Array>,+ mimeType: string,+ chunkSize: number = DEFAULT_CHUNK_SIZE,+ onProgress?: (progress: UploadProgress) => void+ ): Promise<DriveFileResource> {+ const aligned = Math.max(+ GOOGLE_DRIVE_CHUNK_ALIGNMENT,+ Math.floor(chunkSize / GOOGLE_DRIVE_CHUNK_ALIGNMENT) * GOOGLE_DRIVE_CHUNK_ALIGNMENT+ );+ const sessionUri = await this.initiateSession(metadata, 0, mimeType, true);++ const buffers = [new Uint8Array(aligned), new Uint8Array(aligned)];+ let current = 0;+ let filled = 0;+ let held: number | null = null;+ let offset = 0;+ let finalResource: DriveFileResource | undefined;++ const send = async (+ index: number,+ length: number,+ total: number | null+ ): Promise<void> => {+ if (length === 0 && total === null) return;++ const marker = total === null ? "*" : String(total);+ const range =+ length === 0+ ? `bytes */${total}`+ : `bytes ${offset}-${offset + length - 1}/${marker}`;++ const accessToken = await this.oauth.getAccessToken();+ const response = await fetch(sessionUri, {+ method: "PUT",+ headers: {+ Authorization: `Bearer ${accessToken}`,+ "Content-Range": range+ },+ body: buffers[index]!.subarray(0, length)+ });++ if (response.status === 200 || response.status === 201) {+ offset += length;+ finalResource = (await response.json()) as DriveFileResource;+ onProgress?.({ bytesUploaded: offset, totalBytes: offset, percentage: 100 });+ return;+ }++ if (response.status !== 308) {+ let data: unknown;+ try {+ data = await response.json();+ } catch {+ data = await response.text();+ }+ throw mapGoogleDriveError({ status: response.status, data });+ }++ // Trust the server's committed range over our own bookkeeping.+ const committed = response.headers.get("Range");+ offset = committed ? Number(committed.split("-").pop()) + 1 : offset + length;+ onProgress?.({ bytesUploaded: offset, totalBytes: undefined, percentage: undefined });+ };++ for await (const chunk of source) {+ const bytes = chunk instanceof Uint8Array ? chunk : new Uint8Array(chunk);+ let position = 0;+ while (position < bytes.byteLength) {+ const take = Math.min(aligned - filled, bytes.byteLength - position);+ buffers[current]!.set(bytes.subarray(position, position + take), filled);+ filled += take;+ position += take;++ if (filled === aligned) {+ // A full buffer cannot be final while more data may follow, so+ // release the previously held one and hold this instead.+ if (held !== null) await send(held, aligned, null);+ held = current;+ current = 1 - current;+ filled = 0;+ }+ }+ }++ // The stream is exhausted, so whatever remains is genuinely final.+ if (held !== null && filled === 0) {+ await send(held, aligned, offset + aligned);+ } else if (held !== null) {+ await send(held, aligned, null);+ await send(current, filled, offset + filled);+ } else {+ await send(current, filled, filled);+ }++ if (!finalResource) {+ throw mapGoogleDriveError({+ status: 500,+ data: {+ error: {+ message:+ "Google Drive accepted every chunk but never returned the file resource."+ }+ }+ });+ }+ return finalResource;+ }+/*** Opens a resumable session without sending any bytes, and returns its URI.*@@ -68,7 +190,8 @@ export class ResumableUploader {private async initiateSession(metadata: Record<string, unknown>,totalBytes: number,- mimeType: string+ mimeType: string,+ unknownLength = false): Promise<string> {const url = `${DRIVE_UPLOAD_BASE}/files?uploadType=resumable&fields=id,name,mimeType,size,parents,createdTime,modifiedTime,md5Checksum,appProperties`;@@ -78,9 +201,13 @@ export class ResumableUploader {const headers = new Headers({Authorization: `Bearer ${accessToken}`,"Content-Type": "application/json; charset=UTF-8",- "X-Upload-Content-Type": mimeType,- "X-Upload-Content-Length": String(totalBytes)+ "X-Upload-Content-Type": mimeType});+ // Declaring a length we do not have would make Drive reject the final+ // chunk, so a stream simply omits the header.+ if (!unknownLength) {+ headers.set("X-Upload-Content-Length", String(totalBytes));+ }const response = await fetch(url, {method: "POST",diff --git a/packages/google-drive/tests/api/streaming.test.ts b/packages/google-drive/tests/api/streaming.test.tsnew file mode 100644index 0000000..0302ea6--- /dev/null+++ b/packages/google-drive/tests/api/streaming.test.ts@@ -0,0 +1,132 @@+/**+ * Google Drive resumable streaming: the Content-Range protocol.+ *+ * Drive accepts `bytes {start}-{end}/*` while the total is unknown and requires+ * the real total on the final chunk. A chunk therefore cannot be classified+ * until we know whether more data follows, which is why the uploader keeps a+ * one-chunk lookahead. These tests pin the exact header sequence, because+ * getting it wrong fails at the *end* of a large upload rather than the start.+ *+ * The Python SDK asserts the same sequence in tests/test_gdrive_streaming.py.+ */+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";+import { ResumableUploader } from "../../src/api/resumable.js";++const ALIGN = 256 * 1024;+const RANGE_RE = /^bytes (?:(\d+)-(\d+)|\*)\/(\d+|\*)$/;++class TestUploader extends ResumableUploader {+ // Skip the real initiate call; the session URI is not what is under test.+ protected async initiateSession(): Promise<string> {+ return "https://upload.example/session";+ }+}++function driveFetch(seen: [string, number][]) {+ let received = 0;+ return vi.fn(async (_url: string, init: any) => {+ const raw = String(init.headers["Content-Range"]);+ const match = RANGE_RE.exec(raw);+ expect(match, `malformed Content-Range: ${raw}`).not.toBeNull();++ const [, start, end, total] = match!;+ const length = init.body ? (init.body as Uint8Array).byteLength : 0;+ seen.push([raw, length]);++ if (start !== undefined) {+ expect(Number(start), "chunks must be contiguous").toBe(received);+ expect(Number(end) - Number(start) + 1, "range must match body").toBe(length);+ if (total === "*") {+ expect(length % ALIGN, "a non-final chunk must be 256 KiB aligned").toBe(0);+ }+ received += length;+ }++ if (total === "*") {+ return {+ status: 308,+ headers: new Headers({ Range: `bytes=0-${received - 1}` })+ };+ }++ expect(Number(total), "final chunk must declare the real total").toBe(received);+ return {+ status: 200,+ headers: new Headers(),+ json: async () => ({ id: "f1", name: "s.bin", size: String(received) })+ };+ });+}++async function stream(total: number, seen: [string, number][]): Promise<void> {+ const uploader = new TestUploader({ getAccessToken: async () => "token" } as never);+ async function* source(): AsyncGenerator<Uint8Array> {+ const step = 100_000; // deliberately not chunk-aligned+ for (let offset = 0; offset < total; offset += step) {+ yield new Uint8Array(Math.min(step, total - offset)).fill(100);+ }+ }+ await uploader.uploadStream({ name: "s.bin" }, source(), "application/octet-stream", ALIGN);+}++describe("Drive resumable streaming", () => {+ const originalFetch = global.fetch;+ let seen: [string, number][];++ beforeEach(() => {+ seen = [];+ global.fetch = driveFetch(seen) as never;+ });++ afterEach(() => {+ global.fetch = originalFetch;+ vi.restoreAllMocks();+ });++ it.each([+ ["empty", 0, 1],+ ["one byte", 1, 1],+ ["just under one chunk", ALIGN - 1, 1],+ ["exactly one chunk", ALIGN, 1],+ ["exactly two chunks", ALIGN * 2, 2],+ ["two chunks plus one byte", ALIGN * 2 + 1, 3],+ ["many chunks, ragged tail", ALIGN * 7 + 12345, 8]+ ])("sends %s as the right number of chunks", async (_label, total, expected) => {+ await stream(total as number, seen);++ expect(seen.length).toBe(expected);+ });++ it("marks only the last chunk with a real total", async () => {+ await stream(ALIGN * 3 + 99, seen);++ const ranges = seen.map(([header]) => header);+ for (const header of ranges.slice(0, -1)) expect(header.endsWith("/*")).toBe(true);+ expect(ranges.at(-1)).toBe(`bytes ${ALIGN * 3}-${ALIGN * 3 + 98}/${ALIGN * 3 + 99}`);+ });++ it("sends a single full chunk as final, not as unknown-length", async () => {+ // The lookahead exists for exactly this case: a naive implementation ships+ // the full buffer as non-final and then has nothing left to finalise with.+ await stream(ALIGN, seen);++ expect(seen).toEqual([[`bytes 0-${ALIGN - 1}/${ALIGN}`, ALIGN]]);+ });++ it("finalises an empty stream without a byte range", async () => {+ await stream(0, seen);++ expect(seen).toEqual([["bytes */0", 0]]);+ });++ it("keeps offsets contiguous across chunks", async () => {+ await stream(ALIGN * 4 + 7, seen);++ let offset = 0;+ for (const [header, length] of seen) {+ expect(header.startsWith(`bytes ${offset}-`)).toBe(true);+ offset += length;+ }+ expect(offset).toBe(ALIGN * 4 + 7);+ });+});diff --git a/python/src/byoc/providers/gdrive/_http.py b/python/src/byoc/providers/gdrive/_http.pyindex 1f83b3f..f3314d5 100644--- a/python/src/byoc/providers/gdrive/_http.py+++ b/python/src/byoc/providers/gdrive/_http.py@@ -33,6 +33,9 @@ FILE_FIELDS = ()# Drive requires resumable chunks to be a multiple of 256 KiB (except the last).+_UPLOAD_FRAME_SIZE = 64 * 1024+"""Bytes handed to httpx at a time. Small enough that CPython reuses the block."""+CHUNK_ALIGNMENT = 256 * 1024DEFAULT_CHUNK_SIZE = 8 * 1024 * 1024# Above this, upload resumably so a dropped connection does not restart from zero.@@ -418,6 +421,121 @@ class DriveHttpClient:"Resumable upload ended without a completion response.", provider="google-drive")+ async def stream_chunks(+ self,+ upload_url: str,+ source: AsyncIterator[bytes],+ *,+ chunk_size: int = DEFAULT_CHUNK_SIZE,+ on_progress: Callable[[UploadProgress], None] | None = None,+ ) -> dict[str, Any]:+ """Send an async iterator to a resumable session without buffering it.++ Drive accepts ``Content-Range: bytes {start}-{end}/*`` while the total+ is still unknown, and requires the real total on the final chunk. That+ forces a one-chunk lookahead: a chunk cannot be sent as non-final until+ we know more data follows, and cannot be sent as final until we know+ none does. Two buffers alternate so the reader always has somewhere to+ write while a full chunk waits to be classified.++ Memory is therefore bounded at two chunks, not at the object size.+ """+ aligned = max(CHUNK_ALIGNMENT, (chunk_size // CHUNK_ALIGNMENT) * CHUNK_ALIGNMENT)++ buffers = [bytearray(aligned), bytearray(aligned)]+ views = [memoryview(buffers[0]), memoryview(buffers[1])]+ current = 0+ filled = 0++ held: int | None = None # index of a full buffer awaiting classification+ offset = 0 # bytes Drive has committed++ async def send(index: int, length: int, total: int | None) -> dict[str, Any] | None:+ """PUT one chunk. ``total`` set means this is the final chunk."""+ nonlocal offset+ if length == 0 and total is None:+ return None++ end = offset + length - 1+ marker = str(total) if total is not None else "*"+ # An empty object has no byte range at all.+ span = f"bytes */{total}" if length == 0 else f"bytes {offset}-{end}/{marker}"++ async def framed() -> AsyncIterator[bytes]:+ """Feed the chunk to httpx in small pieces.++ httpx accepts only bytes or an iterable of them, so the chunk+ cannot be handed over as a zero-copy view. Copying the whole+ 8 MiB chunk instead makes peak memory climb with object size,+ because CPython does not reuse freed blocks that large.+ Slicing it keeps every allocation in the size class the+ allocator does reuse. Content-Length is explicit, or httpx+ falls back to chunked transfer-encoding.+ """+ for cursor in range(0, length, _UPLOAD_FRAME_SIZE):+ stop = min(cursor + _UPLOAD_FRAME_SIZE, length)+ yield bytes(views[index][cursor:stop])++ response = await self._http().put(+ upload_url,+ headers={"Content-Length": str(length), "Content-Range": span},+ content=framed(),+ )++ if response.status_code in (200, 201):+ offset += length+ if on_progress:+ on_progress(+ UploadProgress(+ bytes_uploaded=offset, total_bytes=offset, percentage=100.0+ )+ )+ resource: dict[str, Any] = response.json()+ return resource++ if response.status_code != 308:+ raise self.map_error(response)++ # Trust the server's committed range over our own bookkeeping.+ committed = response.headers.get("range")+ offset = int(committed.split("-")[-1]) + 1 if committed else offset + length+ if on_progress:+ on_progress(UploadProgress(bytes_uploaded=offset))+ return None++ async for chunk in source:+ position = 0+ while position < len(chunk):+ take = min(aligned - filled, len(chunk) - position)+ buffers[current][filled : filled + take] = chunk[position : position + take]+ filled += take+ position += take++ if filled == aligned:+ # A full buffer cannot be final while more data may follow,+ # so release the previously held one and hold this instead.+ if held is not None:+ await send(held, aligned, None)+ held = current+ current = 1 - current+ filled = 0++ # The stream is exhausted, so whatever remains is genuinely final.+ if held is not None and filled == 0:+ result = await send(held, aligned, offset + aligned)+ elif held is not None:+ await send(held, aligned, None)+ result = await send(current, filled, offset + filled)+ else:+ result = await send(current, filled, filled)++ if result is None:+ raise ProviderUnavailableError(+ "Google Drive accepted every chunk but never returned the file resource.",+ provider="google-drive",+ )+ return result+async def stream_download(self, file_id: str) -> AsyncIterator[bytes]:"""Stream a file's content without buffering it whole."""headers = await self._headers()diff --git a/python/src/byoc/providers/gdrive/adapter.py b/python/src/byoc/providers/gdrive/adapter.pyindex 1c2cfbe..b74de75 100644--- a/python/src/byoc/providers/gdrive/adapter.py+++ b/python/src/byoc/providers/gdrive/adapter.py@@ -125,14 +125,16 @@ class GoogleDriveProvider:if not normalized:raise InvalidInputError("Upload requires a file path.", provider=PROVIDER_ID)+ payload: bytes | Noneif isinstance(data, str):payload = data.encode("utf-8")elif isinstance(data, bytes | bytearray | memoryview):payload = bytes(data)else:- raise InvalidInputError(- "Google Drive upload requires bytes or str.", provider=PROVIDER_ID- )+ # An async iterator streams into the resumable session, so the+ # object never has to fit in memory.+ payload = None+ stream = datamime = (options.mime_type if options else None) or "application/octet-stream"parent_id = await self.resolver.resolve_parent_folder_id(normalized, create=True)@@ -147,19 +149,31 @@ class GoogleDriveProvider:resumable = options.resumable if options and options.resumable is not None else Nonewants_resumable = (- resumable+ # A stream has no length to compare, and chunked transfer is the+ # only way to send one, so it is always resumable.+ True+ if payload is None+ else resumableif resumable is not Noneelse (len(payload) > RESUMABLE_THRESHOLD or bool(options and options.on_progress)))- if wants_resumable:+ chunk_size = options.chunk_size if options and options.chunk_size else 8 * 1024 * 1024++ if payload is None:+ upload_url = await self.http.start_resumable_upload(metadata, mime)+ resource = await self.http.stream_chunks(+ upload_url,+ stream,+ chunk_size=chunk_size,+ on_progress=options.on_progress if options else None,+ )+ elif wants_resumable:upload_url = await self.http.start_resumable_upload(metadata, mime)resource = await self.http.upload_chunks(upload_url,payload,- chunk_size=(- options.chunk_size if options and options.chunk_size else 8 * 1024 * 1024- ),+ chunk_size=chunk_size,on_progress=options.on_progress if options else None,)else:diff --git a/python/tests/test_gdrive_streaming.py b/python/tests/test_gdrive_streaming.pynew file mode 100644index 0000000..bc7bccb--- /dev/null+++ b/python/tests/test_gdrive_streaming.py@@ -0,0 +1,142 @@+"""Google Drive resumable streaming: the Content-Range protocol.++Drive accepts ``bytes {start}-{end}/*`` while the total is unknown and requires+the real total on the final chunk. A chunk therefore cannot be classified until+we know whether more data follows, which is why the uploader keeps a one-chunk+lookahead. These tests pin the exact header sequence, because getting it wrong+fails at the *end* of a large upload rather than the start.+"""++from __future__ import annotations++import json+import re+from collections.abc import AsyncIterator++import httpx+import pytest++from byoc.providers.gdrive._http import DriveHttpClient++ALIGN = 256 * 1024+RANGE_RE = re.compile(r"^bytes (?:(\d+)-(\d+)|\*)/(\d+|\*)$")+++class _FakeOAuth:+ async def get_access_token(self) -> str:+ return "token"+++def _drive_transport(seen: list[tuple[str, int]]) -> httpx.MockTransport:+ """A Drive that records Content-Range and enforces the protocol."""+ received = bytearray()++ def handle(request: httpx.Request) -> httpx.Response:+ raw = request.headers.get("content-range", "")+ match = RANGE_RE.match(raw)+ assert match, f"malformed Content-Range: {raw!r}"++ start, end, total = match.group(1), match.group(2), match.group(3)+ body = request.content+ seen.append((raw, len(body)))++ if start is not None:+ assert int(start) == len(received), "chunks must be contiguous"+ assert int(end) - int(start) + 1 == len(body), "range must match body"+ if total == "*":+ assert len(body) % ALIGN == 0, "a non-final chunk must be 256 KiB aligned"+ received.extend(body)++ if total == "*":+ return httpx.Response(308, headers={"Range": f"bytes=0-{len(received) - 1}"})++ assert int(total) == len(received), "final chunk must declare the real total"+ return httpx.Response(+ 200,+ content=json.dumps(+ {"id": "f1", "name": "s.bin", "size": str(len(received))}+ ).encode(),+ headers={"content-type": "application/json"},+ )++ return httpx.MockTransport(handle)+++async def _stream(total: int, seen: list[tuple[str, int]]) -> dict[str, object]:+ client = DriveHttpClient(_FakeOAuth()) # type: ignore[arg-type]+ client._client = httpx.AsyncClient(transport=_drive_transport(seen))++ async def source() -> AsyncIterator[bytes]:+ step = 100_000 # deliberately not chunk-aligned+ for offset in range(0, total, step):+ yield b"d" * min(step, total - offset)++ return await client.stream_chunks("https://upload/session", source(), chunk_size=ALIGN)+++ ("total", "expected_chunks"),+ [+ (0, 1), # empty object still needs one finalising request+ (1, 1),+ (ALIGN - 1, 1),+ (ALIGN, 1), # exactly one chunk is final, never sent as "*"+ (ALIGN * 2, 2),+ (ALIGN * 2 + 1, 3),+ (ALIGN * 7 + 12345, 8),+ ],+)+async def test_chunk_count_matches_the_payload_shape(+ total: int, expected_chunks: int+) -> None:+ seen: list[tuple[str, int]] = []++ resource = await _stream(total, seen)++ assert resource["id"] == "f1"+ assert len(seen) == expected_chunks+++async def test_only_the_last_chunk_declares_a_total() -> None:+ """Every earlier chunk must say `*`, or Drive rejects the upload."""+ seen: list[tuple[str, int]] = []++ await _stream(ALIGN * 3 + 99, seen)++ ranges = [header for header, _ in seen]+ assert all(header.endswith("/*") for header in ranges[:-1]), ranges+ assert ranges[-1].endswith(f"/{ALIGN * 3 + 99}"), ranges[-1]+++async def test_a_single_full_chunk_is_sent_as_final_not_as_unknown() -> None:+ """The lookahead exists for this case.++ A payload of exactly one chunk fills the buffer, and a naive implementation+ ships it as non-final and then has nothing left to finalise with.+ """+ seen: list[tuple[str, int]] = []++ await _stream(ALIGN, seen)++ assert len(seen) == 1+ assert seen[0][0] == f"bytes 0-{ALIGN - 1}/{ALIGN}"+++async def test_an_empty_stream_finalises_without_a_byte_range() -> None:+ seen: list[tuple[str, int]] = []++ await _stream(0, seen)++ assert seen == [("bytes */0", 0)]+++async def test_offsets_are_contiguous_across_chunks() -> None:+ seen: list[tuple[str, int]] = []++ await _stream(ALIGN * 4 + 7, seen)++ offset = 0+ for header, length in seen:+ assert header.startswith(f"bytes {offset}-"), header+ offset += length+ assert offset == ALIGN * 4 + 7