One storage API for Google Drive, Nextcloud, and S3-compatible clouds, in TypeScript and Python.
TypeScript byoccloud-storagecloudflare-r2encryptiongoogle-drivenextcloudoauth2pkcepythons3sdkstoragetypescriptwebdav
test(gdrive): validate the streaming path, not just the buffered one
Ajay Ramineni635ff6c79e93
diff --git a/python/scripts/validate_gdrive_live.py b/python/scripts/validate_gdrive_live.pyindex 2a84ff4..c539f86 100644--- a/python/scripts/validate_gdrive_live.py+++ b/python/scripts/validate_gdrive_live.py@@ -33,6 +33,7 @@ import sysimport timeimport uuidimport webbrowser+from collections.abc import AsyncIteratorfrom http.server import BaseHTTPRequestHandler, HTTPServerfrom threading import Threadfrom urllib.parse import parse_qs, urlparse@@ -216,6 +217,78 @@ async def main() -> int:check("resumable content round-trip", await storage.read_bytes("large/blob.bin") == big))+ # The check above passes `bytes`, which takes the buffered resumable+ # path. Streaming is a different code path entirely: it opens the+ # session without a declared length and has to classify each chunk as+ # final or not on the fly. Passing one does not exercise the other.+ streamed_size = 5 * 256 * 1024 + 1234 # several full chunks, ragged tail+ streamed_seen: list[int] = []++ async def streamed_source() -> AsyncIterator[bytes]:+ step = 100_000 # deliberately not chunk-aligned+ remaining = streamed_size+ while remaining > 0:+ size = min(step, remaining)+ remaining -= size+ yield b"s" * size++ written = await storage.upload(+ "large/streamed.bin",+ streamed_source(),+ UploadOptions(+ chunk_size=chunk_size,+ on_progress=lambda p: streamed_seen.append(p.bytes_uploaded),+ ),+ )+ results.append(+ check(+ "streamed upload reports the right size",+ written.size == streamed_size,+ f"{written.size} bytes, expected {streamed_size}",+ )+ )+ results.append(+ check(+ "streamed upload reports progress",+ bool(streamed_seen) and streamed_seen == sorted(streamed_seen),+ f"{len(streamed_seen)} events, last={streamed_seen[-1] if streamed_seen else 0}",+ )+ )+ results.append(+ check(+ "streamed content round-trip",+ await storage.read_bytes("large/streamed.bin") == b"s" * streamed_size,+ )+ )++ # A single full chunk is the case the lookahead exists for: it fills the+ # buffer exactly, and a naive implementation ships it as non-final and+ # is then left with nothing to finalise the upload with.+ exact_seen: list[int] = []++ async def exact_source() -> AsyncIterator[bytes]:+ yield b"e" * chunk_size++ exact = await storage.upload(+ "large/exact.bin",+ exact_source(),+ UploadOptions(+ chunk_size=chunk_size,+ on_progress=lambda p: exact_seen.append(p.bytes_uploaded),+ ),+ )+ results.append(+ check(+ "streamed upload of exactly one chunk",+ exact.size == chunk_size+ and await storage.read_bytes("large/exact.bin") == b"e" * chunk_size,+ f"{exact.size} bytes",+ )+ )++ await storage.delete("large/streamed.bin")+ await storage.delete("large/exact.bin")+await storage.delete("docs/hello.md")results.append(check("delete removes the file", not await storage.exists("docs/hello.md")))