One storage API for Google Drive, Nextcloud, and S3-compatible clouds, in TypeScript and Python.
TypeScript byoccloud-storagecloudflare-r2encryptiongoogle-drivenextcloudoauth2pkcepythons3sdkstoragetypescriptwebdav
"""Google Drive API v3 HTTP client.
Owns Drive's wire concerns: multipart and resumable upload framing, the
``fields`` projections, and mapping Drive's error shapes onto BYOC errors.
"""
from __future__ import annotations
import json
from collections.abc import AsyncIterator, Callable
from datetime import datetime
from typing import Any
import httpx
from ...errors import (
AuthRequiredError,
ObjectNotFoundError,
PermissionDeniedError,
ProviderUnavailableError,
QuotaExceededError,
RateLimitedError,
StorageError,
)
from ...types import StorageObject, StorageQuota, UploadProgress
from ._resolver import FOLDER_MIME_TYPE, VIRTUAL_PATH_PROPERTY
API_BASE = "https://www.googleapis.com/drive/v3"
UPLOAD_BASE = "https://www.googleapis.com/upload/drive/v3"
FILE_FIELDS = (
"id,name,mimeType,size,parents,createdTime,modifiedTime,md5Checksum,trashed,appProperties"
)
# Drive requires resumable chunks to be a multiple of 256 KiB (except the last).
_UPLOAD_FRAME_SIZE = 64 * 1024
"""Bytes handed to httpx at a time. Small enough that CPython reuses the block."""
CHUNK_ALIGNMENT = 256 * 1024
DEFAULT_CHUNK_SIZE = 8 * 1024 * 1024
# Above this, upload resumably so a dropped connection does not restart from zero.
RESUMABLE_THRESHOLD = 5 * 1024 * 1024
def _parse_time(value: str | None) -> datetime | None:
if not value:
return None
try:
return datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return None
class DriveHttpClient:
"""Authenticated Drive API client.
Args:
token_provider: Async callable returning a valid access token. Called
per request so a refresh mid-session is picked up automatically.
client: An ``httpx.AsyncClient`` to reuse.
"""
def __init__(
self,
token_provider: Callable[[], Any],
*,
client: httpx.AsyncClient | None = None,
) -> None:
self._token_provider = token_provider
self._client = client
self._owns_client = client is None
def _http(self) -> httpx.AsyncClient:
if self._client is None:
self._client = httpx.AsyncClient(timeout=httpx.Timeout(30.0, read=300.0))
self._owns_client = True
return self._client
async def aclose(self) -> None:
if self._owns_client and self._client is not None:
await self._client.aclose()
self._client = None
async def _headers(self, extra: dict[str, str] | None = None) -> dict[str, str]:
token = await self._token_provider()
return {"Authorization": f"Bearer {token}", **(extra or {})}
# -- error mapping -----------------------------------------------------
def map_error(self, response: httpx.Response) -> StorageError:
status = response.status_code
reason = ""
message = ""
try:
body = response.json().get("error", {})
message = str(body.get("message", ""))
errors = body.get("errors") or []
if errors:
reason = str(errors[0].get("reason", ""))
except (ValueError, AttributeError):
message = response.text[:200]
if status == 401:
return AuthRequiredError(
f"Google Drive authentication failed: {message}",
provider="google-drive",
status_code=status,
)
if status == 404:
return ObjectNotFoundError(
f"Google Drive resource not found: {message}",
provider="google-drive",
status_code=status,
)
if status == 403:
# 403 is overloaded: quota and rate limits share it with real denials.
if reason in ("storageQuotaExceeded", "quotaExceeded"):
return QuotaExceededError(
f"Google Drive quota exceeded: {message}",
provider="google-drive",
status_code=status,
)
if reason in ("rateLimitExceeded", "userRateLimitExceeded"):
return RateLimitedError(
f"Google Drive rate limit exceeded: {message}",
provider="google-drive",
status_code=status,
)
return PermissionDeniedError(
f"Google Drive permission denied: {message}",
provider="google-drive",
status_code=status,
)
if status == 429:
return RateLimitedError(
f"Google Drive rate limit exceeded: {message}",
provider="google-drive",
status_code=status,
)
return ProviderUnavailableError(
f"Google Drive error (HTTP {status}): {message}",
provider="google-drive",
status_code=status,
retryable=status >= 500,
)
def to_storage_object(self, resource: dict[str, Any], path: str) -> StorageObject:
"""Convert a Drive file resource into a provider-neutral StorageObject."""
is_folder = resource.get("mimeType") == FOLDER_MIME_TYPE
size = resource.get("size")
properties = resource.get("appProperties") or {}
return StorageObject(
id=f"gdrive_{resource['id']}",
path=path,
name=str(resource.get("name", "")),
provider="google-drive",
provider_id=str(resource["id"]),
type="folder" if is_folder else "file",
size=int(size) if size is not None and not is_folder else None,
mime_type=str(resource.get("mimeType")) if resource.get("mimeType") else None,
checksum=str(resource["md5Checksum"]) if resource.get("md5Checksum") else None,
created_at=_parse_time(resource.get("createdTime")),
updated_at=_parse_time(resource.get("modifiedTime")),
metadata={k: v for k, v in properties.items() if k != VIRTUAL_PATH_PROPERTY},
)
# -- metadata operations ----------------------------------------------
async def list_files(self, query: str, page_size: int = 100) -> list[dict[str, Any]]:
"""Run a Drive query, following pagination."""
results: list[dict[str, Any]] = []
page_token: str | None = None
while True:
params = {
"q": query,
"pageSize": str(min(page_size, 1000)),
"fields": f"nextPageToken,files({FILE_FIELDS})",
"spaces": "drive",
}
if page_token:
params["pageToken"] = page_token
response = await self._http().get(
f"{API_BASE}/files", params=params, headers=await self._headers()
)
if response.is_error:
raise self.map_error(response)
payload = response.json()
results.extend(payload.get("files", []))
page_token = payload.get("nextPageToken")
if not page_token or len(results) >= page_size:
break
return results
async def get_file(self, file_id: str) -> dict[str, Any]:
response = await self._http().get(
f"{API_BASE}/files/{file_id}",
params={"fields": FILE_FIELDS},
headers=await self._headers(),
)
if response.is_error:
raise self.map_error(response)
resource: dict[str, Any] = response.json()
return resource
async def create_folder(self, name: str, parent_id: str) -> str:
body = {"name": name, "mimeType": FOLDER_MIME_TYPE, "parents": [parent_id]}
response = await self._http().post(
f"{API_BASE}/files",
params={"fields": "id"},
headers=await self._headers({"Content-Type": "application/json"}),
content=json.dumps(body).encode("utf-8"),
)
if response.is_error:
raise self.map_error(response)
return str(response.json()["id"])
async def copy_file(
self, file_id: str, new_name: str, parent_id: str
) -> dict[str, Any]:
"""Server-side copy. Drive duplicates the blob without re-uploading."""
body = {"name": new_name, "parents": [parent_id]}
response = await self._http().post(
f"{API_BASE}/files/{file_id}/copy",
params={"fields": FILE_FIELDS},
headers=await self._headers({"Content-Type": "application/json"}),
content=json.dumps(body).encode("utf-8"),
)
if response.is_error:
raise self.map_error(response)
resource: dict[str, Any] = response.json()
return resource
async def move_file(
self, file_id: str, new_parent_id: str, old_parent_id: str, new_name: str
) -> dict[str, Any]:
"""Reparent and rename in one PATCH.
Drive models a move as an edit to the file's parent list rather than a
rename, so this is instantaneous regardless of file size.
"""
params = {
"addParents": new_parent_id,
"removeParents": old_parent_id,
"fields": FILE_FIELDS,
}
response = await self._http().patch(
f"{API_BASE}/files/{file_id}",
params=params,
headers=await self._headers({"Content-Type": "application/json"}),
content=json.dumps({"name": new_name}).encode("utf-8"),
)
if response.is_error:
raise self.map_error(response)
resource: dict[str, Any] = response.json()
return resource
async def delete_file(self, file_id: str) -> None:
response = await self._http().delete(
f"{API_BASE}/files/{file_id}", headers=await self._headers()
)
if response.is_error and response.status_code != 404:
raise self.map_error(response)
async def download_file(self, file_id: str) -> bytes:
response = await self._http().get(
f"{API_BASE}/files/{file_id}",
params={"alt": "media"},
headers=await self._headers(),
)
if response.is_error:
raise self.map_error(response)
return response.content
async def get_quota(self) -> StorageQuota:
response = await self._http().get(
f"{API_BASE}/about",
params={"fields": "storageQuota"},
headers=await self._headers(),
)
if response.is_error:
raise self.map_error(response)
quota = response.json().get("storageQuota", {})
used = int(quota.get("usage", 0))
limit = quota.get("limit")
total = int(limit) if limit is not None else None
return StorageQuota(
used=used, total=total, available=(total - used) if total is not None else None
)
# -- uploads -----------------------------------------------------------
def build_metadata(
self,
*,
name: str,
parent_id: str,
virtual_path: str,
mime_type: str | None,
extra: dict[str, str] | None = None,
) -> dict[str, Any]:
"""Build the file metadata, including the shared virtual-path property."""
return {
"name": name,
"parents": [parent_id],
**({"mimeType": mime_type} if mime_type else {}),
"appProperties": {VIRTUAL_PATH_PROPERTY: virtual_path, **(extra or {})},
}
async def multipart_upload(
self, metadata: dict[str, Any], payload: bytes, mime_type: str
) -> dict[str, Any]:
"""Upload metadata and content in a single multipart/related request."""
boundary = "byoc-boundary-Xk3Jd9Qm2Lp7"
body = b"".join(
[
f"--{boundary}\r\nContent-Type: application/json; charset=UTF-8\r\n\r\n".encode(),
json.dumps(metadata).encode("utf-8"),
f"\r\n--{boundary}\r\nContent-Type: {mime_type}\r\n\r\n".encode(),
payload,
f"\r\n--{boundary}--".encode(),
]
)
response = await self._http().post(
f"{UPLOAD_BASE}/files",
params={"uploadType": "multipart", "fields": FILE_FIELDS},
headers=await self._headers(
{"Content-Type": f"multipart/related; boundary={boundary}"}
),
content=body,
)
if response.is_error:
raise self.map_error(response)
resource: dict[str, Any] = response.json()
return resource
async def start_resumable_upload(self, metadata: dict[str, Any], mime_type: str) -> str:
"""Begin a resumable session and return its upload URL."""
response = await self._http().post(
f"{UPLOAD_BASE}/files",
params={"uploadType": "resumable", "fields": FILE_FIELDS},
headers=await self._headers(
{
"Content-Type": "application/json; charset=UTF-8",
"X-Upload-Content-Type": mime_type,
}
),
content=json.dumps(metadata).encode("utf-8"),
)
if response.is_error:
raise self.map_error(response)
location: str | None = response.headers.get("location")
if not location:
raise ProviderUnavailableError(
"Google Drive did not return a resumable upload URL.", provider="google-drive"
)
return str(location)
async def upload_chunks(
self,
upload_url: str,
payload: bytes,
*,
chunk_size: int = DEFAULT_CHUNK_SIZE,
on_progress: Callable[[UploadProgress], None] | None = None,
) -> dict[str, Any]:
"""Send a payload in 256 KiB-aligned chunks to a resumable session."""
aligned = max(CHUNK_ALIGNMENT, (chunk_size // CHUNK_ALIGNMENT) * CHUNK_ALIGNMENT)
total = len(payload)
offset = 0
if total == 0:
# `while offset < total` never runs for an empty payload, so without
# this the session is opened and then abandoned: zero requests are
# sent and the caller sees "ended without a completion response"
# rather than an empty file. Drive finalises an empty object with a
# byte range of `*`. TypeScript already guarded this; Python did not.
response = await self._http().put(
upload_url,
headers={"Content-Length": "0", "Content-Range": f"bytes */{total}"},
content=b"",
)
if response.is_error:
raise self.map_error(response)
if on_progress:
on_progress(
UploadProgress(bytes_uploaded=0, total_bytes=0, percentage=100.0)
)
empty: dict[str, Any] = response.json()
return empty
while offset < total:
end = min(offset + aligned, total)
chunk = payload[offset:end]
response = await self._http().put(
upload_url,
headers={
"Content-Length": str(len(chunk)),
"Content-Range": f"bytes {offset}-{end - 1}/{total}",
},
content=chunk,
)
# 308 Resume Incomplete is the success signal for a non-final chunk.
if response.status_code in (200, 201):
if on_progress:
on_progress(
UploadProgress(
bytes_uploaded=total, total_bytes=total, percentage=100.0
)
)
resource: dict[str, Any] = response.json()
return resource
if response.status_code != 308:
raise self.map_error(response)
# Trust the server's committed range over our own bookkeeping.
committed = response.headers.get("range")
offset = int(committed.split("-")[-1]) + 1 if committed else end
if on_progress:
on_progress(
UploadProgress(
bytes_uploaded=offset,
total_bytes=total,
percentage=round(offset / total * 100, 2) if total else 100.0,
)
)
raise ProviderUnavailableError(
"Resumable upload ended without a completion response.", provider="google-drive"
)
async def stream_chunks(
self,
upload_url: str,
source: AsyncIterator[bytes],
*,
chunk_size: int = DEFAULT_CHUNK_SIZE,
on_progress: Callable[[UploadProgress], None] | None = None,
) -> dict[str, Any]:
"""Send an async iterator to a resumable session without buffering it.
Drive accepts ``Content-Range: bytes {start}-{end}/*`` while the total
is still unknown, and requires the real total on the final chunk. That
forces a one-chunk lookahead: a chunk cannot be sent as non-final until
we know more data follows, and cannot be sent as final until we know
none does. Two buffers alternate so the reader always has somewhere to
write while a full chunk waits to be classified.
Memory is therefore bounded at two chunks, not at the object size.
"""
aligned = max(CHUNK_ALIGNMENT, (chunk_size // CHUNK_ALIGNMENT) * CHUNK_ALIGNMENT)
buffers = [bytearray(aligned), bytearray(aligned)]
views = [memoryview(buffers[0]), memoryview(buffers[1])]
current = 0
filled = 0
held: int | None = None # index of a full buffer awaiting classification
offset = 0 # bytes Drive has committed
async def send(index: int, length: int, total: int | None) -> dict[str, Any] | None:
"""PUT one chunk. ``total`` set means this is the final chunk."""
nonlocal offset
if length == 0 and total is None:
return None
end = offset + length - 1
marker = str(total) if total is not None else "*"
# An empty object has no byte range at all.
span = f"bytes */{total}" if length == 0 else f"bytes {offset}-{end}/{marker}"
async def framed() -> AsyncIterator[bytes]:
"""Feed the chunk to httpx in small pieces.
httpx accepts only bytes or an iterable of them, so the chunk
cannot be handed over as a zero-copy view. Copying the whole
8 MiB chunk instead makes peak memory climb with object size,
because CPython does not reuse freed blocks that large.
Slicing it keeps every allocation in the size class the
allocator does reuse. Content-Length is explicit, or httpx
falls back to chunked transfer-encoding.
"""
for cursor in range(0, length, _UPLOAD_FRAME_SIZE):
stop = min(cursor + _UPLOAD_FRAME_SIZE, length)
yield bytes(views[index][cursor:stop])
response = await self._http().put(
upload_url,
headers={"Content-Length": str(length), "Content-Range": span},
content=framed(),
)
if response.status_code in (200, 201):
offset += length
if on_progress:
on_progress(
UploadProgress(
bytes_uploaded=offset, total_bytes=offset, percentage=100.0
)
)
resource: dict[str, Any] = response.json()
return resource
if response.status_code != 308:
raise self.map_error(response)
# Trust the server's committed range over our own bookkeeping.
committed = response.headers.get("range")
offset = int(committed.split("-")[-1]) + 1 if committed else offset + length
if on_progress:
on_progress(UploadProgress(bytes_uploaded=offset))
return None
async for chunk in source:
position = 0
while position < len(chunk):
take = min(aligned - filled, len(chunk) - position)
buffers[current][filled : filled + take] = chunk[position : position + take]
filled += take
position += take
if filled == aligned:
# A full buffer cannot be final while more data may follow,
# so release the previously held one and hold this instead.
if held is not None:
await send(held, aligned, None)
held = current
current = 1 - current
filled = 0
# The stream is exhausted, so whatever remains is genuinely final.
if held is not None and filled == 0:
result = await send(held, aligned, offset + aligned)
elif held is not None:
await send(held, aligned, None)
result = await send(current, filled, offset + filled)
else:
result = await send(current, filled, filled)
if result is None:
raise ProviderUnavailableError(
"Google Drive accepted every chunk but never returned the file resource.",
provider="google-drive",
)
return result
async def stream_download(self, file_id: str) -> AsyncIterator[bytes]:
"""Stream a file's content without buffering it whole."""
headers = await self._headers()
async with self._http().stream(
"GET", f"{API_BASE}/files/{file_id}", params={"alt": "media"}, headers=headers
) as response:
if response.is_error:
await response.aread()
raise self.map_error(response)
async for chunk in response.aiter_bytes():
yield chunk