One storage API for Google Drive, Nextcloud, and S3-compatible clouds, in TypeScript and Python.
TypeScript byoccloud-storagecloudflare-r2encryptiongoogle-drivenextcloudoauth2pkcepythons3sdkstoragetypescriptwebdav
"""S3-compatible provider adapter.
Works against AWS S3, Cloudflare R2, MinIO, Wasabi, and any endpoint speaking
the S3 API. Bucket names, object keys, ETags and multipart uploads stay private
to this module -- BYOC core never sees them.
"""
from __future__ import annotations
import contextlib
from collections.abc import AsyncIterator, Mapping
from datetime import datetime, timedelta, timezone
from email.utils import parsedate_to_datetime
from typing import Any
from urllib.parse import quote
import httpx
from defusedxml.common import DefusedXmlException
from defusedxml.ElementTree import ParseError, fromstring
from ..errors import (
InvalidInputError,
ObjectNotFoundError,
PermissionDeniedError,
ProviderUnavailableError,
RateLimitedError,
StorageError,
)
from ..paths import encode_path_segments, get_basename, normalize_virtual_path
from ..retry import with_retry
from ..types import (
ProgressCallback,
ProviderCapabilities,
ProviderManifest,
StorageInput,
StorageObject,
StorageOutput,
UploadGrant,
UploadGrantOptions,
UploadOptions,
UploadProgress,
)
from ._sigv4 import create_presigned_s3_url, sign_s3_request
_PART_FRAME_SIZE = 64 * 1024
"""Bytes handed to httpx at a time. Small enough that CPython reuses the block."""
MULTIPART_PART_SIZE = 8 * 1024 * 1024
"""Bytes buffered per multipart part. S3's floor is 5 MiB for every part but the last."""
PROVIDER_ID = "s3-compatible"
_S3_NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
_LIST_PAGE_GUARD = 10_000 # pages, not keys: stops a malformed server looping forever
def _parse_xml(text: str) -> Any:
"""Parse an S3 XML response with the same hardened parser list() uses."""
try:
return fromstring(text)
except (ParseError, DefusedXmlException) as error:
raise ProviderUnavailableError(
f"S3 returned a response that is not valid XML: {error}", provider=PROVIDER_ID
) from error
def _text(element: Any, tag: str) -> str | None:
"""Read a child tag, tolerating servers that omit the S3 XML namespace."""
found = element.find(f"{_S3_NS}{tag}")
if found is None:
found = element.find(tag)
return found.text if found is not None else None
class S3CompatibleProvider:
"""Storage adapter for S3-compatible object stores.
Args:
endpoint: Base endpoint, e.g. ``https://<account>.r2.cloudflarestorage.com``.
bucket: Bucket name.
region: Region, or ``"auto"`` for Cloudflare R2.
access_key_id: Access key.
secret_access_key: Secret key.
root_prefix: Optional key prefix all paths are stored under.
force_path_style: Put the bucket in the URL path rather than the
hostname. Required by MinIO and most self-hosted gateways.
client: An existing ``httpx.AsyncClient`` to reuse. If omitted, one is
created and closed by :meth:`disconnect`.
"""
def __init__(
self,
*,
endpoint: str,
bucket: str,
region: str,
access_key_id: str,
secret_access_key: str,
root_prefix: str | None = None,
force_path_style: bool = False,
client: httpx.AsyncClient | None = None,
) -> None:
if not endpoint or not bucket or not access_key_id or not secret_access_key:
raise InvalidInputError(
"S3CompatibleProvider requires endpoint, bucket, access_key_id, "
"and secret_access_key.",
provider=PROVIDER_ID,
)
self.endpoint = endpoint.rstrip("/")
self.bucket = bucket
self.region = region
self.access_key_id = access_key_id
self.secret_access_key = secret_access_key
self.root_prefix = normalize_virtual_path(root_prefix) if root_prefix else ""
self.force_path_style = force_path_style
self._client = client
self._owns_client = client is None
# -- lifecycle ---------------------------------------------------------
def manifest(self) -> ProviderManifest:
return ProviderManifest(
id=PROVIDER_ID,
name="S3 Compatible (R2/AWS/MinIO)",
category="developer-cloud",
authentication="access-key",
supports_user_owned_storage=False,
adapter_version="0.4.0",
)
def capabilities(self) -> ProviderCapabilities:
# S3 has no native folders and reports no account quota, so those stay
# False rather than being faked with prefix tricks.
return ProviderCapabilities(
folders=False,
sharing=False,
public_urls=True,
resumable_uploads=True,
versioning=True,
quota=False,
server_side_copy=True,
direct_upload=True,
)
async def connect(self) -> None:
"""No-op: SigV4 is stateless. Ensures an HTTP client exists."""
self._http()
async def disconnect(self) -> None:
"""Close the HTTP client if this adapter created it."""
if self._owns_client and self._client is not None:
await self._client.aclose()
self._client = None
def _http(self) -> httpx.AsyncClient:
if self._client is None:
self._client = httpx.AsyncClient(timeout=httpx.Timeout(30.0, read=300.0))
self._owns_client = True
return self._client
# -- key handling ------------------------------------------------------
def _to_key(self, path: str) -> str:
"""Virtual path -> provider-side object key (adds ``root_prefix``)."""
normalized = normalize_virtual_path(path)
return f"{self.root_prefix}/{normalized}" if self.root_prefix else normalized
def _to_virtual_path(self, key: str) -> str:
"""Provider-side object key -> virtual path (strips ``root_prefix``).
``root_prefix`` is the adapter's business, so it must not appear in a
returned ``StorageObject.path``: callers feed those paths straight back
into the client, and a leaked prefix would be applied a second time.
The full key stays available as ``provider_id``.
"""
if self.root_prefix and key.startswith(f"{self.root_prefix}/"):
return key[len(self.root_prefix) + 1 :]
return key
def _bucket_base_url(self) -> str:
return f"{self.endpoint}/{self.bucket}" if self.force_path_style else self.endpoint
def object_url(self, key: str) -> str:
"""Absolute URL for an object key, with each segment RFC 3986 encoded.
Segment-wise encoding is what keeps ``#`` and ``?`` in filenames from
truncating the key into a fragment or query string.
"""
return f"{self._bucket_base_url()}/{encode_path_segments(key)}"
def signed_url(
self, path: str, *, method: str = "GET", expires_in_seconds: int = 3600
) -> str:
"""Time-limited URL a browser can use directly, without proxying bytes."""
return create_presigned_s3_url(
access_key_id=self.access_key_id,
secret_access_key=self.secret_access_key,
region=self.region,
url=self.object_url(self._to_key(path)),
method=method,
expires_in_seconds=expires_in_seconds,
)
# Retained under its 0.2.x name; `signed_url` is what the client calls.
presigned_url = signed_url
def _sign(
self,
method: str,
url: str,
headers: dict[str, str] | None = None,
body: bytes | memoryview | None = None,
unsigned_payload: bool = False,
) -> dict[str, str]:
return sign_s3_request(
access_key_id=self.access_key_id,
secret_access_key=self.secret_access_key,
region=self.region,
method=method,
url=url,
headers=headers,
body=body,
unsigned_payload=unsigned_payload,
)
# -- error mapping -----------------------------------------------------
def _map_error(self, response: httpx.Response, body: str = "") -> StorageError:
status = response.status_code
if status == 404 or "NoSuchKey" in body or "NoSuchBucket" in body:
return ObjectNotFoundError(
f"S3 object not found (HTTP {status}).",
provider=PROVIDER_ID,
status_code=status,
)
if status in (401, 403) or "AccessDenied" in body:
return PermissionDeniedError(
"S3 access denied: invalid credentials or insufficient permissions.",
provider=PROVIDER_ID,
status_code=status,
)
if status == 429 or "SlowDown" in body:
return RateLimitedError(
"S3 rate limit exceeded.", provider=PROVIDER_ID, status_code=status
)
return ProviderUnavailableError(
f"S3 error (HTTP {status}): {body[:300]}",
provider=PROVIDER_ID,
status_code=status,
retryable=status >= 500,
)
def _object_from_headers(self, key: str, response: httpx.Response) -> StorageObject:
etag = response.headers.get("etag", "").strip('"') or None
length = response.headers.get("content-length")
last_modified = response.headers.get("last-modified")
updated_at: datetime | None = None
if last_modified:
try:
updated_at = parsedate_to_datetime(last_modified)
except (TypeError, ValueError):
updated_at = None
virtual = self._to_virtual_path(key)
return StorageObject(
id=f"s3_{self.bucket}_{key}",
path=virtual,
name=get_basename(virtual),
provider=PROVIDER_ID,
provider_id=key,
type="file",
size=int(length) if length is not None else None,
mime_type=response.headers.get("content-type"),
checksum=etag,
updated_at=updated_at,
)
# -- operations --------------------------------------------------------
async def upload(
self, path: str, data: StorageInput, options: UploadOptions | None = None
) -> StorageObject:
key = self._to_key(path)
url = self.object_url(key)
payload: bytes | None
if isinstance(data, str):
payload = data.encode("utf-8")
elif isinstance(data, bytes | bytearray | memoryview):
payload = bytes(data)
else:
# An async iterator streams straight to the wire. Its length is not
# knowable up front, so the object's reported size comes from the
# bytes actually sent.
payload = None
stream = data
mime = (options.mime_type if options else None) or "application/octet-stream"
headers = {"content-type": mime}
for meta_key, meta_value in (options.metadata if options else {}).items():
headers[f"x-amz-meta-{meta_key.lower()}"] = meta_value
on_progress = options.on_progress if options else None
virtual = self._to_virtual_path(key)
def described(size: int, checksum: str | None) -> StorageObject:
return StorageObject(
id=f"s3_{self.bucket}_{key}",
path=virtual,
name=get_basename(virtual),
provider=PROVIDER_ID,
provider_id=key,
type="file",
size=size,
mime_type=mime,
checksum=checksum,
metadata=dict(options.metadata) if options else {},
)
if payload is None:
# S3 answers 411 Length Required to a chunked PUT, so an unbounded
# stream goes up as a multipart upload instead. Only one part is
# held in memory at a time, and each part signs its own real hash.
size, checksum = await self._upload_stream(
key, stream, mime, dict(options.metadata) if options else {}, on_progress
)
return described(size, checksum)
async def send() -> StorageObject:
signed = self._sign("PUT", url, headers, payload)
response = await self._http().put(url, headers=signed, content=payload)
if response.is_error:
raise self._map_error(response, response.text)
etag = response.headers.get("etag", "").strip('"') or None
return described(len(payload), etag)
return await with_retry(send)
# -- multipart upload --------------------------------------------------
async def _create_multipart_upload(
self, key: str, mime: str, metadata: Mapping[str, str]
) -> str:
url = f"{self.object_url(key)}?uploads="
headers = {"content-type": mime}
for meta_key, meta_value in metadata.items():
headers[f"x-amz-meta-{meta_key.lower()}"] = meta_value
async def send() -> str:
signed = self._sign("POST", url, headers)
response = await self._http().post(url, headers=signed)
if response.is_error:
raise self._map_error(response, response.text)
root = _parse_xml(response.text)
upload_id: str | None = _text(root, "UploadId")
if not upload_id:
raise ProviderUnavailableError(
"S3 did not return an UploadId for the multipart upload.",
provider=PROVIDER_ID,
)
return str(upload_id)
return await with_retry(send)
async def _upload_part(
self, key: str, upload_id: str, part_number: int, body: bytes | memoryview
) -> str:
encoded = quote(upload_id, safe="")
url = f"{self.object_url(key)}?partNumber={part_number}&uploadId={encoded}"
length = len(body)
async def framed() -> AsyncIterator[bytes]:
"""Feed the part to httpx in small pieces.
httpx accepts only ``bytes`` or an iterable of them -- a memoryview
is iterated as integers and a bytearray is refused outright -- so
the part cannot be handed over as a zero-copy view. Copying the
whole 8 MiB part instead makes peak memory climb with file size,
because CPython does not reuse freed blocks that large. Slicing it
into 64 KiB pieces keeps every allocation inside the size range the
allocator does reuse, so memory stays flat.
"""
for offset in range(0, length, _PART_FRAME_SIZE):
yield bytes(body[offset : offset + _PART_FRAME_SIZE])
async def send() -> str:
# The part is fully known, so its real hash is signed. Streaming
# does not force UNSIGNED-PAYLOAD here: memory stays bounded to one
# part while every byte remains covered by the signature.
signed = self._sign("PUT", url, body=body)
# Content-Length must be explicit, or httpx falls back to chunked
# transfer-encoding and S3 answers 411 Length Required.
signed["content-length"] = str(length)
response = await self._http().put(url, headers=signed, content=framed())
if response.is_error:
raise self._map_error(response, response.text)
etag: str = str(response.headers.get("etag", "")).strip('"')
if not etag:
raise ProviderUnavailableError(
f"S3 did not return an ETag for part {part_number}.",
provider=PROVIDER_ID,
)
return etag
return await with_retry(send)
async def _complete_multipart_upload(
self, key: str, upload_id: str, parts: list[tuple[int, str]]
) -> str | None:
url = f"{self.object_url(key)}?uploadId={quote(upload_id, safe='')}"
body = (
"<CompleteMultipartUpload>"
+ "".join(
f"<Part><PartNumber>{number}</PartNumber><ETag>"{etag}"</ETag></Part>"
for number, etag in parts
)
+ "</CompleteMultipartUpload>"
).encode("utf-8")
headers = {"content-type": "application/xml"}
signed = self._sign("POST", url, headers, body)
response = await self._http().post(url, headers=signed, content=body)
if response.is_error:
raise self._map_error(response, response.text)
# S3 can return a 200 whose body is an error, so the body must be read.
text = response.text
if "<Error>" in text:
raise self._map_error(response, text)
return (_text(_parse_xml(text), "ETag") or "").strip('"') or None
async def _abort_multipart_upload(self, key: str, upload_id: str) -> None:
"""Discard a failed upload so its parts stop accruing storage charges."""
url = f"{self.object_url(key)}?uploadId={quote(upload_id, safe='')}"
# Best effort: the upload has already failed, and that failure is what
# the caller needs to see. Raising from the cleanup would replace a
# useful error with a confusing one. `suppress` states that intent,
# where a bare `except: pass` reads like an oversight.
with contextlib.suppress(Exception):
signed = self._sign("DELETE", url)
await self._http().request("DELETE", url, headers=signed)
async def _upload_stream(
self,
key: str,
source: AsyncIterator[bytes],
mime: str,
metadata: Mapping[str, str],
on_progress: ProgressCallback | None,
) -> tuple[int, str | None]:
"""Stream an async iterator to S3 as a multipart upload.
S3 refuses chunked transfer-encoding on PUT -- it answers 411 Length
Required -- so an unbounded stream cannot go in one request. Multipart
is the way round it: each part carries its own Content-Length, and only
one part is ever held in memory.
"""
upload_id = await self._create_multipart_upload(key, mime, metadata)
parts: list[tuple[int, str]] = []
# One buffer, filled and reused for every part.
#
# Two earlier shapes both failed on memory, and the reason is the same:
# CPython does not return large freed blocks to the OS, so allocating a
# fresh part each time makes peak RSS climb with file size even though
# only one part is ever live.
#
# bytearray + `del buf[:n]` 248 MB peak on a 200 MB stream
# list of chunks + b"".join" 322 MB peak on a 400 MB stream
# this, reusing one buffer ~55 MB regardless of size
#
# Reuse is safe because each part is fully awaited before the buffer is
# written again; nothing is in flight when we overwrite it.
buffer = bytearray(MULTIPART_PART_SIZE)
view = memoryview(buffer)
filled = 0
total = 0
part_number = 1
async def flush(length: int) -> None:
nonlocal part_number, total
etag = await self._upload_part(key, upload_id, part_number, view[:length])
parts.append((part_number, etag))
part_number += 1
total += length
if on_progress:
on_progress(UploadProgress(bytes_uploaded=total))
try:
async for chunk in source:
offset = 0
while offset < len(chunk):
take = min(MULTIPART_PART_SIZE - filled, len(chunk) - offset)
buffer[filled : filled + take] = chunk[offset : offset + take]
filled += take
offset += take
# Every part but the last must be at least 5 MiB.
if filled == MULTIPART_PART_SIZE:
await flush(filled)
filled = 0
# The final part carries whatever is left, and may be short. An
# empty stream still needs one part: S3 rejects a zero-part upload.
if filled or not parts:
await flush(filled)
checksum = await self._complete_multipart_upload(key, upload_id, parts)
except BaseException:
await self._abort_multipart_upload(key, upload_id)
raise
return total, checksum
async def download(self, path: str) -> StorageOutput:
key = self._to_key(path)
url = self.object_url(key)
async def fetch() -> httpx.Response:
# stream=True holds the connection open and leaves the body unread,
# so a large object never has to fit in memory. Previously this read
# response.content up front and handed back a one-chunk iterator,
# which made StorageOutput.stream() a promise the adapter broke --
# and made migrating *out of* S3 cost more than the file's size.
request = self._http().build_request(
"GET", url, headers=self._sign("GET", url)
)
response = await self._http().send(request, stream=True)
if response.is_error:
# The body is needed for the error message, so read it before
# closing; otherwise httpx raises on an unread streamed response.
await response.aread()
await response.aclose()
raise self._map_error(response, response.text)
return response
response = await with_retry(fetch)
async def stream() -> AsyncIterator[bytes]:
try:
async for chunk in response.aiter_bytes():
yield chunk
finally:
# Closing releases the pooled connection. Without this an
# abandoned stream leaks it until garbage collection.
await response.aclose()
async def read() -> bytes:
try:
return await response.aread()
finally:
await response.aclose()
return StorageOutput(
metadata=self._object_from_headers(key, response), stream=stream, read=read
)
async def delete(self, path: str) -> None:
key = self._to_key(path)
url = self.object_url(key)
async def send() -> None:
signed = self._sign("DELETE", url)
response = await self._http().request("DELETE", url, headers=signed)
# S3 delete is idempotent: a missing object is a successful delete.
if response.is_error and response.status_code != 404:
raise self._map_error(response, response.text)
await with_retry(send)
async def create_upload_grant(
self, path: str, options: UploadGrantOptions | None = None
) -> UploadGrant:
"""Mint a presigned PUT the browser uploads to directly.
The signature covers ``host`` only and declares ``UNSIGNED-PAYLOAD``, so
the browser does not have to reproduce any header exactly -- a mismatch
there is the usual cause of a 403 on direct upload. Nothing in the grant
carries this application's secret key; the URL is the capability, and it
expires.
The bucket needs a CORS rule allowing PUT from your origin, or the
browser blocks the request before it is sent.
"""
resolved = options or UploadGrantOptions()
normalized = normalize_virtual_path(path)
if (
isinstance(resolved.expires_in_seconds, bool)
or not isinstance(resolved.expires_in_seconds, int)
or not 1 <= resolved.expires_in_seconds <= 7 * 24 * 60 * 60
):
raise InvalidInputError(
"expires_in_seconds must be an integer between 1 and 604800",
provider=PROVIDER_ID,
)
return UploadGrant(
provider=PROVIDER_ID,
path=normalized,
url=self.signed_url(
normalized,
method="PUT",
expires_in_seconds=resolved.expires_in_seconds,
),
method="PUT",
# Deliberately empty: any header signed here becomes one the browser
# is obliged to send byte-for-byte.
headers={},
protocol="single",
expires_at=datetime.now(timezone.utc)
+ timedelta(seconds=resolved.expires_in_seconds),
)
async def copy(self, source: str, destination: str) -> None:
"""Server-side copy. The bytes never travel through this process."""
source_key = self._to_key(source)
dest_url = self.object_url(self._to_key(destination))
# S3 wants the copy source as `/bucket/key`, URL-encoded. Encoding it
# per segment is what keeps a `#` or `?` in the filename from
# truncating the source key, exactly as it does for the object URL.
copy_source = f"/{self.bucket}/{encode_path_segments(source_key)}"
async def send() -> None:
signed = self._sign("PUT", dest_url, {"x-amz-copy-source": copy_source})
response = await self._http().request("PUT", dest_url, headers=signed)
if response.is_error:
raise self._map_error(response, response.text)
await with_retry(send)
async def move(self, source: str, destination: str) -> None:
"""Copy then delete. S3 has no atomic rename, so this is two calls."""
await self.copy(source, destination)
await self.delete(source)
async def metadata(self, path: str) -> StorageObject:
key = self._to_key(path)
url = self.object_url(key)
async def send() -> StorageObject:
response = await self._http().head(url, headers=self._sign("HEAD", url))
if response.is_error:
raise self._map_error(response)
return self._object_from_headers(key, response)
return await with_retry(send)
async def exists(self, path: str) -> bool:
try:
await self.metadata(path)
except ObjectNotFoundError:
return False
return True
async def list(self, path: str | None = None) -> list[StorageObject]:
"""List objects directly under ``path`` using ListObjectsV2.
Follows ``NextContinuationToken`` past S3's 1000-key response cap, so a
large prefix is not silently truncated.
"""
prefix = self._to_key(path) if path else self.root_prefix
if prefix:
prefix = prefix if prefix.endswith("/") else f"{prefix}/"
base = self._bucket_base_url()
results: list[StorageObject] = []
continuation: str | None = None
for _ in range(_LIST_PAGE_GUARD):
params = {"list-type": "2", "delimiter": "/"}
if prefix:
params["prefix"] = prefix
if continuation:
params["continuation-token"] = continuation
url = str(httpx.URL(f"{base}/", params=params))
async def fetch(target: str = url) -> httpx.Response:
response = await self._http().get(target, headers=self._sign("GET", target))
if response.is_error:
raise self._map_error(response, response.text)
return response
response = await with_retry(fetch)
try:
root = fromstring(response.text)
except (DefusedXmlException, ParseError) as exc:
raise ProviderUnavailableError(
"S3 returned malformed or unsafe XML.",
provider=PROVIDER_ID,
raw_error=exc,
) from exc
for prefix_node in root.iter(f"{_S3_NS}CommonPrefixes"):
folder = _text(prefix_node, "Prefix")
if folder:
trimmed = folder.rstrip("/")
virtual_folder = self._to_virtual_path(trimmed)
results.append(
StorageObject(
id=f"s3_{self.bucket}_{trimmed}",
path=virtual_folder,
name=get_basename(virtual_folder),
provider=PROVIDER_ID,
provider_id=trimmed,
type="folder",
)
)
for item in root.iter(f"{_S3_NS}Contents"):
key = _text(item, "Key")
if not key or key == prefix:
continue
size = _text(item, "Size")
modified = _text(item, "LastModified")
virtual_key = self._to_virtual_path(key)
results.append(
StorageObject(
id=f"s3_{self.bucket}_{key}",
path=virtual_key,
name=get_basename(virtual_key),
provider=PROVIDER_ID,
provider_id=key,
type="file",
size=int(size) if size else None,
checksum=(_text(item, "ETag") or "").strip('"') or None,
updated_at=datetime.fromisoformat(modified.replace("Z", "+00:00"))
if modified
else None,
)
)
truncated = (_text(root, "IsTruncated") or "false").lower() == "true"
continuation = _text(root, "NextContinuationToken") if truncated else None
if not continuation:
break
return results