Skip to content

Official Codebase of 3DCodeVerse

Python
3.1 KB
Raw Blame
"""R2 (S3-compatible) object storage — thin boto3 wrapper used by `3dcode-data`.

Contributors upload straight from their own machine to the staging bucket; the admin
side pulls/lists/deletes the same way. Keys are ``<source>/<project>/<relpath>``.
"""

from __future__ import annotations

import mimetypes
from dataclasses import dataclass
from pathlib import Path

import boto3
from botocore.config import Config as BotoConfig

from .config import Config


@dataclass
class Uploaded:
    files: int
    bytes: int


class R2:
    def __init__(self, cfg: Config):
        cfg.require_r2()
        self.bucket = cfg.bucket
        self._s3 = boto3.client(
            "s3",
            endpoint_url=cfg.endpoint,
            aws_access_key_id=cfg.access_key_id,
            aws_secret_access_key=cfg.secret_access_key,
            region_name="auto",
            config=BotoConfig(signature_version="s3v4", retries={"max_attempts": 5}),
        )

    def put_file(self, local: Path, key: str) -> int:
        ctype = mimetypes.guess_type(str(local))[0] or "application/octet-stream"
        self._s3.upload_file(str(local), self.bucket, key, ExtraArgs={"ContentType": ctype})
        return local.stat().st_size

    def upload_tree(self, local_dir: Path, key_prefix: str) -> Uploaded:
        """Upload every file under ``local_dir`` to ``key_prefix/<relpath>``."""
        files = bytes_ = 0
        for path in sorted(local_dir.rglob("*")):
            if path.is_file():
                rel = path.relative_to(local_dir).as_posix()
                bytes_ += self.put_file(path, f"{key_prefix.rstrip('/')}/{rel}")
                files += 1
        return Uploaded(files, bytes_)

    def download_tree(self, prefix: str, dest_dir: Path) -> Uploaded:
        """Download every object under ``prefix`` into ``dest_dir/<relpath>`` (zero egress on R2)."""
        prefix = prefix.rstrip("/") + "/"
        files = bytes_ = 0
        for key in self.list(prefix):
            rel = key[len(prefix):]
            if not rel:
                continue
            out = dest_dir / rel
            out.parent.mkdir(parents=True, exist_ok=True)
            self._s3.download_file(self.bucket, key, str(out))
            bytes_ += out.stat().st_size
            files += 1
        return Uploaded(files, bytes_)

    def list(self, prefix: str = "") -> list[str]:
        keys, token = [], None
        while True:
            kw = {"Bucket": self.bucket, "Prefix": prefix}
            if token:
                kw["ContinuationToken"] = token
            r = self._s3.list_objects_v2(**kw)
            keys += [o["Key"] for o in r.get("Contents", [])]
            if not r.get("IsTruncated"):
                return keys
            token = r["NextContinuationToken"]

    def get_text(self, key: str) -> str:
        return self._s3.get_object(Bucket=self.bucket, Key=key)["Body"].read().decode()

    def delete_prefix(self, prefix: str) -> int:
        keys = self.list(prefix)
        for i in range(0, len(keys), 1000):
            batch = [{"Key": k} for k in keys[i:i + 1000]]
            self._s3.delete_objects(Bucket=self.bucket, Delete={"Objects": batch})
        return len(keys)