One storage API for Google Drive, Nextcloud, and S3-compatible clouds, in TypeScript and Python.
TypeScript byoccloud-storagecloudflare-r2encryptiongoogle-drivenextcloudoauth2pkcepythons3sdkstoragetypescriptwebdav
"""WebDAV provider adapter.
Targets Nextcloud, ownCloud, Synology NAS, and any RFC 4918 server. WebDAV
collections, multistatus XML, and HTTP method verbs stay private to this module.
Unlike S3, WebDAV has real folders, so an upload creates missing ancestors with
MKCOL before writing.
"""
from __future__ import annotations
import base64
import contextlib
from collections.abc import AsyncIterator
from datetime import datetime
from email.utils import parsedate_to_datetime
from typing import TYPE_CHECKING
from urllib.parse import unquote, urlsplit
import httpx
from defusedxml.common import DefusedXmlException
from defusedxml.ElementTree import ParseError, fromstring
if TYPE_CHECKING:
# Runtime parsing always uses defusedxml; this import is only a type annotation.
from xml.etree.ElementTree import Element # nosec B405
from ..errors import (
ConflictError,
InvalidInputError,
ObjectNotFoundError,
PermissionDeniedError,
ProviderUnavailableError,
RateLimitedError,
StorageError,
)
from ..paths import encode_path_segments, get_basename, get_dirname, normalize_virtual_path
from ..retry import with_retry
from ..types import (
ProviderCapabilities,
ProviderManifest,
StorageInput,
StorageObject,
StorageOutput,
StorageQuota,
UploadOptions,
UploadProgress,
)
PROVIDER_ID = "webdav"
_DAV = "{DAV:}"
_PROPFIND_BODY = """<?xml version="1.0" encoding="utf-8"?>
<d:propfind xmlns:d="DAV:">
<d:prop>
<d:resourcetype/>
<d:getcontentlength/>
<d:getcontenttype/>
<d:getlastmodified/>
<d:getetag/>
</d:prop>
</d:propfind>"""
_QUOTA_BODY = """<?xml version="1.0" encoding="utf-8"?>
<d:propfind xmlns:d="DAV:">
<d:prop>
<d:quota-available-bytes/>
<d:quota-used-bytes/>
</d:prop>
</d:propfind>"""
def _prop(response_el: Element, tag: str) -> str | None:
"""Read a DAV property from a multistatus <response> element."""
for propstat in response_el.findall(f"{_DAV}propstat"):
status = propstat.findtext(f"{_DAV}status") or ""
if "200" not in status:
continue
found = propstat.find(f"{_DAV}prop/{_DAV}{tag}")
if found is not None and found.text is not None:
return found.text
return None
def _is_collection(response_el: Element) -> bool:
for propstat in response_el.findall(f"{_DAV}propstat"):
resource_type = propstat.find(f"{_DAV}prop/{_DAV}resourcetype")
if resource_type is not None and resource_type.find(f"{_DAV}collection") is not None:
return True
return False
def _parse_xml(payload: str) -> Element:
try:
return fromstring(payload)
except (DefusedXmlException, ParseError) as exc:
raise ProviderUnavailableError(
"WebDAV returned malformed or unsafe XML.",
provider=PROVIDER_ID,
raw_error=exc,
) from exc
class WebDAVProvider:
"""Storage adapter for WebDAV servers.
Args:
endpoint: Collection URL, e.g.
``https://cloud.example.com/remote.php/dav/files/alex/``.
username: Username for Basic auth.
password: Password for Basic auth. Use an app password, not the account
password -- app passwords are scoped and independently revocable.
token: Bearer token, as an alternative to username/password.
root_folder: Folder inside the endpoint that all paths live under.
client: An existing ``httpx.AsyncClient`` to reuse.
"""
def __init__(
self,
*,
endpoint: str,
username: str | None = None,
password: str | None = None,
token: str | None = None,
root_folder: str | None = "BYOC",
client: httpx.AsyncClient | None = None,
) -> None:
if not endpoint:
raise InvalidInputError(
"WebDAVProvider requires an 'endpoint' URL.", provider=PROVIDER_ID
)
self.endpoint = endpoint.rstrip("/")
self.username = username
self.password = password
self.token = token
self.root_folder = normalize_virtual_path(root_folder) if root_folder else ""
self._base_path = urlsplit(self.endpoint).path.rstrip("/")
self._client = client
self._owns_client = client is None
# -- lifecycle ---------------------------------------------------------
def manifest(self) -> ProviderManifest:
return ProviderManifest(
id=PROVIDER_ID,
name="Nextcloud / WebDAV",
category="self-hosted",
authentication="oauth2" if self.token else "basic",
supports_user_owned_storage=True,
adapter_version="0.4.0",
)
def capabilities(self) -> ProviderCapabilities:
# WebDAV has real collections and RFC 4331 quota, but no resumable
# upload protocol: large files go as a single PUT.
return ProviderCapabilities(
folders=True,
sharing=False,
public_urls=False,
resumable_uploads=False,
versioning=False,
quota=True,
server_side_copy=True,
# WebDAV authenticates every request with Basic credentials, so
# there is no URL a browser can be handed without also handing it
# the password.
direct_upload=False,
)
async def connect(self) -> None:
"""Ensure the root folder exists, tolerating one that already does."""
if self.root_folder:
with contextlib.suppress(StorageError):
await self._mkcol(self.root_folder)
async def disconnect(self) -> None:
if self._owns_client and self._client is not None:
await self._client.aclose()
self._client = None
def _http(self) -> httpx.AsyncClient:
if self._client is None:
self._client = httpx.AsyncClient(timeout=httpx.Timeout(30.0, read=300.0))
self._owns_client = True
return self._client
def _headers(self, extra: dict[str, str] | None = None) -> dict[str, str]:
headers = dict(extra or {})
if self.token:
headers["Authorization"] = f"Bearer {self.token}"
elif self.username is not None and self.password is not None:
raw = f"{self.username}:{self.password}".encode()
headers["Authorization"] = f"Basic {base64.b64encode(raw).decode('ascii')}"
return headers
# -- path handling -----------------------------------------------------
def _to_remote(self, path: str) -> str:
"""Virtual path -> path relative to the endpoint (adds ``root_folder``)."""
normalized = normalize_virtual_path(path)
if not self.root_folder:
return normalized
return f"{self.root_folder}/{normalized}" if normalized else self.root_folder
def _to_virtual_path(self, remote: str) -> str:
"""Endpoint-relative path -> virtual path (strips ``root_folder``).
``root_folder`` is the adapter's business and must not appear in a
returned ``StorageObject.path``, or a caller feeding that path back in
would have the folder applied twice.
"""
if self.root_folder and remote.startswith(f"{self.root_folder}/"):
return remote[len(self.root_folder) + 1 :]
if remote == self.root_folder:
return ""
return remote
def _url(self, remote_path: str) -> str:
encoded = encode_path_segments(remote_path)
return f"{self.endpoint}/{encoded}" if encoded else self.endpoint
def _href_to_remote(self, href: str) -> str:
"""Turn a multistatus href into an endpoint-relative path.
Servers return hrefs either absolute or path-only, always percent-encoded.
"""
path = unquote(urlsplit(href).path)
if self._base_path and path.startswith(self._base_path):
path = path[len(self._base_path) :]
return path.strip("/")
# -- error mapping -----------------------------------------------------
def _map_error(self, response: httpx.Response) -> StorageError:
status = response.status_code
if status == 404:
return ObjectNotFoundError(
"WebDAV resource not found (HTTP 404).", provider=PROVIDER_ID, status_code=404
)
if status in (401, 403):
return PermissionDeniedError(
"WebDAV access denied: check credentials or use an app password.",
provider=PROVIDER_ID,
status_code=status,
)
if status in (405, 409, 412):
return ConflictError(
f"WebDAV conflict (HTTP {status}): the parent collection may not exist.",
provider=PROVIDER_ID,
status_code=status,
)
if status == 429:
return RateLimitedError(
"WebDAV rate limit exceeded.", provider=PROVIDER_ID, status_code=status
)
return ProviderUnavailableError(
f"WebDAV error (HTTP {status}): {response.text[:300]}",
provider=PROVIDER_ID,
status_code=status,
retryable=status >= 500,
)
def _object_from_response(self, remote: str, response: httpx.Response) -> StorageObject:
virtual = self._to_virtual_path(remote)
length = response.headers.get("content-length")
last_modified = response.headers.get("last-modified")
updated_at: datetime | None = None
if last_modified:
try:
updated_at = parsedate_to_datetime(last_modified)
except (TypeError, ValueError):
updated_at = None
return StorageObject(
id=f"webdav_{remote}",
path=virtual,
name=get_basename(virtual) or get_basename(remote),
provider=PROVIDER_ID,
provider_id=remote,
type="file",
size=int(length) if length is not None else None,
mime_type=response.headers.get("content-type"),
checksum=(response.headers.get("etag") or "").strip('"') or None,
updated_at=updated_at,
)
def _object_from_propfind(self, remote: str, entry: Element) -> StorageObject:
"""Build a StorageObject from one multistatus <response> element."""
is_folder = _is_collection(entry)
size = _prop(entry, "getcontentlength")
modified = _prop(entry, "getlastmodified")
updated_at: datetime | None = None
if modified:
try:
updated_at = parsedate_to_datetime(modified)
except (TypeError, ValueError):
updated_at = None
virtual = self._to_virtual_path(remote)
return StorageObject(
id=f"webdav_{remote}",
path=virtual,
name=get_basename(virtual) or get_basename(remote),
provider=PROVIDER_ID,
provider_id=remote,
type="folder" if is_folder else "file",
size=int(size) if size and not is_folder else None,
mime_type=_prop(entry, "getcontenttype"),
checksum=(_prop(entry, "getetag") or "").strip('"') or None,
updated_at=updated_at,
)
# -- WebDAV verbs ------------------------------------------------------
async def _mkcol(self, remote: str) -> None:
url = self._url(remote)
response = await self._http().request("MKCOL", url, headers=self._headers())
# 405 means the collection already exists, which is the desired state.
if response.is_error and response.status_code != 405:
raise self._map_error(response)
async def _ensure_parents(self, remote_dir: str) -> None:
"""Create each missing ancestor collection, outermost first."""
if not remote_dir:
return
segments = remote_dir.split("/")
for index in range(1, len(segments) + 1):
try:
await self._mkcol("/".join(segments[:index]))
except StorageError:
# A parent that already exists is fine; a real failure surfaces
# on the PUT that follows.
continue
# -- operations --------------------------------------------------------
async def upload(
self, path: str, data: StorageInput, options: UploadOptions | None = None
) -> StorageObject:
remote = self._to_remote(path)
await self._ensure_parents(get_dirname(remote))
payload: bytes | None
if isinstance(data, str):
payload = data.encode("utf-8")
elif isinstance(data, bytes | bytearray | memoryview):
payload = bytes(data)
else:
# An async iterator is sent with chunked transfer encoding, so the
# payload never exists in memory as a whole.
payload = None
stream = data
mime = (options.mime_type if options else None) or "application/octet-stream"
url = self._url(remote)
sent = 0
on_progress = options.on_progress if options else None
async def counted(source: AsyncIterator[bytes]) -> AsyncIterator[bytes]:
"""Pass chunks through, tracking how many bytes actually went out."""
nonlocal sent
async for chunk in source:
sent += len(chunk)
if on_progress:
on_progress(UploadProgress(bytes_uploaded=sent))
yield chunk
async def send() -> StorageObject:
nonlocal sent
headers = self._headers({"content-type": mime})
if payload is not None:
response = await self._http().put(url, headers=headers, content=payload)
size = len(payload)
else:
sent = 0
response = await self._http().put(
url, headers=headers, content=counted(stream)
)
size = sent
if response.is_error:
raise self._map_error(response)
return StorageObject(
id=f"webdav_{remote}",
path=self._to_virtual_path(remote),
name=get_basename(remote),
provider=PROVIDER_ID,
provider_id=remote,
type="file",
size=size,
mime_type=mime,
checksum=(response.headers.get("etag") or "").strip('"') or None,
)
# A stream can only be consumed once, so a retry would resend nothing.
if payload is None:
return await send()
return await with_retry(send)
async def download(self, path: str) -> StorageOutput:
remote = self._to_remote(path)
url = self._url(remote)
async def fetch() -> httpx.Response:
# stream=True leaves the body unread so a large object never has to
# fit in memory; the response stays open until the caller consumes
# it. This previously read response.content up front and yielded it
# as a single chunk, so stream() buffered exactly what it promised
# not to.
request = self._http().build_request("GET", url, headers=self._headers())
response = await self._http().send(request, stream=True)
if response.is_error:
await response.aread()
await response.aclose()
raise self._map_error(response)
return response
response = await with_retry(fetch)
async def stream() -> AsyncIterator[bytes]:
try:
async for chunk in response.aiter_bytes():
yield chunk
finally:
await response.aclose()
async def read() -> bytes:
try:
return await response.aread()
finally:
await response.aclose()
return StorageOutput(
metadata=self._object_from_response(remote, response), stream=stream, read=read
)
async def delete(self, path: str) -> None:
remote = self._to_remote(path)
url = self._url(remote)
async def send() -> None:
response = await self._http().request("DELETE", url, headers=self._headers())
if response.is_error and response.status_code != 404:
raise self._map_error(response)
await with_retry(send)
async def metadata(self, path: str) -> StorageObject:
"""Read properties with ``PROPFIND`` at ``Depth: 0``.
PROPFIND rather than HEAD: it is the canonical WebDAV way to read
properties, returns size/type/etag/mtime uniformly across servers, and
avoids a real interop hazard -- some servers attach an HTML error body
to a HEAD 404, which desynchronizes a keep-alive connection and makes
the *next* request on it fail with a bogus parse error.
"""
remote = self._to_remote(path)
url = self._url(remote)
async def send() -> StorageObject:
response = await self._http().request(
"PROPFIND",
url,
headers=self._headers({"Depth": "0", "content-type": "application/xml"}),
content=_PROPFIND_BODY.encode("utf-8"),
)
if response.is_error:
raise self._map_error(response)
root = _parse_xml(response.text)
entry = root.find(f"{_DAV}response")
if entry is None:
raise ObjectNotFoundError(
f"WebDAV resource not found: {path}", provider=PROVIDER_ID, status_code=404
)
return self._object_from_propfind(remote, entry)
return await with_retry(send)
async def exists(self, path: str) -> bool:
try:
await self.metadata(path)
except ObjectNotFoundError:
return False
return True
async def list(self, path: str | None = None) -> list[StorageObject]:
"""List one level below ``path`` via PROPFIND with ``Depth: 1``."""
remote = self._to_remote(path or "")
url = self._url(remote)
async def fetch() -> httpx.Response:
response = await self._http().request(
"PROPFIND",
url,
headers=self._headers({"Depth": "1", "content-type": "application/xml"}),
content=_PROPFIND_BODY.encode("utf-8"),
)
if response.is_error:
raise self._map_error(response)
return response
response = await with_retry(fetch)
root = _parse_xml(response.text)
results: list[StorageObject] = []
for entry in root.findall(f"{_DAV}response"):
href = entry.findtext(f"{_DAV}href") or ""
entry_remote = self._href_to_remote(href)
# PROPFIND includes the collection itself; only children are listed.
if entry_remote == remote or not entry_remote:
continue
results.append(self._object_from_propfind(entry_remote, entry))
return results
async def create_folder(self, path: str) -> StorageObject:
remote = self._to_remote(path)
await self._ensure_parents(remote)
return StorageObject(
id=f"webdav_{remote}",
path=self._to_virtual_path(remote),
name=get_basename(remote),
provider=PROVIDER_ID,
provider_id=remote,
type="folder",
)
async def move(self, source: str, destination: str) -> None:
"""Server-side move (RFC 4918 MOVE), no bytes transferred."""
target = self._to_remote(destination)
await self._ensure_parents(get_dirname(target))
response = await self._http().request(
"MOVE",
self._url(self._to_remote(source)),
headers=self._headers({"Destination": self._url(target), "Overwrite": "T"}),
)
if response.is_error:
raise self._map_error(response)
async def copy(self, source: str, destination: str) -> None:
"""Server-side copy (RFC 4918 COPY), no bytes transferred."""
target = self._to_remote(destination)
await self._ensure_parents(get_dirname(target))
response = await self._http().request(
"COPY",
self._url(self._to_remote(source)),
headers=self._headers({"Destination": self._url(target), "Overwrite": "T"}),
)
if response.is_error:
raise self._map_error(response)
async def quota(self) -> StorageQuota:
"""Report quota via RFC 4331 properties, where the server supplies them."""
response = await self._http().request(
"PROPFIND",
self._url(self._to_remote("")),
headers=self._headers({"Depth": "0", "content-type": "application/xml"}),
content=_QUOTA_BODY.encode("utf-8"),
)
if response.is_error:
raise self._map_error(response)
root = _parse_xml(response.text)
entry = root.find(f"{_DAV}response")
if entry is None:
return StorageQuota(used=0)
used_text = _prop(entry, "quota-used-bytes")
available_text = _prop(entry, "quota-available-bytes")
used = int(used_text) if used_text and used_text.lstrip("-").isdigit() else 0
available: int | None = None
if available_text and available_text.lstrip("-").isdigit():
candidate = int(available_text)
# Negative values are RFC 4331 sentinels for "unlimited"/"unknown".
available = candidate if candidate >= 0 else None
total = used + available if available is not None else None
return StorageQuota(used=used, total=total, available=available)