Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(temp-upload): hour-bucketed shared uploads with best-effort cleanup (#4479)

Jiahui Zhou550ef796757f
diff --git a/openviking/server/config.py b/openviking/server/config.py
index 6b46ca94b..211780b8e 100644
--- a/openviking/server/config.py
+++ b/openviking/server/config.py
@@ -300,6 +300,11 @@ class TempUploadConfig(BaseModel):
default_mode: Literal["local", "shared"] = "local"
shared_max_size_bytes: int = 512 * 1024 * 1024
ttl_seconds: int = Field(12 * 60 * 60, ge=0)
+ # When True, shared-upload cleanup also removes directories whose names are
+ # not valid upload ids (missing/garbled `<ms-ts>-<uuid>` format). Off by
+ # default so a malformed entry never triggers an unexpected delete; enable
+ # to reclaim junk directories that would otherwise be skipped forever.
+ cleanup_invalid_dirs: bool = False
model_config = {"extra": "forbid"}
diff --git a/openviking/server/temp_upload_store.py b/openviking/server/temp_upload_store.py
index 801ade169..00241a6cd 100644
--- a/openviking/server/temp_upload_store.py
+++ b/openviking/server/temp_upload_store.py
@@ -4,6 +4,8 @@
from __future__ import annotations
+import asyncio
+import calendar
import json
import logging
import os
@@ -13,6 +15,8 @@ import uuid
from contextlib import suppress
from dataclasses import dataclass
from pathlib import Path
+from queue import Full, Queue
+from threading import Lock, Thread
from typing import Any, Optional
from openviking.server.config import ServerConfig, TempUploadConfig
@@ -24,10 +28,56 @@ from openviking_cli.utils.config.open_viking_config import get_openviking_config
_CHUNK_SIZE = 1024 * 1024
_SHARED_UPLOAD_ROOT = "viking://upload"
+_SHARED_CLEANUP_QUEUE_MAX_SIZE = 100
+# Seconds in one hour; shared uploads are grouped into `YYYYMMDDHH` (UTC) buckets
+# so the upload root never holds a single huge flat directory.
+_SHARED_BUCKET_SECONDS = 3600
+_SHARED_CLEANUP_QUEUE: Queue[tuple["TempUploadStore", RequestContext, str]] = Queue(
+ maxsize=_SHARED_CLEANUP_QUEUE_MAX_SIZE
+)
+_SHARED_CLEANUP_WORKER_STARTED = False
+_SHARED_CLEANUP_WORKER_LOCK = Lock()
+
+# Module-level throttle state. TempUploadStore is not a singleton (HTTP router
+# and MCP endpoint call TempUploadStore.build(...) per request), so throttle
+# state must live at module scope rather than on the instance.
+#
+# _SHARED_CLEANUP_DUE_AT maps account_id -> Unix epoch seconds: the expiry time
+# of the earliest still-valid upload observed by the last cleanup. Before that
+# instant there is nothing new to expire, so submitting cleanup is pointless.
+# _SHARED_CLEANUP_PENDING holds account_ids already queued or executing, so we
+# never pile duplicate jobs for one account onto the single-worker queue.
+_SHARED_CLEANUP_DUE_AT: dict[str, float] = {}
+_SHARED_CLEANUP_PENDING: set[str] = set()
+_SHARED_CLEANUP_STATE_LOCK = Lock()
logger = logging.getLogger(__name__)
+def _write_json(path: Path, data: dict[str, Any]) -> None:
+ with path.open("w", encoding="utf-8") as f:
+ json.dump(data, f)
+
+
+def _write_bytes(path: str, content: bytes) -> None:
+ with open(path, "wb") as f:
+ f.write(content)
+
+
+def _open_binary_for_write(path: str | Path):
+ return open(path, "wb")
+
+
+def _close_file(file_obj: Any) -> None:
+ file_obj.close()
+
+
+def _create_temp_file(*, prefix: str, suffix: str) -> str:
+ fd, temp_path = tempfile.mkstemp(prefix=prefix, suffix=suffix)
+ os.close(fd)
+ return temp_path
+
+
@dataclass
class ResolvedTempUpload:
mode: str
@@ -38,18 +88,34 @@ class ResolvedTempUpload:
async def cleanup(self) -> None:
if self.mode == "shared" and self.local_path:
with suppress(FileNotFoundError):
- os.unlink(self.local_path)
+ await asyncio.to_thread(os.unlink, self.local_path)
def get_temp_upload_config(server_config: ServerConfig) -> TempUploadConfig:
return server_config.temp_upload
-def _shared_content_uri(upload_id: str) -> str:
- return f"{_SHARED_UPLOAD_ROOT}/{upload_id}/content"
+def _shared_bucket_start(bucket: str) -> Optional[float]:
+ """Return the epoch-seconds start of a `YYYYMMDDHH` (UTC) bucket, else None."""
+ if len(bucket) != 10 or not bucket.isdigit():
+ return None
+ try:
+ parsed = time.strptime(bucket, "%Y%m%d%H")
+ except ValueError:
+ return None
+ return calendar.timegm(parsed)
+
+
+def _shared_content_uri(bucket: str, leaf: str) -> str:
+ return f"{_SHARED_UPLOAD_ROOT}/{bucket}/{leaf}/content"
-def _shared_meta_uri(upload_id: str) -> str:
+def _shared_meta_uri(bucket: str, leaf: str) -> str:
+ return f"{_SHARED_UPLOAD_ROOT}/{bucket}/{leaf}/meta"
+
+
+def _legacy_shared_meta_uri(upload_id: str) -> str:
+ """Flat pre-bucket meta URI, kept for read compatibility during TTL."""
return f"{_SHARED_UPLOAD_ROOT}/{upload_id}/meta"
@@ -63,10 +129,49 @@ def _parse_shared_temp_file_id(temp_file_id: str) -> Optional[str]:
def _new_shared_upload_id() -> str:
- return f"{time.time_ns() // 1_000_000:013d}-{uuid.uuid4().hex}"
+ """Return a new bucketed upload id: ``<YYYYMMDDHH-UTC>-<uuid hex>``.
+
+ The 10-digit hour prefix is both the storage bucket and a marker of the new
+ bucketed format. Legacy ids start with a 13-digit millisecond timestamp
+ instead, so the two are distinguishable by the first segment's length.
+ """
+ return f"{time.strftime('%Y%m%d%H', time.gmtime())}-{uuid.uuid4().hex}"
+
+
+def _shared_bucket_from_upload_id(upload_id: str) -> Optional[str]:
+ """Return the bucket for a NEW-format upload id, else None.
+
+ New ids are ``<YYYYMMDDHH>-<32-hex>``; the bucket is the literal 10-digit
+ hour prefix. Legacy ids (``<13-digit-ms>-<uuid>``) and anything malformed
+ return None.
+ """
+ split = _split_shared_upload_id(upload_id)
+ return split[0] if split is not None else None
+
+
+def _split_shared_upload_id(upload_id: str) -> Optional[tuple[str, str]]:
+ """Split a NEW-format upload id into ``(bucket, leaf)``, else None.
+
+ ``bucket`` is the 10-digit hour prefix (the storage bucket); ``leaf`` is the
+ uuid hex used as the in-bucket directory name, so the bucket prefix is not
+ repeated on the child directory. Legacy/malformed ids return None.
+ """
+ prefix, sep, rest = upload_id.partition("-")
+ if not sep or len(prefix) != 10 or not prefix.isdigit():
+ return None
+ if len(rest) != 32 or any(char not in "0123456789abcdef" for char in rest):
+ return None
+ if _shared_bucket_start(prefix) is None:
+ return None
+ return prefix, rest
def _shared_upload_created_at(upload_id: str) -> Optional[float]:
+ """Return the creation epoch for a LEGACY flat upload id, else None.
+
+ Legacy ids are ``<13-digit-ms-timestamp>-<32-hex>``; new bucketed ids use a
+ 10-digit hour prefix and return None here (their expiry is bucket-level).
+ """
timestamp_ms, separator, nonce = upload_id.partition("-")
if (
not separator
@@ -81,26 +186,34 @@ def _shared_upload_created_at(upload_id: str) -> Optional[float]:
async def _stream_upload_to_local_temp(upload_file: Any, max_size_bytes: int) -> tuple[str, int]:
suffix = Path(upload_file.filename or "upload.tmp").suffix or ".tmp"
- fd, temp_path = tempfile.mkstemp(prefix="ov_http_upload_", suffix=suffix)
- os.close(fd)
+ temp_path = await asyncio.to_thread(
+ _create_temp_file, prefix="ov_http_upload_", suffix=suffix
+ )
total = 0
+ f = None
try:
- with open(temp_path, "wb") as f:
- while True:
- chunk = await upload_file.read(_CHUNK_SIZE)
- if not chunk:
- break
- total += len(chunk)
- if total > max_size_bytes:
- raise InvalidArgumentError(
- f"Upload exceeds size limit ({max_size_bytes} bytes)."
- )
- f.write(chunk)
+ f = await asyncio.to_thread(_open_binary_for_write, temp_path)
+ while True:
+ chunk = await upload_file.read(_CHUNK_SIZE)
+ if not chunk:
+ break
+ total += len(chunk)
+ if total > max_size_bytes:
+ raise InvalidArgumentError(
+ f"Upload exceeds size limit ({max_size_bytes} bytes)."
+ )
+ # UploadFile reads already yield to the event loop. Local disk writes do
+ # not, so run each bounded write in the default executor rather than
+ # stalling the Core worker event loop on slow local storage.
+ await asyncio.to_thread(f.write, chunk)
return temp_path, total
except Exception:
with suppress(FileNotFoundError):
- os.unlink(temp_path)
+ await asyncio.to_thread(os.unlink, temp_path)
raise
+ finally:
+ if f is not None:
+ await asyncio.to_thread(_close_file, f)
class TempUploadStore:
@@ -138,33 +251,35 @@ class TempUploadStore:
) -> ResolvedTempUpload:
shared_id = _parse_shared_temp_file_id(temp_file_id)
if shared_id is None:
- return self._resolve_local(temp_file_id)
+ return await asyncio.to_thread(self._resolve_local, temp_file_id)
return await self._resolve_shared(temp_file_id, shared_id, ctx)
async def _save_local(self, upload_file: Any) -> str:
config = get_openviking_config()
temp_dir = config.storage.get_upload_temp_dir()
- self._cleanup_local_temp_files(temp_dir)
+ await asyncio.to_thread(self._cleanup_local_temp_files, temp_dir)
file_ext = Path(upload_file.filename).suffix if upload_file.filename else ".tmp"
temp_filename = f"upload_{uuid.uuid4().hex}{file_ext}"
temp_file_path = temp_dir / temp_filename
total = 0
- with open(temp_file_path, "wb") as f:
+ f = await asyncio.to_thread(_open_binary_for_write, temp_file_path)
+ try:
while True:
chunk = await upload_file.read(_CHUNK_SIZE)
if not chunk:
break
total += len(chunk)
if total > self.temp_cfg.shared_max_size_bytes:
- f.close()
with suppress(FileNotFoundError):
- temp_file_path.unlink()
+ await asyncio.to_thread(temp_file_path.unlink)
raise InvalidArgumentError(
f"Upload exceeds size limit ({self.temp_cfg.shared_max_size_bytes} bytes)."
)
- f.write(chunk)
+ await asyncio.to_thread(f.write, chunk)
+ finally:
+ await asyncio.to_thread(_close_file, f)
if upload_file.filename:
meta_path = temp_dir / f"{temp_filename}.ov_upload.meta"
@@ -172,22 +287,21 @@ class TempUploadStore:
"original_filename": upload_file.filename,
"upload_time": time.time(),
}
- with open(meta_path, "w", encoding="utf-8") as f:
- json.dump(meta, f)
+ await asyncio.to_thread(_write_json, meta_path, meta)
return temp_filename
async def _save_shared(self, upload_file: Any, ctx: RequestContext) -> str:
- await self._cleanup_shared_uploads(ctx)
temp_path, total_size = await _stream_upload_to_local_temp(
upload_file, self.temp_cfg.shared_max_size_bytes
)
upload_id = _new_shared_upload_id()
temp_file_id = f"shared_{upload_id}"
+ bucket, leaf = _split_shared_upload_id(upload_id)
vfs = get_viking_fs()
internal_ctx = self._internal_ctx(ctx)
- content_uri = _shared_content_uri(upload_id)
- meta_uri = _shared_meta_uri(upload_id)
+ content_uri = _shared_content_uri(bucket, leaf)
+ meta_uri = _shared_meta_uri(bucket, leaf)
meta = {
"version": 2,
"temp_file_id": temp_file_id,
@@ -201,22 +315,121 @@ class TempUploadStore:
}
try:
- with open(temp_path, "rb") as f:
- content = f.read()
- await vfs.write_file_bytes(content_uri, content, ctx=internal_ctx)
- await vfs.write_file(meta_uri, json.dumps(meta, ensure_ascii=False), ctx=internal_ctx)
+ content = await asyncio.to_thread(Path(temp_path).read_bytes)
+ await vfs.write_file_bytes(
+ content_uri, content, ctx=internal_ctx, auto_pathlock=False
+ )
+ await vfs.write_file(
+ meta_uri,
+ json.dumps(meta, ensure_ascii=False),
+ ctx=internal_ctx,
+ auto_pathlock=False,
+ )
+ self._schedule_shared_cleanup(ctx)
return temp_file_id
except Exception:
with suppress(Exception):
- await vfs.rm(
- f"{_SHARED_UPLOAD_ROOT}/{upload_id}",
+ await vfs.remove_files(
+ f"{_SHARED_UPLOAD_ROOT}/{bucket}/{leaf}",
recursive=True,
ctx=internal_ctx,
+ auto_pathlock=False,
)
raise
finally:
with suppress(FileNotFoundError):
- os.unlink(temp_path)
+ await asyncio.to_thread(os.unlink, temp_path)
+
+ def _schedule_shared_cleanup(self, ctx: RequestContext) -> None:
+ """Enqueue a best-effort shared-upload cleanup off the request path.
+
+ Decisions are made under the state lock before touching the executor:
+ skip when TTL is disabled, when a job for this account is already
+ pending/running, or when the earliest known expiry (``due_at``) is still
+ in the future (nothing new can have expired yet). Otherwise mark the
+ account pending and enqueue exactly one job.
+ """
+ ttl_seconds = self.temp_cfg.ttl_seconds
+ if ttl_seconds == 0:
+ return
+
+ account_id = ctx.account_id
+ now = time.time()
+ with _SHARED_CLEANUP_STATE_LOCK:
+ if account_id in _SHARED_CLEANUP_PENDING:
+ logger.debug(
+ "[TempUpload] Shared cleanup already pending account=%s", account_id
+ )
+ return
+ due_at = _SHARED_CLEANUP_DUE_AT.get(account_id)
+ if due_at is not None and now < due_at:
+ logger.debug(
+ "[TempUpload] Shared cleanup not due account=%s due_at=%s now=%s",
+ account_id,
+ due_at,
+ now,
+ )
+ return
+ _SHARED_CLEANUP_PENDING.add(account_id)
+ try:
+ _SHARED_CLEANUP_QUEUE.put_nowait((self, ctx, account_id))
+ except Full:
+ _SHARED_CLEANUP_PENDING.discard(account_id)
+ logger.warning(
+ "[TempUpload] Shared cleanup queue full account=%s queue_max_size=%s",
+ account_id,
+ _SHARED_CLEANUP_QUEUE_MAX_SIZE,
+ )
+ return
+
+ self._ensure_shared_cleanup_worker()
+
+ @staticmethod
+ def _ensure_shared_cleanup_worker() -> None:
+ global _SHARED_CLEANUP_WORKER_STARTED
+ with _SHARED_CLEANUP_WORKER_LOCK:
+ if _SHARED_CLEANUP_WORKER_STARTED:
+ return
+ Thread(
+ target=TempUploadStore._run_shared_cleanup_worker,
+ name="ov-shared-upload-cleanup",
+ daemon=True,
+ ).start()
+ _SHARED_CLEANUP_WORKER_STARTED = True
+
+ @staticmethod
+ def _run_shared_cleanup_worker() -> None:
+ while True:
+ store, ctx, account_id = _SHARED_CLEANUP_QUEUE.get()
+ started_at = time.monotonic()
+ try:
+ listed_count, scanned_count, removed_count, next_due_at = asyncio.run(
+ store._cleanup_shared_uploads(ctx)
+ )
+ logger.info(
+ "[TempUpload] Shared cleanup completed account=%s elapsed_ms=%.1f "
+ "listed=%s scanned=%s removed=%s next_cleanup_at=%s",
+ account_id,
+ (time.monotonic() - started_at) * 1000.0,
+ listed_count,
+ scanned_count,
+ removed_count,
+ next_due_at,
+ )
+ except Exception:
+ logger.warning(
+ "[TempUpload] Shared cleanup failed account=%s",
+ account_id,
+ exc_info=True,
+ )
+ finally:
+ # Always release the pending slot so the next request for this
+ # account can retry, regardless of list/parse/remove outcome.
+ # due_at is owned by _cleanup_shared_uploads (updated on success,
+ # cleared on failure), so the worker only manages pending here.
+ with _SHARED_CLEANUP_STATE_LOCK:
+ _SHARED_CLEANUP_PENDING.discard(account_id)
+ _SHARED_CLEANUP_QUEUE.task_done()
def _resolve_local(self, temp_file_id: str) -> ResolvedTempUpload:
upload_temp_dir = get_openviking_config().storage.get_upload_temp_dir()
@@ -282,15 +495,15 @@ class TempUploadStore:
raise PermissionDeniedError("Temporary upload is invalid: content missing.")
file_ext = meta.get("file_ext") or ".tmp"
- fd, temp_path = tempfile.mkstemp(prefix="ov_shared_upload_", suffix=file_ext)
- os.close(fd)
+ temp_path = await asyncio.to_thread(
+ _create_temp_file, prefix="ov_shared_upload_", suffix=file_ext
+ )
try:
content = await vfs.read_file_bytes(content_uri, ctx=internal_ctx)
- with open(temp_path, "wb") as f:
- f.write(content)
+ await asyncio.to_thread(_write_bytes, temp_path, content)
except Exception:
with suppress(FileNotFoundError):
- os.unlink(temp_path)
+ await asyncio.to_thread(os.unlink, temp_path)
raise
return ResolvedTempUpload(
mode="shared",
@@ -302,10 +515,22 @@ class TempUploadStore:
async def _read_shared_meta(self, upload_id: str, ctx: RequestContext) -> dict[str, Any]:
vfs = get_viking_fs()
internal_ctx = self._internal_ctx(ctx)
+ # The upload id itself tells us the layout: a new id carries a 10-digit
+ # hour bucket prefix (stored under bucket/<uuid>/), while a legacy id
+ # starts with a 13-digit ms timestamp and lives in the old flat path.
+ # So we read exactly one path, no fallback.
+ split = _split_shared_upload_id(upload_id)
+ if split is not None:
+ bucket, leaf = split
+ meta_uri = _shared_meta_uri(bucket, leaf)
+ else:
+ meta_uri = _legacy_shared_meta_uri(upload_id)
try:
- data = json.loads(await vfs.read_file(_shared_meta_uri(upload_id), ctx=internal_ctx))
+ data = json.loads(await vfs.read_file(meta_uri, ctx=internal_ctx))
except Exception as exc:
- raise PermissionDeniedError("Temporary upload metadata is invalid or missing.") from exc
+ raise PermissionDeniedError(
+ "Temporary upload metadata is invalid or missing."
+ ) from exc
if not isinstance(data, dict):
raise PermissionDeniedError("Temporary upload metadata is invalid or missing.")
return data
@@ -321,70 +546,218 @@ class TempUploadStore:
if meta.get("account") != ctx.account_id:
raise PermissionDeniedError("Temporary upload does not belong to current account.")
- async def _cleanup_shared_uploads(self, ctx: RequestContext) -> None:
+ async def _cleanup_shared_uploads(
+ self, ctx: RequestContext
+ ) -> tuple[int, int, int, Optional[float]]:
+ """Best-effort cleanup of the shared upload root.
+
+ The upload root holds three kinds of top-level directories, and each kind
+ is processed as its own independent oldest-first group so that a backlog
+ of one kind never blocks another (e.g. a large legacy backlog must not
+ starve bucket cleanup):
+
+ - ``YYYYMMDDHH`` hour buckets (the current layout). A bucket expires at
+ ``bucket_start + 3600 + ttl`` (once its newest possible upload has
+ expired) and is removed whole with a single recursive ``rm``.
+ - ``<13-digit-ms>-<uuid>`` legacy flat uploads (pre-bucket). Each expires
+ at ``created_at + ttl`` and is removed individually.
+ - anything else: malformed/foreign directories. These are left alone
+ unless ``temp_upload.cleanup_invalid_dirs`` is enabled, in which case
+ they are removed best-effort. Legacy flat uploads are never treated as
+ invalid.
+
+ Within each group, entries are visited oldest-first (name-ascending),
+ expired ones are removed, and the walk stops at the first still-valid
+ entry recording its expiry. The account ``due_at`` throttle is the
+ earliest expiry across the bucket and legacy groups; a removal failure in
+ either group clears ``due_at`` so the next request retries promptly.
+ Every deletion logs its target and elapsed time.
+
+ Returns ``(listed, scanned, removed, next_due_at)``.
+ """
+ account_id = ctx.account_id
if self.temp_cfg.ttl_seconds == 0:
- return
+ self._clear_due_at(account_id)
+ return 0, 0, 0, None
vfs = get_viking_fs()
internal_ctx = self._internal_ctx(ctx)
try:
- uploads = await vfs.ls(
+ entries = await vfs.ls(
_SHARED_UPLOAD_ROOT,
show_all_hidden=True,
node_limit=LS_ALL_NODES,
+ sort_by="name",
+ sort_order="asc",
ctx=internal_ctx,
)
except Exception:
+ # Listing failed: drop due_at so the next request retries instead of
+ # being throttled behind a stale expiry, then re-raise for logging.
+ self._clear_due_at(account_id)
logger.warning(
"Shared temp upload cleanup list failed account=%s",
- ctx.account_id,
+ account_id,
exc_info=True,
)
- return
+ raise
+ listed_count = len(entries)
now = time.time()
- cutoff = now - self.temp_cfg.ttl_seconds
+ ttl_seconds = self.temp_cfg.ttl_seconds
+ cleanup_invalid = getattr(self.temp_cfg, "cleanup_invalid_dirs", False)
logger.debug(
- "Shared temp upload cleanup account=%s ttl_seconds=%s upload_count=%s "
- "now=%s cutoff=%s",
- ctx.account_id,
- self.temp_cfg.ttl_seconds,
- len(uploads),
+ "Shared temp upload cleanup account=%s ttl_seconds=%s listed_count=%s now=%s",
+ account_id,
+ ttl_seconds,
+ listed_count,
now,
- cutoff,
)
- for upload in uploads:
- if not upload.get("isDir"):
+
+ # Classify the single listing into independent groups (order preserved,
+ # so each group stays name-ascending / oldest-first).
+ buckets: list[tuple[str, float]] = [] # (uri, bucket_expiry)
+ legacy: list[tuple[str, float]] = [] # (uri, upload_expiry)
+ invalid: list[str] = [] # uri
+ for entry in entries:
+ if not entry.get("isDir"):
continue
- uri = str(upload.get("uri") or "").rstrip("/")
- upload_id = uri.removeprefix(f"{_SHARED_UPLOAD_ROOT}/")
- if not uri.startswith(f"{_SHARED_UPLOAD_ROOT}/") or "/" in upload_id:
+ uri = str(entry.get("uri") or "").rstrip("/")
+ name = uri.removeprefix(f"{_SHARED_UPLOAD_ROOT}/")
+ if not uri.startswith(f"{_SHARED_UPLOAD_ROOT}/") or "/" in name:
continue
- created_at = _shared_upload_created_at(upload_id)
- if created_at is None:
- logger.debug("Shared temp upload cleanup skipped malformed uri=%s", uri)
+ bucket_start = _shared_bucket_start(name)
+ if bucket_start is not None:
+ buckets.append((uri, bucket_start + _SHARED_BUCKET_SECONDS + ttl_seconds))
continue
- age_seconds = now - created_at
- expired = created_at < cutoff
+ created_at = _shared_upload_created_at(name)
+ if created_at is not None:
+ legacy.append((uri, created_at + ttl_seconds))
+ continue
+ invalid.append(uri)
+
+ # Process the bucket and legacy groups independently, oldest-first.
+ bucket_scanned, bucket_removed, bucket_due, bucket_failed = (
+ await self._cleanup_expiry_group(vfs, internal_ctx, buckets, now, kind="bucket")
+ )
+ legacy_scanned, legacy_removed, legacy_due, legacy_failed = (
+ await self._cleanup_expiry_group(vfs, internal_ctx, legacy, now, kind="flat")
+ )
+
+ # Invalid/foreign dirs: only when explicitly enabled; never blocks the
+ # expiry groups and failures are logged inside the helper.
+ invalid_removed = 0
+ if cleanup_invalid:
+ for uri in invalid:
+ if await self._remove_shared_dir(vfs, uri, internal_ctx, kind="invalid"):
+ invalid_removed += 1
+ elif invalid:
logger.debug(
- "Shared temp upload cleanup candidate uri=%s created_at=%s "
- "age_seconds=%s expired=%s",
- uri,
- created_at,
- age_seconds,
- expired,
+ "Shared temp upload cleanup skipped %s malformed dirs account=%s",
+ len(invalid),
+ account_id,
)
- if not expired:
- continue
- try:
- await vfs.rm(uri, recursive=True, ctx=internal_ctx)
- except Exception:
- logger.warning(
- "Shared temp upload cleanup remove failed uri=%s",
+
+ scanned_count = bucket_scanned + legacy_scanned
+ removed_count = bucket_removed + legacy_removed + invalid_removed
+
+ # due_at is the earliest live expiry across both groups. A removal
+ # failure in either group means there is still expired data we could not
+ # delete, so clear due_at to let the next request retry promptly.
+ dues = [d for d in (bucket_due, legacy_due) if d is not None]
+ if bucket_failed or legacy_failed:
+ self._clear_due_at(account_id)
+ next_due_at: Optional[float] = None
+ elif dues:
+ next_due_at = min(dues)
+ self._set_due_at(account_id, next_due_at)
+ else:
+ self._clear_due_at(account_id)
+ next_due_at = None
+
+ return listed_count, scanned_count, removed_count, next_due_at
+
+ async def _cleanup_expiry_group(
+ self,
+ vfs: Any,
+ internal_ctx: RequestContext,
+ items: list[tuple[str, float]],
+ now: float,
+ *,
+ kind: str,
+ ) -> tuple[int, int, Optional[float], bool]:
+ """Delete expired entries of one oldest-first group.
+
+ ``items`` is ``(uri, expiry)`` in name-ascending (oldest-first) order.
+ Deletes leading expired entries and stops at the first still-valid one,
+ returning ``(scanned, removed, live_expiry, failed)`` where
+ ``live_expiry`` is the expiry of the first still-valid entry (the group's
+ contribution to ``due_at``) or None, and ``failed`` is True if a deletion
+ failed (caller should not throttle so the next run retries).
+ """
+ scanned = 0
+ removed = 0
+ for uri, expiry in items:
+ scanned += 1
+ if expiry > now:
+ logger.debug(
+ "Shared temp upload cleanup reached live %s uri=%s expires_at=%s",
+ kind,
uri,
- exc_info=True,
+ expiry,
)
- else:
- logger.debug("Shared temp upload cleanup removed uri=%s", uri)
+ return scanned, removed, expiry, False
+ if not await self._remove_shared_dir(vfs, uri, internal_ctx, kind=kind):
+ return scanned, removed, None, True
+ removed += 1
+ return scanned, removed, None, False
+
+ @staticmethod
+ async def _remove_shared_dir(
+ vfs: Any,
+ uri: str,
+ internal_ctx: RequestContext,
+ *,
+ kind: str,
+ ) -> bool:
+ """Recursively delete one shared-upload directory, logging elapsed time.
+
+ Returns True on success, False on failure (logged as a warning). Uses
+ ``auto_pathlock=False`` since shared uploads are never concurrently
+ mutated and cleanup is best-effort. Deletes AGFS storage only via
+ ``remove_files`` because raw temporary uploads never have vector-index
+ records, so skipping vector-store cleanup avoids pointless round-trips.
+ """
+ started_at = time.monotonic()
+ try:
+ await vfs.remove_files(
+ uri, recursive=True, ctx=internal_ctx, auto_pathlock=False
+ )
+ except Exception:
+ logger.warning(
+ "[TempUpload] cleanup remove failed kind=%s uri=%s elapsed_ms=%.1f",
+ kind,
+ uri,
+ (time.monotonic() - started_at) * 1000.0,
+ exc_info=True,
+ )
+ return False
+ logger.info(
+ "[TempUpload] cleanup removed kind=%s uri=%s elapsed_ms=%.1f",
+ kind,
+ uri,
+ (time.monotonic() - started_at) * 1000.0,
+ )
+ return True
+
+ @staticmethod
+ def _set_due_at(account_id: str, due_at: float) -> None:
+ with _SHARED_CLEANUP_STATE_LOCK:
+ _SHARED_CLEANUP_DUE_AT[account_id] = due_at
+
+ @staticmethod
+ def _clear_due_at(account_id: str) -> None:
+ with _SHARED_CLEANUP_STATE_LOCK:
+ _SHARED_CLEANUP_DUE_AT.pop(account_id, None)
def _cleanup_local_temp_files(self, temp_dir: Path) -> None:
if self.temp_cfg.ttl_seconds == 0:
diff --git a/openviking/storage/viking_fs/_ops.py b/openviking/storage/viking_fs/_ops.py
index 0f03e4e5e..b9a79cebe 100644
--- a/openviking/storage/viking_fs/_ops.py
+++ b/openviking/storage/viking_fs/_ops.py
@@ -134,6 +134,7 @@ class _OpsMixin:
recursive: bool = False,
ctx: Optional[RequestContext] = None,
lease_ref: Dict[str, Any] | None = None,
+ auto_pathlock: bool = True,
) -> Dict[str, Any]:
"""Delete file/directory + recursively update vector index.
@@ -144,6 +145,12 @@ class _OpsMixin:
Raises ResourceBusyError when the target is locked by an ongoing
operation (e.g. semantic processing).
+ When ``auto_pathlock`` is False and no outer ``lease_ref`` is supplied,
+ the VikingFS-level tree/exact lease is skipped and the underlying AGFS
+ delete runs with automatic pathlock disabled. Callers must guarantee the
+ target is not concurrently mutated (e.g. best-effort shared upload
+ cleanup that only deletes already-expired directories).
+
Returns:
Dict with 'estimated_deleted_count' indicating the estimated number
of nodes deleted from vector index.
@@ -202,8 +209,12 @@ class _OpsMixin:
recursive = False
lock_method = self._async_agfs.pathlock_acquire_exact
+ # When an outer lease is supplied we always honor it. Otherwise callers
+ # can opt out of the VikingFS-level lease via auto_pathlock=False, in
+ # which case the AGFS delete also runs with automatic pathlock disabled.
+ skip_lock = lease_ref is None and not auto_pathlock
lease = lease_ref
- if lease is None:
+ if lease is None and not skip_lock:
try:
lease = await lock_method(path)
except LockAcquisitionError:
@@ -231,6 +242,7 @@ class _OpsMixin:
path,
recursive=recursive,
fs_ctx=self._pathlock_fs_ctx(ctx, lease),
+ auto_pathlock=auto_pathlock,
)
except AGFSDirectoryNotEmptyError:
raise FailedPreconditionError(
@@ -250,9 +262,40 @@ class _OpsMixin:
result = {"estimated_deleted_count": estimated_count}
return result
finally:
- if lease_ref is None:
+ if lease_ref is None and lease is not None:
await self._async_agfs.pathlock_release(lease)
+ async def remove_files(
+ self,
+ uri: str,
+ recursive: bool = False,
+ ctx: Optional[RequestContext] = None,
+ lease_ref: Dict[str, Any] | None = None,
+ auto_pathlock: bool = True,
+ ) -> Dict[str, Any]:
+ """Delete a file/directory from AGFS storage only, skipping vector-index cleanup.
+
+ Unlike :meth:`rm`, this never touches the vector store. Use it for URIs
+ that carry no vector-index records — e.g. raw temporary uploads under
+ ``viking://upload`` — so cleanup avoids pointless ``delete_by_filter``
+ round-trips. Despite the plural name it deletes a single URI (a file or,
+ with ``recursive=True``, a directory subtree). URI safety and account
+ isolation are still enforced by ``_uri_to_path``; this method performs no
+ ACL checks, so callers must scope the URI themselves.
+
+ ``auto_pathlock`` / ``lease_ref`` are forwarded to AGFS exactly like the
+ underlying delete in :meth:`rm`. Callers that pass ``auto_pathlock=False``
+ must guarantee the target is not concurrently mutated (best-effort
+ cleanup of already-expired, uniquely-named upload directories).
+ """
+ path = self._uri_to_path(uri, ctx=ctx)
+ return await self._async_agfs.rm(
+ path,
+ recursive=recursive,
+ fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref),
+ auto_pathlock=auto_pathlock,
+ )
+
async def mv(
self,
old_uri: str,
@@ -1019,8 +1062,14 @@ class _OpsMixin:
content: Union[str, bytes],
ctx: Optional[RequestContext] = None,
lease_ref: Dict[str, Any] | None = None,
+ auto_pathlock: bool = True,
) -> None:
- """Write file directly. Encryption lock handled internally by EncryptionWrappedFS."""
+ """Write file directly. Encryption lock handled internally by EncryptionWrappedFS.
+
+ When ``auto_pathlock`` is False the underlying AGFS write runs with
+ automatic pathlock disabled. Only safe for URIs that are never written
+ concurrently (e.g. unique-per-request shared upload directories).
+ """
await self._ensure_access(uri, ctx, action=AclAction.WRITE)
path = self._uri_to_path(uri, ctx=ctx)
await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref)
@@ -1028,7 +1077,12 @@ class _OpsMixin:
if isinstance(content, str):
content = content.encode("utf-8")
- await self._async_agfs.write(path, content, fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref))
+ await self._async_agfs.write(
+ path,
+ content,
+ fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref),
+ auto_pathlock=auto_pathlock,
+ )
async def read_file(
self,
@@ -1131,13 +1185,24 @@ class _OpsMixin:
content: bytes,
ctx: Optional[RequestContext] = None,
lease_ref: Dict[str, Any] | None = None,
+ auto_pathlock: bool = True,
) -> None:
- """Write single binary file. Encryption lock handled internally by EncryptionWrappedFS."""
+ """Write single binary file. Encryption lock handled internally by EncryptionWrappedFS.
+
+ When ``auto_pathlock`` is False the underlying AGFS write runs with
+ automatic pathlock disabled. Only safe for URIs that are never written
+ concurrently (e.g. unique-per-request shared upload directories).
+ """
await self._ensure_access(uri, ctx, action=AclAction.WRITE)
path = self._uri_to_path(uri, ctx=ctx)
await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref)
- await self._async_agfs.write(path, content, fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref))
+ await self._async_agfs.write(
+ path,
+ content,
+ fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref),
+ auto_pathlock=auto_pathlock,
+ )
async def append_file(
self,
diff --git a/tests/server/test_temp_upload_store_async_io.py b/tests/server/test_temp_upload_store_async_io.py
new file mode 100644
index 000000000..e0754936b
--- /dev/null
+++ b/tests/server/test_temp_upload_store_async_io.py
@@ -0,0 +1,604 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Event-loop safety and cleanup-throttle tests for temporary HTTP uploads."""
+
+import time
+import uuid
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from openviking.server import temp_upload_store
+
+
+class _UploadFile:
+ filename = "upload.md"
+
+ def __init__(self, chunks: list[bytes]):
+ self._chunks = iter(chunks)
+
+ async def read(self, _size: int) -> bytes:
+ return next(self._chunks, b"")
+
+
+def _make_upload_id(created_at: float) -> str:
+ """Build a valid shared upload id whose embedded timestamp is created_at."""
+ return f"{int(created_at * 1000):013d}-{uuid.uuid4().hex}"
+
+
+def _bucket_entry(bucket: str) -> dict:
+ """A top-level YYYYMMDDHH bucket directory entry."""
+ return {"isDir": True, "uri": f"{temp_upload_store._SHARED_UPLOAD_ROOT}/{bucket}"}
+
+
+def _bucket_for(created_at: float) -> str:
+ return time.strftime("%Y%m%d%H", time.gmtime(created_at))
+
+
+def _flat_entry(created_at: float) -> dict:
+ """A legacy flat <ms-ts>-<uuid> upload directory entry (pre-bucket)."""
+ upload_id = _make_upload_id(created_at)
+ return {"isDir": True, "uri": f"{temp_upload_store._SHARED_UPLOAD_ROOT}/{upload_id}"}
+
+
+# Backwards-compatible alias used by older tests below.
+def _upload_entry(created_at: float) -> dict:
+ return _flat_entry(created_at)
+
+
+def _reset_state() -> None:
+ temp_upload_store._SHARED_CLEANUP_DUE_AT.clear()
+ temp_upload_store._SHARED_CLEANUP_PENDING.clear()
+ while not temp_upload_store._SHARED_CLEANUP_QUEUE.empty():
+ temp_upload_store._SHARED_CLEANUP_QUEUE.get_nowait()
+ temp_upload_store._SHARED_CLEANUP_QUEUE.task_done()
+
+
+async def test_save_local_offloads_cleanup_writes_and_metadata(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+):
+ calls: list[str] = []
+ real_to_thread = temp_upload_store.asyncio.to_thread
+
+ async def recording_to_thread(func, /, *args, **kwargs):
+ calls.append(getattr(func, "__name__", type(func).__name__))
+ return await real_to_thread(func, *args, **kwargs)
+
+ config = SimpleNamespace(
+ storage=SimpleNamespace(get_upload_temp_dir=lambda: tmp_path),
+ temp_upload=SimpleNamespace(ttl_seconds=3600, shared_max_size_bytes=1024),
+ )
+ monkeypatch.setattr(temp_upload_store, "get_openviking_config", lambda: config)
+ monkeypatch.setattr(temp_upload_store.asyncio, "to_thread", recording_to_thread)
+
+ store = temp_upload_store.TempUploadStore(config)
+ temp_file_id = await store.save_upload(_UploadFile([b"hello ", b"world"]), "local", object())
+
+ assert (tmp_path / temp_file_id).read_bytes() == b"hello world"
+ assert (tmp_path / f"{temp_file_id}.ov_upload.meta").is_file()
+ assert "_cleanup_local_temp_files" in calls
+ assert "_create_temp_file" in calls
+ assert "_open_binary_for_write" in calls
+ assert "write" in calls
+ assert "_write_json" in calls
+ assert "_close_file" in calls
+
+
+async def test_resolve_local_offloads_filesystem_validation(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+):
+ calls: list[str] = []
+ real_to_thread = temp_upload_store.asyncio.to_thread
+
+ async def recording_to_thread(func, /, *args, **kwargs):
+ calls.append(getattr(func, "__name__", type(func).__name__))
+ return await real_to_thread(func, *args, **kwargs)
+
+ config = SimpleNamespace(storage=SimpleNamespace(get_upload_temp_dir=lambda: tmp_path))
+ uploaded = tmp_path / "upload.md"
+ uploaded.write_text("hello", encoding="utf-8")
+ monkeypatch.setattr(temp_upload_store, "get_openviking_config", lambda: config)
+ monkeypatch.setattr(temp_upload_store.asyncio, "to_thread", recording_to_thread)
+
+ store = temp_upload_store.TempUploadStore(SimpleNamespace(temp_upload=SimpleNamespace()))
+ resolved = await store.resolve_for_consume("upload.md", object())
+
+ assert resolved.local_path == str(uploaded)
+ assert "_resolve_local" in calls
+
+
+def test_schedule_deduplicates_while_pending(monkeypatch: pytest.MonkeyPatch):
+ store = temp_upload_store.TempUploadStore(
+ SimpleNamespace(temp_upload=SimpleNamespace(ttl_seconds=12 * 60 * 60))
+ )
+ monkeypatch.setattr(store, "_ensure_shared_cleanup_worker", lambda: None)
+ _reset_state()
+
+ store._schedule_shared_cleanup(SimpleNamespace(account_id="account-a"))
+ store._schedule_shared_cleanup(SimpleNamespace(account_id="account-a"))
+
+ assert temp_upload_store._SHARED_CLEANUP_QUEUE.qsize() == 1
+ assert "account-a" in temp_upload_store._SHARED_CLEANUP_PENDING
+
+
+def test_schedule_skips_when_due_at_in_future(monkeypatch: pytest.MonkeyPatch):
+ store = temp_upload_store.TempUploadStore(
+ SimpleNamespace(temp_upload=SimpleNamespace(ttl_seconds=12 * 60 * 60))
+ )
+ monkeypatch.setattr(store, "_ensure_shared_cleanup_worker", lambda: None)
+ _reset_state()
+ # due_at far in the future: nothing new can have expired, so skip.
+ temp_upload_store._SHARED_CLEANUP_DUE_AT["account-a"] = time.time() + 3600
+
+ store._schedule_shared_cleanup(SimpleNamespace(account_id="account-a"))
+
+ assert temp_upload_store._SHARED_CLEANUP_QUEUE.qsize() == 0
+
+
+def test_schedule_submits_when_due_at_elapsed(monkeypatch: pytest.MonkeyPatch):
+ store = temp_upload_store.TempUploadStore(
+ SimpleNamespace(temp_upload=SimpleNamespace(ttl_seconds=12 * 60 * 60))
+ )
+ monkeypatch.setattr(store, "_ensure_shared_cleanup_worker", lambda: None)
+ _reset_state()
+ temp_upload_store._SHARED_CLEANUP_DUE_AT["account-a"] = time.time() - 1
+
+ store._schedule_shared_cleanup(SimpleNamespace(account_id="account-a"))
+
+ assert temp_upload_store._SHARED_CLEANUP_QUEUE.qsize() == 1
+
+
+def test_schedule_releases_pending_when_queue_is_full(monkeypatch: pytest.MonkeyPatch):
+ store = temp_upload_store.TempUploadStore(
+ SimpleNamespace(temp_upload=SimpleNamespace(ttl_seconds=12 * 60 * 60))
+ )
+ monkeypatch.setattr(store, "_ensure_shared_cleanup_worker", lambda: None)
+ monkeypatch.setattr(
+ temp_upload_store,
+ "_SHARED_CLEANUP_QUEUE",
+ temp_upload_store.Queue(maxsize=1),
+ )
+ temp_upload_store._SHARED_CLEANUP_DUE_AT.clear()
+ temp_upload_store._SHARED_CLEANUP_PENDING.clear()
+ temp_upload_store._SHARED_CLEANUP_QUEUE.put_nowait((store, SimpleNamespace(), "occupied"))
+
+ store._schedule_shared_cleanup(SimpleNamespace(account_id="account-a"))
+
+ assert "account-a" not in temp_upload_store._SHARED_CLEANUP_PENDING
+
+
+async def test_cleanup_stops_at_first_live_upload(monkeypatch: pytest.MonkeyPatch):
+ store = temp_upload_store.TempUploadStore(
+ SimpleNamespace(temp_upload=SimpleNamespace(ttl_seconds=3600))
+ )
+ _reset_state()
+ now = time.time()
+ expired_a = _upload_entry(now - 7200)
+ expired_b = _upload_entry(now - 5400)
+ live_created = now - 60
+ live = _upload_entry(live_created)
+
+ removed: list[str] = []
+ list_calls: list = []
+
+ class _FakeVfs:
+ async def ls(self, uri, show_all_hidden=False, node_limit=None,
+ sort_by=None, sort_order="asc", ctx=None):
+ list_calls.append((sort_by, sort_order))
+ return [expired_a, expired_b, live]
+
+ async def remove_files(self, uri, recursive=False, ctx=None, auto_pathlock=True):
+ assert auto_pathlock is False
+ removed.append(uri)
+
+ monkeypatch.setattr(temp_upload_store, "get_viking_fs", lambda: _FakeVfs())
+
+ listed, scanned, removed_count, next_due = await store._cleanup_shared_uploads(
+ SimpleNamespace(account_id="account-a", user=SimpleNamespace())
+ )
+
+ # Listing is requested name-ascending (oldest-first).
+ assert list_calls == [("name", "asc")]
+ assert removed == [expired_a["uri"], expired_b["uri"]]
+ assert removed_count == 2
+ assert scanned == 3
+ assert listed == 3
+ # due_at is the live upload's expiry, and it is stored on the account.
+ assert next_due == pytest.approx(live_created + 3600)
+ assert temp_upload_store._SHARED_CLEANUP_DUE_AT["account-a"] == pytest.approx(
+ live_created + 3600
+ )
+
+
+async def test_cleanup_full_page_expired_clears_due_at(monkeypatch: pytest.MonkeyPatch):
+ store = temp_upload_store.TempUploadStore(
+ SimpleNamespace(temp_upload=SimpleNamespace(ttl_seconds=3600))
+ )
+ _reset_state()
+ now = time.time()
+ entries = [_upload_entry(now - 7200), _upload_entry(now - 5400)]
+
+ removed: list[str] = []
+
+ class _FakeVfs:
+ async def ls(self, uri, show_all_hidden=False, node_limit=None,
+ sort_by=None, sort_order="asc", ctx=None):
+ return entries
+
+ async def remove_files(self, uri, recursive=False, ctx=None, auto_pathlock=True):
+ removed.append(uri)
+
+ monkeypatch.setattr(temp_upload_store, "get_viking_fs", lambda: _FakeVfs())
+
+ listed, scanned, removed_count, next_due = await store._cleanup_shared_uploads(
+ SimpleNamespace(account_id="account-a", user=SimpleNamespace())
+ )
+
+ assert removed_count == 2
+ assert listed == 2
+ # Everything expired: due_at cleared for a fresh listing next time.
+ assert next_due is None
+ assert "account-a" not in temp_upload_store._SHARED_CLEANUP_DUE_AT
+
+
+async def test_cleanup_remove_failure_clears_due_at_and_stops(monkeypatch: pytest.MonkeyPatch):
+ store = temp_upload_store.TempUploadStore(
+ SimpleNamespace(temp_upload=SimpleNamespace(ttl_seconds=3600))
+ )
+ _reset_state()
+ now = time.time()
+ first = _upload_entry(now - 7200)
+ second = _upload_entry(now - 5400)
+ temp_upload_store._SHARED_CLEANUP_DUE_AT["account-a"] = now + 999
+
+ removed: list[str] = []
+
+ class _FakeVfs:
+ async def ls(self, uri, show_all_hidden=False, node_limit=None,
+ sort_by=None, sort_order="asc", ctx=None):
+ return [first, second]
+
+ async def remove_files(self, uri, recursive=False, ctx=None, auto_pathlock=True):
+ raise RuntimeError("boom")
+ monkeypatch.setattr(temp_upload_store, "get_viking_fs", lambda: _FakeVfs())
+
+ listed, scanned, removed_count, next_due = await store._cleanup_shared_uploads(
+ SimpleNamespace(account_id="account-a", user=SimpleNamespace())
+ )
+
+ assert removed_count == 0
+ assert next_due is None
+ assert "account-a" not in temp_upload_store._SHARED_CLEANUP_DUE_AT
+
+
+async def test_cleanup_skips_invalid_dirs_by_default(monkeypatch: pytest.MonkeyPatch):
+ store = temp_upload_store.TempUploadStore(
+ SimpleNamespace(temp_upload=SimpleNamespace(ttl_seconds=3600))
+ )
+ _reset_state()
+ now = time.time()
+ invalid = {"isDir": True, "uri": f"{temp_upload_store._SHARED_UPLOAD_ROOT}/not-an-id"}
+ expired = _upload_entry(now - 7200)
+
+ removed: list[str] = []
+
+ class _FakeVfs:
+ async def ls(self, uri, show_all_hidden=False, node_limit=None,
+ sort_by=None, sort_order="asc", ctx=None):
+ return [invalid, expired]
+
+ async def remove_files(self, uri, recursive=False, ctx=None, auto_pathlock=True):
+ removed.append(uri)
+
+ monkeypatch.setattr(temp_upload_store, "get_viking_fs", lambda: _FakeVfs())
+
+ listed, scanned, removed_count, _ = await store._cleanup_shared_uploads(
+ SimpleNamespace(account_id="account-a", user=SimpleNamespace())
+ )
+
+ # Invalid dir is left untouched; only the valid expired upload is removed.
+ assert removed == [expired["uri"]]
+ assert removed_count == 1
+ assert scanned == 1
+ assert listed == 2
+
+
+async def test_cleanup_removes_invalid_dirs_when_enabled(monkeypatch: pytest.MonkeyPatch):
+ store = temp_upload_store.TempUploadStore(
+ SimpleNamespace(
+ temp_upload=SimpleNamespace(ttl_seconds=3600, cleanup_invalid_dirs=True)
+ )
+ )
+ _reset_state()
+ now = time.time()
+ invalid = {"isDir": True, "uri": f"{temp_upload_store._SHARED_UPLOAD_ROOT}/not-an-id"}
+ expired = _upload_entry(now - 7200)
+ live_created = now - 60
+ live = _upload_entry(live_created)
+
+ removed: list[str] = []
+
+ class _FakeVfs:
+ async def ls(self, uri, show_all_hidden=False, node_limit=None,
+ sort_by=None, sort_order="asc", ctx=None):
+ return [invalid, expired, live]
+
+ async def remove_files(self, uri, recursive=False, ctx=None, auto_pathlock=True):
+ assert auto_pathlock is False
+ removed.append(uri)
+
+ monkeypatch.setattr(temp_upload_store, "get_viking_fs", lambda: _FakeVfs())
+
+ listed, scanned, removed_count, next_due = await store._cleanup_shared_uploads(
+ SimpleNamespace(account_id="account-a", user=SimpleNamespace())
+ )
+
+ # Invalid dir removed alongside the expired legacy upload; the legacy group
+ # stops at the first live upload. Groups are processed independently, so the
+ # removal order across groups is not asserted here.
+ assert set(removed) == {invalid["uri"], expired["uri"]}
+ assert removed_count == 2
+ assert scanned == 2
+ assert listed == 3
+ assert next_due == pytest.approx(live_created + 3600)
+
+
+async def test_cleanup_invalid_dir_remove_failure_does_not_stop_scan(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ store = temp_upload_store.TempUploadStore(
+ SimpleNamespace(
+ temp_upload=SimpleNamespace(ttl_seconds=3600, cleanup_invalid_dirs=True)
+ )
+ )
+ _reset_state()
+ now = time.time()
+ invalid = {"isDir": True, "uri": f"{temp_upload_store._SHARED_UPLOAD_ROOT}/not-an-id"}
+ expired = _upload_entry(now - 7200)
+
+ removed: list[str] = []
+
+ class _FakeVfs:
+ async def ls(self, uri, show_all_hidden=False, node_limit=None,
+ sort_by=None, sort_order="asc", ctx=None):
+ return [invalid, expired]
+
+ async def remove_files(self, uri, recursive=False, ctx=None, auto_pathlock=True):
+ if uri == invalid["uri"]:
+ raise RuntimeError("boom")
+ removed.append(uri)
+
+ monkeypatch.setattr(temp_upload_store, "get_viking_fs", lambda: _FakeVfs())
+
+ listed, scanned, removed_count, next_due = await store._cleanup_shared_uploads(
+ SimpleNamespace(account_id="account-a", user=SimpleNamespace())
+ )
+
+ # A failed invalid-dir remove is logged but must not abort the expiry scan:
+ # the valid expired upload is still removed and due_at is cleared normally.
+ assert removed == [expired["uri"]]
+ assert removed_count == 1
+ assert next_due is None
+ assert "account-a" not in temp_upload_store._SHARED_CLEANUP_DUE_AT
+
+
+async def test_cleanup_removes_expired_buckets_and_stops_at_live(monkeypatch: pytest.MonkeyPatch):
+ store = temp_upload_store.TempUploadStore(
+ SimpleNamespace(temp_upload=SimpleNamespace(ttl_seconds=3600))
+ )
+ _reset_state()
+ now = time.time()
+ # Two fully-expired hour buckets and one still-live bucket, oldest-first.
+ old_bucket = _bucket_for(now - 3 * 3600)
+ older_bucket = _bucket_for(now - 5 * 3600)
+ live_bucket = _bucket_for(now) # current hour: not yet past start+3600+ttl
+ entries = [_bucket_entry(older_bucket), _bucket_entry(old_bucket), _bucket_entry(live_bucket)]
+
+ removed: list[str] = []
+
+ class _FakeVfs:
+ async def ls(self, uri, show_all_hidden=False, node_limit=None,
+ sort_by=None, sort_order="asc", ctx=None):
+ return entries
+
+ async def remove_files(self, uri, recursive=False, ctx=None, auto_pathlock=True):
+ assert auto_pathlock is False
+ assert recursive is True
+ removed.append(uri)
+
+ monkeypatch.setattr(temp_upload_store, "get_viking_fs", lambda: _FakeVfs())
+
+ listed, scanned, removed_count, next_due = await store._cleanup_shared_uploads(
+ SimpleNamespace(account_id="account-a", user=SimpleNamespace())
+ )
+
+ # Both expired buckets removed whole; stops at the live bucket.
+ assert removed == [_bucket_entry(older_bucket)["uri"], _bucket_entry(old_bucket)["uri"]]
+ assert removed_count == 2
+ assert listed == 3
+ # due_at is the live bucket's expiry (bucket_start + 3600 + ttl).
+ live_start = temp_upload_store._shared_bucket_start(live_bucket)
+ assert next_due == pytest.approx(live_start + 3600 + 3600)
+
+
+async def test_cleanup_removes_expired_legacy_flat_uploads(monkeypatch: pytest.MonkeyPatch):
+ store = temp_upload_store.TempUploadStore(
+ SimpleNamespace(temp_upload=SimpleNamespace(ttl_seconds=3600))
+ )
+ _reset_state()
+ now = time.time()
+ expired_flat = _flat_entry(now - 7200)
+ live_flat = _flat_entry(now - 60)
+
+ removed: list[str] = []
+
+ class _FakeVfs:
+ async def ls(self, uri, show_all_hidden=False, node_limit=None,
+ sort_by=None, sort_order="asc", ctx=None):
+ return [expired_flat, live_flat]
+
+ async def remove_files(self, uri, recursive=False, ctx=None, auto_pathlock=True):
+ removed.append(uri)
+
+ monkeypatch.setattr(temp_upload_store, "get_viking_fs", lambda: _FakeVfs())
+
+ listed, scanned, removed_count, next_due = await store._cleanup_shared_uploads(
+ SimpleNamespace(account_id="account-a", user=SimpleNamespace())
+ )
+
+ # Legacy flat uploads are still reclaimed by their own creation-time TTL.
+ assert removed == [expired_flat["uri"]]
+ assert removed_count == 1
+
+
+async def test_cleanup_buckets_not_starved_by_live_legacy(monkeypatch: pytest.MonkeyPatch):
+ # Legacy ids (13-digit prefix) sort before buckets (10-digit prefix), so in
+ # a single mixed listing a still-live legacy upload appears first. Bucket and
+ # legacy groups are processed independently, so an expired bucket must still
+ # be removed even though the legacy group stopped at a live entry.
+ store = temp_upload_store.TempUploadStore(
+ SimpleNamespace(temp_upload=SimpleNamespace(ttl_seconds=3600))
+ )
+ _reset_state()
+ now = time.time()
+ live_legacy = _flat_entry(now - 60) # legacy, still valid
+ expired_bucket = _bucket_entry(_bucket_for(now - 5 * 3600)) # bucket, expired
+
+ removed: list[str] = []
+
+ class _FakeVfs:
+ async def ls(self, uri, show_all_hidden=False, node_limit=None,
+ sort_by=None, sort_order="asc", ctx=None):
+ # Emulate name-ascending: legacy (1...) before bucket (2...).
+ return [live_legacy, expired_bucket]
+
+ async def remove_files(self, uri, recursive=False, ctx=None, auto_pathlock=True):
+ removed.append(uri)
+
+ monkeypatch.setattr(temp_upload_store, "get_viking_fs", lambda: _FakeVfs())
+
+ listed, scanned, removed_count, next_due = await store._cleanup_shared_uploads(
+ SimpleNamespace(account_id="account-a", user=SimpleNamespace())
+ )
+
+ # The expired bucket is removed; the live legacy upload is preserved and its
+ # expiry becomes due_at (earliest live expiry across groups).
+ assert removed == [expired_bucket["uri"]]
+ assert removed_count == 1
+ assert next_due is not None
+
+
+async def test_cleanup_never_treats_legacy_flat_as_invalid(monkeypatch: pytest.MonkeyPatch):
+ store = temp_upload_store.TempUploadStore(
+ SimpleNamespace(
+ temp_upload=SimpleNamespace(ttl_seconds=3600, cleanup_invalid_dirs=True)
+ )
+ )
+ _reset_state()
+ now = time.time()
+ invalid = {"isDir": True, "uri": f"{temp_upload_store._SHARED_UPLOAD_ROOT}/not-an-id"}
+ live_flat = _flat_entry(now - 60) # valid legacy id, not yet expired
+
+ removed: list[str] = []
+
+ class _FakeVfs:
+ async def ls(self, uri, show_all_hidden=False, node_limit=None,
+ sort_by=None, sort_order="asc", ctx=None):
+ return [invalid, live_flat]
+
+ async def remove_files(self, uri, recursive=False, ctx=None, auto_pathlock=True):
+ removed.append(uri)
+
+ monkeypatch.setattr(temp_upload_store, "get_viking_fs", lambda: _FakeVfs())
+
+ listed, scanned, removed_count, next_due = await store._cleanup_shared_uploads(
+ SimpleNamespace(account_id="account-a", user=SimpleNamespace())
+ )
+
+ # Only the invalid dir is removed; the live legacy upload is preserved and
+ # sets due_at (it is a valid, unexpired upload), never deleted as invalid.
+ assert removed == [invalid["uri"]]
+ assert removed_count == 1
+ assert next_due is not None
+
+
+def test_new_upload_id_is_bucketed_and_legacy_is_not():
+ # New id: 10-digit hour prefix -> bucketed.
+ new_id = temp_upload_store._new_shared_upload_id()
+ prefix = new_id.split("-", 1)[0]
+ assert len(prefix) == 10
+ assert temp_upload_store._shared_bucket_from_upload_id(new_id) == prefix
+ # A new id has no legacy creation timestamp.
+ assert temp_upload_store._shared_upload_created_at(new_id) is None
+
+ # Legacy id: 13-digit ms prefix -> not bucketed, but has a creation time.
+ legacy_id = _make_upload_id(time.time())
+ assert temp_upload_store._shared_bucket_from_upload_id(legacy_id) is None
+ assert temp_upload_store._shared_upload_created_at(legacy_id) is not None
+
+
+async def test_read_shared_meta_uses_bucket_path_for_new_id(monkeypatch: pytest.MonkeyPatch):
+ store = temp_upload_store.TempUploadStore(SimpleNamespace(temp_upload=SimpleNamespace()))
+ new_id = temp_upload_store._new_shared_upload_id()
+ bucket, leaf = temp_upload_store._split_shared_upload_id(new_id)
+ expected_uri = temp_upload_store._shared_meta_uri(bucket, leaf)
+
+ read_uris: list[str] = []
+
+ class _FakeVfs:
+ async def read_file(self, uri, ctx=None):
+ read_uris.append(uri)
+ return '{"temp_file_id": "shared_%s"}' % new_id
+
+ monkeypatch.setattr(temp_upload_store, "get_viking_fs", lambda: _FakeVfs())
+
+ meta = await store._read_shared_meta(
+ new_id, SimpleNamespace(account_id="a", user=SimpleNamespace())
+ )
+ # Exactly one read, against the bucketed path (no legacy fallback probe).
+ assert read_uris == [expected_uri]
+ # In-bucket leaf is the uuid only, without repeating the bucket prefix.
+ assert expected_uri == f"{temp_upload_store._SHARED_UPLOAD_ROOT}/{bucket}/{leaf}/meta"
+ assert leaf == new_id.split("-", 1)[1]
+ assert bucket not in leaf
+ assert meta["temp_file_id"] == f"shared_{new_id}"
+
+
+async def test_read_shared_meta_uses_flat_path_for_legacy_id(monkeypatch: pytest.MonkeyPatch):
+ store = temp_upload_store.TempUploadStore(SimpleNamespace(temp_upload=SimpleNamespace()))
+ legacy_id = _make_upload_id(time.time())
+ expected_uri = temp_upload_store._legacy_shared_meta_uri(legacy_id)
+
+ read_uris: list[str] = []
+
+ class _FakeVfs:
+ async def read_file(self, uri, ctx=None):
+ read_uris.append(uri)
+ return '{"temp_file_id": "shared_%s"}' % legacy_id
+
+ monkeypatch.setattr(temp_upload_store, "get_viking_fs", lambda: _FakeVfs())
+
+ meta = await store._read_shared_meta(
+ legacy_id, SimpleNamespace(account_id="a", user=SimpleNamespace())
+ )
+ # Exactly one read, against the legacy flat path.
+ assert read_uris == [expected_uri]
+ assert meta["temp_file_id"] == f"shared_{legacy_id}"
+
+
+