Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
69.2 KB
Raw Blame
#!/usr/bin/env -S uv run --script
# /// script
# requires-python = ">=3.11"
# dependencies = ["httpx>=0.25.0,<1"]
# ///
"""Standalone bulk ingestion and arXiv benchmark client for OpenRAG.

The script intentionally imports no OpenRAG application or SDK modules. This keeps it
usable from a checkout, a Kubernetes job, or a machine that only has ``uv`` installed.
"""

from __future__ import annotations

import argparse
import asyncio
import fnmatch
import hashlib
import json
import mimetypes
import os
import re
import shutil
import subprocess
import sys
import tarfile
import time
import xml.etree.ElementTree as ET
from collections.abc import Callable, Sequence
from contextlib import ExitStack
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from urllib.error import HTTPError
from urllib.parse import unquote, urlencode, urlparse, urlunparse
from urllib.request import Request, urlopen
from uuid import uuid4

import httpx

TERMINAL_TASK_STATUSES = {"completed", "failed", "skipped"}
TRANSIENT_HTTP_STATUSES = {502, 503, 504}
DEFAULT_RUNS_DIR = Path("~/.openrag/bulk/runs")
DEFAULT_ARXIV_DIR = Path("~/.openrag/benchmarks/arxiv")
ARXIV_API_URL = "https://export.arxiv.org/api/query"
ATOM = "{http://www.w3.org/2005/Atom}"
DEFAULT_ARXIV_CATEGORY = "cs.CL"
DEFAULT_ARXIV_DATE_FROM = "2025-01-01"
DEFAULT_ARXIV_DATE_TO = "2025-12-31"
DEFAULT_ARXIV_SOURCE = "s3"
DEFAULT_ARXIV_S3_URI = "s3://arxiv/pdf/arXiv_pdf_1001_001.tar"


class ApiError(RuntimeError):
    """An OpenRAG API request failed."""

    def __init__(self, message: str, status_code: int):
        super().__init__(message)
        self.status_code = status_code


class ArxivBenchmarkError(RuntimeError):
    """The arXiv source could not be staged safely."""


@dataclass(frozen=True)
class BulkUploadItem:
    path: Path
    filename: str | None = None
    content_type: str | None = None


@dataclass(frozen=True)
class BulkUploadOptions:
    batch_size: int = 10
    max_inflight: int = 1
    max_submit: int | None = None
    request_timeout: float = 300.0
    poll_request_timeout: float = 10.0
    task_timeout: float = 1800.0
    poll_interval: float = 1.0
    progress_interval: float = 5.0
    summary_interval: float = 60.0
    input_sort: str = "path"
    settings_json: str | None = None
    tweaks_json: str | None = None
    replace_duplicates: bool = True

    def validate(self) -> None:
        positive = {
            "batch_size": self.batch_size,
            "max_inflight": self.max_inflight,
            "request_timeout": self.request_timeout,
            "poll_request_timeout": self.poll_request_timeout,
            "task_timeout": self.task_timeout,
        }
        for name, value in positive.items():
            if value <= 0:
                raise ValueError(f"{name} must be positive")
        if self.max_submit is not None and self.max_submit <= 0:
            raise ValueError("max_submit must be positive")
        for name, value in {
            "poll_interval": self.poll_interval,
            "progress_interval": self.progress_interval,
            "summary_interval": self.summary_interval,
        }.items():
            if value < 0:
                raise ValueError(f"{name} must be non-negative")


@dataclass(frozen=True)
class ArxivPaper:
    arxiv_id: str
    title: str
    summary: str
    authors: tuple[str, ...]
    published: str | None
    updated: str | None
    abs_url: str
    pdf_url: str


@dataclass(frozen=True)
class DownloadedPaper:
    paper: ArxivPaper
    path: Path
    size: int
    sha256: str
    cached: bool = False


class RateLimiter:
    def __init__(self, delay_seconds: float):
        self.delay_seconds = max(delay_seconds, 0.0)
        self.last_request: float | None = None

    def wait(self) -> None:
        if self.last_request is not None:
            remaining = self.delay_seconds - (time.monotonic() - self.last_request)
            if remaining > 0:
                time.sleep(remaining)
        self.last_request = time.monotonic()


class ProgressReporter:
    def __init__(
        self,
        *,
        enabled: bool = True,
        started: float | None = None,
        prefix: str = "openrag bulk",
        stream=None,
    ) -> None:
        self.enabled = enabled
        self.started = time.monotonic() if started is None else started
        self.prefix = prefix
        self.stream = stream or sys.stderr

    def log(self, message: str) -> None:
        if not self.enabled:
            return
        elapsed = time.monotonic() - self.started
        print(f"{self.prefix} [{elapsed:8.1f}s] {message}", file=self.stream, flush=True)


class OpenRAGApiClient:
    """Small API client covering only the two calls bulk ingestion needs."""

    def __init__(
        self,
        *,
        base_url: str,
        api_key: str,
        api_prefix: str = "/api/v1",
        transport: httpx.AsyncBaseTransport | None = None,
    ) -> None:
        root = base_url.rstrip("/")
        prefix = "/" + api_prefix.strip("/") if api_prefix.strip("/") else ""
        self._base_url = root + prefix
        self._http = httpx.AsyncClient(
            headers={"X-API-Key": api_key},
            transport=transport,
        )

    async def close(self) -> None:
        await self._http.aclose()

    async def __aenter__(self) -> OpenRAGApiClient:
        return self

    async def __aexit__(self, *_args: Any) -> None:
        await self.close()

    async def submit_batch(
        self,
        items: Sequence[BulkUploadItem],
        *,
        settings_json: str | None,
        tweaks_json: str | None,
        replace_duplicates: bool,
        timeout: float,
    ) -> dict[str, Any]:
        data: dict[str, str] = {
            "replace_duplicates": str(replace_duplicates).lower(),
        }
        if settings_json is not None:
            data["settings"] = settings_json
        if tweaks_json is not None:
            data["tweaks"] = tweaks_json

        with ExitStack() as stack:
            files = [
                (
                    "file",
                    (
                        item.filename or item.path.name,
                        stack.enter_context(item.path.open("rb")),
                        item.content_type
                        or mimetypes.guess_type(item.filename or item.path.name)[0]
                        or "application/octet-stream",
                    ),
                )
                for item in items
            ]
            response = await self._http.post(
                f"{self._base_url}/documents/ingest",
                data=data,
                files=files,
                timeout=timeout,
            )
        return _response_json(response)

    async def get_task_status(self, task_id: str, *, timeout: float) -> dict[str, Any]:
        response = await self._http.get(
            f"{self._base_url}/tasks/{task_id}",
            timeout=timeout,
        )
        return _response_json(response)


SummaryCallback = Callable[[dict[str, Any]], Any]


def collect_upload_items(
    paths: Sequence[str],
    *,
    include: Sequence[str] | None = None,
    exclude: Sequence[str] | None = None,
    recursive: bool = True,
) -> list[BulkUploadItem]:
    include_patterns = list(include or ["*"])
    exclude_patterns = list(exclude or [])
    items: list[BulkUploadItem] = []
    seen: set[Path] = set()

    for raw_path in paths:
        path = Path(raw_path).expanduser()
        if path.is_file():
            root = path.parent
            candidates = [path]
        elif path.is_dir():
            root = path
            iterator = path.rglob("*") if recursive else path.glob("*")
            candidates = [candidate for candidate in iterator if candidate.is_file()]
        else:
            raise FileNotFoundError(f"bulk upload path not found: {raw_path}")

        for candidate in sorted(candidates):
            resolved = candidate.resolve()
            if resolved in seen:
                continue
            relative = candidate.name if path.is_file() else str(candidate.relative_to(root))
            if not _matches(relative, candidate.name, include_patterns):
                continue
            if _matches(relative, candidate.name, exclude_patterns):
                continue
            seen.add(resolved)
            items.append(BulkUploadItem(path=resolved))

    return items


def sort_upload_items(items: Sequence[BulkUploadItem], order: str) -> list[BulkUploadItem]:
    if order == "path":
        return sorted(items, key=lambda item: str(item.path))
    if order == "size-asc":
        return sorted(items, key=lambda item: (item.path.stat().st_size, str(item.path)))
    if order == "size-desc":
        return sorted(items, key=lambda item: (-item.path.stat().st_size, str(item.path)))
    raise ValueError(f"unsupported bulk upload sort: {order}")


async def upload_bulk_items(
    *,
    client: Any,
    items: Sequence[BulkUploadItem],
    options: BulkUploadOptions,
    progress: ProgressReporter,
    summary_callback: SummaryCallback | None = None,
) -> dict[str, Any]:
    """Submit batches concurrently, poll each task, and return a run summary."""
    options.validate()
    batches = _chunks(list(items), options.batch_size)
    max_submit = min(options.max_inflight, options.max_submit or options.max_inflight)
    total_bytes = sum(item.path.stat().st_size for item in items)
    summary: dict[str, Any] = {
        "total_files": len(items),
        "total_bytes": total_bytes,
        "total_mb": round(_mb(total_bytes), 3),
        "batch_size": options.batch_size,
        "max_inflight": options.max_inflight,
        "max_submit": max_submit,
        "input_sort": options.input_sort,
        "replace_duplicates": options.replace_duplicates,
        "client_concurrency": {
            "batch_size": options.batch_size,
            "max_inflight_upload_tasks": options.max_inflight,
            "max_inflight_submit_requests": max_submit,
            "request_timeout_seconds": options.request_timeout,
            "poll_request_timeout_seconds": options.poll_request_timeout,
            "task_timeout_seconds": options.task_timeout,
            "poll_interval_seconds": options.poll_interval,
        },
        "batches": [
            _new_batch_result(batch, batch_index=index)
            for index, batch in enumerate(batches, start=1)
        ],
    }
    _notify(summary_callback, summary)
    if not batches:
        return summary

    progress.log(
        f"starting files={len(items)} batches={len(batches)} "
        f"batch_size={options.batch_size} max_inflight={options.max_inflight} "
        f"max_submit={max_submit} input_mb={_mb(total_bytes):.2f}"
    )
    inflight_semaphore = asyncio.Semaphore(options.max_inflight)
    submit_semaphore = asyncio.Semaphore(max_submit)
    finished = asyncio.Event()

    async def process_batch(index: int, batch: list[BulkUploadItem]) -> None:
        result = summary["batches"][index - 1]
        async with inflight_semaphore:
            result.update(status="submitting", phase="multipart_upload")
            _notify(summary_callback, summary)
            submitted_at = time.monotonic()
            try:
                async with submit_semaphore:
                    response = await client.submit_batch(
                        batch,
                        settings_json=options.settings_json,
                        tweaks_json=options.tweaks_json,
                        replace_duplicates=options.replace_duplicates,
                        timeout=options.request_timeout,
                    )
                task_id = str(response["task_id"])
                submit_seconds = round(time.monotonic() - submitted_at, 3)
                result.update(
                    task_id=task_id,
                    status=str(response.get("status") or "accepted"),
                    phase="task_poll",
                    submit_seconds=submit_seconds,
                )
                progress.log(
                    f"batch {index}/{len(batches)} submitted task_id={task_id} "
                    f"files={len(batch)} submit_seconds={submit_seconds:.3f}"
                )
                _notify(summary_callback, summary)
            except Exception as exc:
                result.update(
                    status="failed",
                    phase="multipart_upload",
                    duration_seconds=round(time.monotonic() - submitted_at, 3),
                    error=_error_text(exc),
                )
                progress.log(f"batch {index}/{len(batches)} submit failed: {_error_text(exc)}")
                _notify(summary_callback, summary)
                return

            await _poll_batch(
                client=client,
                result=result,
                batch_count=len(batches),
                options=options,
                progress=progress,
                summary=summary,
                summary_callback=summary_callback,
            )

    async def report_summary() -> None:
        if options.summary_interval == 0:
            return
        while not finished.is_set():
            try:
                await asyncio.wait_for(finished.wait(), timeout=options.summary_interval)
            except TimeoutError:
                progress.log(format_summary_progress(summary))

    reporter = asyncio.create_task(report_summary())
    try:
        await asyncio.gather(
            *(process_batch(index, batch) for index, batch in enumerate(batches, start=1))
        )
    finally:
        finished.set()
        await reporter

    _update_summary_totals(summary)
    _notify(summary_callback, summary)
    return summary


async def _poll_batch(
    *,
    client: Any,
    result: dict[str, Any],
    batch_count: int,
    options: BulkUploadOptions,
    progress: ProgressReporter,
    summary: dict[str, Any],
    summary_callback: SummaryCallback | None,
) -> None:
    task_id = str(result["task_id"])
    started = time.monotonic()
    last_log = 0.0
    last_snapshot: tuple[Any, ...] | None = None
    poll_errors = 0

    while True:
        if time.monotonic() - started >= options.task_timeout:
            result.update(
                status="failed",
                phase="task_poll",
                duration_seconds=round(time.monotonic() - started, 3),
                error=f"task did not finish within {options.task_timeout:g}s",
            )
            progress.log(f"batch {result['batch_index']}/{batch_count} task {task_id} timed out")
            _notify(summary_callback, summary)
            return

        try:
            status = await client.get_task_status(
                task_id,
                timeout=options.poll_request_timeout,
            )
            poll_errors = 0
        except Exception as exc:
            if not _is_transient_error(exc):
                result.update(
                    status="failed",
                    phase="task_poll",
                    duration_seconds=round(time.monotonic() - started, 3),
                    error=_error_text(exc),
                )
                progress.log(
                    f"batch {result['batch_index']}/{batch_count} task {task_id} "
                    f"poll failed: {_error_text(exc)}"
                )
                _notify(summary_callback, summary)
                return
            poll_errors += 1
            now = time.monotonic()
            if _should_log(now, last_log, options.progress_interval):
                progress.log(
                    f"batch {result['batch_index']}/{batch_count} task {task_id} "
                    f"poll_error_count={poll_errors} error={_error_text(exc)}"
                )
                last_log = now
            await asyncio.sleep(options.poll_interval)
            continue

        _update_batch_from_status(result, status)
        _notify(summary_callback, summary)
        snapshot = _status_snapshot(status)
        now = time.monotonic()
        if (
            str(status.get("status")) in TERMINAL_TASK_STATUSES
            or snapshot != last_snapshot
            or _should_log(now, last_log, options.progress_interval)
        ):
            progress.log(
                f"batch {result['batch_index']}/{batch_count} task {task_id} "
                f"{format_task_progress(status)}"
            )
            last_log = now
            last_snapshot = snapshot

        if str(status.get("status")) in TERMINAL_TASK_STATUSES:
            return
        await asyncio.sleep(options.poll_interval)


def bulk_failed(summary: dict[str, Any]) -> bool:
    return any(
        batch.get("status") != "completed" or _int_value(batch.get("failed_files")) > 0
        for batch in summary.get("batches") or []
    )


def format_task_progress(status: dict[str, Any]) -> str:
    values = [f"status={status.get('status')}", f"phase={_task_phase(status)}"]
    for label in (
        "processed_files",
        "total_files",
        "successful_files",
        "failed_files",
        "running_files",
        "pending_files",
    ):
        if status.get(label) is not None:
            values.append(f"{label.removesuffix('_files')}={status[label]}")
    error = _task_error(status)
    if error:
        values.append(f"error={error}")
    return " ".join(values)


def format_summary_progress(summary: dict[str, Any]) -> str:
    statuses: dict[str, int] = {}
    processed = successful = failed = 0
    for batch in summary.get("batches") or []:
        status = str(batch.get("status") or "unknown")
        statuses[status] = statuses.get(status, 0) + 1
        processed += _int_value(batch.get("processed_files"))
        successful += _int_value(batch.get("successful_files"))
        failed += _int_value(batch.get("failed_files"))
    counts = ",".join(f"{key}:{value}" for key, value in sorted(statuses.items()))
    return (
        f"summary batches={counts} files processed={processed}/{summary.get('total_files', 0)} "
        f"successful={successful} failed={failed}"
    )


def persist_summary(
    summary: dict[str, Any],
    *,
    output_dir: Path,
    run_id: str,
    started: float,
) -> dict[str, Any]:
    output_dir.mkdir(parents=True, exist_ok=True)
    summary_path = output_dir / "summary.json"
    summary.update(
        run_id=run_id,
        output_dir=str(output_dir),
        summary_path=str(summary_path),
        elapsed_seconds=round(time.monotonic() - started, 3),
    )
    ordered = _ordered_summary(summary)
    temporary = output_dir / ".summary.json.tmp"
    temporary.write_text(json.dumps(ordered, indent=2) + "\n", encoding="utf-8")
    temporary.replace(summary_path)
    return ordered


def list_runs(runs_dir: Path, *, limit: int = 20) -> dict[str, Any]:
    records: list[dict[str, Any]] = []
    if runs_dir.is_dir():
        paths = sorted(
            (path for path in runs_dir.glob("*/summary.json") if path.is_file()),
            key=lambda path: path.stat().st_mtime,
            reverse=True,
        )
        if limit > 0:
            paths = paths[:limit]
        for path in paths:
            try:
                summary = _load_summary(path)
            except (OSError, ValueError, json.JSONDecodeError):
                continue
            records.append(_run_overview(summary, path))
    return {"runs_dir": str(runs_dir), "count": len(records), "runs": records}


def load_summary_reference(
    runs_dir: Path,
    reference: str | None,
    *,
    detail: bool,
) -> dict[str, Any]:
    value = (reference or "latest").strip()
    if value in {"latest", "current", "recent"}:
        listed = list_runs(runs_dir, limit=1)
        if not listed["runs"]:
            raise FileNotFoundError(f"no runs found under {runs_dir}")
        path = Path(listed["runs"][0]["summary_path"])
    else:
        candidate = Path(value).expanduser()
        if candidate.is_file():
            path = candidate
        elif candidate.is_dir():
            path = candidate / "summary.json"
        else:
            path = runs_dir / value / "summary.json"
    summary = _load_summary(path.resolve())
    if detail:
        return summary
    response = dict(summary)
    response.pop("batches", None)
    response["summary"] = _run_overview(summary, path.resolve())
    return response


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        description="Bulk-load local files or benchmark OpenRAG ingestion.",
    )
    subparsers = parser.add_subparsers(dest="command", required=True)

    bulk = subparsers.add_parser("bulk", help="Bulk-load local files and directories.")
    bulk_subparsers = bulk.add_subparsers(dest="bulk_action", required=True)

    upload = bulk_subparsers.add_parser("upload", help="Upload paths in concurrent batches.")
    upload.add_argument("paths", nargs="+")
    upload.add_argument("--base-url", default=os.getenv("OPENRAG_URL", "http://localhost:3000"))
    upload.add_argument("--api-key", default=os.getenv("OPENRAG_API_KEY"))
    upload.add_argument(
        "--api-prefix",
        default="/api/v1",
        help="Use /v1 when calling the backend directly (default: /api/v1).",
    )
    upload.add_argument("--batch-size", type=int, default=10)
    upload.add_argument("--max-inflight", type=int, default=1)
    upload.add_argument("--max-submit", type=int, default=None)
    upload.add_argument("--sort", choices=("path", "size-asc", "size-desc"), default="path")
    upload.add_argument("--include", action="append", default=None)
    upload.add_argument("--exclude", action="append", default=None)
    upload.add_argument("--recursive", action=argparse.BooleanOptionalAction, default=True)
    upload.add_argument("--settings-json", default=None, help="JSON object or @path to JSON.")
    upload.add_argument("--tweaks-json", default=None, help="JSON object or @path to JSON.")
    upload.add_argument(
        "--replace-duplicates",
        action=argparse.BooleanOptionalAction,
        default=True,
    )
    upload.add_argument("--task-timeout", type=float, default=1800.0)
    upload.add_argument("--request-timeout", type=float, default=300.0)
    upload.add_argument("--poll-request-timeout", type=float, default=10.0)
    upload.add_argument("--poll-interval", type=float, default=1.0)
    upload.add_argument("--progress", action=argparse.BooleanOptionalAction, default=True)
    upload.add_argument("--progress-interval", type=float, default=5.0)
    upload.add_argument("--summary-interval", type=float, default=60.0)
    upload.add_argument("--runs-dir", default=str(DEFAULT_RUNS_DIR))
    upload.add_argument("--output-dir", default=None)
    upload.add_argument("--run-id", default=None)

    list_parser = bulk_subparsers.add_parser("list", help="List locally recorded bulk runs.")
    list_parser.add_argument("--runs-dir", default=str(DEFAULT_RUNS_DIR))
    list_parser.add_argument("--limit", type=int, default=20)

    summary = bulk_subparsers.add_parser("summary", help="Show a locally recorded run.")
    summary.add_argument("run", nargs="?", default=None)
    summary.add_argument("--runs-dir", default=str(DEFAULT_RUNS_DIR))
    summary.add_argument("--detail", action="store_true")

    bench = subparsers.add_parser("bench", help="Run repeatable ingestion benchmarks.")
    bench_subparsers = bench.add_subparsers(dest="benchmark", required=True)
    arxiv = bench_subparsers.add_parser(
        "arxiv",
        help="Download arXiv PDFs and optionally bulk-ingest them.",
    )
    _add_arxiv_args(arxiv)
    arxiv_actions = arxiv.add_subparsers(dest="benchmark_action")
    arxiv_list = arxiv_actions.add_parser("list", help="List arXiv benchmark runs.")
    arxiv_list.add_argument("--runs-dir", default=str(DEFAULT_ARXIV_DIR / "runs"))
    arxiv_list.add_argument("--limit", type=int, default=20)
    arxiv_summary = arxiv_actions.add_parser("summary", help="Show an arXiv benchmark run.")
    arxiv_summary.add_argument("run", nargs="?", default=None)
    arxiv_summary.add_argument("--runs-dir", default=str(DEFAULT_ARXIV_DIR / "runs"))
    arxiv_summary.add_argument("--detail", action="store_true")
    return parser


def _add_arxiv_args(parser: argparse.ArgumentParser) -> None:
    parser.add_argument(
        "--source",
        choices=("s3", "atom"),
        default=DEFAULT_ARXIV_SOURCE,
        help="Use the S3 bulk PDF tarball (default) or the arXiv Atom API.",
    )
    parser.add_argument(
        "--query", default=None, help="Raw Atom API query; overrides category/dates."
    )
    parser.add_argument("--category", default=DEFAULT_ARXIV_CATEGORY)
    parser.add_argument("--date-from", default=DEFAULT_ARXIV_DATE_FROM)
    parser.add_argument("--date-to", default=DEFAULT_ARXIV_DATE_TO)
    parser.add_argument("--max-results", type=int, default=10)
    parser.add_argument("--start", type=int, default=0)
    parser.add_argument(
        "--sort-by",
        choices=("relevance", "lastUpdatedDate", "submittedDate"),
        default="submittedDate",
    )
    parser.add_argument(
        "--sort-order",
        choices=("ascending", "descending"),
        default="descending",
    )
    parser.add_argument("--s3-uri", default=DEFAULT_ARXIV_S3_URI)
    parser.add_argument("--s3-cache-dir", default=None)
    parser.add_argument("--aws-cli", default="aws")
    parser.add_argument("--s3-request-payer", default="requester")
    parser.add_argument("--pdf-cache-dir", default=None)
    parser.add_argument("--failed-download-cache", default=None)
    parser.add_argument("--download-only", action="store_true")
    parser.add_argument("--skip-existing", action=argparse.BooleanOptionalAction, default=True)
    parser.add_argument("--retry-failed-downloads", action="store_true")
    parser.add_argument("--delay-seconds", type=float, default=3.0)
    parser.add_argument("--max-pdf-mb", type=int, default=50)
    parser.add_argument("--user-agent", default="openrag-arxiv-benchmark/0.1")
    parser.add_argument("--base-url", default=os.getenv("OPENRAG_URL", "http://localhost:3000"))
    parser.add_argument("--api-key", default=os.getenv("OPENRAG_API_KEY"))
    parser.add_argument("--api-prefix", default="/api/v1")
    parser.add_argument("--batch-size", type=int, default=5)
    parser.add_argument(
        "--upload-sort",
        choices=("path", "size-asc", "size-desc"),
        default="path",
    )
    parser.add_argument("--max-inflight", type=int, default=1)
    parser.add_argument("--max-submit", type=int, default=None)
    parser.add_argument("--task-timeout", type=float, default=1800.0)
    parser.add_argument("--request-timeout", type=float, default=60.0)
    parser.add_argument("--poll-request-timeout", type=float, default=10.0)
    parser.add_argument("--poll-interval", type=float, default=1.0)
    parser.add_argument("--progress", action=argparse.BooleanOptionalAction, default=True)
    parser.add_argument("--progress-interval", type=float, default=5.0)
    parser.add_argument("--summary-interval", type=float, default=60.0)
    parser.add_argument("--settings-json", default=None, help="JSON object or @path to JSON.")
    parser.add_argument("--tweaks-json", default=None, help="JSON object or @path to JSON.")
    parser.add_argument(
        "--replace-duplicates",
        action=argparse.BooleanOptionalAction,
        default=True,
    )
    parser.add_argument("--runs-dir", default=str(DEFAULT_ARXIV_DIR / "runs"))
    parser.add_argument("--output-dir", default=None)
    parser.add_argument("--run-id", default=None)


async def _upload_main(args: argparse.Namespace) -> int:
    if not args.api_key:
        raise ValueError("an API key is required; set OPENRAG_API_KEY or pass --api-key")
    settings_json = _json_argument(args.settings_json, "settings")
    tweaks_json = _json_argument(args.tweaks_json, "tweaks")
    items = collect_upload_items(
        args.paths,
        include=args.include,
        exclude=args.exclude,
        recursive=args.recursive,
    )
    items = sort_upload_items(items, args.sort)
    if not items:
        raise ValueError("no files matched")

    started = time.monotonic()
    run_id = args.run_id or _new_run_id()
    runs_dir = Path(args.runs_dir).expanduser().resolve()
    output_dir = (
        Path(args.output_dir).expanduser().resolve() if args.output_dir else runs_dir / run_id
    )
    progress = ProgressReporter(enabled=args.progress, started=started)

    def save(summary: dict[str, Any]) -> dict[str, Any]:
        return persist_summary(
            summary,
            output_dir=output_dir,
            run_id=run_id,
            started=started,
        )

    options = BulkUploadOptions(
        batch_size=args.batch_size,
        max_inflight=args.max_inflight,
        max_submit=args.max_submit,
        request_timeout=args.request_timeout,
        poll_request_timeout=args.poll_request_timeout,
        task_timeout=args.task_timeout,
        poll_interval=args.poll_interval,
        progress_interval=args.progress_interval,
        summary_interval=args.summary_interval,
        input_sort=args.sort,
        settings_json=settings_json,
        tweaks_json=tweaks_json,
        replace_duplicates=args.replace_duplicates,
    )
    async with OpenRAGApiClient(
        base_url=args.base_url,
        api_key=args.api_key,
        api_prefix=args.api_prefix,
    ) as client:
        result = await upload_bulk_items(
            client=client,
            items=items,
            options=options,
            progress=progress,
            summary_callback=save,
        )
    result = save(result)
    progress.log(f"complete summary={result['summary_path']}")
    _print_json(result)
    return 1 if bulk_failed(result) else 0


def _validate_arxiv_args(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None:
    positive = {
        "--max-results": args.max_results,
        "--batch-size": args.batch_size,
        "--max-inflight": args.max_inflight,
        "--request-timeout": args.request_timeout,
        "--poll-request-timeout": args.poll_request_timeout,
        "--task-timeout": args.task_timeout,
        "--max-pdf-mb": args.max_pdf_mb,
    }
    for name, value in positive.items():
        if value <= 0:
            parser.error(f"{name} must be positive")
    if args.max_submit is not None and args.max_submit <= 0:
        parser.error("--max-submit must be positive")
    if args.start < 0:
        parser.error("--start must be non-negative")
    for name in ("poll_interval", "progress_interval", "summary_interval", "delay_seconds"):
        if getattr(args, name) < 0:
            parser.error(f"--{name.replace('_', '-')} must be non-negative")
    try:
        if args.query is None:
            args.query = build_arxiv_query(
                category=args.category,
                date_from=args.date_from,
                date_to=args.date_to,
            )
    except ValueError as exc:
        parser.error(str(exc))
    if args.source == "s3" and not str(args.s3_uri or "").strip():
        parser.error("--s3-uri is required when --source=s3")


async def _arxiv_main(args: argparse.Namespace) -> int:
    if not args.download_only and not args.api_key:
        raise ValueError(
            "an API key is required unless --download-only is used; "
            "set OPENRAG_API_KEY or pass --api-key"
        )

    started = time.monotonic()
    run_id = args.run_id or _new_run_id()
    runs_dir = Path(args.runs_dir).expanduser().resolve()
    output_dir = (
        Path(args.output_dir).expanduser().resolve() if args.output_dir else runs_dir / run_id
    )
    base_dir = runs_dir.parent
    pdf_dir = (
        Path(args.pdf_cache_dir).expanduser().resolve() if args.pdf_cache_dir else base_dir / "pdfs"
    )
    failure_cache_path = (
        Path(args.failed_download_cache).expanduser().resolve()
        if args.failed_download_cache
        else pdf_dir / "_failed_downloads.json"
    )
    output_dir.mkdir(parents=True, exist_ok=True)
    pdf_dir.mkdir(parents=True, exist_ok=True)
    progress = ProgressReporter(
        enabled=args.progress,
        started=started,
        prefix="openrag bench arxiv",
    )
    progress.log(f"run_id={run_id} output_dir={output_dir} pdf_cache_dir={pdf_dir}")

    rate = RateLimiter(args.delay_seconds)
    source_ref = str(args.query)
    source_details: dict[str, Any] = {}
    if args.source == "s3":
        s3_cache_dir = (
            Path(args.s3_cache_dir).expanduser().resolve() if args.s3_cache_dir else base_dir / "s3"
        )
        source_ref = str(args.s3_uri)
        progress.log(
            f"staging S3 tarball uri={args.s3_uri} start={args.start} "
            f"max_results={args.max_results}"
        )
        downloads, download_errors, skipped_failures, source_details = stage_s3_papers(
            s3_uri=args.s3_uri,
            s3_cache_dir=s3_cache_dir,
            pdf_dir=pdf_dir,
            aws_cli=args.aws_cli,
            request_payer=args.s3_request_payer,
            start=args.start,
            max_results=args.max_results,
            max_pdf_bytes=args.max_pdf_mb * 1024 * 1024,
            skip_existing=args.skip_existing,
            progress=progress,
        )
        matched_papers = int(source_details.get("selected_pdf_members") or len(downloads))
    else:
        progress.log(
            f"fetching Atom metadata query={args.query!r} start={args.start} "
            f"max_results={args.max_results} sort={args.sort_by}:{args.sort_order}"
        )
        papers = fetch_arxiv_papers(
            query=args.query,
            start=args.start,
            max_results=args.max_results,
            sort_by=args.sort_by,
            sort_order=args.sort_order,
            rate=rate,
            user_agent=args.user_agent,
            timeout=args.request_timeout,
        )
        progress.log(f"metadata fetched matched_papers={len(papers)}")
        failure_cache = load_failed_download_cache(failure_cache_path)
        seeded = seed_failed_download_cache_from_summaries(runs_dir, source_ref=source_ref)
        for key, record in seeded.items():
            failure_cache.setdefault(key, record)
        if seeded:
            save_failed_download_cache(failure_cache_path, failure_cache)
        downloads, download_errors, skipped_failures = download_papers(
            papers,
            pdf_dir=pdf_dir,
            rate=rate,
            user_agent=args.user_agent,
            timeout=args.request_timeout,
            max_pdf_bytes=args.max_pdf_mb * 1024 * 1024,
            skip_existing=args.skip_existing,
            failure_cache=failure_cache,
            failure_cache_path=failure_cache_path,
            retry_failed=args.retry_failed_downloads,
            progress=progress,
        )
        matched_papers = len(papers)

    cached_papers = sum(item.cached for item in downloads)
    downloaded_papers = len(downloads) - cached_papers
    progress.log(
        f"PDF stage complete ready={len(downloads)} cached={cached_papers} "
        f"downloaded={downloaded_papers} skipped_failed={len(skipped_failures)} "
        f"errors={len(download_errors)}"
    )
    summary: dict[str, Any] = {
        "run_id": run_id,
        "source": args.source,
        "source_ref": source_ref,
        "output_dir": str(output_dir),
        "pdf_cache_dir": str(pdf_dir),
        "failed_download_cache_path": str(failure_cache_path),
        "download_only": args.download_only,
        "requested_papers": args.max_results,
        "matched_papers": matched_papers,
        "ready_papers": len(downloads),
        "downloaded_papers": downloaded_papers,
        "network_downloaded_papers": downloaded_papers,
        "cached_papers": cached_papers,
        "download_errors": download_errors,
        "skipped_failed_papers": len(skipped_failures),
        "skipped_failed_downloads": skipped_failures,
        "concurrency": {"client": _arxiv_concurrency_settings(args)},
        "batches": [],
        "state": "downloaded",
    }
    if source_details:
        summary["source_details"] = source_details
    persist_arxiv_summary(summary, output_dir=output_dir, started=started)

    if not args.download_only and downloads:
        settings_json = _json_argument(args.settings_json, "settings")
        tweaks_json = _json_argument(args.tweaks_json, "tweaks")
        items = sort_upload_items(
            [
                BulkUploadItem(
                    path=item.path,
                    content_type="application/pdf",
                )
                for item in downloads
            ],
            args.upload_sort,
        )
        summary["state"] = "uploading"

        def save_bulk(bulk_summary: dict[str, Any]) -> None:
            summary["bulk"] = {
                key: value for key, value in bulk_summary.items() if key != "batches"
            }
            summary["batches"] = bulk_summary.get("batches", [])
            persist_arxiv_summary(summary, output_dir=output_dir, started=started)

        async with OpenRAGApiClient(
            base_url=args.base_url,
            api_key=args.api_key,
            api_prefix=args.api_prefix,
        ) as client:
            bulk_summary = await upload_bulk_items(
                client=client,
                items=items,
                options=BulkUploadOptions(
                    batch_size=args.batch_size,
                    max_inflight=args.max_inflight,
                    max_submit=args.max_submit,
                    request_timeout=args.request_timeout,
                    poll_request_timeout=args.poll_request_timeout,
                    task_timeout=args.task_timeout,
                    poll_interval=args.poll_interval,
                    progress_interval=args.progress_interval,
                    summary_interval=args.summary_interval,
                    input_sort=args.upload_sort,
                    settings_json=settings_json,
                    tweaks_json=tweaks_json,
                    replace_duplicates=args.replace_duplicates,
                ),
                progress=progress,
                summary_callback=save_bulk,
            )
        save_bulk(bulk_summary)

    failed = (
        not downloads
        or bool(download_errors)
        or (not args.download_only and bool(downloads) and bulk_failed(summary))
    )
    summary["state"] = "failed" if failed else "completed"
    result = persist_arxiv_summary(summary, output_dir=output_dir, started=started)
    progress.log(f"complete summary={result['summary_path']}")
    _print_json(result)
    return 1 if failed else 0


def main(argv: Sequence[str] | None = None) -> int:
    parser = build_parser()
    args = parser.parse_args(argv)
    try:
        if args.command == "bulk":
            if args.bulk_action == "list":
                _print_json(
                    list_runs(
                        Path(args.runs_dir).expanduser().resolve(),
                        limit=args.limit,
                    )
                )
                return 0
            if args.bulk_action == "summary":
                _print_json(
                    load_summary_reference(
                        Path(args.runs_dir).expanduser().resolve(),
                        args.run,
                        detail=args.detail,
                    )
                )
                return 0
            return asyncio.run(_upload_main(args))
        if args.command == "bench" and args.benchmark == "arxiv":
            runs_dir = Path(args.runs_dir).expanduser().resolve()
            if args.benchmark_action == "list":
                _print_json(list_runs(runs_dir, limit=args.limit))
                return 0
            if args.benchmark_action == "summary":
                _print_json(
                    load_summary_reference(
                        runs_dir,
                        args.run,
                        detail=args.detail,
                    )
                )
                return 0
            _validate_arxiv_args(parser, args)
            return asyncio.run(_arxiv_main(args))
        parser.error(f"unsupported command: {args.command}")
    except (
        ApiError,
        ArxivBenchmarkError,
        FileNotFoundError,
        ValueError,
        json.JSONDecodeError,
        tarfile.TarError,
    ) as exc:
        print(f"openrag bulk: {exc}", file=sys.stderr)
        return 2
    except KeyboardInterrupt:
        print("openrag bulk: interrupted", file=sys.stderr)
        return 130


def build_arxiv_query(*, category: str, date_from: str, date_to: str) -> str:
    category = category.strip()
    if not category:
        raise ValueError("--category must not be empty")
    start = arxiv_date_bound(date_from, end=False)
    stop = arxiv_date_bound(date_to, end=True)
    if start > stop:
        raise ValueError("--date-from must be before --date-to")
    return f"cat:{category} AND submittedDate:[{start} TO {stop}]"


def arxiv_date_bound(value: str, *, end: bool) -> str:
    value = value.strip()
    if re.fullmatch(r"\d{12}", value):
        return value
    if re.fullmatch(r"\d{8}", value):
        return value + ("2359" if end else "0000")
    match = re.fullmatch(r"(\d{4})-(\d{2})-(\d{2})", value)
    if match:
        return "".join(match.groups()) + ("2359" if end else "0000")
    raise ValueError("arXiv dates must use YYYY-MM-DD, YYYYMMDD, or YYYYMMDDHHMM")


def stage_s3_papers(
    *,
    s3_uri: str,
    s3_cache_dir: Path,
    pdf_dir: Path,
    aws_cli: str,
    request_payer: str,
    start: int,
    max_results: int,
    max_pdf_bytes: int,
    skip_existing: bool,
    progress: ProgressReporter,
) -> tuple[list[DownloadedPaper], list[dict[str, Any]], list[dict[str, Any]], dict[str, Any]]:
    tarball, tarball_downloaded = ensure_s3_tarball(
        s3_uri=s3_uri,
        cache_dir=s3_cache_dir,
        aws_cli=aws_cli,
        request_payer=request_payer,
        skip_existing=skip_existing,
        progress=progress,
    )
    downloads, errors = extract_s3_tarball_papers(
        tarball,
        pdf_dir=pdf_dir,
        start=start,
        max_results=max_results,
        max_pdf_bytes=max_pdf_bytes,
        skip_existing=skip_existing,
        progress=progress,
    )
    details = {
        "s3_uri": s3_uri,
        "tarball_path": str(tarball),
        "tarball_bytes": tarball.stat().st_size,
        "tarball_downloaded": tarball_downloaded,
        "s3_cache_dir": str(s3_cache_dir),
        "selected_pdf_members": len(downloads) + len(errors),
        "ready_pdf_members": len(downloads),
        "failed_pdf_members": len(errors),
    }
    return downloads, errors, [], details


def ensure_s3_tarball(
    *,
    s3_uri: str,
    cache_dir: Path,
    aws_cli: str,
    request_payer: str,
    skip_existing: bool,
    progress: ProgressReporter,
) -> tuple[Path, bool]:
    cache_dir.mkdir(parents=True, exist_ok=True)
    target = cache_dir / tarball_cache_filename(s3_uri)
    if skip_existing and target.is_file() and target.stat().st_size > 0:
        progress.log(f"S3 tarball cached path={target} bytes={target.stat().st_size}")
        return target, False

    temporary = target.with_name(target.name + ".tmp")
    if temporary.exists():
        temporary.unlink()
    parsed = urlparse(s3_uri)
    progress.log(f"S3 tarball downloading source={s3_uri} target={target}")
    if parsed.scheme == "s3":
        command = [aws_cli, "s3", "cp", s3_uri, str(temporary)]
        if request_payer:
            command.extend(["--request-payer", request_payer])
        result = subprocess.run(command, text=True, capture_output=True, check=False)
        if result.returncode != 0:
            detail = (result.stderr or result.stdout or "").strip()
            raise ArxivBenchmarkError(
                f"failed to download arXiv S3 tarball {s3_uri}: {detail or command!r}"
            )
    elif parsed.scheme == "file":
        shutil.copyfile(Path(unquote(parsed.path)).expanduser(), temporary)
    elif parsed.scheme:
        raise ArxivBenchmarkError(f"unsupported arXiv S3 URI scheme: {parsed.scheme}")
    else:
        shutil.copyfile(Path(s3_uri).expanduser(), temporary)
    temporary.replace(target)
    progress.log(f"S3 tarball downloaded path={target} bytes={target.stat().st_size}")
    return target, True


def extract_s3_tarball_papers(
    tarball: Path,
    *,
    pdf_dir: Path,
    start: int,
    max_results: int,
    max_pdf_bytes: int,
    skip_existing: bool,
    progress: ProgressReporter,
) -> tuple[list[DownloadedPaper], list[dict[str, Any]]]:
    pdf_dir.mkdir(parents=True, exist_ok=True)
    downloads: list[DownloadedPaper] = []
    errors: list[dict[str, Any]] = []
    used_filenames: set[str] = set()
    seen_pdf_members = 0
    selected_members = 0
    with tarfile.open(tarball, mode="r:*") as archive:
        for member in archive:
            if not member.isfile() or not member.name.lower().endswith(".pdf"):
                continue
            seen_pdf_members += 1
            if seen_pdf_members <= start:
                continue
            if selected_members >= max_results:
                break
            selected_members += 1
            paper = s3_member_paper(member.name)
            filename = unique_filename(
                pdf_dir,
                pdf_filename(paper),
                used_filenames,
                allow_existing=skip_existing,
            )
            path = pdf_dir / filename
            cached = skip_existing and path.is_file()
            try:
                if not cached:
                    if member.size > max_pdf_bytes:
                        raise ValueError(f"PDF is larger than limit: {member.size} bytes")
                    source = archive.extractfile(member)
                    if source is None:
                        raise ValueError("tar member has no file content")
                    with source, path.open("wb") as output:
                        shutil.copyfileobj(source, output, length=1024 * 1024)
                    if path.stat().st_size > max_pdf_bytes:
                        raise ValueError(f"PDF exceeded limit: {max_pdf_bytes} bytes")
                    progress.log(
                        f"S3 PDF {selected_members}/{max_results}: extracted "
                        f"arxiv_id={paper.arxiv_id} bytes={path.stat().st_size}"
                    )
                else:
                    progress.log(
                        f"S3 PDF {selected_members}/{max_results}: cached "
                        f"arxiv_id={paper.arxiv_id} path={path}"
                    )
                downloads.append(
                    DownloadedPaper(
                        paper=paper,
                        path=path,
                        size=path.stat().st_size,
                        sha256=sha256_file(path),
                        cached=cached,
                    )
                )
            except Exception as exc:
                if not cached and path.exists():
                    path.unlink()
                progress.log(
                    f"S3 PDF {selected_members}/{max_results}: failed member={member.name} "
                    f"error={_error_text(exc)}"
                )
                errors.append(
                    {
                        "arxiv_id": paper.arxiv_id,
                        "pdf_url": paper.pdf_url,
                        "error": _error_text(exc),
                        "tar_member": member.name,
                    }
                )
    progress.log(
        f"S3 tarball scan complete pdf_members_seen={seen_pdf_members} "
        f"selected={selected_members} ready={len(downloads)} errors={len(errors)}"
    )
    return downloads, errors


def s3_member_paper(member_name: str) -> ArxivPaper:
    arxiv_id = arxiv_id_from_pdf_member(member_name)
    return ArxivPaper(
        arxiv_id=arxiv_id,
        title=arxiv_id,
        summary="",
        authors=(),
        published=None,
        updated=None,
        abs_url=f"https://arxiv.org/abs/{arxiv_id}",
        pdf_url=f"https://arxiv.org/pdf/{arxiv_id}",
    )


def arxiv_id_from_pdf_member(member_name: str) -> str:
    filename = Path(member_name).name
    stem = filename[:-4] if filename.lower().endswith(".pdf") else Path(filename).stem
    stem = stem.strip()
    return stem or hashlib.sha256(member_name.encode()).hexdigest()[:16]


def tarball_cache_filename(uri: str) -> str:
    parsed = urlparse(uri)
    name = Path(unquote(parsed.path)).name if parsed.path else Path(uri).name
    name = safe_slug(name, 160)
    digest = hashlib.sha256(uri.encode()).hexdigest()[:12]
    suffix = "".join(Path(name).suffixes)
    stem = name[: -len(suffix)] if suffix else name
    return f"{stem}-{digest}{suffix or '.tar'}"


def fetch_arxiv_papers(
    *,
    query: str,
    start: int,
    max_results: int,
    sort_by: str,
    sort_order: str,
    rate: RateLimiter,
    user_agent: str,
    timeout: float,
) -> list[ArxivPaper]:
    url = f"{ARXIV_API_URL}?" + urlencode(
        {
            "search_query": query,
            "start": str(start),
            "max_results": str(max_results),
            "sortBy": sort_by,
            "sortOrder": sort_order,
        }
    )
    rate.wait()
    try:
        payload = read_url(
            url,
            user_agent=user_agent,
            timeout=timeout,
            accept="application/atom+xml",
        )
    except HTTPError as exc:
        if exc.code == 429:
            raise ArxivBenchmarkError(
                "arXiv returned HTTP 429 for metadata; wait before retrying, or ingest "
                "already-cached PDFs with "
                "`openrag_bulk.py bulk upload <pdf-cache> --include '*.pdf'`"
            ) from exc
        raise
    return parse_arxiv_feed(payload)


def parse_arxiv_feed(payload: bytes) -> list[ArxivPaper]:
    root = ET.fromstring(payload)
    papers: list[ArxivPaper] = []
    for entry in root.findall(f"{ATOM}entry"):
        abs_url = _atom_text(entry, "id")
        title = collapse_ws(_atom_text(entry, "title"))
        summary = collapse_ws(_atom_text(entry, "summary"))
        authors = tuple(
            collapse_ws(_atom_text(author, "name"))
            for author in entry.findall(f"{ATOM}author")
            if collapse_ws(_atom_text(author, "name"))
        )
        pdf_url = ""
        for link in entry.findall(f"{ATOM}link"):
            if link.attrib.get("title") == "pdf" or link.attrib.get("type") == "application/pdf":
                pdf_url = link.attrib.get("href", "")
                break
        if not abs_url or not pdf_url:
            continue
        papers.append(
            ArxivPaper(
                arxiv_id=arxiv_id_from_abs_url(abs_url),
                title=title,
                summary=summary,
                authors=authors,
                published=_atom_text(entry, "published") or None,
                updated=_atom_text(entry, "updated") or None,
                abs_url=https_url(abs_url),
                pdf_url=https_url(pdf_url),
            )
        )
    return papers


def download_papers(
    papers: list[ArxivPaper],
    *,
    pdf_dir: Path,
    rate: RateLimiter,
    user_agent: str,
    timeout: float,
    max_pdf_bytes: int,
    skip_existing: bool,
    failure_cache: dict[str, dict[str, Any]] | None = None,
    failure_cache_path: Path | None = None,
    retry_failed: bool = False,
    progress: ProgressReporter,
) -> tuple[list[DownloadedPaper], list[dict[str, Any]], list[dict[str, Any]]]:
    downloads: list[DownloadedPaper] = []
    errors: list[dict[str, Any]] = []
    skipped_failures: list[dict[str, Any]] = []
    failure_cache = failure_cache if failure_cache is not None else {}
    used_filenames: set[str] = set()
    pdf_dir.mkdir(parents=True, exist_ok=True)
    for index, paper in enumerate(papers, start=1):
        failure_key = failed_download_cache_key(paper)
        filename = unique_filename(
            pdf_dir,
            pdf_filename(paper),
            used_filenames,
            allow_existing=skip_existing,
        )
        path = pdf_dir / filename
        cached = skip_existing and path.is_file()
        cached_failure = failure_cache.get(failure_key)
        if cached_failure and not retry_failed and not cached:
            skipped = public_failed_download_record(cached_failure)
            progress.log(
                f"PDF {index}/{len(papers)}: skipped known failed "
                f"arxiv_id={paper.arxiv_id} error={skipped['error']}"
            )
            skipped_failures.append(skipped)
            continue
        try:
            if not cached:
                progress.log(
                    f"PDF {index}/{len(papers)}: downloading "
                    f"arxiv_id={paper.arxiv_id} url={paper.pdf_url}"
                )
                rate.wait()
                download_pdf(
                    paper.pdf_url,
                    path=path,
                    user_agent=user_agent,
                    timeout=timeout,
                    max_pdf_bytes=max_pdf_bytes,
                )
            else:
                progress.log(
                    f"PDF {index}/{len(papers)}: cached arxiv_id={paper.arxiv_id} path={path}"
                )
            downloads.append(
                DownloadedPaper(
                    paper=paper,
                    path=path,
                    size=path.stat().st_size,
                    sha256=sha256_file(path),
                    cached=cached,
                )
            )
            if failure_key in failure_cache:
                failure_cache.pop(failure_key, None)
                save_failed_download_cache(failure_cache_path, failure_cache)
        except Exception as exc:
            progress.log(
                f"PDF {index}/{len(papers)}: failed arxiv_id={paper.arxiv_id} "
                f"error={_error_text(exc)}"
            )
            record = failed_download_record(paper, exc, previous=failure_cache.get(failure_key))
            failure_cache[failure_key] = record
            save_failed_download_cache(failure_cache_path, failure_cache)
            errors.append(public_failed_download_record(record))
    return downloads, errors, skipped_failures


def download_pdf(
    url: str,
    *,
    path: Path,
    user_agent: str,
    timeout: float,
    max_pdf_bytes: int,
) -> None:
    request = Request(url, headers={"User-Agent": user_agent, "Accept": "application/pdf"})
    total = 0
    try:
        with urlopen(request, timeout=timeout) as response, path.open("wb") as output:
            length = response.headers.get("Content-Length")
            if length and int(length) > max_pdf_bytes:
                raise ValueError(f"PDF is larger than limit: {int(length)} bytes")
            while chunk := response.read(128 * 1024):
                total += len(chunk)
                if total > max_pdf_bytes:
                    raise ValueError(f"PDF exceeded limit: {max_pdf_bytes} bytes")
                output.write(chunk)
    except Exception:
        if path.exists():
            path.unlink()
        raise


def read_url(url: str, *, user_agent: str, timeout: float, accept: str) -> bytes:
    request = Request(url, headers={"User-Agent": user_agent, "Accept": accept})
    with urlopen(request, timeout=timeout) as response:
        return response.read()


def load_failed_download_cache(path: Path) -> dict[str, dict[str, Any]]:
    if not path.is_file():
        return {}
    parsed = json.loads(path.read_text(encoding="utf-8"))
    failures = parsed.get("failures") if isinstance(parsed, dict) else None
    if not isinstance(failures, dict):
        return {}
    return {str(key): value for key, value in failures.items() if isinstance(value, dict)}


def seed_failed_download_cache_from_summaries(
    runs_dir: Path,
    *,
    source_ref: str,
) -> dict[str, dict[str, Any]]:
    if not runs_dir.is_dir():
        return {}
    seeded: dict[str, dict[str, Any]] = {}
    for summary_path in sorted(runs_dir.glob("*/summary.json")):
        try:
            summary = json.loads(summary_path.read_text(encoding="utf-8"))
        except (OSError, json.JSONDecodeError):
            continue
        if not isinstance(summary, dict) or summary.get("source_ref") != source_ref:
            continue
        for record in summary_download_failure_records(summary):
            arxiv_id = str(record.get("arxiv_id") or "")
            if arxiv_id:
                seeded[arxiv_id] = {
                    "arxiv_id": arxiv_id,
                    "pdf_url": str(record.get("pdf_url") or ""),
                    "error": str(record.get("error") or "previous benchmark download failure"),
                    "first_failed_at": str(record.get("first_failed_at") or ""),
                    "last_failed_at": str(record.get("last_failed_at") or ""),
                    "attempts": record.get("attempts") or 1,
                    "seeded_from_summary": str(summary_path),
                }
    return seeded


def summary_download_failure_records(summary: dict[str, Any]) -> list[dict[str, Any]]:
    records: list[dict[str, Any]] = []
    for key in ("download_errors", "skipped_failed_downloads"):
        value = summary.get(key)
        if isinstance(value, list):
            records.extend(item for item in value if isinstance(item, dict))
    return records


def save_failed_download_cache(
    path: Path | None,
    failures: dict[str, dict[str, Any]],
) -> None:
    if path is None:
        return
    path.parent.mkdir(parents=True, exist_ok=True)
    temporary = path.with_name(path.name + ".tmp")
    temporary.write_text(
        json.dumps({"version": 1, "failures": failures}, indent=2, sort_keys=True) + "\n",
        encoding="utf-8",
    )
    temporary.replace(path)


def failed_download_cache_key(paper: ArxivPaper) -> str:
    return paper.arxiv_id or hashlib.sha256(paper.pdf_url.encode()).hexdigest()


def failed_download_record(
    paper: ArxivPaper,
    exc: Exception,
    *,
    previous: dict[str, Any] | None,
) -> dict[str, Any]:
    now = datetime.now(UTC).isoformat()
    first_failed_at = str((previous or {}).get("first_failed_at") or now)
    try:
        attempts = int((previous or {}).get("attempts") or 0) + 1
    except (TypeError, ValueError):
        attempts = 1
    return {
        "arxiv_id": paper.arxiv_id,
        "abs_url": paper.abs_url,
        "pdf_url": paper.pdf_url,
        "title": paper.title,
        "error_type": type(exc).__name__,
        "error": _error_text(exc),
        "first_failed_at": first_failed_at,
        "last_failed_at": now,
        "attempts": attempts,
    }


def public_failed_download_record(record: dict[str, Any]) -> dict[str, Any]:
    return {
        "arxiv_id": str(record.get("arxiv_id") or ""),
        "pdf_url": str(record.get("pdf_url") or ""),
        "error": str(record.get("error") or "unknown download error"),
        "first_failed_at": str(record.get("first_failed_at") or ""),
        "last_failed_at": str(record.get("last_failed_at") or ""),
        "attempts": record.get("attempts") or None,
    }


def persist_arxiv_summary(
    summary: dict[str, Any],
    *,
    output_dir: Path,
    started: float,
) -> dict[str, Any]:
    summary_path = output_dir / "summary.json"
    summary["summary_path"] = str(summary_path)
    summary["elapsed_seconds"] = round(time.monotonic() - started, 3)
    ordered = _ordered_arxiv_summary(summary)
    temporary = output_dir / ".summary.json.tmp"
    temporary.write_text(json.dumps(ordered, indent=2) + "\n", encoding="utf-8")
    temporary.replace(summary_path)
    return ordered


def _ordered_arxiv_summary(summary: dict[str, Any]) -> dict[str, Any]:
    preferred = (
        "run_id",
        "summary_path",
        "output_dir",
        "pdf_cache_dir",
        "failed_download_cache_path",
        "source",
        "source_ref",
        "download_only",
        "requested_papers",
        "matched_papers",
        "ready_papers",
        "downloaded_papers",
        "network_downloaded_papers",
        "cached_papers",
        "download_errors",
        "skipped_failed_papers",
        "skipped_failed_downloads",
        "elapsed_seconds",
        "state",
        "concurrency",
        "bulk",
    )
    ordered = {key: summary[key] for key in preferred if key in summary}
    ordered.update(
        {key: value for key, value in summary.items() if key not in ordered and key != "batches"}
    )
    if "batches" in summary:
        ordered["batches"] = summary["batches"]
    return ordered


def _arxiv_concurrency_settings(args: argparse.Namespace) -> dict[str, Any]:
    source: dict[str, Any] = {
        "source": args.source,
        "max_inflight_requests": 1,
        "request_timeout_seconds": args.request_timeout,
        "skip_existing": args.skip_existing,
    }
    if args.source == "s3":
        source.update(s3_uri=args.s3_uri, s3_request_payer=args.s3_request_payer or None)
    else:
        source.update(
            delay_seconds=args.delay_seconds,
            retry_failed_downloads=args.retry_failed_downloads,
        )
    return {
        "pdf_source": source,
        "upload": {
            "batch_size": args.batch_size,
            "max_inflight_upload_tasks": args.max_inflight,
            "max_inflight_submit_requests": args.max_submit or args.max_inflight,
            "input_sort": args.upload_sort,
            "request_timeout_seconds": args.request_timeout,
            "poll_request_timeout_seconds": args.poll_request_timeout,
            "task_timeout_seconds": args.task_timeout,
            "poll_interval_seconds": args.poll_interval,
        },
    }


def _atom_text(entry: ET.Element, name: str) -> str:
    child = entry.find(f"{ATOM}{name}")
    return child.text.strip() if child is not None and child.text else ""


def collapse_ws(value: str) -> str:
    return re.sub(r"\s+", " ", value).strip()


def arxiv_id_from_abs_url(abs_url: str) -> str:
    return urlparse(abs_url).path.rstrip("/").rsplit("/", 1)[-1]


def https_url(value: str) -> str:
    parsed = urlparse(value)
    return urlunparse(parsed._replace(scheme="https")) if parsed.scheme == "http" else value


def pdf_filename(paper: ArxivPaper) -> str:
    return f"{safe_slug(paper.arxiv_id, 120)}.pdf"


def safe_slug(value: str, max_length: int) -> str:
    slug = re.sub(r"[^A-Za-z0-9._-]+", "-", value).strip(".-_")
    slug = re.sub(r"-{2,}", "-", slug)
    return slug[:max_length].strip(".-_") or "paper"


def unique_filename(
    directory: Path,
    filename: str,
    used: set[str],
    *,
    allow_existing: bool = False,
) -> str:
    stem = Path(filename).stem
    suffix = Path(filename).suffix
    candidate = filename
    index = 2
    while candidate in used or ((directory / candidate).exists() and not allow_existing):
        candidate = f"{stem}-{index}{suffix}"
        index += 1
    used.add(candidate)
    return candidate


def sha256_file(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def _response_json(response: httpx.Response) -> dict[str, Any]:
    if response.is_error:
        try:
            payload = response.json()
            message = payload.get("error") or payload.get("detail") or response.text
        except (ValueError, TypeError):
            message = response.text
        raise ApiError(str(message or f"HTTP {response.status_code}"), response.status_code)
    try:
        payload = response.json()
    except ValueError as exc:
        raise ApiError("OpenRAG returned a non-JSON response", response.status_code) from exc
    if not isinstance(payload, dict):
        raise ApiError("OpenRAG returned a non-object JSON response", response.status_code)
    return payload


def _new_batch_result(batch: Sequence[BulkUploadItem], *, batch_index: int) -> dict[str, Any]:
    byte_count = sum(item.path.stat().st_size for item in batch)
    return {
        "batch_index": batch_index,
        "file_count": len(batch),
        "files": [str(item.path) for item in batch],
        "input_bytes": byte_count,
        "input_mb": round(_mb(byte_count), 3),
        "status": "queued",
        "phase": "queued",
    }


def _update_batch_from_status(result: dict[str, Any], status: dict[str, Any]) -> None:
    result.update(
        status=str(status.get("status") or "unknown"),
        phase=_task_phase(status),
        duration_seconds=status.get("duration_seconds"),
        processed_files=status.get("processed_files"),
        successful_files=status.get("successful_files"),
        failed_files=status.get("failed_files"),
        running_files=status.get("running_files"),
        pending_files=status.get("pending_files"),
        error=_task_error(status) or None,
    )


def _task_phase(status: dict[str, Any]) -> str:
    if status.get("phase"):
        return str(status["phase"])
    phases = {
        str(file_status.get("phase"))
        for file_status in (status.get("files") or {}).values()
        if isinstance(file_status, dict) and file_status.get("phase")
    }
    if len(phases) == 1:
        return phases.pop()
    if phases:
        return "mixed"
    return "complete" if status.get("status") in TERMINAL_TASK_STATUSES else "task_poll"


def _task_error(status: dict[str, Any]) -> str:
    if status.get("error"):
        return _summarize_error(status["error"])
    errors = [
        file_status.get("error")
        for file_status in (status.get("files") or {}).values()
        if isinstance(file_status, dict) and file_status.get("error")
    ]
    if not errors:
        return ""
    first = _summarize_error(errors[0])
    return first if len(errors) == 1 else f"{first} (+{len(errors) - 1} more)"


def _status_snapshot(status: dict[str, Any]) -> tuple[Any, ...]:
    return (
        status.get("status"),
        _task_phase(status),
        status.get("processed_files"),
        status.get("successful_files"),
        status.get("failed_files"),
        status.get("running_files"),
        status.get("pending_files"),
        _task_error(status),
    )


def _update_summary_totals(summary: dict[str, Any]) -> None:
    batches = summary.get("batches") or []
    summary["successful_files"] = sum(_int_value(item.get("successful_files")) for item in batches)
    summary["failed_files"] = sum(_int_value(item.get("failed_files")) for item in batches)
    summary["completed_batches"] = sum(item.get("status") == "completed" for item in batches)
    summary["failed_batches"] = sum(item.get("status") != "completed" for item in batches)


def _run_overview(summary: dict[str, Any], path: Path) -> dict[str, Any]:
    statuses: dict[str, int] = {}
    for batch in summary.get("batches") or []:
        status = str(batch.get("status") or "unknown")
        statuses[status] = statuses.get(status, 0) + 1
    if summary.get("state"):
        state = str(summary["state"])
    elif not statuses:
        state = "unknown"
    elif any(status not in TERMINAL_TASK_STATUSES for status in statuses):
        state = "running"
    elif any(status != "completed" for status in statuses):
        state = "failed"
    else:
        state = "completed"
    return {
        "run_id": str(summary.get("run_id") or path.parent.name),
        "state": state,
        "updated_at": datetime.fromtimestamp(path.stat().st_mtime, tz=UTC).isoformat(),
        "summary_path": str(path),
        "elapsed_seconds": summary.get("elapsed_seconds"),
        "total_files": summary.get("total_files", summary.get("ready_papers")),
        "batch_count": len(summary.get("batches") or []),
        "batch_status_counts": statuses,
    }


def _load_summary(path: Path) -> dict[str, Any]:
    summary = json.loads(path.read_text(encoding="utf-8"))
    if not isinstance(summary, dict):
        raise ValueError(f"summary is not a JSON object: {path}")
    return summary


def _json_argument(value: str | None, label: str) -> str | None:
    if value is None:
        return None
    raw = (
        Path(value[1:]).expanduser().read_text(encoding="utf-8") if value.startswith("@") else value
    )
    parsed = json.loads(raw)
    if not isinstance(parsed, dict):
        raise ValueError(f"{label} JSON must be an object")
    return json.dumps(parsed, separators=(",", ":"))


def _ordered_summary(summary: dict[str, Any]) -> dict[str, Any]:
    preferred = (
        "run_id",
        "summary_path",
        "output_dir",
        "elapsed_seconds",
        "total_files",
        "successful_files",
        "failed_files",
        "completed_batches",
        "failed_batches",
        "batch_size",
        "max_inflight",
        "max_submit",
        "client_concurrency",
    )
    ordered = {key: summary[key] for key in preferred if key in summary}
    ordered.update(
        {key: value for key, value in summary.items() if key not in ordered and key != "batches"}
    )
    if "batches" in summary:
        ordered["batches"] = summary["batches"]
    return ordered


def _notify(callback: SummaryCallback | None, summary: dict[str, Any]) -> None:
    if callback:
        callback(summary)


def _matches(relative: str, name: str, patterns: Sequence[str]) -> bool:
    return any(
        fnmatch.fnmatch(relative, pattern) or fnmatch.fnmatch(name, pattern) for pattern in patterns
    )


def _chunks(items: list[BulkUploadItem], size: int) -> list[list[BulkUploadItem]]:
    return [items[index : index + size] for index in range(0, len(items), size)]


def _should_log(now: float, last_log: float, interval: float) -> bool:
    return interval == 0 or now - last_log >= interval


def _is_transient_error(exc: BaseException) -> bool:
    return isinstance(exc, (httpx.TransportError, TimeoutError)) or (
        isinstance(exc, ApiError) and exc.status_code in TRANSIENT_HTTP_STATUSES
    )


def _error_text(exc: BaseException) -> str:
    return f"{type(exc).__name__}: {_summarize_error(exc)}"


def _summarize_error(value: Any) -> str:
    lines = [line.strip() for line in str(value).splitlines() if line.strip()]
    text = lines[-1] if lines else str(value).strip()
    return text if len(text) <= 320 else text[:317] + "..."


def _int_value(value: Any) -> int:
    try:
        return int(value or 0)
    except (TypeError, ValueError):
        return 0


def _mb(value: int | float) -> float:
    return float(value) / 1_000_000


def _new_run_id() -> str:
    return datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid4().hex[:8]


def _print_json(value: Any) -> None:
    print(json.dumps(value, indent=2))


if __name__ == "__main__":
    raise SystemExit(main())