Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
feat: enforce parse server usage limits and url download size cap (Refs #5501)
myhloli3f84fc2cf33d
diff --git a/mineru/parser/api_server.py b/mineru/parser/api_server.pyindex 2cc9be05..2ccbe779 100644--- a/mineru/parser/api_server.py+++ b/mineru/parser/api_server.py@@ -82,6 +82,8 @@ from .writer import DataWriter_DEFAULT_API_SERVER_TIER: ServerTier = "standard"_API_SERVER_LANGUAGES = PUBLIC_OCR_LANGUAGES_MAX_INLINE_BYTES_DEFAULT = 1024 * 1024+_MAX_FILES_PER_JOB_DEFAULT = 100+_MAX_FILE_SIZE_BYTES_DEFAULT = 209715200_LOCAL_PARSE_OUTPUT_FORMATS: tuple[OutputFormat, ...] = ("markdown","middle_json",@@ -472,7 +474,7 @@ class CallbackConfig(BaseModel):class CreateJobRequest(BaseModel):model_config = _PYDANTIC_CONFIG- files: list[JobFileEntry] = Field(min_length=1)+ files: list[JobFileEntry] = Field(min_length=1, max_length=_MAX_FILES_PER_JOB_DEFAULT)tier: Tier | None = Noneocr_mode: Literal["auto", "txt", "ocr"] = Field(default="auto", description="OCR mode for this parse job: auto-detect, native text, or forced OCR."@@ -592,8 +594,8 @@ class UsageCurrent(BaseModel):class UsageLimits(BaseModel):model_config = _PYDANTIC_CONFIGmax_pages_per_file: int = 1000- max_file_size_bytes: int = 209715200- max_files_per_job: int = 100+ max_file_size_bytes: int = _MAX_FILE_SIZE_BYTES_DEFAULT+ max_files_per_job: int = _MAX_FILES_PER_JOB_DEFAULTmax_concurrent_jobs: int = 1max_file_retention_days: int | None = None@@ -1106,7 +1108,8 @@ class _JobRecord:class JobStore:def __init__(self, concurrency: int = 1) -> None:self._jobs: dict[str, _JobRecord] = {}- self._semaphore = asyncio.Semaphore(max(1, concurrency))+ self._concurrency = max(1, concurrency)+ self._semaphore = asyncio.Semaphore(self._concurrency)self._started_at = JobStore._now()self._tasks: dict[str, asyncio.Task[None]] = {}self._closing = False@@ -1265,7 +1268,7 @@ class JobStore:has_more=(start + limit) < len(recs),)- def build_response(self, rec: _JobRecord, access_level: AccessLevel = "registered") -> JobAsyncResponse:+ def build_response(self, rec: _JobRecord, *, access_level: AccessLevel) -> JobAsyncResponse:return JobAsyncResponse(job_id=rec.id,status=rec.status,@@ -1281,7 +1284,7 @@ class JobStore:)def usage(self, access_level: AccessLevel) -> UsageResponse:- completed = sum(1 for j in self._jobs.values() if j.status in ("completed", "partial"))+ files_processed = sum(1 for j in self._jobs.values() for fr in j.files if fr.status == "completed")processed_page_count = sum(sum(_count_pages_in_range(fr.page_range) for fr in j.files if fr.status == "completed") for j in self._jobs.values())@@ -1290,11 +1293,11 @@ class JobStore:billing_period=UsageBillingPeriod(start=self._started_at),current=UsageCurrent(pages_processed=processed_page_count,- files_processed=completed,+ files_processed=files_processed,jobs_created=len(self._jobs),),limits=UsageLimits(- max_concurrent_jobs=self._semaphore._value,+ max_concurrent_jobs=self._concurrency,),)@@ -1362,6 +1365,7 @@ async def _extract_bytes(allow_local_source: bool = False,max_inline_bytes: int = _MAX_INLINE_BYTES_DEFAULT,allow_http_source: bool = False,+ max_url_bytes: int = _MAX_FILE_SIZE_BYTES_DEFAULT,) -> _ExtractedSource:"""按既有来源策略读取字节,并保留不会由临时文件替代的来源上下文。"""_validate_source_policy(@@ -1377,14 +1381,22 @@ async def _extract_bytes(return _ExtractedSource(file_store.read_blob(rec.sha256sum))if isinstance(source, UrlSource):async with httpx.AsyncClient(timeout=url_timeout) as cli:- r = await cli.get(source.url)- r.raise_for_status()- return _ExtractedSource(- r.content,- source_uri=str(r.url),- transport_encoding=r.charset_encoding,- media_type=(r.headers.get("content-type") or "").partition(";")[0].strip().casefold() or None,- )+ # 流式读取并在超过 max_url_bytes 时中止,避免无上限响应被整包缓冲进内存。+ async with cli.stream("GET", source.url) as r:+ r.raise_for_status()+ chunks: list[bytes] = []+ received = 0+ async for chunk in r.aiter_bytes():+ received += len(chunk)+ if received > max_url_bytes:+ raise ValueError(f"url source exceeds max_url_bytes ({max_url_bytes})")+ chunks.append(chunk)+ return _ExtractedSource(+ b"".join(chunks),+ source_uri=str(r.url),+ transport_encoding=r.charset_encoding,+ media_type=(r.headers.get("content-type") or "").partition(";")[0].strip().casefold() or None,+ )if isinstance(source, InlineSource):return _ExtractedSource(_decode_inline_data(source.data))if isinstance(source, LocalSource):@@ -1411,6 +1423,7 @@ async def _run_job(allow_local_source: bool = False,max_inline_bytes: int = _MAX_INLINE_BYTES_DEFAULT,allow_http_source: bool = False,+ max_url_bytes: int = _MAX_FILE_SIZE_BYTES_DEFAULT,flash_enabled: bool = True,vlm_config: VlmConfig | None = None,) -> None:@@ -1434,6 +1447,7 @@ async def _run_job(allow_local_source=allow_local_source,max_inline_bytes=max_inline_bytes,allow_http_source=allow_http_source,+ max_url_bytes=max_url_bytes,)data = extracted.datasuffix = pathlib.Path(fr.name).suffix@@ -1932,6 +1946,7 @@ async def create_job(allow_local_source_val: bool = request.app.state.allow_local_sourcemax_inline_bytes_val: int = request.app.state.max_inline_bytesallow_http_source_val: bool = request.app.state.allow_http_source+ max_url_bytes_val: int = request.app.state.max_url_bytesflash_enabled_val: bool = request.app.state.flash_enabledimage_analysis_val: bool = request.app.state.image_analysisvlm_config_val: VlmConfig = request.app.state.vlm_config@@ -1947,12 +1962,15 @@ async def create_job(allow_local_source=allow_local_source_val,max_inline_bytes=max_inline_bytes_val,allow_http_source=allow_http_source_val,+ max_url_bytes=max_url_bytes_val,flash_enabled=flash_enabled_val,vlm_config=vlm_config_val,)job_store.start_task(rec, _bg_run)- return JSONResponse(content=job_store.build_response(rec).model_dump(by_alias=True), status_code=202)+ return JSONResponse(+ content=job_store.build_response(rec, access_level=access_level).model_dump(by_alias=True), status_code=202+ )@_router.get(@@ -1965,9 +1983,10 @@ async def create_job(async def get_job(job_id: str = Path(description="Job ID (job_...)"),job_store: JobStore = Depends(_get_job_store),+ access_level: AccessLevel = Depends(_resolve_access_level),) -> JobAsyncResponse:"""Retrieve a job's current status and results."""- return job_store.build_response(job_store.get(job_id))+ return job_store.build_response(job_store.get(job_id), access_level=access_level)@_router.get(@@ -2219,6 +2238,7 @@ def create_app(url_timeout: int = 60,allow_local_source: bool = False,max_inline_bytes: int = _MAX_INLINE_BYTES_DEFAULT,+ max_url_bytes: int = _MAX_FILE_SIZE_BYTES_DEFAULT,allow_http_source: bool = False,api_key: str | None = None,language: str = "ch",@@ -2247,6 +2267,9 @@ def create_app(Whether ``local`` sources may read paths visible to the server process.max_inline_bytes:Maximum decoded bytes accepted for ``inline`` sources.+ max_url_bytes:+ Maximum bytes downloaded for ``url`` sources; larger responses are+ aborted mid-download instead of being buffered into memory.allow_http_source:Whether ``url`` sources may use plain HTTP. HTTPS is always allowed.api_key:@@ -2276,6 +2299,8 @@ def create_app(_upload_dir.mkdir(parents=True, exist_ok=True)if max_inline_bytes < 0:raise ValueError("max_inline_bytes must be non-negative")+ if max_url_bytes < 0:+ raise ValueError("max_url_bytes must be non-negative")language = validate_public_ocr_lang(language)_model_ids, _tiers = _model_ids_and_tiers_for_server_tiers(server_tiers)@@ -2297,6 +2322,7 @@ def create_app(application.state.url_timeout = url_timeoutapplication.state.allow_local_source = allow_local_sourceapplication.state.max_inline_bytes = max_inline_bytes+ application.state.max_url_bytes = max_url_bytesapplication.state.allow_http_source = allow_http_sourceapplication.state.api_key = _api_keyapplication.state.language = language@@ -2361,6 +2387,7 @@ def create_app(application.state.url_timeout = url_timeoutapplication.state.allow_local_source = allow_local_sourceapplication.state.max_inline_bytes = max_inline_bytes+ application.state.max_url_bytes = max_url_bytesapplication.state.allow_http_source = allow_http_sourceapplication.state.api_key = _api_keyapplication.state.language = language@@ -2508,6 +2535,12 @@ def _build_server_log_config(log_level: str) -> dict[str, Any]:type=int,help=f"Maximum decoded bytes for inline sources (default: {_MAX_INLINE_BYTES_DEFAULT})",)+ "--max-url-bytes",+ default=_MAX_FILE_SIZE_BYTES_DEFAULT,+ type=int,+ help=f"Maximum bytes downloaded for url sources (default: {_MAX_FILE_SIZE_BYTES_DEFAULT})",+)@click.option("--allow-http-source",is_flag=True,@@ -2556,6 +2589,7 @@ def main(url_timeout: int,allow_local_source: bool,max_inline_bytes: int,+ max_url_bytes: int,allow_http_source: bool,language: str,disable_image_analysis: bool,@@ -2612,6 +2646,7 @@ def main(url_timeout=url_timeout,allow_local_source=allow_local_source,max_inline_bytes=max_inline_bytes,+ max_url_bytes=max_url_bytes,allow_http_source=allow_http_source,api_key=api_key,language=language,diff --git a/tests/unittest/test_flash_html.py b/tests/unittest/test_flash_html.pyindex 2fc49658..16d4b86d 100644--- a/tests/unittest/test_flash_html.py+++ b/tests/unittest/test_flash_html.py@@ -30,6 +30,26 @@ from mineru.render.structured_content import render_structured_contentfrom mineru.types import BlockType, ImageBlock, ImageBodyBlock, MiddleJson+class _StreamContext:+ """把已构造的 httpx.Response 包装成 cli.stream 返回的异步上下文。"""++ def __init__(self, response: httpx.Response) -> None:+ self._response = response++ async def __aenter__(self) -> httpx.Response:+ return self._response++ async def __aexit__(self, *args: object) -> None:+ return None+++def _url_stream_client(response: httpx.Response) -> AsyncMock:+ client = AsyncMock()+ client.__aenter__.return_value = client+ client.stream = lambda *_args, **_kwargs: _StreamContext(response)+ return client++def _all_raw_blocks(model_pages: list[list[dict[str, object]]]) -> list[dict[str, object]]:"""按页展开 raw model-list,方便断言 HTML 映射结果。"""return [block for page in model_pages for block in page]@@ -115,9 +135,7 @@ def test_html_parse_server_url_preserves_http_declared_charset(tmp_path: Path, mheaders={"Content-Type": "text/html; charset=shift_jis"},request=httpx.Request("GET", url),)- client = AsyncMock()- client.__aenter__.return_value = client- client.get.return_value = response+ client = _url_stream_client(response)monkeypatch.setattr(api_server, "httpx", SimpleNamespace(AsyncClient=lambda **_: client))file_store = FileStore(tmp_path / "api-files")request = CreateJobRequest.model_validate(@@ -165,9 +183,7 @@ def test_html_parse_server_url_accepts_extensionless_text_html(headers={"Content-Type": "text/html; charset=utf-8"},request=httpx.Request("GET", url),)- client = AsyncMock()- client.__aenter__.return_value = client- client.get.return_value = response+ client = _url_stream_client(response)monkeypatch.setattr(api_server, "httpx", SimpleNamespace(AsyncClient=lambda **_: client))file_store = FileStore(tmp_path / "api-files")request = CreateJobRequest.model_validate(@@ -231,9 +247,7 @@ def test_html_parse_server_no_flash_rejects_extensionless_text_html_after_fetch(headers={"Content-Type": "text/html; charset=utf-8"},request=httpx.Request("GET", url),)- client = AsyncMock()- client.__aenter__.return_value = client- client.get.return_value = response+ client = _url_stream_client(response)parse_async_mock = AsyncMock()monkeypatch.setattr(api_server, "httpx", SimpleNamespace(AsyncClient=lambda **_: client))monkeypatch.setattr(api_server, "parse_async", parse_async_mock)@@ -264,6 +278,43 @@ def test_html_parse_server_no_flash_rejects_extensionless_text_html_after_fetch(parse_async_mock.assert_not_awaited()+def test_url_source_aborts_when_exceeding_max_url_bytes(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ """验证 URL 源超过 max_url_bytes 时流式下载中止并使该文件失败。"""+ url = "https://example.com/big.html"+ response = httpx.Response(+ 200,+ content=b"x" * 1024,+ headers={"Content-Type": "text/html; charset=utf-8"},+ request=httpx.Request("GET", url),+ )+ client = _url_stream_client(response)+ monkeypatch.setattr(api_server, "httpx", SimpleNamespace(AsyncClient=lambda **_: client))+ file_store = FileStore(tmp_path / "api-files")+ request = CreateJobRequest.model_validate(+ {+ "files": [{"source": {"type": "url", "url": url}}],+ "tier": "standard",+ "output_formats": ["middle_json"],+ }+ )+ record = api_server.JobStore().create(request, file_store)++ asyncio.run(+ api_server._run_job(+ record,+ request,+ file_store,+ image_analysis=True,+ max_url_bytes=64,+ )+ )++ assert record.status == "failed"+ assert record.files[0].status == "failed"+ assert record.files[0].error is not None+ assert record.files[0].error.message == "url source exceeds max_url_bytes (64)"++def test_html_local_base_images_styles_and_escape_are_bounded(tmp_path: Path) -> None:"""验证本地 base、CSS、栅格图可读取,但父目录逃逸图片只保留说明。"""assets = tmp_path / "assets"diff --git a/tests/unittest/test_parser_api_contract.py b/tests/unittest/test_parser_api_contract.pyindex 488f7bd9..cf3cde9a 100644--- a/tests/unittest/test_parser_api_contract.py+++ b/tests/unittest/test_parser_api_contract.py@@ -2059,6 +2059,87 @@ def test_run_job_preserves_input_file_id_and_keeps_source_undownloadable(assert exc_info.value.status_code == 403+def test_create_job_request_rejects_more_than_max_files_per_job(tmp_path: Path) -> None:+ """超过广告的 max_files_per_job 时拒绝创建请求。"""+ source = str(tmp_path / "demo.pdf")+ entries = [{"source": {"type": "local", "path": source}}] * (api_server._MAX_FILES_PER_JOB_DEFAULT + 1)++ with pytest.raises(ValidationError):+ CreateJobRequest.model_validate({"files": entries, "tier": "flash"})+++def test_job_responses_report_resolved_access_level(+ monkeypatch: pytest.MonkeyPatch,+ tmp_path: Path,+) -> None:+ """job 创建与查询响应携带按 api_key 解析的 access_level,而不是固定 registered。"""++ async def fake_parse_async(*args: object, **kwargs: object) -> ParseResult:+ return ParseResult(middle_json=_full_middle_json(PageInfo(page_idx=0)))++ monkeypatch.setattr("mineru.parser.api_server.parse_async", fake_parse_async)+ source = tmp_path / "demo.html"+ source.write_text("<p>content</p>", encoding="utf-8")+ payload = {+ "files": [{"source": {"type": "local", "path": str(source)}}],+ "tier": "flash",+ "output_formats": ["middle_json"],+ }++ anonymous_app = create_app(upload_dir=str(tmp_path / "api-anonymous"), tier="flash", allow_local_source=True)+ with TestClient(anonymous_app) as client:+ created = client.post("/v1/parse/jobs", json=payload)+ assert created.status_code == 202+ assert created.json()["access_level"] == "anonymous"+ fetched = client.get(f"/v1/parse/jobs/{created.json()['job_id']}")+ assert fetched.status_code == 200+ assert fetched.json()["access_level"] == "anonymous"++ registered_app = create_app(+ upload_dir=str(tmp_path / "api-registered"), tier="flash", allow_local_source=True, api_key="secret-key"+ )+ with TestClient(registered_app) as client:+ created = client.post("/v1/parse/jobs", json=payload, headers={"Authorization": "Bearer secret-key"})+ assert created.status_code == 202+ assert created.json()["access_level"] == "registered"+ fetched = client.get(f"/v1/parse/jobs/{created.json()['job_id']}", headers={"Authorization": "Bearer secret-key"})+ assert fetched.status_code == 200+ assert fetched.json()["access_level"] == "registered"+++def test_usage_counts_completed_files_and_reports_configured_concurrency(tmp_path: Path) -> None:+ """files_processed 按完成的文件计数,max_concurrent_jobs 报告配置上限而非剩余信号量。"""+ file_store = FileStore(tmp_path / "api-files")+ store = api_server.JobStore(concurrency=2)++ def _request(*names: str) -> CreateJobRequest:+ return CreateJobRequest.model_validate(+ {+ "files": [{"source": {"type": "local", "path": str(tmp_path / name)}} for name in names],+ "tier": "flash",+ "output_formats": ["middle_json"],+ }+ )++ rec_a = store.create(_request("a1.html", "a2.html"), file_store)+ rec_a.files[0].status = "completed"+ rec_a.files[1].status = "failed"+ rec_a.status = "partial"+ rec_b = store.create(_request("b1.html"), file_store)+ rec_b.files[0].status = "completed"+ rec_b.status = "completed"++ async def _report_while_busy() -> "api_server.UsageResponse":+ await store._semaphore.acquire()+ return store.usage("anonymous")++ usage = asyncio.run(_report_while_busy())++ assert usage.current.jobs_created == 2+ assert usage.current.files_processed == 2+ assert usage.limits.max_concurrent_jobs == 2++@pytest.mark.parametrize(("request_tier", "response_tier"), [("standard", "standard"), (None, "flash")])def test_api_server_accepts_lightweight_job_without_requested_quality_runtime(monkeypatch: pytest.MonkeyPatch,@@ -2595,7 +2676,7 @@ def test_api_server_standard_jobs_use_requested_tier(tmp_path: Path, monkeypatchmax_inline_bytes=app.state.max_inline_bytes,allow_http_source=app.state.allow_http_source,)- return job_store.build_response(rec).model_dump(by_alias=True)+ return job_store.build_response(rec, access_level="anonymous").model_dump(by_alias=True)default_response = asyncio.run(run_request(