Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
Release lost upstream tasks and retry retained copy cleanup; Refs #5495
myhloli143077ef273f
diff --git a/mineru/kit/router/app.py b/mineru/kit/router/app.pyindex 88a8c3c2..26451972 100644--- a/mineru/kit/router/app.py+++ b/mineru/kit/router/app.py@@ -335,14 +335,42 @@ async def _finalize_terminal_job(route.metadata.pop("upstream_headers", None)+def _mark_missing_job(route: ResourceRoute, pool: WorkerPool) -> dict[str, Any]:+ """上游已丢失任务时保存失败快照并归还负载,避免重启后永久占用名额。"""+ payload = route.metadata.get("payload") or {}+ if payload.get("status") not in {"completed", "partial", "failed", "canceled"}:+ payload["status"] = "failed"+ payload["finished_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())+ for file in payload.get("files") or []:+ if file.get("status") != "completed":+ file["status"] = "failed"+ file["error"] = {+ "type": "engine_error",+ "code": "upstream_job_lost",+ "message": "Upstream task no longer exists",+ }+ progress = payload.get("progress")+ if isinstance(progress, dict):+ progress["failed"] = progress.get("total", 0) - progress.get("completed", 0)+ route.metadata["payload"] = payload+ route.metadata["upstream_lost"] = True+ release_job_load(route, pool)+ return payload++async def _reconcile_jobs_once(registry: ResourceRegistry, pool: WorkerPool) -> None:"""后台查询活动或待清理 Job,并统一执行终态资源收敛。"""for route in registry.list("job", include_expired=True):if not route.metadata.get("active_counted") and not route.metadata.get("copied_inputs"):continue+ headers = dict(route.metadata.get("upstream_headers") or {})+ if route.metadata.get("upstream_lost"):+ await _finalize_terminal_job(+ route, route.metadata["payload"], request=None, pool=pool, registry=registry, headers=headers+ )+ continueworker = pool.get(route.worker_id)generation = worker.generation- headers = dict(route.metadata.get("upstream_headers") or {})try:response = await request_upstream(pool,@@ -354,8 +382,8 @@ async def _reconcile_jobs_once(registry: ResourceRegistry, pool: WorkerPool) ->if worker.generation != generation or route.metadata.get("upstream_lost"):continueif response.status_code >= 400:- cached = route.metadata.get("payload") or {}- if response.status_code == 404 and cached.get("status") in {"completed", "partial", "failed", "canceled"}:+ if response.status_code == 404:+ cached = _mark_missing_job(route, pool)await _finalize_terminal_job(route, cached, request=None, pool=pool, registry=registry, headers=headers)continuepayload = json_or_error(response)@@ -973,6 +1001,10 @@ def create_app(upstream = await request_upstream(pool, worker, "GET", f"/v1/parse/jobs/{route.upstream_id}", request=request)if worker.generation != generation or route.metadata.get("upstream_lost"):return JSONResponse(route.metadata["payload"])+ if upstream.status_code == 404:+ failed = _mark_missing_job(route, pool)+ await _finalize_terminal_job(route, failed, request=request, pool=pool, registry=registry)+ return JSONResponse(failed)result = _successful_json(upstream)if isinstance(result, Response):return resultdiff --git a/tests/unittest/test_resource_retention.py b/tests/unittest/test_resource_retention.pyindex 64979685..60e185ee 100644--- a/tests/unittest/test_resource_retention.py+++ b/tests/unittest/test_resource_retention.py@@ -313,3 +313,58 @@ def test_kit_cli_forwards_explicit_retention(monkeypatch: pytest.MonkeyPatch) ->assert response.exit_code == 0, response.outputargs = forwarded.call_args.kwargs["args"]assert args[args.index("--retention-seconds") + 1] == "0"+++def test_external_worker_lost_task_releases_load_and_keeps_snapshot() -> None:+ """外部 worker 重启丢失内存任务时,Router 不得永久占用名额或丢弃失败历史。"""+ from test_v1_router import _FakeV1Upstream, _make_router+ from mineru.kit.router.app import _reconcile_jobs_once++ upstream = _FakeV1Upstream("worker-a", ("standard",))+ app, _ = _make_router(upstream)+ with TestClient(app) as client:+ created = client.post(+ "/v1/parse/jobs",+ json={"tier": "standard", "files": [{"source": {"type": "inline", "name": "demo.pdf", "data": "cGRm"}}]},+ )+ assert created.status_code == 202+ job_id = created.json()["job_id"]+ route = app.state.registry.get("job", job_id)+ worker = app.state.worker_pool.get(route.worker_id)+ assert worker.active_jobs == 1+ upstream.jobs.clear()+ client.portal.call(_reconcile_jobs_once, app.state.registry, app.state.worker_pool)+ assert worker.active_jobs == 0+ response = client.get(f"/v1/parse/jobs/{job_id}")+ assert response.status_code == 200 and response.json()["status"] == "failed"+ assert response.json()["files"][0]["error"]["code"] == "upstream_job_lost"+++def test_lost_task_retries_failed_cross_worker_copy_cleanup() -> None:+ """上游任务丢失后副本删除失败一次,后续后台轮次仍继续回收并释放 alias。"""+ from test_v1_router import _FakeV1Upstream, _make_router, _upload_file+ from mineru.kit.router.app import _reconcile_jobs_once++ first = _FakeV1Upstream("worker-a", ("standard",))+ second = _FakeV1Upstream("worker-b", ("advanced",))+ app, _ = _make_router(first, second)+ headers = {"authorization": "Bearer lost-copy"}+ with TestClient(app) as client:+ source = _upload_file(client, token="lost-copy", content=b"pdf")+ response = client.post(+ "/v1/parse/jobs",+ headers=headers,+ json={"tier": "advanced", "files": [{"source": {"type": "file_id", "file_id": source}}]},+ )+ assert response.status_code == 202+ route = app.state.registry.get("job", response.json()["job_id"])+ copied = route.metadata["copied_inputs"][0]+ second.jobs.clear()+ second.fail_file_delete_once.add(copied.upstream_file_id)+ client.portal.call(_reconcile_jobs_once, app.state.registry, app.state.worker_pool)+ assert route.metadata["upstream_lost"] and route.metadata["copied_inputs"]+ assert second.files.get(copied.upstream_file_id) is not None+ client.portal.call(_reconcile_jobs_once, app.state.registry, app.state.worker_pool)+ assert not route.metadata.get("copied_inputs")+ assert copied.upstream_file_id not in second.files+ assert app.state.registry.find_upstream("file", copied.owner_scope, copied.worker_id, copied.upstream_file_id) is None