Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

Release lost upstream tasks and retry retained copy cleanup; Refs #5495

myhloli143077ef273f
diff --git a/mineru/kit/router/app.py b/mineru/kit/router/app.py
index 88a8c3c2..26451972 100644
--- a/mineru/kit/router/app.py
+++ b/mineru/kit/router/app.py
@@ -335,14 +335,42 @@ async def _finalize_terminal_job(
route.metadata.pop("upstream_headers", None)
+def _mark_missing_job(route: ResourceRoute, pool: WorkerPool) -> dict[str, Any]:
+ """上游已丢失任务时保存失败快照并归还负载,避免重启后永久占用名额。"""
+ payload = route.metadata.get("payload") or {}
+ if payload.get("status") not in {"completed", "partial", "failed", "canceled"}:
+ payload["status"] = "failed"
+ payload["finished_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+ for file in payload.get("files") or []:
+ if file.get("status") != "completed":
+ file["status"] = "failed"
+ file["error"] = {
+ "type": "engine_error",
+ "code": "upstream_job_lost",
+ "message": "Upstream task no longer exists",
+ }
+ progress = payload.get("progress")
+ if isinstance(progress, dict):
+ progress["failed"] = progress.get("total", 0) - progress.get("completed", 0)
+ route.metadata["payload"] = payload
+ route.metadata["upstream_lost"] = True
+ release_job_load(route, pool)
+ return payload
+
+
async def _reconcile_jobs_once(registry: ResourceRegistry, pool: WorkerPool) -> None:
"""后台查询活动或待清理 Job,并统一执行终态资源收敛。"""
for route in registry.list("job", include_expired=True):
if not route.metadata.get("active_counted") and not route.metadata.get("copied_inputs"):
continue
+ headers = dict(route.metadata.get("upstream_headers") or {})
+ if route.metadata.get("upstream_lost"):
+ await _finalize_terminal_job(
+ route, route.metadata["payload"], request=None, pool=pool, registry=registry, headers=headers
+ )
+ continue
worker = pool.get(route.worker_id)
generation = worker.generation
- headers = dict(route.metadata.get("upstream_headers") or {})
try:
response = await request_upstream(
pool,
@@ -354,8 +382,8 @@ async def _reconcile_jobs_once(registry: ResourceRegistry, pool: WorkerPool) ->
if worker.generation != generation or route.metadata.get("upstream_lost"):
continue
if response.status_code >= 400:
- cached = route.metadata.get("payload") or {}
- if response.status_code == 404 and cached.get("status") in {"completed", "partial", "failed", "canceled"}:
+ if response.status_code == 404:
+ cached = _mark_missing_job(route, pool)
await _finalize_terminal_job(route, cached, request=None, pool=pool, registry=registry, headers=headers)
continue
payload = json_or_error(response)
@@ -973,6 +1001,10 @@ def create_app(
upstream = await request_upstream(pool, worker, "GET", f"/v1/parse/jobs/{route.upstream_id}", request=request)
if worker.generation != generation or route.metadata.get("upstream_lost"):
return JSONResponse(route.metadata["payload"])
+ if upstream.status_code == 404:
+ failed = _mark_missing_job(route, pool)
+ await _finalize_terminal_job(route, failed, request=request, pool=pool, registry=registry)
+ return JSONResponse(failed)
result = _successful_json(upstream)
if isinstance(result, Response):
return result
diff --git a/tests/unittest/test_resource_retention.py b/tests/unittest/test_resource_retention.py
index 64979685..60e185ee 100644
--- a/tests/unittest/test_resource_retention.py
+++ b/tests/unittest/test_resource_retention.py
@@ -313,3 +313,58 @@ def test_kit_cli_forwards_explicit_retention(monkeypatch: pytest.MonkeyPatch) ->
assert response.exit_code == 0, response.output
args = forwarded.call_args.kwargs["args"]
assert args[args.index("--retention-seconds") + 1] == "0"
+
+
+def test_external_worker_lost_task_releases_load_and_keeps_snapshot() -> None:
+ """外部 worker 重启丢失内存任务时,Router 不得永久占用名额或丢弃失败历史。"""
+ from test_v1_router import _FakeV1Upstream, _make_router
+ from mineru.kit.router.app import _reconcile_jobs_once
+
+ upstream = _FakeV1Upstream("worker-a", ("standard",))
+ app, _ = _make_router(upstream)
+ with TestClient(app) as client:
+ created = client.post(
+ "/v1/parse/jobs",
+ json={"tier": "standard", "files": [{"source": {"type": "inline", "name": "demo.pdf", "data": "cGRm"}}]},
+ )
+ assert created.status_code == 202
+ job_id = created.json()["job_id"]
+ route = app.state.registry.get("job", job_id)
+ worker = app.state.worker_pool.get(route.worker_id)
+ assert worker.active_jobs == 1
+ upstream.jobs.clear()
+ client.portal.call(_reconcile_jobs_once, app.state.registry, app.state.worker_pool)
+ assert worker.active_jobs == 0
+ response = client.get(f"/v1/parse/jobs/{job_id}")
+ assert response.status_code == 200 and response.json()["status"] == "failed"
+ assert response.json()["files"][0]["error"]["code"] == "upstream_job_lost"
+
+
+def test_lost_task_retries_failed_cross_worker_copy_cleanup() -> None:
+ """上游任务丢失后副本删除失败一次,后续后台轮次仍继续回收并释放 alias。"""
+ from test_v1_router import _FakeV1Upstream, _make_router, _upload_file
+ from mineru.kit.router.app import _reconcile_jobs_once
+
+ first = _FakeV1Upstream("worker-a", ("standard",))
+ second = _FakeV1Upstream("worker-b", ("advanced",))
+ app, _ = _make_router(first, second)
+ headers = {"authorization": "Bearer lost-copy"}
+ with TestClient(app) as client:
+ source = _upload_file(client, token="lost-copy", content=b"pdf")
+ response = client.post(
+ "/v1/parse/jobs",
+ headers=headers,
+ json={"tier": "advanced", "files": [{"source": {"type": "file_id", "file_id": source}}]},
+ )
+ assert response.status_code == 202
+ route = app.state.registry.get("job", response.json()["job_id"])
+ copied = route.metadata["copied_inputs"][0]
+ second.jobs.clear()
+ second.fail_file_delete_once.add(copied.upstream_file_id)
+ client.portal.call(_reconcile_jobs_once, app.state.registry, app.state.worker_pool)
+ assert route.metadata["upstream_lost"] and route.metadata["copied_inputs"]
+ assert second.files.get(copied.upstream_file_id) is not None
+ client.portal.call(_reconcile_jobs_once, app.state.registry, app.state.worker_pool)
+ assert not route.metadata.get("copied_inputs")
+ assert copied.upstream_file_id not in second.files
+ assert app.state.registry.find_upstream("file", copied.owner_scope, copied.worker_id, copied.upstream_file_id) is None