Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

Reject stale worker responses and classify fatal preload failures; Refs #5188

myhloli6a5e7b2d9f9c
diff --git a/mineru/kit/router/app.py b/mineru/kit/router/app.py
index 95f255d4..88a8c3c2 100644
--- a/mineru/kit/router/app.py
+++ b/mineru/kit/router/app.py
@@ -268,6 +268,7 @@ async def _hydrate_job_output_files(
if route is None or isinstance(route.metadata.get("payload"), dict):
continue
worker = pool.get(route.worker_id)
+ generation = worker.generation
try:
response = await request_upstream(
pool,
@@ -291,6 +292,8 @@ async def _hydrate_job_output_files(
"message": exc.message,
}
continue
+ if worker.generation != generation or registry.find("file", public_file_id) is not route:
+ continue
rewrite_file_payload(metadata, worker, registry, route.owner_scope)
route.metadata.pop("hydration_error", None)
@@ -338,6 +341,7 @@ async def _reconcile_jobs_once(registry: ResourceRegistry, pool: WorkerPool) ->
if not route.metadata.get("active_counted") and not route.metadata.get("copied_inputs"):
continue
worker = pool.get(route.worker_id)
+ generation = worker.generation
headers = dict(route.metadata.get("upstream_headers") or {})
try:
response = await request_upstream(
@@ -347,6 +351,8 @@ async def _reconcile_jobs_once(registry: ResourceRegistry, pool: WorkerPool) ->
f"/v1/parse/jobs/{route.upstream_id}",
headers=headers,
)
+ if worker.generation != generation or route.metadata.get("upstream_lost"):
+ continue
if response.status_code >= 400:
cached = route.metadata.get("payload") or {}
if response.status_code == 404 and cached.get("status") in {"completed", "partial", "failed", "canceled"}:
@@ -963,7 +969,10 @@ def create_app(
if route.metadata.get("upstream_lost"):
return JSONResponse(route.metadata["payload"])
worker = pool.get(route.worker_id)
+ generation = worker.generation
upstream = await request_upstream(pool, worker, "GET", f"/v1/parse/jobs/{route.upstream_id}", request=request)
+ if worker.generation != generation or route.metadata.get("upstream_lost"):
+ return JSONResponse(route.metadata["payload"])
result = _successful_json(upstream)
if isinstance(result, Response):
return result
@@ -975,6 +984,8 @@ def create_app(
pool=pool,
registry=registry,
)
+ if worker.generation != generation or route.metadata.get("upstream_lost"):
+ return JSONResponse(route.metadata["payload"])
return JSONResponse(rewritten, status_code=upstream.status_code)
@application.delete("/v1/parse/jobs/{job_id}")
diff --git a/mineru/kit/router/proxy.py b/mineru/kit/router/proxy.py
index 385cf20e..3a701458 100644
--- a/mineru/kit/router/proxy.py
+++ b/mineru/kit/router/proxy.py
@@ -80,6 +80,7 @@ async def request_upstream(
follow_redirects: bool = False,
) -> httpx.Response:
"""执行普通 upstream 请求,并把连接失败与超时转成稳定 Router 错误。"""
+ generation = worker.generation
outgoing_headers = forwarded_headers(request) if request is not None else {}
outgoing_headers.pop("expect", None)
if json_body is not None:
@@ -94,7 +95,7 @@ async def request_upstream(
headers=outgoing_headers,
follow_redirects=follow_redirects,
)
- if response.status_code == 503:
+ if response.status_code == 503 and worker.generation == generation:
try:
if (response.json().get("error") or {}).get("code") == "engine_dead":
pool.notify_engine_dead(worker)
diff --git a/mineru/parser/api_server.py b/mineru/parser/api_server.py
index c09558e3..32239c1b 100644
--- a/mineru/parser/api_server.py
+++ b/mineru/parser/api_server.py
@@ -2469,6 +2469,8 @@ _MODEL_PRELOAD_DEVICE_ERROR_PATTERNS = (
def _classify_model_preload_error(exc: Exception) -> tuple[str, str]:
message = str(exc).strip() or type(exc).__name__
+ if is_fatal_engine_error(exc):
+ return "engine_dead", message
if isinstance(exc, (ModuleNotFoundError, ImportError)):
return "model_preload_dependency_missing", message
if isinstance(exc, FileNotFoundError):
@@ -2654,6 +2656,8 @@ def create_app(
)
except Exception as exc:
error_code, error_msg = _classify_model_preload_error(exc)
+ if error_code == "engine_dead":
+ job_store.fail_engine(error_msg)
application.state.model_preload_error = _ModelPreloadError(
code=error_code,
message=error_msg,
diff --git a/tests/unittest/test_engine_dead_lifecycle.py b/tests/unittest/test_engine_dead_lifecycle.py
index a2e99a4c..2c5f2437 100644
--- a/tests/unittest/test_engine_dead_lifecycle.py
+++ b/tests/unittest/test_engine_dead_lifecycle.py
@@ -180,3 +180,41 @@ def test_old_generation_failure_snapshot_remains_queryable() -> None:
assert response.json()["status"] == "failed"
assert response.json()["files"][0]["error"]["code"] == "engine_dead"
assert client.delete(f"/v1/parse/jobs/{route.public_id}").status_code == 409
+
+
+def test_old_generation_engine_dead_response_does_not_kill_replacement() -> None:
+ """旧请求迟到报告死亡时,不得把健康的新 generation 再次标记死亡。"""
+ from mineru.kit.router.proxy import request_upstream
+
+ async def scenario() -> None:
+ """响应处理之前模拟替换已发布,验证 notify 的 generation 条件。"""
+ state = workers.WorkerState(
+ "local",
+ "http://old",
+ "local",
+ generation=1,
+ healthy=True,
+ local_worker=SimpleNamespace(process=SimpleNamespace(poll=lambda: None)),
+ )
+
+ def handle(request: httpx.Request) -> httpx.Response:
+ """旧网络请求返回前发布新代,随后返回旧代的 engine_dead。"""
+ state.generation = 2
+ state.base_url = "http://new"
+ state.healthy = True
+ return httpx.Response(503, json={"error": {"code": "engine_dead"}})
+
+ pool = workers.WorkerPool(workers.RouterSettings(local_gpus="none"), transport=httpx.MockTransport(handle))
+ try:
+ response = await request_upstream(pool, state, "GET", "/v1/health")
+ assert response.status_code == 503
+ assert state.healthy and not state.restart_requested
+ finally:
+ await pool.client.aclose()
+
+ asyncio.run(scenario())
+
+
+def test_fatal_preload_uses_engine_dead_code() -> None:
+ """预加载期间死亡与推理期间死亡共享同一稳定错误码。"""
+ assert api_server._classify_model_preload_error(EngineDeadError("dead")) == ("engine_dead", "dead")