Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
fix(api): update expiration handling to support subsecond precision
myhloli0090767138ca
diff --git a/mineru/parser/api_server.py b/mineru/parser/api_server.pyindex 511f29ab..43d59807 100644--- a/mineru/parser/api_server.py+++ b/mineru/parser/api_server.py@@ -8,6 +8,7 @@ import base64import hashlibfrom importlib.resources import files as package_resourcesimport logging+import mathimport osimport pathlibimport re@@ -345,7 +346,7 @@ class _FileRecord:created_at: intpurpose: FilePurposesha256sum: str | None = None- expires_at: int | None = None+ expires_at: float | None = Nonejob_id: str | None = None@@ -398,7 +399,7 @@ class FileStore:def pin_cached_blob(self, sha: str) -> pathlib.Path | None:"""哈希复用检查和取得传输引用保持原子性,过期视图不能被新请求复用。"""with self._lock:- now = int(time.time())+ now = time.time()if not any(rec.sha256sum == sha and (rec.expires_at is None or rec.expires_at > now) for rec in self._files.values()):@@ -416,8 +417,8 @@ class FileStore:if not self._pins[sha]:self._pins.pop(sha)- def finalize_outputs(self, job_id: str, finished: int) -> None:- """产物从任务清理完成时计时,部分失败和取消产生的产物采用相同期限。"""+ def finalize_outputs(self, job_id: str, finished: float) -> None:+ """产物从任务清理完成时计时,保留亚秒精度,避免短保留期被向下取整。"""with self._lock:for rec in self._files.values():if rec.job_id == job_id:@@ -432,7 +433,7 @@ class FileStore:if not self.retention_seconds:returnwith self._lock:- now = int(time.time())+ now = time.time()for file_id, rec in tuple(self._files.items()):if rec.expires_at is not None and rec.expires_at <= now:self._files.pop(file_id)@@ -654,7 +655,7 @@ class FileStore:def get_file(self, file_id: str) -> _FileRecord:with self._lock:rec = self._files.get(file_id)- if rec is None or (rec.expires_at is not None and rec.expires_at <= int(time.time())):+ if rec is None or (rec.expires_at is not None and rec.expires_at <= time.time()):raise_api_error(404,code="file_not_found",@@ -664,11 +665,12 @@ class FileStore:@staticmethoddef _make_file_object(rec: _FileRecord) -> FileObjectModel:+ """公开时间戳保持整秒并向上取整,内部过期判断使用未取整的期限。"""return FileObjectModel(id=rec.id,bytes=rec.bytes,created_at=rec.created_at,- expires_at=rec.expires_at,+ expires_at=math.ceil(rec.expires_at) if rec.expires_at is not None else None,file_name=rec.filename,purpose=rec.purpose,sha256sum=rec.sha256sum,@@ -870,9 +872,12 @@ class ExecutionStore:"""回收句柄并读取异常,覆盖任务首次执行前就被取消的路径。"""self._tasks.pop(rec.id, None)rec.finished_at = execution_timestamp()+ if not rec.retention_held:+ if rec.retention_epoch is None:+ rec.retention_epoch = time.time()+ if rec.file_store is not None:+ rec.file_store.finalize_outputs(rec.id, rec.retention_epoch)if rec.file_store is not None:- if not rec.retention_held:- rec.file_store.finalize_outputs(rec.id, int(rec.retention_epoch or time.time()))rec.file_store.release_inputs(rec.input_blobs)if not task.cancelled():error = task.exception()@@ -959,7 +964,7 @@ class ExecutionStore:rec.retention_held = Falserec.retention_epoch = finishedif rec.file_store is not None:- rec.file_store.finalize_outputs(job_id, int(finished))+ rec.file_store.finalize_outputs(job_id, finished)def get(self, job_id: str) -> _ExecutionRecord:"""读取仍在保留期内的执行,未知标识沿用已有错误合同。"""diff --git a/tests/unittest/test_resource_retention.py b/tests/unittest/test_resource_retention.pyindex be5a87aa..df9f6500 100644--- a/tests/unittest/test_resource_retention.py+++ b/tests/unittest/test_resource_retention.py@@ -129,6 +129,84 @@ def test_output_retention_starts_after_cleanup(tmp_path: Path, clock: list[int])asyncio.run(scenario())[email protected]("parent_held", [False, True])[email protected]("retention_seconds", [0, 1])+def test_output_retention_preserves_subsecond_deadline(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, parent_held: bool, retention_seconds: int+) -> None:+ """完成回调和父任务释放都保留完整 TTL;下载、缓存与回收共享亚秒边界。"""+ now = [2_000_000_000.999]++ def wall_time() -> float:+ """固定完成时间的小数部分,稳定复现向下取整导致的提前过期。"""+ return now[0]++ monkeypatch.setattr(api_server.time, "time", wall_time)++ async def scenario() -> None:+ """沿真实执行完成流程检查短保留期及禁用回收策略。"""+ files = FileStore(tmp_path / "files", retention_seconds=retention_seconds)+ source, _ = _source(files, tmp_path)+ jobs = ExecutionStore(retention_seconds=retention_seconds)+ rec = jobs.create(execution_request(files, {"type": "file_id", "file_id": source}, tier="flash"), files)+ rec.retention_held = parent_held++ async def work() -> None:+ """创建真实产物,交给执行回调或父任务启动保留期。"""+ rec.result_file_id = files.create_file_for_output("out.md", b"result", job_id=rec.id)+ rec.status = "completed"++ try:+ jobs.start_task(rec, work)+ await jobs.wait_for_terminal(rec.id, 2)+ output = rec.result_file_id+ assert output is not None+ if parent_held:+ now[0] += 2+ jobs.collect_expired()+ files.collect_expired()+ assert jobs.get(rec.id) is rec+ assert files.read_file_data(output) == b"result"+ jobs.release_task_result(rec.id, now[0])+ finished = now[0]+ metadata = files._make_file_object(files.get_file(output))+ if not retention_seconds:+ assert metadata.expires_at is None+ now[0] += 100+ jobs.collect_expired()+ files.collect_expired()+ assert jobs.get(rec.id) is rec+ assert files.read_file_data(output) == b"result"+ return++ assert metadata.expires_at == int(finished) + 2+ assert isinstance(metadata.model_dump()["expires_at"], int)+ for elapsed in (0.002, 0.999):+ now[0] = finished + elapsed+ assert jobs.get(rec.id) is rec+ files.collect_expired()+ assert files.read_file_data(output) == b"result"+ sha = files.get_file(output).sha256sum+ assert sha is not None+ assert files.pin_cached_blob(sha) is not None+ files.unpin_cached_blob(sha)++ now[0] = finished + retention_seconds+ with pytest.raises(ApiServerError) as exc:+ files.get_file(output)+ assert exc.value.status_code == 404+ assert files.pin_cached_blob(sha) is None+ jobs.collect_expired()+ files.collect_expired()+ assert rec.id not in jobs._executions+ assert output not in files._files+ assert not files._blob_abs(sha).exists()+ finally:+ await jobs.shutdown()++ asyncio.run(scenario())++def test_cached_preflight_pins_and_last_reference_delete(tmp_path: Path, clock: list[int]) -> None:"""删除最后文件与哈希复用交错时,预检租约保护路径直到提交或拒绝完成。"""store = FileStore(tmp_path / "files", retention_seconds=10)diff --git a/tests/unittest/test_task_api.py b/tests/unittest/test_task_api.pyindex e0fc7c0f..46635d35 100644--- a/tests/unittest/test_task_api.py+++ b/tests/unittest/test_task_api.py@@ -1001,9 +1001,18 @@ def test_expired_upload_rejects_before_reading_body(service: tuple[TestClient, Fassert _job_count(app) == 0-def test_completed_child_retention_starts_after_parent_finishes(service: tuple[TestClient, FastAPI]) -> None:+def test_completed_child_retention_starts_after_parent_finishes(+ service: tuple[TestClient, FastAPI], monkeypatch: pytest.MonkeyPatch+) -> None:"""子结果不能在父任务仍等待上传时过期,父任务完成后才开始统一保留期。"""client, app = service+ now = [int(time.time()) + 0.999]++ def wall_time() -> float:+ """固定墙上时间到整秒边界前,事件循环仍使用真实单调时钟调度。"""+ return now[0]++ monkeypatch.setattr(time, "time", wall_time)app.state.task_store.retention_seconds = 1if hasattr(app.state, "registry"):app.state.registry.retention_seconds = 1@@ -1015,7 +1024,7 @@ def test_completed_child_retention_starts_after_parent_finishes(service: tuple[Tassert client.put(pending_files(task)[0].get("upload").get("url"), content=HTML).status_code == 200ready = _file_done(client, task, 0)result_url = ready["files"][0].get("result_url")- time.sleep(1.05)+ now[0] += 2for worker in _execution_apps(app):worker.state.execution_store.collect_expired()worker.state.file_store.collect_expired()@@ -1030,8 +1039,10 @@ def test_completed_child_retention_starts_after_parent_finishes(service: tuple[Tawait record.operationclient.portal.call(finish_retention)+ assert record.retention_epoch is not None+ now[0] = record.retention_epoch + 0.002assert client.get(result_url).status_code == 200- time.sleep(1.05)+ now[0] = record.retention_epoch + 1assert client.get(task_url(client, task)).status_code == 404assert client.get(result_url).status_code == 404@@ -1128,9 +1139,7 @@ def test_router_ends_local_result_lease_even_if_worker_rejects_release(tmp_path:monkeypatch.setattr(transport, "handle_async_request", reject_release)client.portal.call(maintain_task_result_leases, app.state.registry, app.state.worker_pool)assert all(not route.lease.held for route in app.state.registry.list("job", include_expired=True))- assert all(- not route.lease.held for route in app.state.registry.list("file", include_expired=True)- )+ assert all(not route.lease.held for route in app.state.registry.list("file", include_expired=True))assert result["files"][0].get("result_url")