Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

fix(api): update expiration handling to support subsecond precision

myhloli0090767138ca
diff --git a/mineru/parser/api_server.py b/mineru/parser/api_server.py
index 511f29ab..43d59807 100644
--- a/mineru/parser/api_server.py
+++ b/mineru/parser/api_server.py
@@ -8,6 +8,7 @@ import base64
import hashlib
from importlib.resources import files as package_resources
import logging
+import math
import os
import pathlib
import re
@@ -345,7 +346,7 @@ class _FileRecord:
created_at: int
purpose: FilePurpose
sha256sum: str | None = None
- expires_at: int | None = None
+ expires_at: float | None = None
job_id: str | None = None
@@ -398,7 +399,7 @@ class FileStore:
def pin_cached_blob(self, sha: str) -> pathlib.Path | None:
"""哈希复用检查和取得传输引用保持原子性,过期视图不能被新请求复用。"""
with self._lock:
- now = int(time.time())
+ now = time.time()
if not any(
rec.sha256sum == sha and (rec.expires_at is None or rec.expires_at > now) for rec in self._files.values()
):
@@ -416,8 +417,8 @@ class FileStore:
if not self._pins[sha]:
self._pins.pop(sha)
- def finalize_outputs(self, job_id: str, finished: int) -> None:
- """产物从任务清理完成时计时,部分失败和取消产生的产物采用相同期限。"""
+ def finalize_outputs(self, job_id: str, finished: float) -> None:
+ """产物从任务清理完成时计时,保留亚秒精度,避免短保留期被向下取整。"""
with self._lock:
for rec in self._files.values():
if rec.job_id == job_id:
@@ -432,7 +433,7 @@ class FileStore:
if not self.retention_seconds:
return
with self._lock:
- now = int(time.time())
+ now = time.time()
for file_id, rec in tuple(self._files.items()):
if rec.expires_at is not None and rec.expires_at <= now:
self._files.pop(file_id)
@@ -654,7 +655,7 @@ class FileStore:
def get_file(self, file_id: str) -> _FileRecord:
with self._lock:
rec = self._files.get(file_id)
- if rec is None or (rec.expires_at is not None and rec.expires_at <= int(time.time())):
+ if rec is None or (rec.expires_at is not None and rec.expires_at <= time.time()):
raise_api_error(
404,
code="file_not_found",
@@ -664,11 +665,12 @@ class FileStore:
@staticmethod
def _make_file_object(rec: _FileRecord) -> FileObjectModel:
+ """公开时间戳保持整秒并向上取整,内部过期判断使用未取整的期限。"""
return FileObjectModel(
id=rec.id,
bytes=rec.bytes,
created_at=rec.created_at,
- expires_at=rec.expires_at,
+ expires_at=math.ceil(rec.expires_at) if rec.expires_at is not None else None,
file_name=rec.filename,
purpose=rec.purpose,
sha256sum=rec.sha256sum,
@@ -870,9 +872,12 @@ class ExecutionStore:
"""回收句柄并读取异常,覆盖任务首次执行前就被取消的路径。"""
self._tasks.pop(rec.id, None)
rec.finished_at = execution_timestamp()
+ if not rec.retention_held:
+ if rec.retention_epoch is None:
+ rec.retention_epoch = time.time()
+ if rec.file_store is not None:
+ rec.file_store.finalize_outputs(rec.id, rec.retention_epoch)
if rec.file_store is not None:
- if not rec.retention_held:
- rec.file_store.finalize_outputs(rec.id, int(rec.retention_epoch or time.time()))
rec.file_store.release_inputs(rec.input_blobs)
if not task.cancelled():
error = task.exception()
@@ -959,7 +964,7 @@ class ExecutionStore:
rec.retention_held = False
rec.retention_epoch = finished
if rec.file_store is not None:
- rec.file_store.finalize_outputs(job_id, int(finished))
+ rec.file_store.finalize_outputs(job_id, finished)
def get(self, job_id: str) -> _ExecutionRecord:
"""读取仍在保留期内的执行,未知标识沿用已有错误合同。"""
diff --git a/tests/unittest/test_resource_retention.py b/tests/unittest/test_resource_retention.py
index be5a87aa..df9f6500 100644
--- a/tests/unittest/test_resource_retention.py
+++ b/tests/unittest/test_resource_retention.py
@@ -129,6 +129,84 @@ def test_output_retention_starts_after_cleanup(tmp_path: Path, clock: list[int])
asyncio.run(scenario())
[email protected]("parent_held", [False, True])
[email protected]("retention_seconds", [0, 1])
+def test_output_retention_preserves_subsecond_deadline(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, parent_held: bool, retention_seconds: int
+) -> None:
+ """完成回调和父任务释放都保留完整 TTL;下载、缓存与回收共享亚秒边界。"""
+ now = [2_000_000_000.999]
+
+ def wall_time() -> float:
+ """固定完成时间的小数部分,稳定复现向下取整导致的提前过期。"""
+ return now[0]
+
+ monkeypatch.setattr(api_server.time, "time", wall_time)
+
+ async def scenario() -> None:
+ """沿真实执行完成流程检查短保留期及禁用回收策略。"""
+ files = FileStore(tmp_path / "files", retention_seconds=retention_seconds)
+ source, _ = _source(files, tmp_path)
+ jobs = ExecutionStore(retention_seconds=retention_seconds)
+ rec = jobs.create(execution_request(files, {"type": "file_id", "file_id": source}, tier="flash"), files)
+ rec.retention_held = parent_held
+
+ async def work() -> None:
+ """创建真实产物,交给执行回调或父任务启动保留期。"""
+ rec.result_file_id = files.create_file_for_output("out.md", b"result", job_id=rec.id)
+ rec.status = "completed"
+
+ try:
+ jobs.start_task(rec, work)
+ await jobs.wait_for_terminal(rec.id, 2)
+ output = rec.result_file_id
+ assert output is not None
+ if parent_held:
+ now[0] += 2
+ jobs.collect_expired()
+ files.collect_expired()
+ assert jobs.get(rec.id) is rec
+ assert files.read_file_data(output) == b"result"
+ jobs.release_task_result(rec.id, now[0])
+ finished = now[0]
+ metadata = files._make_file_object(files.get_file(output))
+ if not retention_seconds:
+ assert metadata.expires_at is None
+ now[0] += 100
+ jobs.collect_expired()
+ files.collect_expired()
+ assert jobs.get(rec.id) is rec
+ assert files.read_file_data(output) == b"result"
+ return
+
+ assert metadata.expires_at == int(finished) + 2
+ assert isinstance(metadata.model_dump()["expires_at"], int)
+ for elapsed in (0.002, 0.999):
+ now[0] = finished + elapsed
+ assert jobs.get(rec.id) is rec
+ files.collect_expired()
+ assert files.read_file_data(output) == b"result"
+ sha = files.get_file(output).sha256sum
+ assert sha is not None
+ assert files.pin_cached_blob(sha) is not None
+ files.unpin_cached_blob(sha)
+
+ now[0] = finished + retention_seconds
+ with pytest.raises(ApiServerError) as exc:
+ files.get_file(output)
+ assert exc.value.status_code == 404
+ assert files.pin_cached_blob(sha) is None
+ jobs.collect_expired()
+ files.collect_expired()
+ assert rec.id not in jobs._executions
+ assert output not in files._files
+ assert not files._blob_abs(sha).exists()
+ finally:
+ await jobs.shutdown()
+
+ asyncio.run(scenario())
+
+
def test_cached_preflight_pins_and_last_reference_delete(tmp_path: Path, clock: list[int]) -> None:
"""删除最后文件与哈希复用交错时,预检租约保护路径直到提交或拒绝完成。"""
store = FileStore(tmp_path / "files", retention_seconds=10)
diff --git a/tests/unittest/test_task_api.py b/tests/unittest/test_task_api.py
index e0fc7c0f..46635d35 100644
--- a/tests/unittest/test_task_api.py
+++ b/tests/unittest/test_task_api.py
@@ -1001,9 +1001,18 @@ def test_expired_upload_rejects_before_reading_body(service: tuple[TestClient, F
assert _job_count(app) == 0
-def test_completed_child_retention_starts_after_parent_finishes(service: tuple[TestClient, FastAPI]) -> None:
+def test_completed_child_retention_starts_after_parent_finishes(
+ service: tuple[TestClient, FastAPI], monkeypatch: pytest.MonkeyPatch
+) -> None:
"""子结果不能在父任务仍等待上传时过期,父任务完成后才开始统一保留期。"""
client, app = service
+ now = [int(time.time()) + 0.999]
+
+ def wall_time() -> float:
+ """固定墙上时间到整秒边界前,事件循环仍使用真实单调时钟调度。"""
+ return now[0]
+
+ monkeypatch.setattr(time, "time", wall_time)
app.state.task_store.retention_seconds = 1
if hasattr(app.state, "registry"):
app.state.registry.retention_seconds = 1
@@ -1015,7 +1024,7 @@ def test_completed_child_retention_starts_after_parent_finishes(service: tuple[T
assert client.put(pending_files(task)[0].get("upload").get("url"), content=HTML).status_code == 200
ready = _file_done(client, task, 0)
result_url = ready["files"][0].get("result_url")
- time.sleep(1.05)
+ now[0] += 2
for worker in _execution_apps(app):
worker.state.execution_store.collect_expired()
worker.state.file_store.collect_expired()
@@ -1030,8 +1039,10 @@ def test_completed_child_retention_starts_after_parent_finishes(service: tuple[T
await record.operation
client.portal.call(finish_retention)
+ assert record.retention_epoch is not None
+ now[0] = record.retention_epoch + 0.002
assert client.get(result_url).status_code == 200
- time.sleep(1.05)
+ now[0] = record.retention_epoch + 1
assert client.get(task_url(client, task)).status_code == 404
assert client.get(result_url).status_code == 404
@@ -1128,9 +1139,7 @@ def test_router_ends_local_result_lease_even_if_worker_rejects_release(tmp_path:
monkeypatch.setattr(transport, "handle_async_request", reject_release)
client.portal.call(maintain_task_result_leases, app.state.registry, app.state.worker_pool)
assert all(not route.lease.held for route in app.state.registry.list("job", include_expired=True))
- assert all(
- not route.lease.held for route in app.state.registry.list("file", include_expired=True)
- )
+ assert all(not route.lease.held for route in app.state.registry.list("file", include_expired=True))
assert result["files"][0].get("result_url")