Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

12.7 KB
Raw Blame
"""验证独立响应、服务端公共文件 ID,以及取消后可直接使用的逐文件终态。"""

from __future__ import annotations

import hashlib
import io
import time
import zipfile
from pathlib import Path
from typing import Any

import httpx
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from test_task_api import HTML, SECOND
from test_task_api import service as _service_fixture
from mineru.parser.api_server import FileStore
from mineru.kit.router.resources import ResourceRegistry


service = _service_fixture


def _create(client: TestClient, contents: list[bytes]) -> dict[str, Any]:
    """用同名输入验证服务端按请求顺序分配 ID,不依赖文件名唯一性。"""
    response = client.post(
        "/v1/tasks",
        json={
            "files": [
                {"file_name": "same.html", "bytes": len(content), "sha256sum": hashlib.sha256(content).hexdigest()}
                for content in contents
            ]
        },
    )
    assert response.status_code == 202, response.text
    return response.json()


def _wait_files(client: TestClient, created: dict[str, Any], count: int) -> dict[str, Any]:
    """等待指定数量的成功文件,允许同一父任务仍有尚未上传的输入。"""
    deadline = time.monotonic() + 10
    while time.monotonic() < deadline:
        response = client.get(f"/v1/tasks/{created['task_id']}")
        assert response.status_code == 200, response.text
        task = response.json()
        if sum(item["status"] == "completed" for item in task["files"]) == count:
            return task
        assert all(item["status"] != "failed" for item in task["files"]), task
        time.sleep(0.01)
    raise AssertionError(f"Files did not complete: {task}")


@pytest.mark.parametrize("field", ["client_file_id", "file_id"])
@pytest.mark.parametrize("value", [None, "file-user"])
def test_creation_rejects_all_caller_file_ids(service: tuple[TestClient, FastAPI], field: str, value: str | None) -> None:
    """客户端不能通过新旧字段注入标识,空值也不能绕过请求模型。"""
    client, app = service
    response = client.post("/v1/tasks", json={"files": [{"file_name": "x.html", "bytes": 1, field: value}]})
    assert response.status_code == 400
    assert response.json()["error"]["code"] == "invalid_request"
    assert not app.state.task_store._records


def test_split_responses_keep_ids_and_cancellation_results(service: tuple[TestClient, FastAPI]) -> None:
    """完整 HTTP 链路验证预留 ID、独立回执、同名产物和取消后直接下载。"""
    client, _app = service
    contents = [HTML, SECOND, b"<p>not uploaded</p>"]
    created = _create(client, contents)
    assert set(created) == {"task_id", "status", "files"}
    assert client.delete(f"/v1/tasks/{created['task_id']}").status_code == 405
    assert client.get(f"/v1/tasks/{created['task_id']}").json()["status"] == "waiting-file"
    file_ids = [item["file_id"] for item in created["files"]]
    assert len(set(file_ids)) == 3 and all(value.startswith("file-") for value in file_ids)
    for item in created["files"]:
        assert set(item) == {"file_id", "file_name", "status", "upload"}
        for suffix in ("", "/content"):
            assert client.get(f"/v1/files/{item['file_id']}{suffix}").status_code == 404
        assert client.delete(f"/v1/files/{item['file_id']}").status_code == 405
    detail = client.get(f"/v1/tasks/{created['task_id']}").json()
    assert detail["progress"] == {"completed": 0, "failed": 0, "total": 3}
    assert {"started_at", "finished_at", "error"}.isdisjoint(detail)
    for item, content in zip(created["files"][:2], contents):
        response = client.put(item["upload"]["url"], content=content, headers=item["upload"]["headers"])
        assert response.status_code == 200, response.text
        assert set(response.json()) == {"task_id", "file_id", "status"}
        assert response.json()["file_id"] == item["file_id"]
    ready = _wait_files(client, created, 2)
    assert [item["file_id"] for item in ready["files"]] == file_ids
    response = client.post(f"/v1/tasks/{created['task_id']}" + "/cancel")
    assert response.status_code == 200, response.text
    canceled = response.json()
    assert set(canceled) == {"task_id", "status", "finished_at", "files"}
    assert canceled["status"] == "canceled"
    assert [item["status"] for item in canceled["files"]] == ["completed", "completed", "canceled"]
    assert [item["file_id"] for item in canceled["files"]] == file_ids
    assert set(canceled["files"][2]) == {"file_id", "file_name", "status"}
    for item in canceled["files"][:2]:
        assert set(item) == {"file_id", "file_name", "status", "parse", "result_url"}
        assert item["result_url"].endswith(f"/v1/files/{item['file_id']}/content")
        metadata = client.get(f"/v1/files/{item['file_id']}").json()
        assert metadata["id"] == item["file_id"] and metadata["file_name"] == "same.zip"
        downloaded = client.get(item["result_url"])
        assert downloaded.status_code == 200
        with zipfile.ZipFile(io.BytesIO(downloaded.content)) as archive:
            assert archive.read("markdown.md").strip()
    assert client.post(f"/v1/tasks/{created['task_id']}" + "/cancel").status_code == 409
    assert client.delete(f"/v1/files/{file_ids[0]}").status_code == 405
    client.get(f"/v1/tasks/{created['task_id']}")
    assert client.get(f"/v1/files/{file_ids[0]}").status_code == 200
    assert client.get(canceled["files"][0]["result_url"]).status_code == 200
    assert client.get(f"/v1/files/{file_ids[2]}").status_code == 404


def test_cached_source_uses_new_public_result_id(service: tuple[TestClient, FastAPI]) -> None:
    """源字节免传不复用上次任务的公共文件或产物标识。"""
    client, _app = service
    first = _create(client, [HTML])
    client.put(first["files"][0]["upload"]["url"], content=HTML)
    _wait_files(client, first, 1)
    second = _create(client, [HTML])
    assert second["files"][0]["file_id"] != first["files"][0]["file_id"]
    assert "upload" not in second["files"][0]
    completed = _wait_files(client, second, 1)
    assert completed["files"][0]["result_url"].endswith(f"/{second['files'][0]['file_id']}/content")


def test_openapi_separates_operation_schemas(service: tuple[TestClient, FastAPI]) -> None:
    """OpenAPI 显式区分四类成功响应,查询的可选字段不再声明为必填。"""
    client, _app = service
    schema = client.get("/openapi.json").json()
    for path, method, status, model in (
        ("/v1/tasks", "post", "202", "CreateTaskResponse"),
        ("/v1/tasks/{task_id}", "get", "200", "TaskDetailResponse"),
        ("/v1/tasks/{task_id}/cancel", "post", "200", "CancelTaskResponse"),
        ("/v1/tasks/{task_id}/uploads/{file_id}", "put", "200", "UploadFileResponse"),
    ):
        reference = schema["paths"][path][method]["responses"][status]["content"]["application/json"]["schema"]["$ref"]
        assert reference.endswith("/" + model)
    assert "delete" not in schema["paths"]["/v1/tasks/{task_id}"]
    assert "delete" not in schema["paths"]["/v1/files/{file_id}"]
    models = schema["components"]["schemas"]
    assert "FileDeletionResponse" not in models
    assert set(models["TaskFileDetail"]["required"]) == {"file_id", "file_name", "status"}
    assert set(models["CreateTaskResponse"]["properties"]) == {"task_id", "status", "files"}
    assert "upload" not in models["CanceledTaskFile"]["properties"]


def test_reserved_output_ids_reject_collisions_and_late_publication(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    """故意制造随机碰撞,验证取消、覆盖和已删除结果都不能重新占用旧 ID。"""
    store = FileStore(tmp_path)
    values = iter(["file-first", "file-first", "file-second"])

    def next_id() -> str:
        """把首次碰撞固定为可复现路径,第三个候选 ID 才可分配。"""
        return next(values)

    monkeypatch.setattr(store, "_new_file_id", next_id)
    first, second = store.reserve_file_id(), store.reserve_file_id()
    assert (first, second) == ("file-first", "file-second")
    source = tmp_path / "output.zip"
    source.write_bytes(b"complete output")
    store.discard_file_id(first)
    with pytest.raises(ValueError, match="not reserved"):
        store.create_file_for_output_path("output.zip", source, job_id="job-one", file_id=first)
    assert store.create_file_for_output_path("output.zip", source, job_id="job-two", file_id=second) == second
    with pytest.raises(ValueError, match="already been published"):
        store.create_file_for_output_path("other.zip", source, job_id="job-other", file_id=second)
    store.delete_file(second)
    with pytest.raises(ValueError, match="not reserved"):
        store.create_file_for_output_path("output.zip", source, job_id="job-two", file_id=second)
    store.finalize_outputs("job-two", int(time.time()))
    assert not store._files and not store._pins and not store._reserved_output_ids
    store.forget_file_id(first)
    store.forget_file_id(second)
    assert not store._allocated_file_ids and not store._task_file_ids


def test_router_rejects_rebinding_and_deleted_result_resurrection() -> None:
    """不同 worker 不能覆盖同一公共 ID,已删除映射不能被迟到完成响应复活。"""
    registry = ResourceRegistry()
    public_id = registry.reserve_file_id()
    route = registry.register_file(owner_scope="owner", worker_id="a", upstream_id="file-worker-a", public_id=public_id)
    with pytest.raises(ValueError, match="occupied"):
        registry.register_file(owner_scope="owner", worker_id="b", upstream_id="file-worker-b", public_id=public_id)
    assert registry.find("file", public_id) is route
    registry.remove("file", public_id)
    with pytest.raises(ValueError, match="consumed or discarded"):
        registry.register_file(owner_scope="owner", worker_id="a", upstream_id="file-worker-a", public_id=public_id)
    registry.forget_file_id(public_id)
    assert not registry._allocated_file_ids and not registry._task_file_ids


@pytest.mark.parametrize("service", ["router"], indirect=True)
@pytest.mark.parametrize("changed_field", ["task_id", "file_id"])
def test_router_rejects_mismatched_upload_receipt(
    service: tuple[TestClient, FastAPI], monkeypatch: pytest.MonkeyPatch, changed_field: str
) -> None:
    """下游上传回执串到其他任务或文件时失败,迟到的真实产物也不能注册到公共 ID。"""
    client, app = service
    transport = app.state.test_transport
    original = transport.handle_async_request

    async def mismatched(request: httpx.Request) -> httpx.Response:
        """保留真实 worker 上传和执行,只破坏回执中的身份字段。"""
        response = await original(request)
        if request.method == "PUT" and response.status_code == 200:
            await response.aread()
            payload = response.json()
            payload[changed_field] = "another-resource"
            return httpx.Response(200, json=payload)
        return response

    monkeypatch.setattr(transport, "handle_async_request", mismatched)
    created = _create(client, [HTML])
    file_id = created["files"][0]["file_id"]
    assert client.put(created["files"][0]["upload"]["url"], content=HTML).status_code == 200
    deadline = time.monotonic() + 10
    while time.monotonic() < deadline:
        detail = client.get(f"/v1/tasks/{created['task_id']}").json()
        if detail["status"] == "failed":
            break
        time.sleep(0.01)
    assert detail["status"] == "failed", detail
    assert detail["files"][0]["error"]["code"] == "invalid_upstream_response"
    assert "result_url" not in detail["files"][0]
    assert client.get(f"/v1/files/{file_id}").status_code == 404


def test_task_expiry_releases_only_its_file_id_reservations(service: tuple[TestClient, FastAPI]) -> None:
    """取消任务回收后释放 ID 索引,其他仍等待上传的任务继续持有自己的预留。"""
    client, app = service
    created, other = _create(client, [HTML]), _create(client, [SECOND])
    file_id, other_id = created["files"][0]["file_id"], other["files"][0]["file_id"]
    assert client.post(f"/v1/tasks/{created['task_id']}" + "/cancel").status_code == 200

    async def expire() -> None:
        """在服务所属循环等待取消清理,再模拟保留期结束。"""
        store = app.state.task_store
        record = store._records[created["task_id"]]
        await record.operation
        record.retention_epoch = time.time() - 2
        store.retention_seconds = 1
        store.collect_expired()

    client.portal.call(expire)
    allocator = app.state.file_store if hasattr(app.state, "file_store") else app.state.registry
    assert file_id not in allocator._allocated_file_ids and file_id not in allocator._task_file_ids
    assert other_id in allocator._allocated_file_ids and other_id in allocator._task_file_ids