Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

3.8 KB
Raw Blame
"""Parse worker pool — picks up pending parse batches and executes engine."""

from __future__ import annotations

import asyncio
import logging
import time

from ..services.parse_queue_svc import ParseQueueService
from ..services.parse_execution_svc import ParseExecutionService
from ..types import PARSE_STATUS_FAILED

logger = logging.getLogger("mineru.parse_worker")


class ParseWorkerPool:
    def __init__(self, queue_svc: ParseQueueService, execution_svc: ParseExecutionService, num_workers: int = 2) -> None:
        """显式接收队列与执行服务,保持原有工作池数量和轮询行为。"""
        self.queue_svc = queue_svc
        self.execution_svc = execution_svc
        self.num_workers = num_workers
        self.running = False
        self._tasks: list[asyncio.Task] = []

    async def run(self) -> None:
        self.running = True
        self._tasks = [asyncio.create_task(self._worker(i)) for i in range(self.num_workers)]
        for index, task in enumerate(self._tasks):
            task.add_done_callback(lambda completed, worker_id=index: self._log_task_result(worker_id, completed))

    async def _worker(self, worker_id: int) -> None:
        logger.info(f"Parse worker {worker_id} started")
        processed = 0
        no_task_count = 0

        while self.running:
            try:
                task = await self.queue_svc.acquire_task()
                if task is None:
                    no_task_count += 1
                    if no_task_count == 1 or no_task_count % 20 == 0:
                        q = await self.queue_svc.get_queue_length()
                        logger.debug(f"Parse worker {worker_id}: queue={q}")
                    await asyncio.sleep(0.5)
                    continue
            except Exception as exc:
                logger.error(
                    "Parse worker %s loop error: %s",
                    worker_id,
                    exc,
                    exc_info=(type(exc), exc, exc.__traceback__),
                )
                if self.running:
                    await asyncio.sleep(0.5)
                continue

            no_task_count = 0
            try:
                success = await self.execution_svc.process_doc(task)
                if success:
                    processed += 1
                    if processed % 50 == 0:
                        logger.info(f"Parse worker {worker_id} processed {processed} batches")
            except Exception as exc:
                logger.error(
                    "Parse worker %s error on %s: %s",
                    worker_id,
                    task.get("sha256"),
                    exc,
                    exc_info=(type(exc), exc, exc.__traceback__),
                )
                try:
                    now = int(time.time() * 1000)
                    await self.execution_svc.db.execute(
                        "UPDATE parses SET status=?, error_code=?, error_msg=?, locked_at=NULL, updated_at=? WHERE id=?",
                        (PARSE_STATUS_FAILED, "parse_failed", str(exc)[:500], now, task["id"]),
                    )
                except Exception:
                    pass

        logger.info(f"Parse worker {worker_id} stopped, processed {processed} total")

    async def stop(self) -> None:
        self.running = False
        for t in self._tasks:
            t.cancel()
        if self._tasks:
            await asyncio.gather(*self._tasks, return_exceptions=True)

    @staticmethod
    def _log_task_result(worker_id: int, task: asyncio.Task[None]) -> None:
        if task.cancelled():
            return
        try:
            exc = task.exception()
        except asyncio.CancelledError:
            return
        if exc is not None:
            logger.error(
                "Parse worker %s crashed: %s",
                worker_id,
                exc,
                exc_info=(type(exc), exc, exc.__traceback__),
            )