Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

5.6 KB
Raw Blame
"""Regression: the file_processing worker must not boot the executor stack.

``structure_tool_task`` dispatches all real extraction work to the executor
worker over the PG queue. From the ``executor`` package it needs only two cheap
things: ``ExecutorToolShim`` (a StreamMixin wrapper) and some string constants.

Both used to drag in ``LegacyExecutor`` and every adapter behind it, because
``executor/__init__.py`` eagerly imported ``.worker``, which imported
``executor.executors``, which imported ``LegacyExecutor`` and ran entry-point
discovery. ``structure_tool_task`` makes those imports inside the task function,
so the cost landed on each forked child's FIRST task rather than at startup —
~9s per child, measured on staging (UN-4136).

These pin the import graph, not a duration: a timing assertion would be flaky on
CI, while the thing that actually regresses is an eager import creeping back
into either ``__init__``. Each runs in a fresh interpreter so the rest of the
suite cannot pre-import the stack and mask the regression.
"""

import os
import subprocess
import sys

# Importing these must not pull the executor stack in behind them.
_FILE_PROCESSING_IMPORTS = (
    "from executor.executor_tool_shim import ExecutorToolShim",
    "from executor.executors.constants import PromptServiceConstants",
)

# The expensive modules: ``legacy_executor`` pulls the x2text adapter stack at
# module scope, and ``executor.worker`` builds the Celery app.
_MUST_NOT_LOAD = (
    "executor.worker",
    "executor.executors.legacy_executor",
)

_WORKERS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


# Child snippets use ``raise SystemExit``, never a bare ``assert``: the child
# inherits the parent's environment, and under ``PYTHONOPTIMIZE``/``-O`` every
# assert is stripped — the snippet would print OK, exit 0, and the guard would
# pass having checked nothing.
def _run(snippet: str) -> subprocess.CompletedProcess:
    return subprocess.run(
        [sys.executable, "-c", snippet],
        capture_output=True,
        text=True,
        env={**os.environ, "WORKER_TYPE": "file_processing"},
        cwd=_WORKERS_DIR,
    )


def test_file_processing_imports_do_not_load_the_executor_stack():
    """Neither import may leave the executor worker or LegacyExecutor loaded."""
    code = "\n".join(
        [
            "import sys",
            *_FILE_PROCESSING_IMPORTS,
            f"loaded = [m for m in {_MUST_NOT_LOAD!r} if m in sys.modules]",
            "if loaded: raise SystemExit(f'eagerly imported: {loaded}')",
            "print('OK')",
        ]
    )
    result = _run(code)
    assert result.returncode == 0, (
        "importing ExecutorToolShim / PromptServiceConstants pulled in the "
        f"executor stack.\nstdout: {result.stdout}\nstderr: {result.stderr}"
    )
    assert "OK" in result.stdout


def test_loading_executor_tasks_suppresses_celery_result_logging():
    """The trace logger must be muted by the module the executor actually loads.

    ``celery.app.trace`` logs the full result dict on task success, which for
    ``execute_extraction`` is extracted customer document text. The
    ``setLevel(WARNING)`` that mutes it used to live in ``executor/worker.py``
    and was reached only because ``executor/__init__.py`` eagerly imported
    ``.worker`` — so making that import lazy silently un-suppressed it, and
    every extraction would have logged ~1KB of the payload at INFO.

    Nothing else in the repo sets this level, and no launcher runs
    ``celery -A executor``, so ``executor/tasks.py`` — which ``workers/worker.py``
    exec-loads by path for both executor roles — is the only module that can
    carry it. A fresh interpreter is required: another test importing
    ``executor.worker`` first would mask the regression.
    """
    code = "\n".join(
        [
            "import logging, sys",
            "import executor.tasks",  # exactly what the deployed executor loads
            "lvl = logging.getLogger('celery.app.trace').level",
            "if lvl != logging.WARNING:",
            "    raise SystemExit(f'trace logger not suppressed: {lvl}')",
            # The suppression must not come back via the expensive import.
            "if 'executor.worker' in sys.modules:",
            "    raise SystemExit('pulled in executor.worker')",
            "print('OK')",
        ]
    )
    result = subprocess.run(
        [sys.executable, "-c", code],
        capture_output=True,
        text=True,
        env={**os.environ, "WORKER_TYPE": "executor"},
        cwd=_WORKERS_DIR,
    )
    assert result.returncode == 0, (
        "executor.tasks did not suppress celery.app.trace result logging.\n"
        f"stdout: {result.stdout}\nstderr: {result.stderr}"
    )
    assert "OK" in result.stdout


def test_importing_the_executor_package_does_not_build_the_celery_app():
    """``import executor`` alone must stay cheap.

    ``celery_app`` is still reachable — it resolves on attribute access (PEP
    562) — so this pins laziness, not removal.
    """
    code = "\n".join(
        [
            "import sys, executor",
            "if 'executor.worker' in sys.modules:",
            "    raise SystemExit('executor/__init__ built the app')",
            "if executor.celery_app is None:",
            "    raise SystemExit('celery_app no longer resolves')",
            "if 'executor.worker' not in sys.modules:",
            "    raise SystemExit('attribute access did not load it')",
            "print('OK')",
        ]
    )
    result = _run(code)
    assert result.returncode == 0, f"stdout: {result.stdout}\nstderr: {result.stderr}"
    assert "OK" in result.stdout