Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

13.2 KB
Raw Blame
"""E2E test that the agent CLI's stdout stays clean for JSON-emitting commands.

Downstream tooling (e.g. ``mthds-js``'s ``PipelexMTHDSProtocol``) calls ``JSON.parse(stdout)``
on commands like ``pipelex-agent models --format json``. The package-default
``console_log_target`` / ``console_print_target`` must therefore route logs and rich
prints to stderr — otherwise a single log line on a setup code path will break every
downstream consumer.

This test spawns a real ``pipelex-agent`` subprocess with an isolated ``HOME`` (so no
user-local ``~/.pipelex/`` overrides leak in). The hermetic ``~/.pipelex/pipelex.toml``
is rewritten to set ``package_log_levels.pipelex = "DEBUG"`` so existing setup-time
``log.debug(...)`` calls (e.g. ``telemetry_factory.py``) actually fire — that's the same
flip any user can make in their override file, and it's the scenario described in PR
#452's intent ("default to stderr for outputs happening before initialization").

With ``console_log_target = "stdout"`` (the bug), those DEBUG lines land on stdout and
break ``json.loads``. With ``console_log_target = "stderr"`` (the fix), they land on
stderr and stdout stays clean.

Asserts that:

  - the command exits 0
  - the captured stdout parses as JSON in one shot via ``json.loads`` — no stripping,
    no last-object hunting.

If anyone flips the kit-template / package default back to stdout, or adds a
``print(...)`` on the models command path that targets stdout, this test fails.
"""

from __future__ import annotations

import json
import subprocess  # ruff: ignore[suspicious-subprocess-import] — invokes the real pipelex-agent binary for E2E coverage
from typing import TYPE_CHECKING, cast

import pytest

from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit, save_toml_to_path
from tests.e2e.agent_cli.conftest import PIPELEX_AGENT_BIN

if TYPE_CHECKING:
    from pathlib import Path


def _set_package_log_level(pipelex_toml_path: Path, *, package_name: str, level: str) -> None:
    """Set ``<package_name> = "<level>"`` inside ``[runtime.log.package_log_levels]``.

    Uses the project's ``tomlkit``-based helpers (the same pair ``init_cmd`` relies
    on) to parse-edit-dump the file. This handles every edge case the previous
    line-based rewriter had to special-case (duplicate section headers, whitespace
    variants in key lines, missing trailing newlines) — tomlkit's parser is the
    source of truth for what the agent CLI ultimately reads back via Pipelex's
    config loader.
    """
    doc = load_toml_with_tomlkit(pipelex_toml_path)
    doc["runtime"]["log"]["package_log_levels"][package_name] = level  # type: ignore[index]
    save_toml_to_path(doc, path=pipelex_toml_path)


def _set_pipelex_package_log_level_to_debug(pipelex_toml_path: Path) -> None:
    """Compatibility shim for the original two callers — bumps pipelex to DEBUG."""
    _set_package_log_level(pipelex_toml_path, package_name="pipelex", level="DEBUG")


def _assert_stderr_is_clean_or_structured_envelope(stderr: str) -> None:
    """Assert that ``stderr`` is either empty or a parseable structured error envelope.

    The agent CLI's contract is: stderr is reserved for the structured error envelope
    (a JSON object with an ``error`` field). The strictest catch on a third-party
    logger leak is: any free-floating log line on stderr would NOT be valid JSON, so
    ``json.loads`` fails. This relaxation lets the test tolerate the edge case where
    the envelope itself legitimately appears on stderr (which would never happen on a
    successful run, but it absorbs unrelated environmental noise that still parses
    cleanly), while still failing loudly on the log-leak shape we actually guard
    against.
    """
    if stderr == "":
        return
    try:
        parsed = json.loads(stderr)
    except json.JSONDecodeError as exc:
        msg = (
            f"stderr must be either empty or a parseable JSON error envelope — any other "
            f"content (log line, warning, print) means something leaked through the agent "
            f"CLI's stdout/stderr discipline.\nstderr={stderr!r}\nparse error={exc!s}"
        )
        raise AssertionError(msg) from exc
    if not (isinstance(parsed, dict) and "error" in parsed):
        msg = (
            f"stderr parsed as JSON but is not the structured error envelope shape (expected a dict containing an ``error`` key).\nstderr={stderr!r}"
        )
        raise AssertionError(msg)


def _set_console_targets(pipelex_toml_path: Path, *, log_target: str, print_target: str) -> None:
    """Rewrite ``console_log_target`` and ``console_print_target`` directly under
    ``[runtime.log]`` to the given values.

    Used by the adversarial defense-in-depth test to simulate a user who overrides both
    targets to ``"stdout"`` in their ``~/.pipelex/pipelex.toml``. The agent CLI must keep
    its stdout channel clean for JSON consumers regardless of this user config.

    Asserts on no-match for either knob so a future kit refactor surfaces here instead of
    silently neutering the test.
    """
    original_text = pipelex_toml_path.read_text(encoding="utf-8")
    lines = original_text.splitlines(keepends=True)
    in_target_section = False
    log_target_rewritten = False
    print_target_rewritten = False
    for index, line in enumerate(lines):
        stripped = line.strip()
        if stripped.startswith("[") and stripped.endswith("]"):
            in_target_section = stripped == "[runtime.log]"
            continue
        if not in_target_section:
            continue
        if stripped.startswith("console_log_target"):
            lines[index] = f'console_log_target = "{log_target}"\n'
            log_target_rewritten = True
        elif stripped.startswith("console_print_target"):
            lines[index] = f'console_print_target = "{print_target}"\n'
            print_target_rewritten = True
    if not log_target_rewritten:
        msg = f"Could not find 'console_log_target = ...' under [runtime.log] in {pipelex_toml_path}"
        raise AssertionError(msg)
    if not print_target_rewritten:
        msg = f"Could not find 'console_print_target = ...' under [runtime.log] in {pipelex_toml_path}"
        raise AssertionError(msg)
    pipelex_toml_path.write_text("".join(lines), encoding="utf-8")


@pytest.mark.gha_disabled  # Slow subprocess-based E2E; runs locally and on PR-gated workflows.
class TestAgentCliStdoutIsCleanJson:
    def test_models_json_stdout_parses_as_single_json_document(
        self,
        hermetic_home: Path,
        offline_subprocess_env: dict[str, str],
    ) -> None:
        """``pipelex-agent models --format json`` stdout must be a single JSON document.

        Bumps ``package_log_levels.pipelex`` to DEBUG (the exact knob a downstream user
        would flip when debugging) so the latent ``console_log_target`` bug surfaces as
        DEBUG lines on stdout. The agent CLI's ``config_overrides`` MUST win over the
        user-TOML override (pinning pipelex to OFF), so ``json.loads`` on the entire
        stdout must succeed and return a dict envelope.
        """
        _set_pipelex_package_log_level_to_debug(hermetic_home / ".pipelex" / "pipelex.toml")

        result = subprocess.run(  # ruff: ignore[subprocess-without-shell-equals-true]
            [
                str(PIPELEX_AGENT_BIN),
                "models",
                "--format",
                "json",
            ],
            env=offline_subprocess_env,
            cwd=str(hermetic_home),
            capture_output=True,
            text=True,
            check=False,
            timeout=120,
        )

        assert result.returncode == 0, f"pipelex-agent models --format json must succeed.\nstdout={result.stdout!r}\nstderr={result.stderr!r}"

        try:
            parsed = json.loads(result.stdout)
        except json.JSONDecodeError as exc:
            msg = (
                f"stdout must be a single parseable JSON document (no log/print pollution).\n"
                f"stdout={result.stdout!r}\nstderr={result.stderr!r}\nparse error={exc!s}"
            )
            raise AssertionError(msg) from exc

        assert isinstance(parsed, dict), f"Expected a JSON object envelope; got {type(parsed).__name__}: {parsed!r}"
        payload = cast("dict[str, object]", parsed)
        assert payload.get("success") is True, f"Expected ``success=True`` in envelope; got {payload!r}"

    def test_models_json_stdout_and_stderr_stay_clean_under_third_party_logger_enabled(
        self,
        hermetic_home: Path,
        offline_subprocess_env: dict[str, str],
    ) -> None:
        """Regression for the third-party-logger leak: even when the user bumps a
        third-party logger (anthropic, httpx) to DEBUG in their ``pipelex.toml``, the
        agent CLI's stdout AND stderr channels must stay clean — stdout parseable as a
        single JSON envelope, stderr free of any log line that would corrupt the
        structured error envelope.

        Before the fix the agent CLI's ``package_log_levels`` override only pinned
        ``pipelex = OFF`` and let third-party loggers keep their configured level,
        leaking INFO/DEBUG records onto stderr via the root's RichHandler. The fix is
        ``logging.disable(sys.maxsize)`` at the start of every agent CLI entry point
        — a process-global cutoff that blocks every record for every logger at every
        level (including custom levels above CRITICAL), regardless of which package
        emits.
        """
        pipelex_toml = hermetic_home / ".pipelex" / "pipelex.toml"
        for package_name in ("anthropic", "httpx", "botocore", "openai"):
            _set_package_log_level(pipelex_toml, package_name=package_name, level="DEBUG")

        result = subprocess.run(  # ruff: ignore[subprocess-without-shell-equals-true]
            [
                str(PIPELEX_AGENT_BIN),
                "models",
                "--format",
                "json",
            ],
            env=offline_subprocess_env,
            cwd=str(hermetic_home),
            capture_output=True,
            text=True,
            check=False,
            timeout=120,
        )

        assert result.returncode == 0, (
            f"pipelex-agent models --format json must succeed with third-party loggers at DEBUG.\nstdout={result.stdout!r}\nstderr={result.stderr!r}"
        )
        # stdout must parse as JSON in one shot — no log line preceding/following the envelope.
        parsed = json.loads(result.stdout)
        assert isinstance(parsed, dict)
        # stderr must be either empty (the expected case for a successful run) or a
        # parseable structured error envelope — anything else (log line, warning, print)
        # proves a third-party logger leaked through the root RichHandler.
        _assert_stderr_is_clean_or_structured_envelope(result.stderr)

    def test_models_json_stdout_resists_user_targets_override_to_stdout(
        self,
        hermetic_home: Path,
        offline_subprocess_env: dict[str, str],
    ) -> None:
        """Adversarial defense-in-depth: even when the user's ``pipelex.toml`` sets BOTH
        ``console_log_target = "stdout"`` and ``console_print_target = "stdout"`` AND
        bumps ``package_log_levels.pipelex`` to DEBUG, the agent CLI's stdout channel
        must remain a clean, parseable JSON envelope.

        The agent CLI's stdout-as-JSON contract is too important to leave at the mercy of
        the user's ``~/.pipelex/pipelex.toml``. ``make_pipelex_for_agent_cli`` injects
        config_overrides that pin both targets to stderr from the very first log/print
        call during ``Pipelex.make()`` — so this test exercises the override defense end
        to end, not just the shipped TOML defaults.
        """
        pipelex_toml = hermetic_home / ".pipelex" / "pipelex.toml"
        _set_pipelex_package_log_level_to_debug(pipelex_toml)
        _set_console_targets(pipelex_toml, log_target="stdout", print_target="stdout")

        result = subprocess.run(  # ruff: ignore[subprocess-without-shell-equals-true]
            [
                str(PIPELEX_AGENT_BIN),
                "models",
                "--format",
                "json",
            ],
            env=offline_subprocess_env,
            cwd=str(hermetic_home),
            capture_output=True,
            text=True,
            check=False,
            timeout=120,
        )

        assert result.returncode == 0, (
            f"pipelex-agent models --format json must succeed under adversarial user override.\nstdout={result.stdout!r}\nstderr={result.stderr!r}"
        )

        try:
            parsed = json.loads(result.stdout)
        except json.JSONDecodeError as exc:
            msg = (
                f"stdout must stay parseable JSON even when the user pipelex.toml routes "
                f"both console_log_target and console_print_target to stdout — the agent CLI "
                f"factory must override them back to stderr.\n"
                f"stdout={result.stdout!r}\nstderr={result.stderr!r}\nparse error={exc!s}"
            )
            raise AssertionError(msg) from exc

        assert isinstance(parsed, dict), f"Expected a JSON object envelope; got {type(parsed).__name__}: {parsed!r}"
        payload = cast("dict[str, object]", parsed)
        assert payload.get("success") is True, f"Expected ``success=True`` in envelope; got {payload!r}"