Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
15.0 KB
Raw Blame
"""The live-validation harness's own safety properties.

`scripts/validate_azure_foundry_live.py` is run by hand against a real Foundry
resource, so its failure modes are not otherwise exercised by CI. Two of them
are worth pinning: it must never put a real credential on a terminal, and the
checks that pass by observing a rejection must not pass on an unrelated one.

The leak tests matter because detection alone is not enough. Check 11 finds a
leak *after* the run, by which point anything written to stdout or stderr is
already on screen and, in CI, already in a retained build log.
"""

from __future__ import annotations

import importlib.util
import io
import sys
from pathlib import Path

import pytest

ROOT = Path(__file__).resolve().parent.parent.parent
SCRIPT = ROOT / "scripts" / "validate_azure_foundry_live.py"

REAL_KEY = "primary-credential-must-never-be-printed"
LISTING_DENIED_KEY = "listing-denied-credential-also-secret"


@pytest.fixture
def harness():
    """The script as an importable module, with its secret list reset."""
    spec = importlib.util.spec_from_file_location("azure_foundry_live_validation", SCRIPT)
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    sys.modules[spec.name] = module
    spec.loader.exec_module(module)
    try:
        yield module
    finally:
        module._set_real_secrets()
        sys.modules.pop(spec.name, None)


class TestRedaction:
    def test_every_supplied_credential_is_redacted(self, harness) -> None:
        harness._set_real_secrets(REAL_KEY, LISTING_DENIED_KEY)
        text = f"key={REAL_KEY} other={LISTING_DENIED_KEY}"

        cleaned = harness._redact(text)

        assert REAL_KEY not in cleaned
        assert LISTING_DENIED_KEY not in cleaned
        assert cleaned.count("<redacted>") == 2

    def test_a_short_value_is_not_treated_as_a_secret(self, harness) -> None:
        """Blanking a two-character value would destroy the output instead."""
        harness._set_real_secrets("ab")
        assert harness._redact("a table of absolute values") == "a table of absolute values"

    def test_blank_credentials_are_ignored(self, harness) -> None:
        harness._set_real_secrets("", None or "")
        assert harness._REAL_SECRETS == ()


class TestTeeDoesNotLeakToTheTerminal:
    """The stream forwards redacted text and keeps the original for detection."""

    @pytest.mark.parametrize("secret", [REAL_KEY, LISTING_DENIED_KEY])
    def test_the_real_stream_never_receives_the_secret(self, harness, secret) -> None:
        harness._set_real_secrets(REAL_KEY, LISTING_DENIED_KEY)
        terminal = io.StringIO()
        sink: list[str] = []
        tee = harness._Tee(terminal, sink)

        tee.write(f"litellm debug: authorization=Bearer {secret}\n")

        assert secret not in terminal.getvalue()
        assert "<redacted>" in terminal.getvalue()
        # ...but the capture keeps the original, or check 11 could not detect it.
        assert any(secret in fragment for fragment in sink)


class TestCheckElevenDetectsEitherCredential:
    @pytest.mark.parametrize(
        "leaked, index",
        [(REAL_KEY, 1), (LISTING_DENIED_KEY, 2)],
    )
    def test_a_forced_leak_of_either_credential_fails_the_check(
        self, harness, leaked, index
    ) -> None:
        matrix = harness.Matrix()
        matrix.capture(f"some library wrote {leaked} to stderr")

        harness._check_secret_leak(matrix, (REAL_KEY, LISTING_DENIED_KEY))

        check = matrix.checks[-1]
        assert check.status == "fail"
        assert check.severity == harness.BLOCKER
        assert f"credential #{index}" in check.detail

    def test_the_failure_detail_never_contains_the_secret(self, harness) -> None:
        """Reporting a leak must not be the leak."""
        matrix = harness.Matrix()
        matrix.capture(f"leaked {REAL_KEY} here")

        harness._check_secret_leak(matrix, (REAL_KEY,))

        assert REAL_KEY not in matrix.checks[-1].detail

    def test_a_clean_run_passes_and_says_how_much_it_scanned(self, harness) -> None:
        matrix = harness.Matrix()
        matrix.capture("nothing sensitive at all")

        harness._check_secret_leak(matrix, (REAL_KEY, LISTING_DENIED_KEY))

        check = matrix.checks[-1]
        assert check.status == "pass"
        assert "2 credential(s)" in check.detail

    def test_no_credentials_to_scan_is_a_failure_not_a_pass(self, harness) -> None:
        """An empty secret list would otherwise pass vacuously."""
        matrix = harness.Matrix()

        harness._check_secret_leak(matrix, ())

        assert matrix.checks[-1].status == "fail"


class TestTheReportCannotPrintASecret:
    @pytest.mark.parametrize("secret", [REAL_KEY, LISTING_DENIED_KEY])
    def test_a_secret_echoed_into_a_check_detail_is_redacted_on_print(
        self, harness, secret, capsys
    ) -> None:
        """A provider can echo the credential back in an error body.

        That body ends up in a check's detail, so the report has to redact on
        the way out even though check 11 has already flagged it.
        """
        harness._set_real_secrets(REAL_KEY, LISTING_DENIED_KEY)
        matrix = harness.Matrix()
        matrix.record(
            2,
            "invalid API key is rejected at save time",
            harness.BLOCKER,
            "fail",
            f'upstream echoed the credential: body={{"key": "{secret}"}}',
        )

        harness._print_report(matrix)

        assert secret not in capsys.readouterr().out


class TestNotFoundClassification:
    """A bare 404 is not evidence: those digits appear in ids and counters."""

    @pytest.mark.parametrize(
        "text",
        [
            "DeploymentNotFound",
            "The API deployment for this resource does not exist",
            "model_not_found",
            "HTTP 404",
            "HTTP/1.1 404",
            "status_code: 404",
            "status=404",
            "404 Not Found",
        ],
    )
    def test_a_real_not_found_is_recognised(self, harness, text) -> None:
        assert harness._looks_like_not_found(text)

    @pytest.mark.parametrize(
        "text",
        [
            "request id 8a404b2c",
            "used 404 prompt tokens",
            "completed in 404ms",
            "x-ms-request-id: 404aa1",
            "rate limit exceeded",
        ],
    )
    def test_an_incidental_404_is_not_mistaken_for_one(self, harness, text) -> None:
        assert not harness._looks_like_not_found(text)


class TestFailureCategories:
    """Checks that pass on a rejection must not pass on an unrelated failure."""

    @pytest.mark.parametrize(
        "text",
        [
            "ConnectError: nodename nor servname provided, or not known",
            "Cannot connect to host contoso.services.ai.azure.com:443",
            "ReadTimeout",
            "certificate verify failed",
        ],
    )
    def test_transport_failures_are_identified_as_such(self, harness, text) -> None:
        assert harness._looks_like_connectivity_failure(text)
        assert not harness._looks_like_credential_failure(text)

    @pytest.mark.parametrize(
        "text",
        [
            "Access denied due to invalid subscription key",
            "401 Unauthorized",
            "Incorrect API key provided",
            "HTTP 401",
            "status_code: 401",
        ],
    )
    def test_credential_rejections_are_identified_as_such(self, harness, text) -> None:
        assert harness._looks_like_credential_failure(text)
        assert not harness._looks_like_connectivity_failure(text)

    @pytest.mark.parametrize(
        "text",
        [
            "request id 8a401b2c",
            "completed in 401 ms",
            "prompt_tokens: 401",
        ],
    )
    def test_an_incidental_401_is_not_mistaken_for_a_rejection(self, harness, text) -> None:
        """Same reasoning as the 404 case above.

        Check 2 runs this classifier over 300 characters of response body, so
        a request id carrying those three digits would otherwise let a
        non-credential rejection pass as a credential one.
        """
        assert not harness._looks_like_credential_failure(text)


class TestTheRoutingCheckCanFail:
    """A blocker that always passes is not a check.

    Both paths below return before any network call, which is what makes them
    testable without a live resource.
    """

    @pytest.fixture
    def settings(self):
        return {
            "api_key": REAL_KEY,
            "chat_deployment": "chat-dep",
            "embedding_deployment": "embed-dep",
        }

    @pytest.mark.asyncio
    async def test_the_wrong_surface_is_reported_rather_than_passed(
        self, harness, settings, monkeypatch
    ) -> None:
        """A legacy `/models` URL in AZURE_AI_API_BASE used to satisfy the
        `/openai/v1` blocker, which is the one thing the matrix exists to
        prove."""
        from enhancements.providers.azure import foundry

        monkeypatch.setattr(foundry, "endpoint_profile", lambda _b: "legacy_models")
        monkeypatch.setattr(foundry, "litellm_route", lambda _s: "azure_ai")

        matrix = harness.Matrix()
        await harness._run_endpoint_matrix(
            matrix,
            "resource v1",
            "https://contoso.services.ai.azure.com/models",
            settings,
            base_number=8,
            expected_profile="resource_openai_v1",
        )

        assert [c.status for c in matrix.checks] == ["fail"]
        assert "expected resource_openai_v1" in matrix.checks[0].detail

    @pytest.mark.asyncio
    async def test_an_unclassifiable_endpoint_does_not_abort_the_run(
        self, harness, settings, monkeypatch
    ) -> None:
        """`litellm_route` refuses what it cannot classify, by design.

        Uncaught, that exception escaped `_run_all_checks` and the report was
        never printed — a traceback instead of the matrix, which is the least
        useful thing a release gate can hand an operator.
        """
        from enhancements.providers.azure import foundry

        def _refuse(_stored):
            raise foundry.UnsupportedEndpointError(foundry.UNSUPPORTED_ENDPOINT_MESSAGE)

        monkeypatch.setattr(foundry, "endpoint_profile", lambda _b: "unknown")
        monkeypatch.setattr(foundry, "litellm_route", _refuse)

        matrix = harness.Matrix()
        await harness._run_endpoint_matrix(
            matrix,
            "resource v1",
            "https://contoso.services.ai.azure.com/anthropic/v1",
            settings,
            base_number=8,
            expected_profile="resource_openai_v1",
        )

        assert [c.status for c in matrix.checks] == ["fail"]
        assert "UnsupportedEndpointError" in matrix.checks[0].detail


class TestBlockersRejectUnusableSuccesses:
    """A 200 is not the same as a working deployment.

    Both of these recorded a pass on a successful-but-empty response, which
    made the release matrix green without usable inference — the one outcome
    a blocker here has to rule out.
    """

    @pytest.mark.parametrize("text", ["", "   ", "\n"])
    def test_an_empty_completion_is_not_a_pass(self, harness, text) -> None:
        assert harness._unusable_chat_reply(text)

    def test_a_real_completion_passes(self, harness) -> None:
        assert harness._unusable_chat_reply("OK") == ""

    @pytest.mark.parametrize(
        "vector",
        [
            [],
            None,
            "not-a-vector",
            [0.1, float("nan")],
            [float("inf"), 0.2],
            [0.1, "0.2"],
            [0.1, True],
        ],
    )
    def test_an_unusable_vector_is_not_a_pass(self, harness, vector) -> None:
        """Length alone is not enough: a vector of NaN has the right shape and
        is useless for retrieval, and OpenSearch rejects it at index time
        rather than here."""
        assert harness._unusable_embedding(vector)

    def test_a_real_vector_passes(self, harness) -> None:
        assert harness._unusable_embedding([0.1, -0.2, 3]) == ""


class TestTheEndpointIsNeverPrintedRaw:
    """`_redact` only knows the credentials this run was handed.

    A secret carried *inside* the endpoint is invisible to it, and the
    provider module strips or refuses one only after this line is on screen.
    """

    def test_userinfo_and_query_are_removed(self, harness) -> None:
        shown = harness._safe_endpoint(
            "https://operator:[email protected]/openai/v1?sig=abc123"
        )
        assert "s3cr3t" not in shown
        assert "operator" not in shown
        assert "abc123" not in shown
        assert "sig" not in shown
        assert shown.startswith("https://contoso.services.ai.azure.com/openai/v1")
        assert "userinfo" in shown and "query" in shown

    def test_an_ordinary_endpoint_is_left_alone(self, harness) -> None:
        plain = "https://contoso.services.ai.azure.com/openai/v1"
        assert harness._safe_endpoint(plain) == plain

    def test_a_non_url_is_passed_through(self, harness) -> None:
        assert harness._safe_endpoint("not-a-url") == "not-a-url"


class TestPreboundLogHandlersCannotLeak:
    """Replacing `sys.stderr` does not retarget a handler that already holds it.

    litellm attaches three such handlers at import, and it is the library most
    likely to log a request detail carrying a credential — so without this the
    harness's central promise, that nothing reaches a terminal unredacted,
    does not hold for the most likely source.
    """

    def test_a_handler_bound_before_the_block_is_redirected(self, harness, monkeypatch) -> None:
        import logging

        # The handler must hold the *same object* `sys.stderr` names when
        # `_capturing` is entered — that is the situation litellm creates at
        # import, and the one swapping `sys.stderr` fails to cover. A handler
        # pointed somewhere else entirely is a log file, not a terminal, and
        # is deliberately left alone.
        terminal = io.StringIO()
        monkeypatch.setattr(sys, "stderr", terminal)
        logger = logging.getLogger("test.prebound.handler")
        logger.propagate = False
        handler = logging.StreamHandler(terminal)
        logger.addHandler(handler)
        logger.setLevel(logging.INFO)

        harness._set_real_secrets(REAL_KEY)
        sink: list[str] = []
        try:
            with harness._capturing(sink):
                logger.info("leaking %s now", REAL_KEY)
        finally:
            logger.removeHandler(handler)

        assert REAL_KEY not in terminal.getvalue(), "the secret reached the real stream"
        assert handler.stream is terminal, "the handler's stream was not restored"
        assert any(REAL_KEY in entry for entry in sink), "detection still needs the original"


class TestImageFixture:
    def test_the_probe_image_is_large_enough_to_be_accepted(self, harness) -> None:
        """A 1x1 PNG is valid but some vision deployments reject it on size,
        which would read as an integration failure rather than a fixture one."""
        import struct

        png = harness._sample_png()

        assert png[:8] == b"\x89PNG\r\n\x1a\n"
        width, height = struct.unpack(">II", png[16:24])
        assert width == height == 128