Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
18.9 KB
Raw Blame
"""Langflow provider-error formatting and stream error regressions."""

from __future__ import annotations

import json
from collections.abc import AsyncIterator

import pytest

import agent as agent_module


def test_format_provider_error_message_extracts_embedded_json():
    raw = (
        'Provider request failed: {"error": {"message": '
        '"Incorrect API key provided", "type": "invalid_request_error"}}'
    )
    assert (
        agent_module._format_provider_error_message(Exception(raw)) == "Incorrect API key provided"
    )


def test_format_provider_error_message_extracts_ibm_iam_error_message():
    raw = (
        'Failed to authenticate with IBM Watson: {"errorCode":"BXNIM0415E",'
        '"errorMessage":"Provided API key could not be found.",'
        '"context":{"requestId":"abc","url":"https://iam.cloud.ibm.com"}}'
    )
    assert agent_module._format_provider_error_message(Exception(raw)) == (
        "Provided API key is Invalid."
    )


def test_format_provider_error_message_keeps_plain_text():
    msg = "Rate limit exceeded for watsonx.ai. Please try again later."
    assert agent_module._format_provider_error_message(Exception(msg)) == msg


def test_provider_error_display_text_keeps_partial_answer():
    assert agent_module._provider_error_display_text(
        "Rate limit exceeded", "Partial answer so far"
    ) == ("Partial answer so far\n\nRate limit exceeded")


def test_provider_error_display_text_drops_error_like_partial():
    partial = (
        "Failed to initialize IBM WatsonX embedding model. Error: "
        '{"errorMessage":"Provided API key could not be found."}'
    )
    assert (
        agent_module._provider_error_display_text("Provided API key is Invalid.", partial)
        == "Provided API key is Invalid."
    )


def test_resolve_conversation_store_id_prefers_warm_thread():
    cold_id = agent_module._resolve_conversation_store_id(
        None,
        "resp_cold",
        previous_loaded_from_memory=False,
        mint_if_missing=True,
    )
    # Cold previous ids must not be reused (would overwrite metadata); mint a new id.
    assert cold_id != "resp_cold"
    assert not cold_id.startswith("error_")

    assert (
        agent_module._resolve_conversation_store_id(
            None,
            "resp_warm",
            previous_loaded_from_memory=True,
        )
        == "resp_warm"
    )
    assert (
        agent_module._resolve_conversation_store_id(
            "resp_stream",
            "resp_warm",
            previous_loaded_from_memory=True,
        )
        == "resp_warm"
    )


@pytest.fixture
def store_in_memory(monkeypatch):
    stored: list[tuple[str, str]] = []
    claimed: list[tuple[str, str]] = []

    async def _store(user_id, response_id, conversation_state):
        stored.append((user_id, response_id))
        if user_id not in agent_module.active_conversations:
            agent_module.active_conversations[user_id] = {}
        agent_module.active_conversations[user_id][response_id] = conversation_state

    class _Ownership:
        async def claim_session(self, user_id, session_id, conversation_id=None):
            claimed.append((user_id, session_id))
            return True

        async def get_session_owner(self, session_id):
            return next((user_id for user_id, sid in claimed if sid == session_id), None)

        async def get_conversation_sessions(self, user_id, session_id):
            return []

    monkeypatch.setattr(agent_module, "store_conversation_thread", _store)
    monkeypatch.setattr(
        "services.session_ownership_service.session_ownership_service",
        _Ownership(),
    )
    return {"stored": stored, "claimed": claimed}


async def _collect_error_chunks(stream) -> list[dict]:
    chunks: list[dict] = []
    async for raw in stream:
        chunks.append(json.loads(raw.decode("utf-8")))
    return chunks


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "provider_message",
    [
        "Rate limit exceeded for watsonx.ai. Please try again later.",
        "Invalid API key for Anthropic. Check your credentials.",
        "Permission denied: model access not authorized for this project.",
        "Quota exceeded: you have reached your monthly token limit.",
    ],
)
async def test_langflow_stream_yields_provider_error_chunk(
    monkeypatch, store_in_memory, provider_message: str
):
    async def raise_provider_error(*_args, **_kwargs) -> AsyncIterator[bytes]:
        raise Exception(provider_message)
        yield b""  # pragma: no cover — keep this an async generator

    monkeypatch.setattr(agent_module, "async_stream", raise_provider_error)

    user_id = "test-provider-error-user"
    agent_module.active_conversations.pop(user_id, None)

    chunks = await _collect_error_chunks(
        agent_module.async_langflow_chat_stream(
            langflow_client=object(),
            flow_id="flow-id",
            prompt="hello",
            user_id=user_id,
        )
    )

    assert len(chunks) == 1
    error_chunk = chunks[0]
    assert error_chunk["status"] == "failed"
    assert error_chunk["finish_reason"] == "error"
    assert error_chunk["error"]["message"] == provider_message

    # Failed first turns still get a real store id (UUID, not error_*).
    assert len(store_in_memory["stored"]) == 1
    stored_user, stored_id = store_in_memory["stored"][0]
    assert stored_user == user_id
    assert not stored_id.startswith("error_")
    assert error_chunk.get("id") == stored_id
    assert stored_id in agent_module.active_conversations.get(user_id, {})


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "response_text",
    [
        "Unauthorized absence may be treated as disciplinary.",
        "Unauthorized access to payroll records is prohibited.",
    ],
)
async def test_langflow_stream_keeps_policy_text_with_unauthorized(
    monkeypatch, store_in_memory, response_text
):
    """Ordinary policy language must not turn a successful turn into an error."""

    async def stream_policy(*_args, **_kwargs) -> AsyncIterator[bytes]:
        yield (
            json.dumps(
                {
                    "id": "resp-policy",
                    "delta": {"content": response_text},
                }
            )
            + "\n"
        ).encode("utf-8")

    monkeypatch.setattr(agent_module, "async_stream", stream_policy)

    user_id = "test-policy-text-user"
    agent_module.active_conversations.pop(user_id, None)
    chunks = await _collect_error_chunks(
        agent_module.async_langflow_chat_stream(
            langflow_client=object(),
            flow_id="flow-id",
            prompt="What does the leave policy say?",
            user_id=user_id,
        )
    )

    assert chunks == [
        {
            "id": "resp-policy",
            "delta": {"content": response_text},
        }
    ]
    assert not any(chunk.get("status") == "failed" for chunk in chunks)
    assert (
        agent_module.active_conversations[user_id]["resp-policy"]["messages"][-1]["error"]
        is not True
    )


@pytest.mark.asyncio
async def test_langflow_stream_marks_explanatory_unauthorized_as_error(
    monkeypatch, store_in_memory
):
    response_text = "Unauthorized: Access token is expired"

    async def stream_error_text(*_args, **_kwargs) -> AsyncIterator[bytes]:
        yield (
            json.dumps(
                {
                    "id": "resp-unauthorized",
                    "delta": {"content": response_text},
                }
            )
            + "\n"
        ).encode("utf-8")

    monkeypatch.setattr(agent_module, "async_stream", stream_error_text)

    user_id = "test-unauthorized-error-user"
    agent_module.active_conversations.pop(user_id, None)
    chunks = await _collect_error_chunks(
        agent_module.async_langflow_chat_stream(
            langflow_client=object(),
            flow_id="flow-id",
            prompt="Why did this request fail?",
            user_id=user_id,
        )
    )

    assert chunks[-1] == {
        "status": "failed",
        "finish_reason": "error",
        "error": {"message": response_text},
        "id": "resp-unauthorized",
        "response_id": "resp-unauthorized",
    }
    assert (
        agent_module.active_conversations[user_id]["resp-unauthorized"]["messages"][-1]["error"]
        is True
    )


@pytest.mark.asyncio
async def test_langflow_stream_persists_error_onto_existing_thread(monkeypatch, store_in_memory):
    provider_message = "Invalid API key for Anthropic. Check your credentials."

    async def raise_provider_error(*_args, **_kwargs) -> AsyncIterator[bytes]:
        raise Exception(provider_message)
        yield b""  # pragma: no cover

    monkeypatch.setattr(agent_module, "async_stream", raise_provider_error)

    user_id = "test-provider-error-existing"
    previous_id = "resp_existing_thread"
    agent_module.active_conversations[user_id] = {
        previous_id: {
            "messages": [{"role": "user", "content": "earlier"}],
            "created_at": None,
            "last_activity": None,
        }
    }

    chunks = await _collect_error_chunks(
        agent_module.async_langflow_chat_stream(
            langflow_client=object(),
            flow_id="flow-id",
            prompt="hello",
            user_id=user_id,
            previous_response_id=previous_id,
        )
    )

    assert chunks[0]["error"]["message"] == provider_message
    assert store_in_memory["stored"] == [(user_id, previous_id)]
    assert store_in_memory["claimed"] == [(user_id, previous_id)]
    assert not any(rid.startswith("error_") for _, rid in store_in_memory["stored"])

    stored = agent_module.active_conversations[user_id][previous_id]
    error_messages = [m for m in stored["messages"] if m.get("error")]
    assert len(error_messages) == 1
    assert error_messages[0]["content"] == provider_message


@pytest.mark.asyncio
async def test_langflow_follow_up_after_error_reuses_store_id(monkeypatch, store_in_memory):
    """Retrying in the same chat must not create a second sidebar conversation."""
    provider_message = "Rate limit exceeded for watsonx.ai."
    stream_calls: list[dict] = []

    async def raise_provider_error(*_args, **kwargs) -> AsyncIterator[bytes]:
        stream_calls.append(kwargs)
        raise Exception(provider_message)
        yield b""  # pragma: no cover

    monkeypatch.setattr(agent_module, "async_stream", raise_provider_error)

    user_id = "test-provider-error-follow-up"
    existing_id = "resp_failed_thread"
    agent_module.active_conversations[user_id] = {
        existing_id: {
            "messages": [
                {"role": "user", "content": "first"},
                {
                    "role": "assistant",
                    "content": "Provided API key could not be found.",
                    "error": True,
                },
            ],
            "created_at": None,
            "last_activity": None,
        }
    }

    # Client keeps conversation_id after an error but omits previous_response_id
    # so Langflow starts fresh while OpenRAG appends to the same sidebar thread.
    chunks = await _collect_error_chunks(
        agent_module.async_langflow_chat_stream(
            langflow_client=object(),
            flow_id="flow-id",
            prompt="retry hello",
            user_id=user_id,
            conversation_id=existing_id,
            previous_response_id=None,
        )
    )

    assert chunks[0]["error"]["message"] == provider_message
    assert store_in_memory["stored"] == [(user_id, existing_id)]
    assert list(agent_module.active_conversations[user_id].keys()) == [existing_id]
    # Fresh Langflow session, same OpenRAG conversation id.
    assert stream_calls[0].get("previous_response_id") is None
    stored_messages = agent_module.active_conversations[user_id][existing_id]["messages"]
    assert [m["role"] for m in stored_messages if m["role"] != "system"][-2:] == [
        "user",
        "assistant",
    ]


@pytest.mark.asyncio
async def test_langflow_follow_up_ignores_broken_previous_response_id(monkeypatch, store_in_memory):
    """Even if the client still sends a failed Langflow session id, do not chain it."""
    provider_message = "Invalid API key for Anthropic. Check your credentials."
    stream_calls: list[dict] = []

    async def raise_provider_error(*_args, **kwargs) -> AsyncIterator[bytes]:
        stream_calls.append(kwargs)
        raise Exception(provider_message)
        yield b""  # pragma: no cover

    monkeypatch.setattr(agent_module, "async_stream", raise_provider_error)

    user_id = "test-provider-error-broken-session"
    existing_id = "resp_broken_langflow"
    agent_module.active_conversations[user_id] = {
        existing_id: {
            "messages": [
                {"role": "user", "content": "first"},
                {
                    "role": "assistant",
                    "content": provider_message,
                    "error": True,
                },
            ],
            "created_at": None,
            "last_activity": None,
        }
    }

    chunks = await _collect_error_chunks(
        agent_module.async_langflow_chat_stream(
            langflow_client=object(),
            flow_id="flow-id",
            prompt="retry hello",
            user_id=user_id,
            conversation_id=existing_id,
            previous_response_id=existing_id,
        )
    )

    assert chunks[0]["error"]["message"] == provider_message
    assert stream_calls[0].get("previous_response_id") is None
    assert store_in_memory["stored"] == [(user_id, existing_id)]


@pytest.mark.asyncio
async def test_langflow_stream_skips_persist_for_cold_previous_response_id(
    monkeypatch, store_in_memory
):
    provider_message = "Permission denied: model access not authorized."

    async def raise_provider_error(*_args, **_kwargs) -> AsyncIterator[bytes]:
        raise Exception(provider_message)
        yield b""  # pragma: no cover

    monkeypatch.setattr(agent_module, "async_stream", raise_provider_error)

    user_id = "test-provider-error-cold"
    previous_id = "resp_not_in_memory"
    agent_module.active_conversations.pop(user_id, None)

    chunks = await _collect_error_chunks(
        agent_module.async_langflow_chat_stream(
            langflow_client=object(),
            flow_id="flow-id",
            prompt="hello",
            user_id=user_id,
            previous_response_id=previous_id,
        )
    )

    assert chunks[0]["error"]["message"] == provider_message
    # Cold previous_response_id must not overwrite conversation metadata —
    # persist under a freshly minted id instead.
    assert len(store_in_memory["stored"]) == 1
    stored_user, stored_id = store_in_memory["stored"][0]
    assert stored_user == user_id
    assert stored_id != previous_id
    assert not stored_id.startswith("error_")
    assert previous_id not in agent_module.active_conversations.get(user_id, {})
    assert stored_id in agent_module.active_conversations.get(user_id, {})


@pytest.mark.asyncio
async def test_langflow_stream_treats_credential_dump_content_as_error(
    monkeypatch, store_in_memory
):
    """WatsonX failures are often streamed as assistant text with no exception."""
    dump = (
        "Failed to initialize IBM WatsonX embedding model: Attempt of authenticating "
        "connection to service failed, please validate your credentials. Error: "
        '{"errorCode":"BXNIM0415E","errorMessage":"Provided API key could not be found."} '
        "IBM WatsonX requires additional configuration parameters "
        "(API endpoint URL and project ID). An error occurred while generating a response."
    )

    async def stream_dump(*_args, **_kwargs) -> AsyncIterator[bytes]:
        yield (json.dumps({"id": "resp_dump", "delta": {"content": dump}}) + "\n").encode("utf-8")

    monkeypatch.setattr(agent_module, "async_stream", stream_dump)

    user_id = "test-provider-error-content-dump"
    agent_module.active_conversations.pop(user_id, None)

    chunks = await _collect_error_chunks(
        agent_module.async_langflow_chat_stream(
            langflow_client=object(),
            flow_id="flow-id",
            prompt="hello",
            user_id=user_id,
        )
    )

    assert any(c.get("status") == "failed" for c in chunks)
    error_chunk = next(c for c in chunks if c.get("status") == "failed")
    assert "{" not in error_chunk["error"]["message"]
    assert "Provided API key is Invalid" in error_chunk["error"]["message"]
    assert store_in_memory["stored"] == [(user_id, "resp_dump")] * 2
    assert store_in_memory["claimed"] == [(user_id, "resp_dump")]
    stored = agent_module.active_conversations[user_id]["resp_dump"]
    assert stored["messages"][-1]["error"] is True
    assert "{" not in stored["messages"][-1]["content"]


@pytest.mark.asyncio
async def test_langflow_stream_keeps_partial_text_with_provider_error(monkeypatch, store_in_memory):
    provider_message = "Rate limit exceeded for watsonx.ai."

    async def partial_then_fail(*_args, **_kwargs) -> AsyncIterator[bytes]:
        yield (
            json.dumps(
                {
                    "id": "resp_partial",
                    "delta": {"content": "Here is a partial answer"},
                }
            )
            + "\n"
        ).encode("utf-8")
        raise Exception(provider_message)

    monkeypatch.setattr(agent_module, "async_stream", partial_then_fail)

    user_id = "test-provider-error-partial"
    agent_module.active_conversations.pop(user_id, None)

    chunks = await _collect_error_chunks(
        agent_module.async_langflow_chat_stream(
            langflow_client=object(),
            flow_id="flow-id",
            prompt="hello",
            user_id=user_id,
        )
    )

    assert len(chunks) == 2
    assert chunks[0]["delta"]["content"] == "Here is a partial answer"
    assert chunks[1]["error"]["message"] == (f"Here is a partial answer\n\n{provider_message}")
    assert store_in_memory["stored"] == [(user_id, "resp_partial")] * 2
    assert store_in_memory["claimed"] == [(user_id, "resp_partial")]


@pytest.mark.asyncio
async def test_chat_stream_yields_provider_error_chunk(monkeypatch, store_in_memory):
    provider_message = "Permission denied: model access not authorized for this project."

    async def raise_provider_error(*_args, **_kwargs) -> AsyncIterator[bytes]:
        raise Exception(provider_message)
        yield b""  # pragma: no cover

    monkeypatch.setattr(agent_module, "async_stream", raise_provider_error)

    user_id = "test-chat-provider-error-user"
    agent_module.active_conversations.pop(user_id, None)

    chunks = await _collect_error_chunks(
        agent_module.async_chat_stream(
            async_client=object(),
            prompt="hello",
            user_id=user_id,
            model="gpt-4.1-mini",
        )
    )

    assert len(chunks) == 1
    assert chunks[0]["error"]["message"] == provider_message
    assert len(store_in_memory["stored"]) == 1
    stored_user, stored_id = store_in_memory["stored"][0]
    assert stored_user == user_id
    assert not stored_id.startswith("error_")
    assert chunks[0].get("id") == stored_id
    assert stored_id in agent_module.active_conversations.get(user_id, {})