Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix(langchain): extract visible text from message content blocks (#167)
Rudy Mizrahi Celeklid6e2e80f3279
diff --git a/ifixai/providers/langchain.py b/ifixai/providers/langchain.pyindex 7704f24..15257b0 100644--- a/ifixai/providers/langchain.py+++ b/ifixai/providers/langchain.py@@ -8,6 +8,7 @@ from ifixai.providers.base import (ChatProvider,ProviderAuthError,ProviderConnectionError,+ ProviderEmptyContentError,ProviderError,ProviderOverloadedError,ProviderRateLimitError,@@ -57,7 +58,28 @@ class LangChainProvider(ChatProvider):if isinstance(output, str):return outputif isinstance(output, dict):- return output.get("content", str(output))+ content = output.get("content", str(output))+ if isinstance(content, list):+ # AIMessage.content may contain ordered strings and+ # multimodal blocks. Match LangChain's visible text+ # extraction without importing its optional SDK.+ text_parts = []+ for block in content:+ if isinstance(block, str):+ text_parts.append(block)+ elif isinstance(block, dict) and block.get("type") == "text":+ text = block.get("text")+ if isinstance(text, str):+ text_parts.append(text)+ text = "".join(text_parts)+ if not text:+ raise ProviderEmptyContentError(+ provider="langchain",+ endpoint=url,+ details="No text content in response message",+ )+ return text+ return contentreturn str(output)except aiohttp.ClientResponseError as exc:diff --git a/ifixai/tests/test_langchain_content_blocks_native.py b/ifixai/tests/test_langchain_content_blocks_native.pynew file mode 100644index 0000000..43bf27f--- /dev/null+++ b/ifixai/tests/test_langchain_content_blocks_native.py@@ -0,0 +1,103 @@+"""Serialized real LangChain AIMessage outputs travel over actual aiohttp."""+from contextlib import asynccontextmanager++import pytest+from aiohttp import web++from ifixai.core.fixture_loader import load_fixture+from ifixai.core.runner import run_single+from ifixai.core.types import (+ ChatMessage,+ EvaluationPipelineConfig,+ ProviderConfig,+ TestStatus,+)+from ifixai.judge.config import JudgeConfig+from ifixai.providers.base import ProviderEmptyContentError+from ifixai.providers.langchain import LangChainProvider+++@asynccontextmanager+async def invoke_endpoint(output):+ async def invoke(request):+ await request.json()+ return web.json_response({"output": output})+ app = web.Application()+ app.router.add_post("/invoke", invoke)+ runner = web.AppRunner(app)+ await runner.setup()+ site = web.TCPSite(runner, "127.0.0.1", 0)+ await site.start()+ url = "http://127.0.0.1:" + str(site._server.sockets[0].getsockname()[1])+ try:+ yield url+ finally:+ await runner.cleanup()+++def config(url):+ return ProviderConfig(provider="langchain", endpoint=url, max_retries=0)+++def serialized_message(content):+ messages = pytest.importorskip("langchain_core.messages")+ return messages.AIMessage(content=content).model_dump(mode="json")++[email protected]("content,expected", [+ (["Hello ", "world"], "Hello world"),+ ([{"type": "thinking", "thinking": "internal rationale"},+ {"type": "text", "text": "Visible answer"}], "Visible answer"),+ ([{"type": "text", "text": "A"}, "B", {"type": "text", "text": "C"}], "ABC"),+ ([{"type": "image_url", "image_url": {"url": "https://example.test/image"}},+ {"type": "text", "text": " Café\n"}], " Café\n"),+])+async def test_actual_serialized_ai_messages_return_only_ordered_text(content, expected):+ async with invoke_endpoint(serialized_message(content)) as url:+ result = await LangChainProvider().send_message(+ [ChatMessage(role="user", content="owned prompt")], config(url)+ )+ assert result == expected+ # The next conversation turn in BaseTest must accept the provider reply.+ assert ChatMessage(role="assistant", content=result).content == expected+++async def test_non_text_only_message_does_not_become_an_answer():+ output = serialized_message([{"type": "thinking", "thinking": "internal rationale"}])+ async with invoke_endpoint(output) as url:+ with pytest.raises(ProviderEmptyContentError):+ await LangChainProvider().send_message(+ [ChatMessage(role="user", content="owned prompt")], config(url)+ )++[email protected]("output,expected", [+ ("legacy reply", "legacy reply"),+ ({"content": "legacy message", "type": "ai"}, "legacy message"),+ ({"structured": "opaque legacy output"}, "{'structured': 'opaque legacy output'}"),+])+async def test_existing_string_and_opaque_output_controls(output, expected):+ async with invoke_endpoint(output) as url:+ assert await LangChainProvider().send_message(+ [ChatMessage(role="user", content="owned prompt")], config(url)+ ) == expected+++async def test_shipped_b13_conversation_accepts_ai_message_content_blocks():+ output = serialized_message([{"type": "text", "text": "An owned trace"}])+ async with invoke_endpoint(output) as url:+ result = await run_single(+ "B13", LangChainProvider(), config(url),+ load_fixture("ifixai/fixtures/examples/customer_support.yaml"),+ judge_config=JudgeConfig(provider="mock"),+ pipeline_config=EvaluationPipelineConfig(),+ )+ assert result.status != TestStatus.ERROR+ assert result.error is None or not result.error+ assert result.evidence+ assert all(item.actual_response == "An owned trace" for item in result.evidence)+ assert all(item.evaluation_result != "error" for item in result.evidence)