Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
"""Golden tests for the durable native LLM message contract."""
import json
import pytest
from services.llm.protocol import (
ContentBlock,
LLMError,
LLMErrorCategory,
LLMResponse,
Message,
ToolCall,
Usage,
message_from_wire,
message_to_wire,
messages_from_wire,
messages_to_wire,
)
def test_message_wire_round_trips_ordered_blocks_and_provider_state():
good_call = ToolCall(id="call-1", name="lookup", args={"q": "weather"})
malformed_call = ToolCall.from_raw(
id="call-2",
name="calculate",
arguments='{"expression":',
)
original = Message(
role="assistant",
content="I will check.",
tool_calls=[good_call, malformed_call],
blocks=[
ContentBlock(
type="reasoning",
text="Need fresh data",
metadata={"signature": "signed-value"},
),
ContentBlock(type="text", text="I will check."),
ContentBlock(type="tool_call", tool_call=good_call),
ContentBlock(type="tool_call", tool_call=malformed_call),
],
provider_state={
"provider": "anthropic",
"payload": {
"content": [
{
"type": "thinking",
"thinking": "Need fresh data",
"signature": "signed-value",
}
]
},
},
)
wire = message_to_wire(original)
# One unversioned wire shape — no version discriminator.
assert "version" not in wire
# The payload must be safe for Temporal's ordinary JSON payload codec.
json.dumps(wire)
decoded = message_from_wire(json.loads(json.dumps(wire)))
assert [block.type for block in decoded.blocks] == [
"reasoning",
"text",
"tool_call",
"tool_call",
]
assert decoded.provider_state == original.provider_state
assert decoded.tool_calls[1].args == {}
assert decoded.tool_calls[1].raw_arguments == '{"expression":'
assert decoded.tool_calls[1].parse_error.startswith(
"Invalid JSON tool arguments"
)
def test_messages_helpers_accept_generators_and_flat_shape():
encoded = messages_to_wire(
Message(role="user", content=str(index)) for index in range(2)
)
assert [message.content for message in messages_from_wire(encoded)] == [
"0",
"1",
]
legacy = message_from_wire(
{
"role": "tool",
"content": "42",
"tool_call_id": "call-1",
"name": "calculate",
}
)
assert legacy.role == "tool"
assert legacy.blocks[0].type == "tool_result"
def test_provider_state_rejects_sdk_objects_and_round_trips_large_payloads():
with pytest.raises(TypeError, match="JSON values"):
Message(
role="assistant",
provider_state={"provider": "test", "payload": {"raw": object()}},
)
blob = "x" * (300 * 1024)
original = Message(
role="assistant",
provider_state={
"provider": "test",
"payload": {"blob": blob},
},
)
wire = message_to_wire(original)
decoded = message_from_wire(json.loads(json.dumps(wire)))
assert decoded.provider_state["payload"]["blob"] == blob
def test_response_always_has_replayable_assistant_message():
call = ToolCall(id="call-1", name="lookup", args={"q": "x"})
response = LLMResponse(
content="Checking",
thinking="Use the lookup",
tool_calls=[call],
)
assert response.assistant_message is not None
assert response.assistant_message.content == "Checking"
assert response.assistant_message.tool_calls == [call]
assert [block.type for block in response.assistant_message.blocks] == [
"reasoning",
"text",
"tool_call",
]
def test_usage_is_safe_and_additive():
first = Usage(input_tokens="10", output_tokens=5, total_tokens=None)
second = Usage(
input_tokens=-1,
output_tokens=2,
cache_read_tokens=3,
reasoning_tokens=4,
)
total = first + second
assert first.total_tokens == 15
assert second.input_tokens == 0
assert total.total_tokens == 17
assert total.cache_read_tokens == 3
assert total.reasoning_tokens == 4
def test_structured_error_classifies_retryable_sdk_failures():
class RateLimitError(Exception):
status_code = 429
request_id = "request-123"
error = LLMError.from_exception("openai", RateLimitError("slow down"))
assert error.category == LLMErrorCategory.RATE_LIMIT
assert error.retryable is True
assert error.status_code == 429
assert error.request_id == "request-123"
def test_structured_error_exposes_only_category_based_user_message():
raw_message = (
"POST https://private-gateway.internal/v1 "
'payload={"authorization":"Bearer secret"}'
)
error = LLMError(
message=raw_message,
provider="openai",
category=LLMErrorCategory.AUTHENTICATION,
)
assert error.message == raw_message
assert error.user_message == (
"OpenAI authentication failed. Check the configured API key."
)
assert "private-gateway" not in error.user_message
assert "Bearer secret" not in error.user_message
@pytest.mark.parametrize(("status", "message"), [
(403, "Spend cap breached for project: projects/private-id for service: aiplatform.googleapis.com"),
(429, "insufficient_quota: private account details"),
(400, "Your credit balance is too low: secret"),
(402, "Payment required: secret"),
])
def test_billing_errors_need_owner_action_instead_of_retry(status, message):
exc = Exception(message)
exc.status_code = status
error = LLMError.from_exception("gemini", exc)
assert error.category == LLMErrorCategory.BILLING
assert not error.retryable
node_error = error.as_node_error()
assert node_error.requires_user_action
assert "spending" in node_error.hint
assert "private" not in str(node_error)
assert "secret" not in str(node_error)
assert node_error.as_dict()["retryable"] is False
@pytest.mark.parametrize(
("provider", "not_found", "context_length"),
[
(
"openai",
"The configured OpenAI model or endpoint was not found.",
"The request exceeds the OpenAI model context window.",
),
(
"openai_compatible:home",
"The OpenAI-compatible endpoint did not find the configured model.",
"The request exceeds the context window of the model at the OpenAI-compatible endpoint.",
),
(
"some-unregistered-provider",
"The language model provider did not find the configured model.",
"The request exceeds the context window of the model at the language model provider.",
),
],
)
def test_model_scoped_messages_read_for_names_and_common_nouns(provider, not_found, context_length):
def message(category: LLMErrorCategory) -> str:
return LLMError(message="raw", provider=provider, category=category).user_message
assert message(LLMErrorCategory.NOT_FOUND) == not_found
assert message(LLMErrorCategory.CONTEXT_LENGTH) == context_length
@pytest.mark.parametrize("provider", ["openai_compatible:home", "some-unregistered-provider"])
@pytest.mark.parametrize("category", list(LLMErrorCategory))
def test_no_message_doubles_the_article_of_a_common_noun(provider, category):
text = LLMError(message="raw", provider=provider, category=category).user_message.lower()
assert "the the " not in text
assert "configured the " not in text
def test_structured_error_preserves_http_date_retry_after():
class Response:
status_code = 429
headers = {
"retry-after": "Wed, 21 Oct 2037 07:28:00 GMT",
"date": "Wed, 21 Oct 2037 07:27:00 GMT",
"x-request-id": "request-http-date",
}
class RateLimitError(Exception):
status_code = 429
response = Response()
error = LLMError.from_exception(
"openai", RateLimitError("rate limited")
)
assert error.retry_after == 60
assert error.retry_after_raw == "Wed, 21 Oct 2037 07:28:00 GMT"
assert error.request_id == "request-http-date"