CubeLoop — a Pythonic, async-native agent harness core designed for high performance, readability, and production-grade persistence.
Python agent-frameworkagent-harnessagent-runtimeagentic-aiai-agentsai-frameworkanthropicharnessharness-engineeringharness-frameworklanggraphlanggraph-alternativellmmcpopenaipi-agentpi-agent-corepythonpython-agent
"""Tests for AnthropicProvider's capability-driven thinking + temperature path.
The Anthropic provider always runs through the capability path; when
``capability=None`` the provider falls back to ``_ANTHROPIC_DEFAULT_CAPABILITY``
which mirrors today's wire bytes exactly. See Task 10 of the capability
descriptor plan and ``cubepi/providers/anthropic.py``.
"""
from __future__ import annotations
from typing import Any
import pytest
from cubeloop.providers.anthropic import AnthropicProvider
from cubeloop.providers.base import (
Model,
ReasoningControl,
StreamOptions,
TextContent,
UserMessage,
)
from cubeloop.providers.capability import (
CapabilityDescriptor,
ReasoningCapability,
TemperatureSpec,
)
def _model() -> Model:
return Model(
id="claude-sonnet-test",
provider_id="anthropic",
context_window=200000,
max_tokens=8192,
reasoning=True,
temperature=1.0,
)
async def _capture_anthropic(
p: AnthropicProvider, opts: StreamOptions
) -> dict[str, Any]:
captured: dict[str, Any] = {}
async def listener(kw: dict[str, Any], m: Model) -> None:
captured.update(kw)
p._request_listeners.append(listener)
class _FakeStream:
response = None
async def __aenter__(self) -> "_FakeStream":
return self
async def __aexit__(self, *a: Any) -> bool:
return False
def __aiter__(self) -> Any:
async def gen() -> Any:
return
yield # pragma: no cover - generator marker
return gen()
async def get_final_message(self) -> Any:
from anthropic.types import Message
return Message.model_construct(
id="m_test",
model="claude-sonnet-test",
role="assistant",
content=[],
stop_reason="end_turn",
stop_sequence=None,
type="message",
usage={"input_tokens": 1, "output_tokens": 1},
)
def fake_stream(**kw: Any) -> _FakeStream:
return _FakeStream()
p._client.messages.stream = fake_stream # type: ignore[method-assign,assignment]
s = await p.stream(
model=_model(),
messages=[UserMessage(content=[TextContent(text="hi")])],
options=opts,
)
async for _ in s:
pass
await s.result()
return captured
@pytest.mark.asyncio
async def test_anthropic_legacy_budget_profile_maps_effort_to_budget() -> None:
provider = AnthropicProvider(api_key="x")
payload = await _capture_anthropic(
provider,
StreamOptions(reasoning=ReasoningControl(mode="on", effort="medium")),
)
assert payload["thinking"]["type"] == "enabled"
assert payload["thinking"]["budget_tokens"] == 8192
assert payload["max_tokens"] > payload["thinking"]["budget_tokens"]
@pytest.mark.asyncio
async def test_default_capability_matches_legacy_thinking_off() -> None:
p = AnthropicProvider(api_key="x")
payload = await _capture_anthropic(
p,
StreamOptions(reasoning=ReasoningControl(mode="off")),
)
assert payload["thinking"] == {"type": "disabled"}
# Temperature is allowed when thinking is off.
assert payload.get("temperature") == 1.0
@pytest.mark.asyncio
async def test_default_capability_thinking_medium_writes_budget() -> None:
p = AnthropicProvider(api_key="x")
payload = await _capture_anthropic(
p,
StreamOptions(reasoning=ReasoningControl(mode="on", effort="medium")),
)
assert payload["thinking"]["type"] == "enabled"
# Mirrors the built-in Anthropic legacy-budget profile.
assert payload["thinking"]["budget_tokens"] == 8192
# Anthropic rejects custom temperature with thinking on.
assert "temperature" not in payload
@pytest.mark.asyncio
async def test_custom_capability_overrides_default() -> None:
custom = CapabilityDescriptor(
reasoning=ReasoningCapability(
mode_payloads={
"off": {"thinking": {"type": "disabled"}},
"on": {"thinking": {"type": "enabled"}},
},
effort_path="thinking.budget_tokens",
effort_values={"medium": 99999},
),
temperature=TemperatureSpec(mode="ignored"),
)
p = AnthropicProvider(api_key="x", capability=custom)
payload = await _capture_anthropic(
p,
StreamOptions(reasoning=ReasoningControl(mode="on", effort="medium")),
)
assert payload["thinking"]["budget_tokens"] == 99999
assert "temperature" not in payload
@pytest.mark.asyncio
async def test_model_capability_overrides_take_precedence() -> None:
"""Per-model override beats the instance-level capability."""
override = CapabilityDescriptor(
reasoning=ReasoningCapability(
mode_payloads={"on": {"thinking": {"type": "enabled"}}},
effort_path="thinking.budget_tokens",
effort_values={"medium": 4242},
),
temperature=TemperatureSpec(mode="ignored"),
)
p = AnthropicProvider(
api_key="x",
model_capability_overrides={"claude-sonnet-test": override},
)
payload = await _capture_anthropic(
p,
StreamOptions(reasoning=ReasoningControl(mode="on", effort="medium")),
)
assert payload["thinking"]["budget_tokens"] == 4242
@pytest.mark.asyncio
async def test_custom_high_budget_capability_bumps_max_tokens():
"""Regression for max_tokens/budget_tokens desync: when custom level_budgets
writes a large budget, max_tokens must be
expanded to accommodate it (else Anthropic API rejects with
budget_tokens >= max_tokens)."""
custom = CapabilityDescriptor(
reasoning=ReasoningCapability(
mode_payloads={"on": {"thinking": {"type": "enabled"}}},
effort_path="thinking.budget_tokens",
effort_values={"medium": 50000},
),
)
p = AnthropicProvider(api_key="x", capability=custom)
payload = await _capture_anthropic(
p,
StreamOptions(reasoning=ReasoningControl(mode="on", effort="medium")),
)
assert payload["thinking"]["budget_tokens"] == 50000
# max_tokens must be at least budget + something (model.max_tokens=8192,
# so min(8192+50000, 200000) = 58192). Anthropic rejects budget>=max_tokens.
assert payload["max_tokens"] >= 50000 + 1
@pytest.mark.asyncio
async def test_capability_clamps_budget_when_context_too_small() -> None:
"""Regression: when context_window can't fit max_tokens + budget,
budget is reduced to fit. Anthropic rejects budget >= max_tokens."""
custom = CapabilityDescriptor(
reasoning=ReasoningCapability(
mode_payloads={"on": {"thinking": {"type": "enabled"}}},
effort_path="thinking.budget_tokens",
effort_values={"high": 16384},
),
)
p = AnthropicProvider(api_key="x", capability=custom)
# Tight model: context_window only 10000, max_tokens 8192.
m = Model(
id="claude-tight-test",
provider_id="anthropic",
context_window=10000,
max_tokens=8192,
reasoning=True,
temperature=1.0,
)
captured: dict[str, Any] = {}
async def listener(kw: dict[str, Any], model: Model) -> None:
captured.update(kw)
p._request_listeners.append(listener)
class _FakeStream:
response = None
async def __aenter__(self) -> "_FakeStream":
return self
async def __aexit__(self, *a: Any) -> bool:
return False
def __aiter__(self) -> Any:
async def gen() -> Any:
return
yield # pragma: no cover - generator marker
return gen()
async def get_final_message(self) -> Any:
from anthropic.types import Message
return Message.model_construct(
id="m_test",
model="claude-tight-test",
role="assistant",
content=[],
stop_reason="end_turn",
stop_sequence=None,
type="message",
usage={"input_tokens": 1, "output_tokens": 1},
)
def fake_stream(**kw: Any) -> _FakeStream:
return _FakeStream()
p._client.messages.stream = fake_stream # type: ignore[method-assign,assignment]
s = await p.stream(
model=m,
messages=[UserMessage(content=[TextContent(text="hi")])],
options=StreamOptions(reasoning=ReasoningControl(mode="on", effort="high")),
)
async for _ in s:
pass
await s.result()
# Either budget was reduced below max_tokens, or thinking was disabled.
thinking_block = captured.get("thinking", {})
if thinking_block.get("type") == "enabled":
assert thinking_block["budget_tokens"] < captured["max_tokens"]
@pytest.mark.asyncio
async def test_capability_effort_mapping_controls_budget() -> None:
cap = CapabilityDescriptor(
reasoning=ReasoningCapability(
mode_payloads={"on": {"thinking": {"type": "enabled"}}},
effort_path="thinking.budget_tokens",
effort_values={"medium": 12288},
)
)
p = AnthropicProvider(api_key="x", capability=cap)
payload = await _capture_anthropic(
p,
StreamOptions(reasoning=ReasoningControl(mode="on", effort="medium")),
)
assert payload["thinking"]["type"] == "enabled"
assert payload["thinking"]["budget_tokens"] == 12288
# max_tokens must accommodate the new budget.
assert payload["max_tokens"] >= 12288 + 512 # min_output_tokens floor
@pytest.mark.asyncio
async def test_capability_disables_thinking_when_budget_reduced_to_zero() -> None:
"""When max_tokens floor is below min_output_tokens (1024), the clamp reduces
budget to 0 and the provider must fall back to thinking.type=disabled."""
custom = CapabilityDescriptor(
reasoning=ReasoningCapability(
mode_payloads={"on": {"thinking": {"type": "enabled"}}},
effort_path="thinking.budget_tokens",
effort_values={"high": 16384},
),
)
p = AnthropicProvider(api_key="x", capability=custom)
# context_window=512 leaves no room for any budget at all.
m = Model(
id="claude-tiny-test",
provider_id="anthropic",
context_window=512,
max_tokens=512,
reasoning=True,
temperature=1.0,
)
captured: dict[str, Any] = {}
async def listener(kw: dict[str, Any], model: Model) -> None:
captured.update(kw)
p._request_listeners.append(listener)
class _FakeStream:
response = None
async def __aenter__(self) -> "_FakeStream":
return self
async def __aexit__(self, *a: Any) -> bool:
return False
def __aiter__(self) -> Any:
async def gen() -> Any:
return
yield # pragma: no cover - generator marker
return gen()
async def get_final_message(self) -> Any:
from anthropic.types import Message
return Message.model_construct(
id="m_test",
model="claude-tiny-test",
role="assistant",
content=[],
stop_reason="end_turn",
stop_sequence=None,
type="message",
usage={"input_tokens": 1, "output_tokens": 1},
)
def fake_stream(**kw: Any) -> _FakeStream:
return _FakeStream()
p._client.messages.stream = fake_stream # type: ignore[method-assign,assignment]
s = await p.stream(
model=m,
messages=[UserMessage(content=[TextContent(text="hi")])],
options=StreamOptions(reasoning=ReasoningControl(mode="on", effort="high")),
)
async for _ in s:
pass
await s.result()
assert captured["thinking"] == {"type": "disabled"}