Skip to content

Personal AI, On Personal Devices

Python

fix(channels): wire channel→agent handler and fix Telegram send pipeline (#94)

Prathap Pc2756964a706
diff --git a/pyproject.toml b/pyproject.toml
index 696885aa..d6adcaba 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -14,6 +14,7 @@ dependencies = [
"ddgs>=9.11.4",
"httpx>=0.27",
"openai>=1.30",
+ "python-telegram-bot>=22.6",
"rich>=13",
"tomli>=2.0; python_version < '3.11'",
]
diff --git a/src/openjarvis/channels/telegram.py b/src/openjarvis/channels/telegram.py
index 7097ece5..7cfa2146 100644
--- a/src/openjarvis/channels/telegram.py
+++ b/src/openjarvis/channels/telegram.py
@@ -109,14 +109,13 @@ class TelegramChannel(BaseChannel):
import httpx
url = f"https://api.telegram.org/bot{self._token}/sendMessage"
+ chat_id = conversation_id or channel
payload: Dict[str, Any] = {
- "chat_id": channel,
+ "chat_id": chat_id,
"text": content,
}
if self._parse_mode:
payload["parse_mode"] = self._parse_mode
- if conversation_id:
- payload["reply_to_message_id"] = conversation_id
resp = httpx.post(url, json=payload, timeout=10.0)
if resp.status_code < 300:
@@ -164,6 +163,19 @@ class TelegramChannel(BaseChannel):
message_id=str(msg.message_id),
conversation_id=str(msg.chat.id),
)
+ # Enforce allow-list when configured
+ if self._allowed_chat_ids:
+ _allowed = {
+ cid.strip()
+ for cid in self._allowed_chat_ids.split(",")
+ if cid.strip()
+ }
+ if cm.conversation_id not in _allowed:
+ logger.debug(
+ "Ignoring message from unlisted chat %s",
+ cm.conversation_id,
+ )
+ return
for handler in self._handlers:
try:
handler(cm)
@@ -181,7 +193,7 @@ class TelegramChannel(BaseChannel):
)
app.add_handler(MessageHandler(filters.TEXT, _handle_msg))
- app.run_polling(stop_signals=None)
+ app.run_polling(stop_signals=None, drop_pending_updates=True)
except Exception:
logger.debug("Telegram poll loop error", exc_info=True)
self._status = ChannelStatus.ERROR
diff --git a/src/openjarvis/cli/serve.py b/src/openjarvis/cli/serve.py
index d4b68798..bfd819a6 100644
--- a/src/openjarvis/cli/serve.py
+++ b/src/openjarvis/cli/serve.py
@@ -230,6 +230,20 @@ def serve(
console.print(f"[yellow]Channel failed to start: {exc}[/yellow]")
channel_bridge = None
+ # Wire channel messages → agent / engine (per-chat session isolation)
+ if channel_bridge is not None:
+ from openjarvis.system import JarvisSystem
+
+ _wire_system = JarvisSystem(
+ config=config,
+ bus=bus,
+ engine=engine,
+ engine_key=engine_name,
+ model=model_name,
+ agent_name=agent_key or "",
+ )
+ _wire_system.wire_channel(channel_bridge)
+
# Set up speech backend
speech_backend = None
try:
diff --git a/src/openjarvis/sessions/session.py b/src/openjarvis/sessions/session.py
index f546c964..2809113d 100644
--- a/src/openjarvis/sessions/session.py
+++ b/src/openjarvis/sessions/session.py
@@ -66,7 +66,7 @@ class SessionStore:
) -> None:
self._db_path = Path(db_path)
self._db_path.parent.mkdir(parents=True, exist_ok=True)
- self._conn = sqlite3.connect(str(self._db_path))
+ self._conn = sqlite3.connect(str(self._db_path), check_same_thread=False)
self._max_age_hours = max_age_hours
self._consolidation_threshold = consolidation_threshold
self._create_tables()
diff --git a/src/openjarvis/system.py b/src/openjarvis/system.py
index d1514e81..023c4671 100644
--- a/src/openjarvis/system.py
+++ b/src/openjarvis/system.py
@@ -269,6 +269,85 @@ class JarvisSystem:
logger.warning("Failed to build tool %r: %s", name, exc)
return tools
+ def wire_channel(self, channel_bridge: Any) -> None:
+ """Register a message handler on *channel_bridge* that routes every
+ incoming message through this system (agent or engine) and replies.
+
+ Sessions are isolated per ``"<channel>:<conversation_id>"`` key so
+ each chat retains its own history.
+
+ Parameters
+ ----------
+ channel_bridge:
+ A connected :class:`~openjarvis.channels._stubs.BaseChannel`
+ instance whose ``on_message`` method accepts a callable.
+ """
+ from openjarvis.agents._stubs import AgentContext
+ from openjarvis.core.types import Message, Role
+ from openjarvis.sessions.session import SessionStore
+
+ if self.session_store is None:
+ from pathlib import Path
+ self.session_store = SessionStore(
+ db_path=Path(self.config.sessions.db_path).expanduser(),
+ max_age_hours=self.config.sessions.max_age_hours,
+ consolidation_threshold=self.config.sessions.consolidation_threshold,
+ )
+
+ _system = self # capture for closure
+
+ def _on_channel_message(cm) -> None:
+ session_key = f"{cm.channel}:{cm.conversation_id}"
+ session = _system.session_store.get_or_create(
+ session_key,
+ channel=cm.channel,
+ channel_user_id=cm.sender,
+ )
+
+ # Rebuild prior conversation turns into AgentContext
+ ctx = AgentContext()
+ for sm in session.messages:
+ try:
+ role = Role(sm.role)
+ except ValueError:
+ role = Role.USER
+ ctx.conversation.add(Message(role=role, content=sm.content))
+
+ reply = ""
+ try:
+ if _system.agent_name and _system.agent_name != "none":
+ result = _system.ask(
+ cm.content, context=False,
+ agent=_system.agent_name,
+ )
+ reply = result.get("content", "")
+ else:
+ result = _system.ask(cm.content, context=False)
+ reply = result.get("content", "")
+ except Exception:
+ logger.exception("Channel message handler error")
+ reply = "Sorry, I encountered an error processing your message."
+
+ try:
+ _system.session_store.save_message(
+ session.session_id, "user", cm.content, channel=cm.channel,
+ )
+ _system.session_store.save_message(
+ session.session_id, "assistant", reply, channel=cm.channel,
+ )
+ except Exception:
+ logger.debug("Session save error", exc_info=True)
+
+ if reply:
+ try:
+ channel_bridge.send(
+ cm.channel, reply, conversation_id=cm.conversation_id,
+ )
+ except Exception:
+ logger.exception("Channel send error")
+
+ channel_bridge.on_message(_on_channel_message)
+
def close(self) -> None:
"""Release resources."""
if self.scheduler and hasattr(self.scheduler, "stop"):
diff --git a/tests/a2a/test_a2a.py b/tests/a2a/test_a2a.py
index e3560e2d..b17175d8 100644
--- a/tests/a2a/test_a2a.py
+++ b/tests/a2a/test_a2a.py
@@ -78,6 +78,7 @@ class TestA2AResponse:
def test_from_json(self):
import json
+
data = json.dumps({"jsonrpc": "2.0", "result": "ok", "id": "3"})
resp = A2AResponse.from_json(data)
assert resp.result == "ok"
@@ -88,24 +89,28 @@ class TestA2AServer:
def test_task_send(self):
card = AgentCard(name="Test")
server = A2AServer(card, handler=lambda x: f"Echo: {x}")
- response = server.handle_request({
- "jsonrpc": "2.0",
- "method": "tasks/send",
- "params": {"input": "Hello"},
- "id": "1",
- })
+ response = server.handle_request(
+ {
+ "jsonrpc": "2.0",
+ "method": "tasks/send",
+ "params": {"input": "Hello"},
+ "id": "1",
+ }
+ )
assert response["result"]["state"] == "completed"
assert "Echo: Hello" in response["result"]["output"]
def test_task_send_with_message_format(self):
card = AgentCard(name="Test")
server = A2AServer(card, handler=lambda x: f"Got: {x}")
- response = server.handle_request({
- "jsonrpc": "2.0",
- "method": "tasks/send",
- "params": {"message": {"role": "user", "parts": [{"text": "Hi"}]}},
- "id": "1",
- })
+ response = server.handle_request(
+ {
+ "jsonrpc": "2.0",
+ "method": "tasks/send",
+ "params": {"message": {"role": "user", "parts": [{"text": "Hi"}]}},
+ "id": "1",
+ }
+ )
assert response["result"]["state"] == "completed"
assert "Got: Hi" in response["result"]["output"]
@@ -113,62 +118,74 @@ class TestA2AServer:
card = AgentCard(name="Test")
server = A2AServer(card, handler=lambda x: x)
# First send a task
- send_resp = server.handle_request({
- "jsonrpc": "2.0",
- "method": "tasks/send",
- "params": {"input": "test"},
- "id": "1",
- })
+ send_resp = server.handle_request(
+ {
+ "jsonrpc": "2.0",
+ "method": "tasks/send",
+ "params": {"input": "test"},
+ "id": "1",
+ }
+ )
task_id = send_resp["result"]["id"]
# Now get it
- get_resp = server.handle_request({
- "jsonrpc": "2.0",
- "method": "tasks/get",
- "params": {"id": task_id},
- "id": "2",
- })
+ get_resp = server.handle_request(
+ {
+ "jsonrpc": "2.0",
+ "method": "tasks/get",
+ "params": {"id": task_id},
+ "id": "2",
+ }
+ )
assert get_resp["result"]["id"] == task_id
def test_task_get_not_found(self):
card = AgentCard(name="Test")
server = A2AServer(card)
- response = server.handle_request({
- "jsonrpc": "2.0",
- "method": "tasks/get",
- "params": {"id": "nonexistent"},
- "id": "1",
- })
+ response = server.handle_request(
+ {
+ "jsonrpc": "2.0",
+ "method": "tasks/get",
+ "params": {"id": "nonexistent"},
+ "id": "1",
+ }
+ )
assert "error" in response
def test_task_cancel(self):
card = AgentCard(name="Test")
server = A2AServer(card, handler=lambda x: x)
- send_resp = server.handle_request({
- "jsonrpc": "2.0",
- "method": "tasks/send",
- "params": {"input": "test"},
- "id": "1",
- })
+ send_resp = server.handle_request(
+ {
+ "jsonrpc": "2.0",
+ "method": "tasks/send",
+ "params": {"input": "test"},
+ "id": "1",
+ }
+ )
task_id = send_resp["result"]["id"]
- cancel_resp = server.handle_request({
- "jsonrpc": "2.0",
- "method": "tasks/cancel",
- "params": {"id": task_id},
- "id": "2",
- })
+ cancel_resp = server.handle_request(
+ {
+ "jsonrpc": "2.0",
+ "method": "tasks/cancel",
+ "params": {"id": task_id},
+ "id": "2",
+ }
+ )
assert cancel_resp["result"]["state"] == "canceled"
def test_unknown_method(self):
card = AgentCard(name="Test")
server = A2AServer(card)
- response = server.handle_request({
- "jsonrpc": "2.0",
- "method": "unknown/method",
- "params": {},
- "id": "1",
- })
+ response = server.handle_request(
+ {
+ "jsonrpc": "2.0",
+ "method": "unknown/method",
+ "params": {},
+ "id": "1",
+ }
+ )
assert "error" in response
assert response["error"]["code"] == -32601
@@ -176,12 +193,14 @@ class TestA2AServer:
bus = EventBus(record_history=True)
card = AgentCard(name="Test")
server = A2AServer(card, handler=lambda x: x, bus=bus)
- server.handle_request({
- "jsonrpc": "2.0",
- "method": "tasks/send",
- "params": {"input": "test"},
- "id": "1",
- })
+ server.handle_request(
+ {
+ "jsonrpc": "2.0",
+ "method": "tasks/send",
+ "params": {"input": "test"},
+ "id": "1",
+ }
+ )
event_types = {e.event_type for e in bus.history}
assert EventType.A2A_TASK_RECEIVED in event_types
assert EventType.A2A_TASK_COMPLETED in event_types
@@ -193,10 +212,12 @@ class TestA2AServer:
raise ValueError("boom")
server = A2AServer(card, handler=bad_handler)
- response = server.handle_request({
- "jsonrpc": "2.0",
- "method": "tasks/send",
- "params": {"input": "test"},
- "id": "1",
- })
+ response = server.handle_request(
+ {
+ "jsonrpc": "2.0",
+ "method": "tasks/send",
+ "params": {"input": "test"},
+ "id": "1",
+ }
+ )
assert response["result"]["state"] == "failed"
diff --git a/tests/agents/test_base_agent.py b/tests/agents/test_base_agent.py
index b1c0313e..cbfa6cc5 100644
--- a/tests/agents/test_base_agent.py
+++ b/tests/agents/test_base_agent.py
@@ -87,7 +87,11 @@ class TestBaseAgentInit:
bus = EventBus()
engine = MagicMock()
agent = _ConcreteAgent(
- engine, "m", bus=bus, temperature=0.1, max_tokens=256,
+ engine,
+ "m",
+ bus=bus,
+ temperature=0.1,
+ max_tokens=256,
)
assert agent._temperature == 0.1
assert agent._max_tokens == 256
@@ -176,7 +180,9 @@ class TestBuildMessages:
conv.add(Message(role=Role.USER, content="prev"))
ctx = AgentContext(conversation=conv)
messages = agent._build_messages(
- "new", ctx, system_prompt="System.",
+ "new",
+ ctx,
+ system_prompt="System.",
)
assert len(messages) == 3
assert messages[0].role == Role.SYSTEM
diff --git a/tests/agents/test_budget.py b/tests/agents/test_budget.py
index 77325414..e8e35349 100644
--- a/tests/agents/test_budget.py
+++ b/tests/agents/test_budget.py
@@ -20,8 +20,7 @@ def test_budget_exceeded_sets_status(tmp_path):
assert updated["status"] == "budget_exceeded"
budget_events = [
- e for e in bus.history
- if e.event_type == EventType.AGENT_BUDGET_EXCEEDED
+ e for e in bus.history if e.event_type == EventType.AGENT_BUDGET_EXCEEDED
]
assert len(budget_events) == 1
mgr.close()
@@ -54,7 +53,8 @@ def test_budget_unlimited_skips_check(tmp_path):
mgr.start_tick(agent["id"])
result = AgentResult(
- content="done", metadata={"cost": 999.99, "tokens_used": 1000000},
+ content="done",
+ metadata={"cost": 999.99, "tokens_used": 1000000},
)
executor._finalize_tick(agent["id"], result, error=None, duration=1.0)
diff --git a/tests/agents/test_claude_code.py b/tests/agents/test_claude_code.py
index ec05e729..6eabfa37 100644
--- a/tests/agents/test_claude_code.py
+++ b/tests/agents/test_claude_code.py
@@ -141,16 +141,19 @@ class TestClaudeCodeRun:
def test_successful_run(self):
agent = self._make_agent()
- output = _wrap_output({
- "content": "Hello from Claude Code!",
- "tool_results": [],
- "metadata": {"message_count": 3},
- })
+ output = _wrap_output(
+ {
+ "content": "Hello from Claude Code!",
+ "tool_results": [],
+ "metadata": {"message_count": 3},
+ }
+ )
proc = _mock_proc(stdout=output)
with (
patch.object(
- agent, "_ensure_runner",
+ agent,
+ "_ensure_runner",
return_value="/fake/runner",
),
patch("subprocess.run", return_value=proc),
@@ -165,22 +168,25 @@ class TestClaudeCodeRun:
def test_run_with_tool_results(self):
agent = self._make_agent()
- output = _wrap_output({
- "content": "I read the file.",
- "tool_results": [
- {
- "tool_name": "Read",
- "content": "file contents",
- "success": True,
- },
- ],
- "metadata": {},
- })
+ output = _wrap_output(
+ {
+ "content": "I read the file.",
+ "tool_results": [
+ {
+ "tool_name": "Read",
+ "content": "file contents",
+ "success": True,
+ },
+ ],
+ "metadata": {},
+ }
+ )
proc = _mock_proc(stdout=output)
with (
patch.object(
- agent, "_ensure_runner",
+ agent,
+ "_ensure_runner",
return_value="/fake/runner",
),
patch("subprocess.run", return_value=proc),
@@ -200,20 +206,24 @@ class TestClaudeCodeRun:
allowed_tools=["Read", "Write"],
system_prompt="Be helpful.",
)
- output = _wrap_output({
- "content": "ok",
- "tool_results": [],
- "metadata": {},
- })
+ output = _wrap_output(
+ {
+ "content": "ok",
+ "tool_results": [],
+ "metadata": {},
+ }
+ )
proc = _mock_proc(stdout=output)
with (
patch.object(
- agent, "_ensure_runner",
+ agent,
+ "_ensure_runner",
return_value="/fake/runner",
),
patch(
- "subprocess.run", return_value=proc,
+ "subprocess.run",
+ return_value=proc,
) as mock_run,
):
agent.run("Do something")
@@ -230,12 +240,14 @@ class TestClaudeCodeRun:
def test_timeout_handling(self):
agent = self._make_agent(timeout=5)
exc = subprocess.TimeoutExpired(
- cmd="node", timeout=5,
+ cmd="node",
+ timeout=5,
)
with (
patch.object(
- agent, "_ensure_runner",
+ agent,
+ "_ensure_runner",
return_value="/fake/runner",
),
patch("subprocess.run", side_effect=exc),
@@ -249,12 +261,14 @@ class TestClaudeCodeRun:
def test_nonzero_exit_code(self):
agent = self._make_agent()
proc = _mock_proc(
- returncode=1, stderr="ENOENT: module not found",
+ returncode=1,
+ stderr="ENOENT: module not found",
)
with (
patch.object(
- agent, "_ensure_runner",
+ agent,
+ "_ensure_runner",
return_value="/fake/runner",
),
patch("subprocess.run", return_value=proc),
@@ -273,7 +287,8 @@ class TestClaudeCodeRun:
with (
patch.object(
- agent, "_ensure_runner",
+ agent,
+ "_ensure_runner",
return_value="/fake/runner",
),
patch("subprocess.run", return_value=proc),
@@ -291,7 +306,8 @@ class TestClaudeCodeRun:
with (
patch.object(
- agent, "_ensure_runner",
+ agent,
+ "_ensure_runner",
return_value="/fake/runner",
),
patch("subprocess.run", return_value=proc),
@@ -312,18 +328,24 @@ class TestClaudeCodeEvents:
engine = MagicMock()
engine.engine_id = "mock"
agent = ClaudeCodeAgent(
- engine, "test-model", bus=bus, api_key="k",
+ engine,
+ "test-model",
+ bus=bus,
+ api_key="k",
+ )
+ output = _wrap_output(
+ {
+ "content": "hi",
+ "tool_results": [],
+ "metadata": {},
+ }
)
- output = _wrap_output({
- "content": "hi",
- "tool_results": [],
- "metadata": {},
- })
proc = _mock_proc(stdout=output)
with (
patch.object(
- agent, "_ensure_runner",
+ agent,
+ "_ensure_runner",
return_value="/fake/runner",
),
patch("subprocess.run", return_value=proc),
@@ -339,18 +361,24 @@ class TestClaudeCodeEvents:
engine = MagicMock()
engine.engine_id = "mock"
agent = ClaudeCodeAgent(
- engine, "test-model", bus=bus, api_key="k",
+ engine,
+ "test-model",
+ bus=bus,
+ api_key="k",
+ )
+ output = _wrap_output(
+ {
+ "content": "hi",
+ "tool_results": [],
+ "metadata": {},
+ }
)
- output = _wrap_output({
- "content": "hi",
- "tool_results": [],
- "metadata": {},
- })
proc = _mock_proc(stdout=output)
with (
patch.object(
- agent, "_ensure_runner",
+ agent,
+ "_ensure_runner",
return_value="/fake/runner",
),
patch("subprocess.run", return_value=proc),
@@ -358,8 +386,7 @@ class TestClaudeCodeEvents:
agent.run("test input")
start_events = [
- e for e in bus.history
- if e.event_type == EventType.AGENT_TURN_START
+ e for e in bus.history if e.event_type == EventType.AGENT_TURN_START
]
assert len(start_events) == 1
assert start_events[0].data["agent"] == "claude_code"
@@ -370,13 +397,17 @@ class TestClaudeCodeEvents:
engine = MagicMock()
engine.engine_id = "mock"
agent = ClaudeCodeAgent(
- engine, "test-model", bus=bus, api_key="k",
+ engine,
+ "test-model",
+ bus=bus,
+ api_key="k",
)
proc = _mock_proc(returncode=1, stderr="error")
with (
patch.object(
- agent, "_ensure_runner",
+ agent,
+ "_ensure_runner",
return_value="/fake/runner",
),
patch("subprocess.run", return_value=proc),
@@ -449,11 +480,7 @@ class TestParseOutput:
"tool_results": [],
"metadata": {},
}
- stdout = (
- "some debug output\n"
- + _wrap_output(payload)
- + "\nmore output"
- )
+ stdout = "some debug output\n" + _wrap_output(payload) + "\nmore output"
content, tools, meta = ClaudeCodeAgent._parse_output(
stdout,
)
@@ -485,7 +512,9 @@ class TestClaudeCodeDefaults:
engine = MagicMock()
engine.engine_id = "mock"
agent = ClaudeCodeAgent(
- engine, "test-model", api_key="explicit-key",
+ engine,
+ "test-model",
+ api_key="explicit-key",
)
assert agent._api_key == "explicit-key"
@@ -499,7 +528,9 @@ class TestClaudeCodeDefaults:
engine = MagicMock()
engine.engine_id = "mock"
agent = ClaudeCodeAgent(
- engine, "test-model", timeout=60,
+ engine,
+ "test-model",
+ timeout=60,
)
assert agent._timeout == 60
@@ -507,18 +538,23 @@ class TestClaudeCodeDefaults:
engine = MagicMock()
engine.engine_id = "mock"
agent = ClaudeCodeAgent(
- engine, "test-model", api_key="k",
+ engine,
+ "test-model",
+ api_key="k",
+ )
+ output = _wrap_output(
+ {
+ "content": "ok",
+ "tool_results": [],
+ "metadata": {},
+ }
)
- output = _wrap_output({
- "content": "ok",
- "tool_results": [],
- "metadata": {},
- })
proc = _mock_proc(stdout=output)
with (
patch.object(
- agent, "_ensure_runner",
+ agent,
+ "_ensure_runner",
return_value="/fake/runner",
),
patch("subprocess.run", return_value=proc),
diff --git a/tests/agents/test_continuation.py b/tests/agents/test_continuation.py
index 0d596384..4bd3b131 100644
--- a/tests/agents/test_continuation.py
+++ b/tests/agents/test_continuation.py
@@ -43,39 +43,47 @@ class ContinuationAgent(BaseAgent):
class TestContinuation:
def test_no_continuation_needed(self):
- engine = MockEngine([
- {"content": "Hello world", "finish_reason": "stop"},
- ])
+ engine = MockEngine(
+ [
+ {"content": "Hello world", "finish_reason": "stop"},
+ ]
+ )
agent = ContinuationAgent(engine, "test-model")
result = agent.run("Hi")
assert result.content == "Hello world"
def test_single_continuation(self):
- engine = MockEngine([
- {"content": "Part 1...", "finish_reason": "length"},
- {"content": " Part 2.", "finish_reason": "stop"},
- ])
+ engine = MockEngine(
+ [
+ {"content": "Part 1...", "finish_reason": "length"},
+ {"content": " Part 2.", "finish_reason": "stop"},
+ ]
+ )
agent = ContinuationAgent(engine, "test-model")
result = agent.run("Hi")
assert result.content == "Part 1... Part 2."
def test_multiple_continuations(self):
- engine = MockEngine([
- {"content": "A", "finish_reason": "length"},
- {"content": "B", "finish_reason": "length"},
- {"content": "C", "finish_reason": "stop"},
- ])
+ engine = MockEngine(
+ [
+ {"content": "A", "finish_reason": "length"},
+ {"content": "B", "finish_reason": "length"},
+ {"content": "C", "finish_reason": "stop"},
+ ]
+ )
agent = ContinuationAgent(engine, "test-model")
result = agent.run("Hi")
assert result.content == "ABC"
def test_max_continuations_respected(self):
- engine = MockEngine([
- {"content": "A", "finish_reason": "length"},
- {"content": "B", "finish_reason": "length"},
- {"content": "C", "finish_reason": "length"}, # 3rd continuation
- {"content": "D", "finish_reason": "stop"},
- ])
+ engine = MockEngine(
+ [
+ {"content": "A", "finish_reason": "length"},
+ {"content": "B", "finish_reason": "length"},
+ {"content": "C", "finish_reason": "length"}, # 3rd continuation
+ {"content": "D", "finish_reason": "stop"},
+ ]
+ )
agent = ContinuationAgent(engine, "test-model")
# Default max_continuations=2, so should stop after 2 continuations
messages = agent._build_messages("Hi")
@@ -84,9 +92,11 @@ class TestContinuation:
assert content == "ABC" # A + B + C, but not D
def test_empty_finish_reason(self):
- engine = MockEngine([
- {"content": "Done", "finish_reason": ""},
- ])
+ engine = MockEngine(
+ [
+ {"content": "Done", "finish_reason": ""},
+ ]
+ )
agent = ContinuationAgent(engine, "test-model")
result = agent.run("Hi")
assert result.content == "Done"
diff --git a/tests/agents/test_executor_error_detail.py b/tests/agents/test_executor_error_detail.py
index aa4fbd81..87aa0bed 100644
--- a/tests/agents/test_executor_error_detail.py
+++ b/tests/agents/test_executor_error_detail.py
@@ -1,4 +1,5 @@
"""Tests for structured error_detail in executor traces."""
+
from openjarvis.agents.errors import EscalateError, FatalError, RetryableError
from openjarvis.agents.executor import AgentExecutor
from openjarvis.core.events import EventBus
@@ -6,6 +7,7 @@ from openjarvis.core.events import EventBus
def test_build_error_detail_fatal(tmp_path):
from openjarvis.agents.manager import AgentManager
+
mgr = AgentManager(db_path=str(tmp_path / "agents.db"))
exe = AgentExecutor(manager=mgr, event_bus=EventBus())
error = FatalError("401 unauthorized")
@@ -17,6 +19,7 @@ def test_build_error_detail_fatal(tmp_path):
def test_build_error_detail_retryable(tmp_path):
from openjarvis.agents.manager import AgentManager
+
mgr = AgentManager(db_path=str(tmp_path / "agents.db"))
exe = AgentExecutor(manager=mgr, event_bus=EventBus())
error = RetryableError("connection timed out")
@@ -27,6 +30,7 @@ def test_build_error_detail_retryable(tmp_path):
def test_build_error_detail_escalate(tmp_path):
from openjarvis.agents.manager import AgentManager
+
mgr = AgentManager(db_path=str(tmp_path / "agents.db"))
exe = AgentExecutor(manager=mgr, event_bus=EventBus())
error = EscalateError("agent needs help")
diff --git a/tests/agents/test_learning_integration.py b/tests/agents/test_learning_integration.py
index a8292e35..e889d9bf 100644
--- a/tests/agents/test_learning_integration.py
+++ b/tests/agents/test_learning_integration.py
@@ -50,11 +50,14 @@ def test_scheduler_tracks_tick_count_for_learning(tmp_path):
executor = AgentExecutor(mgr, bus)
scheduler = AgentScheduler(mgr, executor, event_bus=bus)
- agent = mgr.create_agent("tick-counter", config={
- "learning_enabled": True,
- "learning_schedule": "every_3_ticks",
- "schedule_type": "manual",
- })
+ agent = mgr.create_agent(
+ "tick-counter",
+ config={
+ "learning_enabled": True,
+ "learning_schedule": "every_3_ticks",
+ "schedule_type": "manual",
+ },
+ )
# Simulate 3 ticks completing
for _ in range(3):
@@ -62,8 +65,7 @@ def test_scheduler_tracks_tick_count_for_learning(tmp_path):
# Should have triggered learning
learning_events = [
- e for e in bus.history
- if e.event_type == EventType.AGENT_LEARNING_STARTED
+ e for e in bus.history if e.event_type == EventType.AGENT_LEARNING_STARTED
]
assert len(learning_events) == 1
assert learning_events[0].data["agent_id"] == agent["id"]
@@ -85,17 +87,19 @@ def test_scheduler_no_learning_when_disabled(tmp_path):
executor = AgentExecutor(mgr, bus)
scheduler = AgentScheduler(mgr, executor, event_bus=bus)
- agent = mgr.create_agent("no-learning", config={
- "learning_enabled": False,
- "learning_schedule": "every_3_ticks",
- })
+ agent = mgr.create_agent(
+ "no-learning",
+ config={
+ "learning_enabled": False,
+ "learning_schedule": "every_3_ticks",
+ },
+ )
for _ in range(5):
scheduler._on_tick_completed(agent["id"])
learning_events = [
- e for e in bus.history
- if e.event_type == EventType.AGENT_LEARNING_STARTED
+ e for e in bus.history if e.event_type == EventType.AGENT_LEARNING_STARTED
]
assert len(learning_events) == 0
mgr.close()
diff --git a/tests/agents/test_loop_guard.py b/tests/agents/test_loop_guard.py
index 69ea11f6..3b15f3c0 100644
--- a/tests/agents/test_loop_guard.py
+++ b/tests/agents/test_loop_guard.py
@@ -8,6 +8,7 @@ from openjarvis.core.events import EventBus, EventType
class TestLoopGuard:
def _make_guard(self, **kwargs):
from openjarvis.agents.loop_guard import LoopGuard, LoopGuardConfig
+
kwargs.setdefault("warn_before_block", False)
config = LoopGuardConfig(**kwargs)
bus = EventBus(record_history=True)
@@ -55,8 +56,7 @@ class TestLoopGuard:
guard.check_call("x", '{"a": 1}')
guard.check_call("x", '{"a": 1}')
events = [
- e for e in bus.history
- if e.event_type == EventType.LOOP_GUARD_TRIGGERED
+ e for e in bus.history if e.event_type == EventType.LOOP_GUARD_TRIGGERED
]
assert len(events) == 1
@@ -70,6 +70,7 @@ class TestLoopGuard:
def test_context_compression_no_overflow(self):
from openjarvis.core.types import Message, Role
+
guard, _ = self._make_guard(max_context_messages=100)
messages = [Message(role=Role.USER, content=f"msg {i}") for i in range(10)]
result = guard.compress_context(messages)
@@ -77,42 +78,43 @@ class TestLoopGuard:
def test_context_compression_with_overflow(self):
from openjarvis.core.types import Message, Role
+
guard, _ = self._make_guard(max_context_messages=10)
- messages = [
- Message(role=Role.SYSTEM, content="sys"),
- ] + [
- Message(role=Role.USER, content=f"msg {i}")
- for i in range(50)
- ] + [
- Message(role=Role.TOOL, content=f"result {i}", tool_call_id=f"t{i}")
- for i in range(50)
- ]
+ messages = (
+ [
+ Message(role=Role.SYSTEM, content="sys"),
+ ]
+ + [Message(role=Role.USER, content=f"msg {i}") for i in range(50)]
+ + [
+ Message(role=Role.TOOL, content=f"result {i}", tool_call_id=f"t{i}")
+ for i in range(50)
+ ]
+ )
result = guard.compress_context(messages)
assert len(result) <= 10
def test_context_compression_stage4_uses_current_state(self):
"""Stage 4 should derive from compressed state."""
from openjarvis.core.types import Message, Role
+
guard, _ = self._make_guard(max_context_messages=5)
- messages = [
- Message(role=Role.SYSTEM, content="sys"),
- ] + [
- Message(role=Role.USER, content=f"msg {i}")
- for i in range(100)
- ] + [
- Message(
- role=Role.TOOL,
- content=f"result {i}",
- tool_call_id=f"t{i}",
- )
- for i in range(100)
- ]
+ messages = (
+ [
+ Message(role=Role.SYSTEM, content="sys"),
+ ]
+ + [Message(role=Role.USER, content=f"msg {i}") for i in range(100)]
+ + [
+ Message(
+ role=Role.TOOL,
+ content=f"result {i}",
+ tool_call_id=f"t{i}",
+ )
+ for i in range(100)
+ ]
+ )
result = guard.compress_context(messages)
assert len(result) == 5
- system_count = sum(
- 1 for m in result
- if getattr(m, 'role', None) == 'system'
- )
+ system_count = sum(1 for m in result if getattr(m, "role", None) == "system")
assert system_count == 1
def test_check_response_returns_unblocked(self):
@@ -122,6 +124,7 @@ class TestLoopGuard:
def test_disabled_loop_guard(self):
from openjarvis.agents.loop_guard import LoopGuard, LoopGuardConfig
+
config = LoopGuardConfig(enabled=False)
guard = LoopGuard(config)
# Even though we'd normally block, disabled guard shouldn't
diff --git a/tests/agents/test_loop_guard_warn.py b/tests/agents/test_loop_guard_warn.py
index 534082ad..24495766 100644
--- a/tests/agents/test_loop_guard_warn.py
+++ b/tests/agents/test_loop_guard_warn.py
@@ -7,7 +7,9 @@ from openjarvis.agents.loop_guard import LoopGuard, LoopGuardConfig, LoopVerdict
def test_warn_before_block_first_cycle_warns():
config = LoopGuardConfig(
- enabled=True, max_identical_calls=2, warn_before_block=True,
+ enabled=True,
+ max_identical_calls=2,
+ warn_before_block=True,
)
guard = LoopGuard(config)
# Simulate the Rust backend blocking on the second identical call
@@ -25,7 +27,9 @@ def test_warn_before_block_first_cycle_warns():
def test_warn_before_block_second_cycle_blocks():
config = LoopGuardConfig(
- enabled=True, max_identical_calls=2, warn_before_block=True,
+ enabled=True,
+ max_identical_calls=2,
+ warn_before_block=True,
)
guard = LoopGuard(config)
mock_rust = MagicMock()
@@ -47,7 +51,9 @@ def test_warn_before_block_second_cycle_blocks():
def test_default_behavior_unchanged():
config = LoopGuardConfig(
- enabled=True, max_identical_calls=2, warn_before_block=False,
+ enabled=True,
+ max_identical_calls=2,
+ warn_before_block=False,
)
guard = LoopGuard(config)
mock_rust = MagicMock()
diff --git a/tests/agents/test_manager_recover.py b/tests/agents/test_manager_recover.py
index 81a3defd..dfe1c830 100644
--- a/tests/agents/test_manager_recover.py
+++ b/tests/agents/test_manager_recover.py
@@ -1,4 +1,5 @@
"""Tests for AgentManager.recover_agent() always resetting status."""
+
import pytest
from openjarvis.agents.manager import AgentManager
diff --git a/tests/agents/test_monitor_operative.py b/tests/agents/test_monitor_operative.py
index 48820969..dedb0826 100644
--- a/tests/agents/test_monitor_operative.py
+++ b/tests/agents/test_monitor_operative.py
@@ -23,6 +23,7 @@ class TestMonitorOperativeAgent:
# Import triggers registration; re-register after autouse fixture
# clears the registry (same pattern as test_monitor.py)
import openjarvis.agents.monitor_operative # noqa: F401
+
if not AgentRegistry.contains("monitor_operative"):
AgentRegistry.register_value("monitor_operative", MonitorOperativeAgent)
assert AgentRegistry.contains("monitor_operative")
@@ -46,7 +47,8 @@ class TestMonitorOperativeAgent:
def test_custom_strategies(self) -> None:
engine = _make_engine()
agent = MonitorOperativeAgent(
- engine, "test-model",
+ engine,
+ "test-model",
memory_extraction="scratchpad",
observation_compression="none",
retrieval_strategy="keyword",
diff --git a/tests/agents/test_native_openhands.py b/tests/agents/test_native_openhands.py
index 8a37f880..c4fdc36b 100644
--- a/tests/agents/test_native_openhands.py
+++ b/tests/agents/test_native_openhands.py
@@ -38,6 +38,7 @@ class _CodeInterpreterStub(BaseTool):
# Simple simulation: if it contains print(), capture the content
if "print(" in code:
import re
+
match = re.search(r"print\((.+?)\)", code)
if match:
try:
@@ -134,13 +135,12 @@ class TestNativeOpenHandsAgent:
engine = MagicMock()
engine.engine_id = "mock"
engine.generate.side_effect = [
- _engine_response(
- "Let me calculate:\n```python\nprint(2+2)\n```"
- ),
+ _engine_response("Let me calculate:\n```python\nprint(2+2)\n```"),
_engine_response("The result is 4."),
]
agent = NativeOpenHandsAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CodeInterpreterStub()],
)
result = agent.run("What is 2+2?")
@@ -159,7 +159,8 @@ class TestNativeOpenHandsAgent:
_engine_response("First was 2, second was 9."),
]
agent = NativeOpenHandsAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CodeInterpreterStub()],
)
result = agent.run("Two calculations")
@@ -175,7 +176,8 @@ class TestNativeOpenHandsAgent:
"More code:\n```python\nprint('hello')\n```"
)
agent = NativeOpenHandsAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CodeInterpreterStub()],
max_turns=3,
)
@@ -205,7 +207,8 @@ class TestNativeOpenHandsAgent:
_engine_response("Done."),
]
agent = NativeOpenHandsAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CodeInterpreterStub()],
bus=bus,
)
@@ -238,13 +241,12 @@ class TestNativeOpenHandsAgent:
engine = MagicMock()
engine.engine_id = "mock"
engine.generate.side_effect = [
- _engine_response(
- 'Action: calculator\nAction Input: {"expression": "7*6"}'
- ),
+ _engine_response('Action: calculator\nAction Input: {"expression": "7*6"}'),
_engine_response("The answer is 42."),
]
agent = NativeOpenHandsAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CalculatorStub()],
)
result = agent.run("What is 7 times 6?")
@@ -285,7 +287,8 @@ class TestNativeOpenHandsAgent:
engine.engine_id = "mock"
engine.generate.return_value = _engine_response("Ok")
agent = NativeOpenHandsAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CodeInterpreterStub(), _CalculatorStub()],
)
agent.run("Hello")
@@ -301,7 +304,8 @@ class TestNativeOpenHandsAgent:
engine.engine_id = "mock"
engine.generate.return_value = _engine_response("Ok")
agent = NativeOpenHandsAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CodeInterpreterStub(), _CalculatorStub()],
)
agent.run("Hello")
@@ -321,7 +325,8 @@ class TestNativeOpenHandsAgent:
"Still working:\n```python\nx = 1\n```"
)
agent = NativeOpenHandsAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CodeInterpreterStub()],
max_turns=2,
)
@@ -339,7 +344,8 @@ class TestNativeOpenHandsAgent:
_engine_response("Got 42."),
]
agent = NativeOpenHandsAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CodeInterpreterStub()],
)
agent.run("Print 42")
@@ -358,8 +364,7 @@ class TestNativeOpenHandsAgent:
agent = NativeOpenHandsAgent(engine, "test-model", bus=bus)
agent.run("test input")
start_events = [
- e for e in bus.history
- if e.event_type == EventType.AGENT_TURN_START
+ e for e in bus.history if e.event_type == EventType.AGENT_TURN_START
]
assert len(start_events) == 1
assert start_events[0].data["agent"] == "native_openhands"
@@ -389,13 +394,12 @@ class TestNativeOpenHandsAgent:
engine = MagicMock()
engine.engine_id = "mock"
engine.generate.side_effect = [
- _engine_response(
- '<tool_call>calculator\n$expression=7*6</calculator>'
- ),
+ _engine_response("<tool_call>calculator\n$expression=7*6</calculator>"),
_engine_response("The answer is 42."),
]
agent = NativeOpenHandsAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CalculatorStub()],
)
result = agent.run("What is 7 times 6?")
@@ -408,9 +412,7 @@ class TestNativeOpenHandsAgent:
engine = MagicMock()
engine.engine_id = "mock"
engine.generate.side_effect = [
- _engine_response(
- 'Action: calculator\nAction Input: {"expression": "1+1"}'
- ),
+ _engine_response('Action: calculator\nAction Input: {"expression": "1+1"}'),
_engine_response("Done."),
]
# Make calculator return very long output
@@ -512,7 +514,8 @@ class TestUrlExpansion:
import httpx
monkeypatch.setattr(
- httpx, "get",
+ httpx,
+ "get",
MagicMock(side_effect=Exception("Connection error")),
)
text, expanded = NativeOpenHandsAgent._expand_urls(
diff --git a/tests/agents/test_native_react.py b/tests/agents/test_native_react.py
index 8a3797b1..6ce065da 100644
--- a/tests/agents/test_native_react.py
+++ b/tests/agents/test_native_react.py
@@ -111,8 +111,8 @@ class TestNativeReActParsing:
def test_parse_thought_action(self):
parse = self._parser()
text = (
- 'Thought: I need to calculate 2+2.\n'
- 'Action: calculator\n'
+ "Thought: I need to calculate 2+2.\n"
+ "Action: calculator\n"
'Action Input: {"expression": "2+2"}'
)
result = parse(text)
@@ -157,8 +157,8 @@ class TestNativeReActParsing:
def test_parse_case_insensitive_thought_action(self):
parse = self._parser()
text = (
- 'thought: I need to calculate 2+2.\n'
- 'action: calculator\n'
+ "thought: I need to calculate 2+2.\n"
+ "action: calculator\n"
'action input: {"expression": "2+2"}'
)
result = parse(text)
@@ -199,18 +199,18 @@ class TestNativeReActAgent:
engine.engine_id = "mock"
engine.generate.side_effect = [
_engine_response(
- 'Thought: I need to calculate.\n'
- 'Action: calculator\n'
+ "Thought: I need to calculate.\n"
+ "Action: calculator\n"
'Action Input: {"expression": "2+2"}'
),
- _engine_response(
- "Thought: The result is 4.\nFinal Answer: 4"
- ),
+ _engine_response("Thought: The result is 4.\nFinal Answer: 4"),
]
bus = EventBus(record_history=True)
agent = NativeReActAgent(
- engine, "test-model",
- tools=[_CalculatorStub()], bus=bus,
+ engine,
+ "test-model",
+ tools=[_CalculatorStub()],
+ bus=bus,
)
result = agent.run("What is 2+2?")
assert result.content == "4"
@@ -225,7 +225,7 @@ class TestNativeReActAgent:
engine.engine_id = "mock"
engine.generate.side_effect = [
_engine_response(
- 'Thought: Calculate.\nAction: calculator\n'
+ "Thought: Calculate.\nAction: calculator\n"
'Action Input: {"expression": "3*7"}'
),
_engine_response("Thought: Done.\nFinal Answer: 21"),
@@ -241,21 +241,22 @@ class TestNativeReActAgent:
engine.engine_id = "mock"
engine.generate.side_effect = [
_engine_response(
- 'Thought: Step 1.\nAction: calculator\n'
+ "Thought: Step 1.\nAction: calculator\n"
'Action Input: {"expression": "1+1"}'
),
_engine_response(
- 'Thought: Step 2.\nAction: calculator\n'
+ "Thought: Step 2.\nAction: calculator\n"
'Action Input: {"expression": "2+2"}'
),
_engine_response(
- 'Thought: Step 3.\nAction: think\n'
+ "Thought: Step 3.\nAction: think\n"
'Action Input: {"thought": "combining results"}'
),
_engine_response("Thought: All done.\nFinal Answer: Complete."),
]
agent = NativeReActAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CalculatorStub(), _ThinkStub()],
)
result = agent.run("Multi step")
@@ -268,11 +269,12 @@ class TestNativeReActAgent:
engine = MagicMock()
engine.engine_id = "mock"
engine.generate.return_value = _engine_response(
- 'Thought: Keep going.\nAction: calculator\n'
+ "Thought: Keep going.\nAction: calculator\n"
'Action Input: {"expression": "1+1"}'
)
agent = NativeReActAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CalculatorStub()],
max_turns=3,
)
@@ -287,12 +289,10 @@ class TestNativeReActAgent:
engine.engine_id = "mock"
engine.generate.side_effect = [
_engine_response(
- 'Thought: Use a tool.\nAction: nonexistent\n'
- 'Action Input: {}'
+ "Thought: Use a tool.\nAction: nonexistent\nAction Input: {}"
),
_engine_response(
- "Thought: Error occurred.\n"
- "Final Answer: Could not run tool."
+ "Thought: Error occurred.\nFinal Answer: Could not run tool."
),
]
agent = NativeReActAgent(engine, "test-model", tools=[_CalculatorStub()])
@@ -322,14 +322,16 @@ class TestNativeReActAgent:
engine.engine_id = "mock"
engine.generate.side_effect = [
_engine_response(
- 'Thought: Calc.\nAction: calculator\n'
+ "Thought: Calc.\nAction: calculator\n"
'Action Input: {"expression": "1+1"}'
),
_engine_response("Thought: Done.\nFinal Answer: 2"),
]
agent = NativeReActAgent(
- engine, "test-model",
- tools=[_CalculatorStub()], bus=bus,
+ engine,
+ "test-model",
+ tools=[_CalculatorStub()],
+ bus=bus,
)
agent.run("Calc")
event_types = [e.event_type for e in bus.history]
@@ -365,7 +367,7 @@ class TestNativeReActAgent:
engine.engine_id = "mock"
engine.generate.side_effect = [
_engine_response(
- 'Thought: Let me reason.\nAction: think\n'
+ "Thought: Let me reason.\nAction: think\n"
'Action Input: {"thought": "The user wants a greeting"}'
),
_engine_response("Thought: Now I know.\nFinal Answer: Greetings!"),
@@ -403,7 +405,7 @@ class TestNativeReActAgent:
engine.engine_id = "mock"
engine.generate.side_effect = [
_engine_response(
- 'Thought: Calc.\nAction: calculator\n'
+ "Thought: Calc.\nAction: calculator\n"
'Action Input: {"expression": "5+5"}'
),
_engine_response("Thought: Got it.\nFinal Answer: 10"),
@@ -427,7 +429,8 @@ class TestNativeReActAgent:
"Thought: Done.\nFinal Answer: ok"
)
agent = NativeReActAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CalculatorStub(), _ThinkStub()],
)
agent.run("Hello")
@@ -455,11 +458,11 @@ class TestNativeReActAgent:
engine = MagicMock()
engine.engine_id = "mock"
engine.generate.return_value = _engine_response(
- 'Thought: Go.\nAction: calculator\n'
- 'Action Input: {"expression": "1"}'
+ 'Thought: Go.\nAction: calculator\nAction Input: {"expression": "1"}'
)
agent = NativeReActAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CalculatorStub()],
max_turns=1,
)
@@ -478,14 +481,12 @@ class TestNativeReActAgent:
agent = NativeReActAgent(engine, "test-model", bus=bus)
agent.run("test input")
start_events = [
- e for e in bus.history
- if e.event_type == EventType.AGENT_TURN_START
+ e for e in bus.history if e.event_type == EventType.AGENT_TURN_START
]
assert len(start_events) == 1
assert start_events[0].data["agent"] == "native_react"
assert start_events[0].data["input"] == "test input"
-
def test_system_prompt_enriched_descriptions(self):
"""System prompt should include parameter schemas, not just names."""
engine = MagicMock()
@@ -494,7 +495,8 @@ class TestNativeReActAgent:
"Thought: Done.\nFinal Answer: ok"
)
agent = NativeReActAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CalculatorStub(), _ThinkStub()],
)
agent.run("Hello")
@@ -514,16 +516,15 @@ class TestNativeReActAgent:
engine.engine_id = "mock"
engine.generate.side_effect = [
_engine_response(
- 'thought: I need to calculate.\n'
- 'action: calculator\n'
+ "thought: I need to calculate.\n"
+ "action: calculator\n"
'action input: {"expression": "2+2"}'
),
- _engine_response(
- "thought: The result is 4.\nfinal answer: 4"
- ),
+ _engine_response("thought: The result is 4.\nfinal answer: 4"),
]
agent = NativeReActAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CalculatorStub()],
)
result = agent.run("What is 2+2?")
diff --git a/tests/agents/test_orchestrator.py b/tests/agents/test_orchestrator.py
index e27f5027..f3c753d0 100644
--- a/tests/agents/test_orchestrator.py
+++ b/tests/agents/test_orchestrator.py
@@ -114,11 +114,13 @@ def _make_engine_multi_tool() -> MagicMock:
"content": "",
"tool_calls": [
{
- "id": "call_1", "name": "calculator",
+ "id": "call_1",
+ "name": "calculator",
"arguments": '{"expression":"2+2"}',
},
{
- "id": "call_2", "name": "think",
+ "id": "call_2",
+ "name": "think",
"arguments": '{"thought":"thinking..."}',
},
],
@@ -158,7 +160,9 @@ class TestOrchestratorAgent:
def test_single_tool_call(self):
engine = _make_engine_with_tool_call()
agent = OrchestratorAgent(
- engine, "test-model", tools=[_CalculatorStub()],
+ engine,
+ "test-model",
+ tools=[_CalculatorStub()],
)
result = agent.run("What is 2+2?")
assert result.content == "The answer is 4."
@@ -170,7 +174,8 @@ class TestOrchestratorAgent:
def test_multiple_tool_calls_same_turn(self):
engine = _make_engine_multi_tool()
agent = OrchestratorAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CalculatorStub(), _ThinkStub()],
)
result = agent.run("Think and calculate.")
@@ -193,7 +198,9 @@ class TestOrchestratorAgent:
def test_tools_passed_to_engine(self):
engine = _make_engine_no_tools()
agent = OrchestratorAgent(
- engine, "test-model", tools=[_CalculatorStub()],
+ engine,
+ "test-model",
+ tools=[_CalculatorStub()],
)
agent.run("Hello")
call_kwargs = engine.generate.call_args[1]
@@ -221,7 +228,8 @@ class TestOrchestratorAgent:
"finish_reason": "tool_calls",
}
agent = OrchestratorAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CalculatorStub()],
max_turns=3,
)
@@ -236,7 +244,9 @@ class TestOrchestratorAgent:
final_content="Handled.",
)
agent = OrchestratorAgent(
- engine, "test-model", tools=[_CalculatorStub()],
+ engine,
+ "test-model",
+ tools=[_CalculatorStub()],
)
result = agent.run("Use unknown tool")
assert result.content == "Handled."
@@ -281,7 +291,10 @@ class TestOrchestratorAgent:
bus = EventBus(record_history=True)
engine = _make_engine_with_tool_call()
agent = OrchestratorAgent(
- engine, "test-model", tools=[_CalculatorStub()], bus=bus,
+ engine,
+ "test-model",
+ tools=[_CalculatorStub()],
+ bus=bus,
)
agent.run("Calc 2+2")
event_types = [e.event_type for e in bus.history]
@@ -292,7 +305,9 @@ class TestOrchestratorAgent:
"""After tool call, messages include assistant + tool messages."""
engine = _make_engine_with_tool_call()
agent = OrchestratorAgent(
- engine, "test-model", tools=[_CalculatorStub()],
+ engine,
+ "test-model",
+ tools=[_CalculatorStub()],
)
agent.run("What is 2+2?")
# Second call should include accumulated messages
@@ -305,7 +320,9 @@ class TestOrchestratorAgent:
def test_tool_message_has_tool_call_id(self):
engine = _make_engine_with_tool_call(tool_call_id="abc123")
agent = OrchestratorAgent(
- engine, "test-model", tools=[_CalculatorStub()],
+ engine,
+ "test-model",
+ tools=[_CalculatorStub()],
)
agent.run("What is 2+2?")
second_call = engine.generate.call_args_list[1]
@@ -317,7 +334,9 @@ class TestOrchestratorAgent:
def test_no_bus_works(self):
engine = _make_engine_with_tool_call()
agent = OrchestratorAgent(
- engine, "test-model", tools=[_CalculatorStub()],
+ engine,
+ "test-model",
+ tools=[_CalculatorStub()],
)
result = agent.run("What is 2+2?")
assert result.content == "The answer is 4."
@@ -335,10 +354,13 @@ class TestOrchestratorAgent:
engine.generate.side_effect = [
{
"content": "",
- "tool_calls": [{
- "id": "c1", "name": "calculator",
- "arguments": '{"expression":"2+2"}',
- }],
+ "tool_calls": [
+ {
+ "id": "c1",
+ "name": "calculator",
+ "arguments": '{"expression":"2+2"}',
+ }
+ ],
"usage": {
"prompt_tokens": 5,
"completion_tokens": 3,
@@ -349,10 +371,13 @@ class TestOrchestratorAgent:
},
{
"content": "",
- "tool_calls": [{
- "id": "c2", "name": "calculator",
- "arguments": '{"expression":"4*3"}',
- }],
+ "tool_calls": [
+ {
+ "id": "c2",
+ "name": "calculator",
+ "arguments": '{"expression":"4*3"}',
+ }
+ ],
"usage": {
"prompt_tokens": 15,
"completion_tokens": 3,
@@ -373,7 +398,9 @@ class TestOrchestratorAgent:
},
]
agent = OrchestratorAgent(
- engine, "test-model", tools=[_CalculatorStub()],
+ engine,
+ "test-model",
+ tools=[_CalculatorStub()],
)
result = agent.run("Calculate")
assert result.turns == 3
@@ -384,7 +411,9 @@ class TestOrchestratorAgent:
def test_tool_result_latency_tracked(self):
engine = _make_engine_with_tool_call()
agent = OrchestratorAgent(
- engine, "test-model", tools=[_CalculatorStub()],
+ engine,
+ "test-model",
+ tools=[_CalculatorStub()],
)
result = agent.run("What is 2+2?")
assert result.tool_results[0].latency_seconds >= 0
@@ -403,7 +432,8 @@ class TestOrchestratorAgent:
"finish_reason": "tool_calls",
}
agent = OrchestratorAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CalculatorStub()],
max_turns=1,
)
@@ -433,7 +463,8 @@ class TestOrchestratorAgent:
"finish_reason": "tool_calls",
}
agent = OrchestratorAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CalculatorStub()],
max_turns=2,
)
@@ -456,7 +487,9 @@ class TestOrchestratorStructuredMode:
"finish_reason": "stop",
}
agent = OrchestratorAgent(
- engine, "test-model", mode="structured",
+ engine,
+ "test-model",
+ mode="structured",
)
result = agent.run("What is the capital of France?")
assert result.content == "Paris"
@@ -471,7 +504,7 @@ class TestOrchestratorStructuredMode:
{
"content": (
"THOUGHT: Need to calculate.\n"
- 'TOOL: calculator\n'
+ "TOOL: calculator\n"
'INPUT: {"expression":"2+2"}'
),
"usage": {
@@ -483,10 +516,7 @@ class TestOrchestratorStructuredMode:
"finish_reason": "stop",
},
{
- "content": (
- "THOUGHT: Got 4.\n"
- "FINAL_ANSWER: The answer is 4."
- ),
+ "content": ("THOUGHT: Got 4.\nFINAL_ANSWER: The answer is 4."),
"usage": {
"prompt_tokens": 20,
"completion_tokens": 10,
@@ -497,7 +527,8 @@ class TestOrchestratorStructuredMode:
},
]
agent = OrchestratorAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CalculatorStub()],
mode="structured",
)
@@ -519,7 +550,8 @@ class TestOrchestratorStructuredMode:
"finish_reason": "stop",
}
agent = OrchestratorAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CalculatorStub()],
mode="structured",
)
@@ -591,7 +623,10 @@ class TestOrchestratorParallelTools:
]
agent = OrchestratorAgent(
- engine, "test-model", tools=[_SlowTool()], parallel_tools=True,
+ engine,
+ "test-model",
+ tools=[_SlowTool()],
+ parallel_tools=True,
)
t0 = time.time()
result = agent.run("Do things")
@@ -610,7 +645,8 @@ class TestOrchestratorParallelTools:
"""parallel_tools=False runs tools sequentially."""
engine = _make_engine_multi_tool()
agent = OrchestratorAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[_CalculatorStub(), _ThinkStub()],
parallel_tools=False,
)
@@ -622,7 +658,9 @@ class TestOrchestratorParallelTools:
"""Single tool call should not use parallel path even if parallel_tools=True."""
engine = _make_engine_with_tool_call()
agent = OrchestratorAgent(
- engine, "test-model", tools=[_CalculatorStub()],
+ engine,
+ "test-model",
+ tools=[_CalculatorStub()],
parallel_tools=True,
)
result = agent.run("What is 2+2?")
diff --git a/tests/agents/test_rlm.py b/tests/agents/test_rlm.py
index 1e1c280d..132e6246 100644
--- a/tests/agents/test_rlm.py
+++ b/tests/agents/test_rlm.py
@@ -197,9 +197,7 @@ class TestRLMSubLMCalls:
engine.generate.side_effect = [
{
"content": (
- "```python\n"
- "result = llm_query('What is 2+2?')\n"
- "FINAL(result)\n```"
+ "```python\nresult = llm_query('What is 2+2?')\nFINAL(result)\n```"
),
"usage": {
"prompt_tokens": 5,
@@ -236,10 +234,7 @@ class TestRLMMultiTurn:
engine.generate.side_effect = [
# Turn 1: code that sets a variable
{
- "content": (
- "```python\n"
- "x = 10\nprint(f'x = {x}')\n```"
- ),
+ "content": ("```python\nx = 10\nprint(f'x = {x}')\n```"),
"usage": {
"prompt_tokens": 5,
"completion_tokens": 10,
@@ -250,9 +245,7 @@ class TestRLMMultiTurn:
},
# Turn 2: code that uses the variable and terminates
{
- "content": (
- "```python\ny = x * 2\nFINAL(y)\n```"
- ),
+ "content": ("```python\ny = x * 2\nFINAL(y)\n```"),
"usage": {
"prompt_tokens": 20,
"completion_tokens": 10,
@@ -335,9 +328,7 @@ class TestRLMSubLMWithTools:
# Root LM: code that calls llm_query
{
"content": (
- "```python\n"
- "result = llm_query('Calculate 2+2')\n"
- "FINAL(result)\n```"
+ "```python\nresult = llm_query('Calculate 2+2')\nFINAL(result)\n```"
),
"usage": {
"prompt_tokens": 5,
diff --git a/tests/agents/test_simple.py b/tests/agents/test_simple.py
index fecf2fdd..e3195d3b 100644
--- a/tests/agents/test_simple.py
+++ b/tests/agents/test_simple.py
@@ -91,10 +91,7 @@ class TestSimpleAgent:
agent = SimpleAgent(engine, "test-model", bus=bus)
agent.run("test input")
evts = bus.history
- start = [
- e for e in evts
- if e.event_type == EventType.AGENT_TURN_START
- ][0]
+ start = [e for e in evts if e.event_type == EventType.AGENT_TURN_START][0]
assert start.data["agent"] == "simple"
assert start.data["input"] == "test input"
diff --git a/tests/agents/test_stall.py b/tests/agents/test_stall.py
index 3d9f59d4..18e932b1 100644
--- a/tests/agents/test_stall.py
+++ b/tests/agents/test_stall.py
@@ -18,10 +18,13 @@ def test_activity_tracking_updates_last_activity_at(tmp_path):
agent = mgr.create_agent("stall-test")
def fake_invoke(agent_dict):
- bus.publish(EventType.TOOL_CALL_START, {
- "agent": agent_dict["id"],
- "tool": "web_search",
- })
+ bus.publish(
+ EventType.TOOL_CALL_START,
+ {
+ "agent": agent_dict["id"],
+ "tool": "web_search",
+ },
+ )
return AgentResult(content="done", metadata={})
with patch.object(executor, "_invoke_agent", side_effect=fake_invoke):
@@ -44,10 +47,13 @@ def test_activity_tracking_filters_by_agent_id(tmp_path):
def fake_invoke(agent_dict):
# Emit event for agent_b while agent_a is executing
- bus.publish(EventType.TOOL_CALL_START, {
- "agent": agent_b["id"],
- "tool": "web_search",
- })
+ bus.publish(
+ EventType.TOOL_CALL_START,
+ {
+ "agent": agent_b["id"],
+ "tool": "web_search",
+ },
+ )
return AgentResult(content="done", metadata={})
with patch.object(executor, "_invoke_agent", side_effect=fake_invoke):
@@ -67,10 +73,13 @@ def test_reconcile_detects_stalled_agent(tmp_path):
scheduler = AgentScheduler(mgr, executor, event_bus=bus)
- agent = mgr.create_agent("stall-me", config={
- "timeout_seconds": 10,
- "max_stall_retries": 3,
- })
+ agent = mgr.create_agent(
+ "stall-me",
+ config={
+ "timeout_seconds": 10,
+ "max_stall_retries": 3,
+ },
+ )
mgr.update_agent(agent["id"], status="running", last_activity_at=time.time() - 30)
scheduler._reconcile()
@@ -79,8 +88,7 @@ def test_reconcile_detects_stalled_agent(tmp_path):
assert updated["stall_retries"] == 1
stall_events = [
- e for e in bus.history
- if e.event_type == EventType.AGENT_STALL_DETECTED
+ e for e in bus.history if e.event_type == EventType.AGENT_STALL_DETECTED
]
assert len(stall_events) == 1
mgr.close()
@@ -114,12 +122,19 @@ def test_reconcile_retries_exhausted_sets_error(tmp_path):
scheduler = AgentScheduler(mgr, executor, event_bus=bus)
- agent = mgr.create_agent("exhausted", config={
- "timeout_seconds": 10,
- "max_stall_retries": 2,
- })
- mgr.update_agent(agent["id"], status="running",
- last_activity_at=time.time() - 30, stall_retries=2)
+ agent = mgr.create_agent(
+ "exhausted",
+ config={
+ "timeout_seconds": 10,
+ "max_stall_retries": 2,
+ },
+ )
+ mgr.update_agent(
+ agent["id"],
+ status="running",
+ last_activity_at=time.time() - 30,
+ stall_retries=2,
+ )
scheduler._reconcile()
diff --git a/tests/agents/test_suggest_action.py b/tests/agents/test_suggest_action.py
index 004ed738..aa816c2d 100644
--- a/tests/agents/test_suggest_action.py
+++ b/tests/agents/test_suggest_action.py
@@ -1,4 +1,5 @@
"""Tests for suggest_action helper."""
+
from openjarvis.agents.errors import FatalError, RetryableError, suggest_action
diff --git a/tests/agents/test_trace_recording.py b/tests/agents/test_trace_recording.py
index 51397ab9..e1c07b68 100644
--- a/tests/agents/test_trace_recording.py
+++ b/tests/agents/test_trace_recording.py
@@ -21,17 +21,23 @@ def test_executor_records_trace(tmp_path):
agent = mgr.create_agent("trace-test")
def fake_invoke(agent_dict):
- bus.publish(EventType.TOOL_CALL_START, {
- "agent": agent_dict["id"],
- "tool": "web_search",
- "args": {"query": "test"},
- })
- bus.publish(EventType.TOOL_CALL_END, {
- "agent": agent_dict["id"],
- "tool": "web_search",
- "result": "search results...",
- "duration": 0.5,
- })
+ bus.publish(
+ EventType.TOOL_CALL_START,
+ {
+ "agent": agent_dict["id"],
+ "tool": "web_search",
+ "args": {"query": "test"},
+ },
+ )
+ bus.publish(
+ EventType.TOOL_CALL_END,
+ {
+ "agent": agent_dict["id"],
+ "tool": "web_search",
+ "result": "search results...",
+ "duration": 0.5,
+ },
+ )
return AgentResult(content="found it", metadata={"tokens_used": 100})
with patch.object(executor, "_invoke_agent", side_effect=fake_invoke):
@@ -62,7 +68,9 @@ def test_executor_records_error_trace(tmp_path):
agent = mgr.create_agent("error-trace")
with patch.object(
- executor, "_invoke_agent", side_effect=FatalError("boom"),
+ executor,
+ "_invoke_agent",
+ side_effect=FatalError("boom"),
):
executor.execute_tick(agent["id"])
diff --git a/tests/bench/test_energy.py b/tests/bench/test_energy.py
index 37a3cdb5..d5602b23 100644
--- a/tests/bench/test_energy.py
+++ b/tests/bench/test_energy.py
@@ -78,7 +78,10 @@ class TestEnergyBenchmark:
b = EnergyBenchmark()
result = b.run(
- engine, "test-model", num_samples=3, warmup_samples=0,
+ engine,
+ "test-model",
+ num_samples=3,
+ warmup_samples=0,
energy_monitor=monitor,
)
diff --git a/tests/bench/test_latency.py b/tests/bench/test_latency.py
index 109da0f6..ea03a3df 100644
--- a/tests/bench/test_latency.py
+++ b/tests/bench/test_latency.py
@@ -56,8 +56,12 @@ class TestLatencyBenchmark:
b = LatencyBenchmark()
result = b.run(engine, "test-model", num_samples=3)
expected_keys = {
- "mean_latency", "p50_latency", "p95_latency",
- "min_latency", "max_latency", "std_latency",
+ "mean_latency",
+ "p50_latency",
+ "p95_latency",
+ "min_latency",
+ "max_latency",
+ "std_latency",
}
assert set(result.metrics.keys()) == expected_keys
diff --git a/tests/channels/test_channel_config.py b/tests/channels/test_channel_config.py
index 005fa09a..0e7ad81d 100644
--- a/tests/channels/test_channel_config.py
+++ b/tests/channels/test_channel_config.py
@@ -66,7 +66,9 @@ class TestChannelConfig:
class TestTomlLoading:
def _write_toml(self, content: str) -> Path:
f = tempfile.NamedTemporaryFile(
- mode="w", suffix=".toml", delete=False,
+ mode="w",
+ suffix=".toml",
+ delete=False,
)
f.write(content)
f.flush()
diff --git a/tests/channels/test_channel_registry.py b/tests/channels/test_channel_registry.py
index d8e992c4..70f8a1df 100644
--- a/tests/channels/test_channel_registry.py
+++ b/tests/channels/test_channel_registry.py
@@ -25,8 +25,12 @@ class TestChannelRegistry:
pass
def send(
- self, channel, content,
- *, conversation_id="", metadata=None,
+ self,
+ channel,
+ content,
+ *,
+ conversation_id="",
+ metadata=None,
) -> bool:
return True
@@ -56,8 +60,12 @@ class TestChannelRegistry:
pass
def send(
- self, channel, content,
- *, conversation_id="", metadata=None,
+ self,
+ channel,
+ content,
+ *,
+ conversation_id="",
+ metadata=None,
) -> bool:
return True
@@ -99,8 +107,12 @@ class TestChannelRegistry:
pass
def send(
- self, channel, content,
- *, conversation_id="", metadata=None,
+ self,
+ channel,
+ content,
+ *,
+ conversation_id="",
+ metadata=None,
) -> bool:
return True
diff --git a/tests/channels/test_channels_phase21.py b/tests/channels/test_channels_phase21.py
index 822a17ef..fe8d46ce 100644
--- a/tests/channels/test_channels_phase21.py
+++ b/tests/channels/test_channels_phase21.py
@@ -153,10 +153,13 @@ class TestChannelImportError:
class TestLineChannel:
def test_env_fallback(self):
- with patch.dict("os.environ", {
- "LINE_CHANNEL_ACCESS_TOKEN": "env-tok",
- "LINE_CHANNEL_SECRET": "env-sec",
- }):
+ with patch.dict(
+ "os.environ",
+ {
+ "LINE_CHANNEL_ACCESS_TOKEN": "env-tok",
+ "LINE_CHANNEL_SECRET": "env-sec",
+ },
+ ):
ch = LineChannel()
assert ch._channel_access_token == "env-tok"
assert ch._channel_secret == "env-sec"
@@ -164,10 +167,13 @@ class TestLineChannel:
class TestViberChannel:
def test_env_fallback(self):
- with patch.dict("os.environ", {
- "VIBER_AUTH_TOKEN": "env-tok",
- "VIBER_BOT_NAME": "TestBot",
- }):
+ with patch.dict(
+ "os.environ",
+ {
+ "VIBER_AUTH_TOKEN": "env-tok",
+ "VIBER_BOT_NAME": "TestBot",
+ },
+ ):
ch = ViberChannel()
assert ch._auth_token == "env-tok"
assert ch._name == "TestBot"
@@ -175,21 +181,27 @@ class TestViberChannel:
class TestMessengerChannel:
def test_env_fallback(self):
- with patch.dict("os.environ", {
- "MESSENGER_ACCESS_TOKEN": "env-tok",
- }):
+ with patch.dict(
+ "os.environ",
+ {
+ "MESSENGER_ACCESS_TOKEN": "env-tok",
+ },
+ ):
ch = MessengerChannel()
assert ch._access_token == "env-tok"
class TestRedditChannel:
def test_env_fallback(self):
- with patch.dict("os.environ", {
- "REDDIT_CLIENT_ID": "cid",
- "REDDIT_CLIENT_SECRET": "csec",
- "REDDIT_USERNAME": "user",
- "REDDIT_PASSWORD": "pass",
- }):
+ with patch.dict(
+ "os.environ",
+ {
+ "REDDIT_CLIENT_ID": "cid",
+ "REDDIT_CLIENT_SECRET": "csec",
+ "REDDIT_USERNAME": "user",
+ "REDDIT_PASSWORD": "pass",
+ },
+ ):
ch = RedditChannel()
assert ch._client_id == "cid"
assert ch._client_secret == "csec"
@@ -199,10 +211,13 @@ class TestRedditChannel:
class TestMastodonChannel:
def test_env_fallback(self):
- with patch.dict("os.environ", {
- "MASTODON_API_BASE_URL": "https://m.social",
- "MASTODON_ACCESS_TOKEN": "env-tok",
- }):
+ with patch.dict(
+ "os.environ",
+ {
+ "MASTODON_API_BASE_URL": "https://m.social",
+ "MASTODON_ACCESS_TOKEN": "env-tok",
+ },
+ ):
ch = MastodonChannel()
assert ch._api_base_url == "https://m.social"
assert ch._access_token == "env-tok"
@@ -210,12 +225,15 @@ class TestMastodonChannel:
class TestXMPPChannel:
def test_env_fallback(self):
- with patch.dict("os.environ", {
- "XMPP_JID": "[email protected]",
- "XMPP_PASSWORD": "pass",
- "XMPP_SERVER": "xmpp.example.com",
- "XMPP_PORT": "5223",
- }):
+ with patch.dict(
+ "os.environ",
+ {
+ "XMPP_JID": "[email protected]",
+ "XMPP_PASSWORD": "pass",
+ "XMPP_SERVER": "xmpp.example.com",
+ "XMPP_PORT": "5223",
+ },
+ ):
ch = XMPPChannel()
assert ch._jid == "[email protected]"
assert ch._password == "pass"
@@ -225,22 +243,28 @@ class TestXMPPChannel:
class TestRocketChatChannel:
def test_env_fallback(self):
- with patch.dict("os.environ", {
- "ROCKETCHAT_URL": "https://rc.example.com",
- "ROCKETCHAT_USER": "bot",
- "ROCKETCHAT_PASSWORD": "pass",
- }):
+ with patch.dict(
+ "os.environ",
+ {
+ "ROCKETCHAT_URL": "https://rc.example.com",
+ "ROCKETCHAT_USER": "bot",
+ "ROCKETCHAT_PASSWORD": "pass",
+ },
+ ):
ch = RocketChatChannel()
assert ch._url == "https://rc.example.com"
assert ch._user == "bot"
assert ch._password == "pass"
def test_token_auth_env(self):
- with patch.dict("os.environ", {
- "ROCKETCHAT_URL": "https://rc.example.com",
- "ROCKETCHAT_AUTH_TOKEN": "tok",
- "ROCKETCHAT_USER_ID": "uid",
- }):
+ with patch.dict(
+ "os.environ",
+ {
+ "ROCKETCHAT_URL": "https://rc.example.com",
+ "ROCKETCHAT_AUTH_TOKEN": "tok",
+ "ROCKETCHAT_USER_ID": "uid",
+ },
+ ):
ch = RocketChatChannel()
assert ch._auth_token == "tok"
assert ch._user_id == "uid"
@@ -248,32 +272,41 @@ class TestRocketChatChannel:
class TestZulipChannel:
def test_env_fallback(self):
- with patch.dict("os.environ", {
- "ZULIP_EMAIL": "[email protected]",
- "ZULIP_API_KEY": "key",
- "ZULIP_SITE": "https://z.com",
- }):
+ with patch.dict(
+ "os.environ",
+ {
+ "ZULIP_EMAIL": "[email protected]",
+ "ZULIP_API_KEY": "key",
+ "ZULIP_SITE": "https://z.com",
+ },
+ ):
ch = ZulipChannel()
assert ch._email == "[email protected]"
assert ch._api_key == "key"
assert ch._site == "https://z.com"
def test_zuliprc_env(self):
- with patch.dict("os.environ", {
- "ZULIP_RC": "/path/to/zuliprc",
- }):
+ with patch.dict(
+ "os.environ",
+ {
+ "ZULIP_RC": "/path/to/zuliprc",
+ },
+ ):
ch = ZulipChannel()
assert ch._zuliprc == "/path/to/zuliprc"
class TestTwitchChannel:
def test_env_fallback(self):
- with patch.dict("os.environ", {
- "TWITCH_ACCESS_TOKEN": "env-tok",
- "TWITCH_CLIENT_ID": "env-cid",
- "TWITCH_NICK": "env-nick",
- "TWITCH_CHANNELS": "chan1,chan2",
- }):
+ with patch.dict(
+ "os.environ",
+ {
+ "TWITCH_ACCESS_TOKEN": "env-tok",
+ "TWITCH_CLIENT_ID": "env-cid",
+ "TWITCH_NICK": "env-nick",
+ "TWITCH_CHANNELS": "chan1,chan2",
+ },
+ ):
ch = TwitchChannel()
assert ch._access_token == "env-tok"
assert ch._client_id == "env-cid"
@@ -283,10 +316,13 @@ class TestTwitchChannel:
class TestNostrChannel:
def test_env_fallback(self):
- with patch.dict("os.environ", {
- "NOSTR_PRIVATE_KEY": "aa" * 32,
- "NOSTR_RELAYS": "wss://r1.example.com,wss://r2.example.com",
- }):
+ with patch.dict(
+ "os.environ",
+ {
+ "NOSTR_PRIVATE_KEY": "aa" * 32,
+ "NOSTR_RELAYS": "wss://r1.example.com,wss://r2.example.com",
+ },
+ ):
ch = NostrChannel()
assert ch._private_key == "aa" * 32
assert len(ch._relays) == 2
diff --git a/tests/channels/test_email_channel.py b/tests/channels/test_email_channel.py
index fc515b98..0aa5e8b6 100644
--- a/tests/channels/test_email_channel.py
+++ b/tests/channels/test_email_channel.py
@@ -26,7 +26,8 @@ class TestRegistration:
def test_channel_id(self):
ch = EmailChannel(
- smtp_host="smtp.example.com", username="[email protected]",
+ smtp_host="smtp.example.com",
+ username="[email protected]",
)
assert ch.channel_id == "email"
@@ -62,10 +63,13 @@ class TestInit:
assert ch._use_tls is False
def test_env_var_fallback(self):
- with patch.dict(os.environ, {
- "EMAIL_USERNAME": "[email protected]",
- "EMAIL_PASSWORD": "env-pass",
- }):
+ with patch.dict(
+ os.environ,
+ {
+ "EMAIL_USERNAME": "[email protected]",
+ "EMAIL_PASSWORD": "env-pass",
+ },
+ ):
ch = EmailChannel()
assert ch._username == "[email protected]"
assert ch._password == "env-pass"
@@ -206,7 +210,8 @@ class TestOnMessage:
class TestDisconnect:
def test_disconnect(self):
ch = EmailChannel(
- smtp_host="smtp.example.com", username="[email protected]",
+ smtp_host="smtp.example.com",
+ username="[email protected]",
)
ch._status = ChannelStatus.CONNECTED
ch.disconnect()
diff --git a/tests/channels/test_google_chat.py b/tests/channels/test_google_chat.py
index b04ec86b..082cb7a1 100644
--- a/tests/channels/test_google_chat.py
+++ b/tests/channels/test_google_chat.py
@@ -25,7 +25,9 @@ class TestRegistration:
assert ChannelRegistry.contains("google_chat")
def test_channel_id(self):
- ch = GoogleChatChannel(webhook_url="https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy")
+ ch = GoogleChatChannel(
+ webhook_url="https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy"
+ )
assert ch.channel_id == "google_chat"
@@ -36,27 +38,48 @@ class TestInit:
assert ch._status == ChannelStatus.DISCONNECTED
def test_constructor_url(self):
- ch = GoogleChatChannel(webhook_url="https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy")
- assert ch._webhook_url == "https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy"
+ ch = GoogleChatChannel(
+ webhook_url="https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy"
+ )
+ assert (
+ ch._webhook_url
+ == "https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy"
+ )
def test_env_var_fallback(self):
- with patch.dict(os.environ, {
- "GOOGLE_CHAT_WEBHOOK_URL": "https://chat.googleapis.com/v1/spaces/env/messages?key=env",
- }):
+ with patch.dict(
+ os.environ,
+ {
+ "GOOGLE_CHAT_WEBHOOK_URL": "https://chat.googleapis.com/v1/spaces/env/messages?key=env",
+ },
+ ):
ch = GoogleChatChannel()
- assert ch._webhook_url == "https://chat.googleapis.com/v1/spaces/env/messages?key=env"
+ assert (
+ ch._webhook_url
+ == "https://chat.googleapis.com/v1/spaces/env/messages?key=env"
+ )
def test_constructor_overrides_env(self):
- with patch.dict(os.environ, {
- "GOOGLE_CHAT_WEBHOOK_URL": "https://chat.googleapis.com/v1/spaces/env/messages?key=env",
- }):
- ch = GoogleChatChannel(webhook_url="https://chat.googleapis.com/v1/spaces/explicit/messages?key=explicit")
- assert ch._webhook_url == "https://chat.googleapis.com/v1/spaces/explicit/messages?key=explicit"
+ with patch.dict(
+ os.environ,
+ {
+ "GOOGLE_CHAT_WEBHOOK_URL": "https://chat.googleapis.com/v1/spaces/env/messages?key=env",
+ },
+ ):
+ ch = GoogleChatChannel(
+ webhook_url="https://chat.googleapis.com/v1/spaces/explicit/messages?key=explicit"
+ )
+ assert (
+ ch._webhook_url
+ == "https://chat.googleapis.com/v1/spaces/explicit/messages?key=explicit"
+ )
class TestSend:
def test_send_success(self):
- ch = GoogleChatChannel(webhook_url="https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy")
+ ch = GoogleChatChannel(
+ webhook_url="https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy"
+ )
mock_response = MagicMock()
mock_response.status_code = 200
@@ -67,7 +90,9 @@ class TestSend:
mock_post.assert_called_once()
def test_send_failure(self):
- ch = GoogleChatChannel(webhook_url="https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy")
+ ch = GoogleChatChannel(
+ webhook_url="https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy"
+ )
mock_response = MagicMock()
mock_response.status_code = 400
@@ -78,7 +103,9 @@ class TestSend:
assert result is False
def test_send_exception(self):
- ch = GoogleChatChannel(webhook_url="https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy")
+ ch = GoogleChatChannel(
+ webhook_url="https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy"
+ )
with patch("httpx.post", side_effect=ConnectionError("refused")):
result = ch.send("space", "Hello!")
@@ -108,13 +135,17 @@ class TestSend:
class TestListChannels:
def test_list_channels(self):
- ch = GoogleChatChannel(webhook_url="https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy")
+ ch = GoogleChatChannel(
+ webhook_url="https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy"
+ )
assert ch.list_channels() == ["google_chat"]
class TestStatus:
def test_disconnected_initially(self):
- ch = GoogleChatChannel(webhook_url="https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy")
+ ch = GoogleChatChannel(
+ webhook_url="https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy"
+ )
assert ch.status() == ChannelStatus.DISCONNECTED
def test_no_url_connect_error(self):
@@ -125,7 +156,9 @@ class TestStatus:
class TestOnMessage:
def test_on_message(self):
- ch = GoogleChatChannel(webhook_url="https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy")
+ ch = GoogleChatChannel(
+ webhook_url="https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy"
+ )
handler = MagicMock()
ch.on_message(handler)
assert handler in ch._handlers
@@ -133,7 +166,9 @@ class TestOnMessage:
class TestDisconnect:
def test_disconnect(self):
- ch = GoogleChatChannel(webhook_url="https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy")
+ ch = GoogleChatChannel(
+ webhook_url="https://chat.googleapis.com/v1/spaces/xxx/messages?key=yyy"
+ )
ch._status = ChannelStatus.CONNECTED
ch.disconnect()
assert ch.status() == ChannelStatus.DISCONNECTED
diff --git a/tests/channels/test_irc_channel.py b/tests/channels/test_irc_channel.py
index cc0392c8..e754f682 100644
--- a/tests/channels/test_irc_channel.py
+++ b/tests/channels/test_irc_channel.py
@@ -45,12 +45,15 @@ class TestInit:
assert ch._password == "pass123"
def test_env_var_fallback(self):
- with patch.dict(os.environ, {
- "IRC_SERVER": "irc.env.com",
- "IRC_NICK": "envbot",
- "IRC_PASSWORD": "envpass",
- "IRC_PORT": "6697",
- }):
+ with patch.dict(
+ os.environ,
+ {
+ "IRC_SERVER": "irc.env.com",
+ "IRC_NICK": "envbot",
+ "IRC_PASSWORD": "envpass",
+ "IRC_PORT": "6697",
+ },
+ ):
ch = IRCChannel()
assert ch._server == "irc.env.com"
assert ch._nick == "envbot"
@@ -58,11 +61,14 @@ class TestInit:
assert ch._port == 6697
def test_constructor_overrides_env(self):
- with patch.dict(os.environ, {
- "IRC_SERVER": "irc.env.com",
- "IRC_NICK": "envbot",
- "IRC_PASSWORD": "envpass",
- }):
+ with patch.dict(
+ os.environ,
+ {
+ "IRC_SERVER": "irc.env.com",
+ "IRC_NICK": "envbot",
+ "IRC_PASSWORD": "envpass",
+ },
+ ):
ch = IRCChannel(
server="irc.explicit.com",
nick="explicit",
diff --git a/tests/channels/test_signal_channel.py b/tests/channels/test_signal_channel.py
index 38f6303d..34c83606 100644
--- a/tests/channels/test_signal_channel.py
+++ b/tests/channels/test_signal_channel.py
@@ -42,19 +42,25 @@ class TestInit:
assert ch._phone_number == "+1234567890"
def test_env_var_fallback(self):
- with patch.dict(os.environ, {
- "SIGNAL_API_URL": "http://env-signal:8080",
- "SIGNAL_PHONE_NUMBER": "+9876543210",
- }):
+ with patch.dict(
+ os.environ,
+ {
+ "SIGNAL_API_URL": "http://env-signal:8080",
+ "SIGNAL_PHONE_NUMBER": "+9876543210",
+ },
+ ):
ch = SignalChannel()
assert ch._api_url == "http://env-signal:8080"
assert ch._phone_number == "+9876543210"
def test_constructor_overrides_env(self):
- with patch.dict(os.environ, {
- "SIGNAL_API_URL": "http://env-signal:8080",
- "SIGNAL_PHONE_NUMBER": "+9876543210",
- }):
+ with patch.dict(
+ os.environ,
+ {
+ "SIGNAL_API_URL": "http://env-signal:8080",
+ "SIGNAL_PHONE_NUMBER": "+9876543210",
+ },
+ ):
ch = SignalChannel(
api_url="http://explicit:8080",
phone_number="+1111111111",
diff --git a/tests/channels/test_stubs.py b/tests/channels/test_stubs.py
index bdc487bd..18e92184 100644
--- a/tests/channels/test_stubs.py
+++ b/tests/channels/test_stubs.py
@@ -95,8 +95,12 @@ class TestBaseChannel:
pass
def send(
- self, channel, content,
- *, conversation_id="", metadata=None,
+ self,
+ channel,
+ content,
+ *,
+ conversation_id="",
+ metadata=None,
) -> bool:
return True
diff --git a/tests/channels/test_telegram.py b/tests/channels/test_telegram.py
index ae01dfb8..0e348060 100644
--- a/tests/channels/test_telegram.py
+++ b/tests/channels/test_telegram.py
@@ -140,3 +140,125 @@ class TestDisconnect:
ch._status = ChannelStatus.CONNECTED
ch.disconnect()
assert ch.status() == ChannelStatus.DISCONNECTED
+
+
+class TestAllowedChatIds:
+ """Tests for the allowed_chat_ids enforcement in _poll_loop."""
+
+ def _make_update(self, chat_id: str, text: str = "hello"):
+ """Build a minimal fake python-telegram-bot Update object."""
+ msg = MagicMock()
+ msg.text = text
+ msg.message_id = 1
+ msg.from_user.id = chat_id
+ msg.chat.id = chat_id
+ update = MagicMock()
+ update.message = msg
+ return update
+
+ def _invoke_handle_msg(self, ch: TelegramChannel, chat_id: str, text: str = "hi"):
+ """Simulate _poll_loop dispatching a message without starting a thread."""
+ from openjarvis.channels._stubs import ChannelMessage
+
+ cm = ChannelMessage(
+ channel="telegram",
+ sender=chat_id,
+ content=text,
+ message_id="1",
+ conversation_id=chat_id,
+ )
+ # Directly exercise the allow-list logic (mirrors _handle_msg body)
+ if ch._allowed_chat_ids:
+ _allowed = {
+ cid.strip() for cid in ch._allowed_chat_ids.split(",") if cid.strip()
+ }
+ if cm.conversation_id not in _allowed:
+ return False # would return inside _handle_msg
+ for handler in ch._handlers:
+ handler(cm)
+ return True
+
+ def test_no_allowlist_accepts_any(self):
+ """When allowed_chat_ids is empty every chat is dispatched."""
+ ch = TelegramChannel(bot_token="tok", allowed_chat_ids="")
+ handler = MagicMock()
+ ch.on_message(handler)
+ dispatched = self._invoke_handle_msg(ch, "99999")
+ assert dispatched is True
+ handler.assert_called_once()
+
+ def test_allowlist_passes_listed_chat(self):
+ """A chat ID present in the allow-list is dispatched to handlers."""
+ ch = TelegramChannel(bot_token="tok", allowed_chat_ids="111,222")
+ handler = MagicMock()
+ ch.on_message(handler)
+ dispatched = self._invoke_handle_msg(ch, "111")
+ assert dispatched is True
+ handler.assert_called_once()
+
+ def test_allowlist_blocks_unlisted_chat(self):
+ """A chat ID not in the allow-list is silently dropped (not dispatched)."""
+ ch = TelegramChannel(bot_token="tok", allowed_chat_ids="111,222")
+ handler = MagicMock()
+ ch.on_message(handler)
+ dispatched = self._invoke_handle_msg(ch, "999")
+ assert dispatched is False
+ handler.assert_not_called()
+
+ def test_allowlist_trims_whitespace(self):
+ """Spaces around IDs in the allow-list are handled gracefully."""
+ ch = TelegramChannel(bot_token="tok", allowed_chat_ids=" 111 , 222 ")
+ handler = MagicMock()
+ ch.on_message(handler)
+ dispatched = self._invoke_handle_msg(ch, "111")
+ assert dispatched is True
+ handler.assert_called_once()
+
+
+class TestChannelAgentWiring:
+ """Tests for the channel → agent handler wired in serve.py."""
+
+ def test_on_message_handler_invoked_on_message(self):
+ """on_message callback registered on a channel is called when a message
+ arrives."""
+ ch = TelegramChannel(bot_token="tok")
+ received = []
+ ch.on_message(lambda cm: received.append(cm))
+
+ from openjarvis.channels._stubs import ChannelMessage
+
+ cm = ChannelMessage(
+ channel="telegram",
+ sender="42",
+ content="ping",
+ message_id="1",
+ conversation_id="42",
+ )
+ for h in ch._handlers:
+ h(cm)
+
+ assert len(received) == 1
+ assert received[0].content == "ping"
+
+ def test_multiple_handlers_all_invoked(self):
+ """Both handlers registered via on_message are called for the same message."""
+ ch = TelegramChannel(bot_token="tok")
+ calls_a: list = []
+ calls_b: list = []
+ ch.on_message(lambda cm: calls_a.append(cm))
+ ch.on_message(lambda cm: calls_b.append(cm))
+
+ from openjarvis.channels._stubs import ChannelMessage
+
+ cm = ChannelMessage(
+ channel="telegram",
+ sender="1",
+ content="x",
+ message_id="1",
+ conversation_id="1",
+ )
+ for h in ch._handlers:
+ h(cm)
+
+ assert len(calls_a) == 1
+ assert len(calls_b) == 1
diff --git a/tests/channels/test_whatsapp.py b/tests/channels/test_whatsapp.py
index 3b0a2a30..a4e118f5 100644
--- a/tests/channels/test_whatsapp.py
+++ b/tests/channels/test_whatsapp.py
@@ -42,19 +42,25 @@ class TestInit:
assert ch._phone_number_id == "12345"
def test_env_var_fallback(self):
- with patch.dict(os.environ, {
- "WHATSAPP_ACCESS_TOKEN": "env-token",
- "WHATSAPP_PHONE_NUMBER_ID": "env-id",
- }):
+ with patch.dict(
+ os.environ,
+ {
+ "WHATSAPP_ACCESS_TOKEN": "env-token",
+ "WHATSAPP_PHONE_NUMBER_ID": "env-id",
+ },
+ ):
ch = WhatsAppChannel()
assert ch._token == "env-token"
assert ch._phone_number_id == "env-id"
def test_constructor_overrides_env(self):
- with patch.dict(os.environ, {
- "WHATSAPP_ACCESS_TOKEN": "env-token",
- "WHATSAPP_PHONE_NUMBER_ID": "env-id",
- }):
+ with patch.dict(
+ os.environ,
+ {
+ "WHATSAPP_ACCESS_TOKEN": "env-token",
+ "WHATSAPP_PHONE_NUMBER_ID": "env-id",
+ },
+ ):
ch = WhatsAppChannel(
access_token="explicit-token",
phone_number_id="explicit-id",
diff --git a/tests/channels/test_whatsapp_baileys.py b/tests/channels/test_whatsapp_baileys.py
index 72e64035..9d969fce 100644
--- a/tests/channels/test_whatsapp_baileys.py
+++ b/tests/channels/test_whatsapp_baileys.py
@@ -259,13 +259,15 @@ class TestReaderLoop:
bus = EventBus(record_history=True)
ch = WhatsAppBaileysChannel(bus=bus)
- ch._handle_bridge_event({
- "type": "message",
- "jid": "[email protected]",
- "sender": "[email protected]",
- "text": "Bus test",
- "message_id": "msg-002",
- })
+ ch._handle_bridge_event(
+ {
+ "type": "message",
+ "jid": "[email protected]",
+ "sender": "[email protected]",
+ "text": "Bus test",
+ "message_id": "msg-002",
+ }
+ )
event_types = [e.event_type for e in bus.history]
assert EventType.CHANNEL_MESSAGE_RECEIVED in event_types
@@ -276,13 +278,16 @@ class TestReaderLoop:
lines = [
json.dumps({"type": "status", "status": "connected"}) + "\n",
- json.dumps({
- "type": "message",
- "jid": "j",
- "sender": "s",
- "text": "t",
- "message_id": "m",
- }) + "\n",
+ json.dumps(
+ {
+ "type": "message",
+ "jid": "j",
+ "sender": "s",
+ "text": "t",
+ "message_id": "m",
+ }
+ )
+ + "\n",
]
mock_proc = MagicMock()
@@ -316,11 +321,13 @@ class TestReaderLoop:
ch.on_message(bad_handler)
# Should not raise.
- ch._handle_bridge_event({
- "type": "message",
- "jid": "j",
- "sender": "s",
- "text": "t",
- "message_id": "m",
- })
+ ch._handle_bridge_event(
+ {
+ "type": "message",
+ "jid": "j",
+ "sender": "s",
+ "text": "t",
+ "message_id": "m",
+ }
+ )
bad_handler.assert_called_once()
diff --git a/tests/cli/test_add_cmd.py b/tests/cli/test_add_cmd.py
index 1348f2cf..06b8e2d4 100644
--- a/tests/cli/test_add_cmd.py
+++ b/tests/cli/test_add_cmd.py
@@ -42,7 +42,8 @@ class TestAddCmd:
mcp_dir = tmp_path / "mcp"
with mock.patch("openjarvis.cli.add_cmd._MCP_CONFIG_DIR", mcp_dir):
result = CliRunner().invoke(
- add, ["github", "--key", "test_token"],
+ add,
+ ["github", "--key", "test_token"],
)
assert result.exit_code == 0
@@ -55,6 +56,6 @@ class TestAddCmd:
def test_mcp_templates_complete(self) -> None:
required_fields = {"command", "args", "env_key", "description"}
for name, tmpl in _MCP_TEMPLATES.items():
- assert required_fields.issubset(
- tmpl.keys()
- ), f"Template '{name}' missing fields: {required_fields - tmpl.keys()}"
+ assert required_fields.issubset(tmpl.keys()), (
+ f"Template '{name}' missing fields: {required_fields - tmpl.keys()}"
+ )
diff --git a/tests/cli/test_agent_cmd.py b/tests/cli/test_agent_cmd.py
index 3e0fa94b..8a71c579 100644
--- a/tests/cli/test_agent_cmd.py
+++ b/tests/cli/test_agent_cmd.py
@@ -80,8 +80,19 @@ class TestNewAgentCommands:
result = CliRunner().invoke(cli, ["agents", "--help"])
assert result.exit_code == 0
cmds = (
- "launch", "start", "stop", "run", "status", "logs",
- "daemon", "watch", "recover", "errors", "ask", "instruct", "messages",
+ "launch",
+ "start",
+ "stop",
+ "run",
+ "status",
+ "logs",
+ "daemon",
+ "watch",
+ "recover",
+ "errors",
+ "ask",
+ "instruct",
+ "messages",
)
for cmd in cmds:
assert cmd in result.output, f"Missing command: {cmd}"
diff --git a/tests/cli/test_ask_agent.py b/tests/cli/test_ask_agent.py
index 577aa499..622c2910 100644
--- a/tests/cli/test_ask_agent.py
+++ b/tests/cli/test_ask_agent.py
@@ -123,7 +123,8 @@ def agent_setup():
patch.object(_ask_mod, "get_engine", return_value=("mock", engine)),
patch.object(_ask_mod, "discover_engines", return_value=[("mock", engine)]),
patch.object(
- _ask_mod, "discover_models",
+ _ask_mod,
+ "discover_models",
return_value={"mock": ["test-model"]},
),
patch.object(_ask_mod, "register_builtin_models"),
@@ -152,6 +153,7 @@ def mock_setup():
patch.object(_ask_mod, "merge_discovered_models"),
):
from openjarvis.core.config import JarvisConfig
+
mock_cfg.return_value = JarvisConfig()
mock_ge.return_value = ("mock", engine)
mock_de.return_value = [("mock", engine)]
@@ -175,7 +177,8 @@ class TestAskAgentOption:
def test_agent_orchestrator_no_tools(self, runner, mock_setup):
result = runner.invoke(
- cli, ["ask", "--agent", "orchestrator", "Hello"],
+ cli,
+ ["ask", "--agent", "orchestrator", "Hello"],
)
assert result.exit_code == 0
@@ -183,8 +186,11 @@ class TestAskAgentOption:
result = runner.invoke(
cli,
[
- "ask", "--agent", "orchestrator",
- "--tools", "calculator,think",
+ "ask",
+ "--agent",
+ "orchestrator",
+ "--tools",
+ "calculator,think",
"What is 2+2?",
],
)
@@ -192,7 +198,8 @@ class TestAskAgentOption:
def test_agent_json_output(self, runner, mock_setup):
result = runner.invoke(
- cli, ["ask", "--agent", "simple", "--json", "Hello"],
+ cli,
+ ["ask", "--agent", "simple", "--json", "Hello"],
)
assert result.exit_code == 0
assert '"content"' in result.output
@@ -200,7 +207,8 @@ class TestAskAgentOption:
def test_unknown_agent(self, runner, mock_setup):
result = runner.invoke(
- cli, ["ask", "--agent", "nonexistent", "Hello"],
+ cli,
+ ["ask", "--agent", "nonexistent", "Hello"],
)
assert result.exit_code != 0
@@ -211,13 +219,15 @@ class TestAskAgentOption:
def test_agent_simple_with_model(self, runner, mock_setup):
result = runner.invoke(
- cli, ["ask", "--agent", "simple", "-m", "test-model", "Hello"],
+ cli,
+ ["ask", "--agent", "simple", "-m", "test-model", "Hello"],
)
assert result.exit_code == 0
def test_agent_simple_with_temperature(self, runner, mock_setup):
result = runner.invoke(
- cli, ["ask", "--agent", "simple", "-t", "0.1", "Hello"],
+ cli,
+ ["ask", "--agent", "simple", "-t", "0.1", "Hello"],
)
assert result.exit_code == 0
@@ -240,7 +250,8 @@ class TestAskAgentOption:
agent_setup.config.agent.tools = agent_tools
result = runner.invoke(
- cli, ["ask", "--agent", "confirming_agent", "Hello"],
+ cli,
+ ["ask", "--agent", "confirming_agent", "Hello"],
)
assert result.exit_code == 0
diff --git a/tests/cli/test_ask_context.py b/tests/cli/test_ask_context.py
index afd3773b..5e4ed5ed 100644
--- a/tests/cli/test_ask_context.py
+++ b/tests/cli/test_ask_context.py
@@ -11,9 +11,7 @@ from openjarvis.cli import cli
def test_ask_no_context_flag():
"""The --no-context flag is accepted."""
- result = CliRunner().invoke(
- cli, ["ask", "--no-context", "--help"]
- )
+ result = CliRunner().invoke(cli, ["ask", "--no-context", "--help"])
# --help should succeed regardless
assert result.exit_code == 0
@@ -26,7 +24,8 @@ def test_ask_has_no_context_option():
def test_get_memory_backend_returns_none_when_empty(
- tmp_path, monkeypatch,
+ tmp_path,
+ monkeypatch,
):
"""_get_memory_backend returns None when no docs indexed."""
from openjarvis.core.config import JarvisConfig, MemoryConfig
@@ -47,7 +46,8 @@ def test_get_memory_backend_returns_none_when_empty(
def test_get_memory_backend_returns_backend_with_docs(
- tmp_path, monkeypatch,
+ tmp_path,
+ monkeypatch,
):
"""_get_memory_backend returns a backend when docs exist."""
from openjarvis.core.config import JarvisConfig, MemoryConfig
diff --git a/tests/cli/test_ask_e2e.py b/tests/cli/test_ask_e2e.py
index 18839a50..336f58b4 100644
--- a/tests/cli/test_ask_e2e.py
+++ b/tests/cli/test_ask_e2e.py
@@ -30,9 +30,7 @@ def _mock_engine_response():
}
-def _patch_ask(
- monkeypatch, tmp_path, *, engine_result=None, no_engine=False
-):
+def _patch_ask(monkeypatch, tmp_path, *, engine_result=None, no_engine=False):
"""Set up common mocks for ask tests."""
cfg = JarvisConfig()
cfg.telemetry.db_path = str(tmp_path / "telemetry.db")
@@ -40,74 +38,55 @@ def _patch_ask(
monkeypatch.setattr(_ask_mod, "load_config", lambda: cfg)
if no_engine:
- monkeypatch.setattr(
- _ask_mod, "get_engine", lambda *a, **kw: None
- )
+ monkeypatch.setattr(_ask_mod, "get_engine", lambda *a, **kw: None)
else:
fake_engine = mock.MagicMock()
fake_engine.engine_id = "mock"
fake_engine.health.return_value = True
- fake_engine.generate.return_value = (
- engine_result or _mock_engine_response()
- )
+ fake_engine.generate.return_value = engine_result or _mock_engine_response()
fake_engine.list_models.return_value = ["test-model"]
monkeypatch.setattr(
- _ask_mod, "get_engine",
+ _ask_mod,
+ "get_engine",
lambda *a, **kw: ("mock", fake_engine),
)
monkeypatch.setattr(
- _ask_mod, "discover_engines",
+ _ask_mod,
+ "discover_engines",
lambda c: [("mock", fake_engine)],
)
monkeypatch.setattr(
- _ask_mod, "discover_models",
+ _ask_mod,
+ "discover_models",
lambda e: {"mock": ["test-model"]},
)
class TestAskCommand:
- def test_basic_response(
- self, monkeypatch, tmp_path: Path
- ) -> None:
+ def test_basic_response(self, monkeypatch, tmp_path: Path) -> None:
_patch_ask(monkeypatch, tmp_path)
- result = CliRunner().invoke(
- cli, ["ask", "What is 2+2?"]
- )
+ result = CliRunner().invoke(cli, ["ask", "What is 2+2?"])
assert result.exit_code == 0
assert "The answer is 4" in result.output
- def test_no_engine_error(
- self, monkeypatch, tmp_path: Path
- ) -> None:
+ def test_no_engine_error(self, monkeypatch, tmp_path: Path) -> None:
_patch_ask(monkeypatch, tmp_path, no_engine=True)
- result = CliRunner().invoke(
- cli, ["ask", "Hello"]
- )
+ result = CliRunner().invoke(cli, ["ask", "Hello"])
assert result.exit_code != 0
- def test_model_override(
- self, monkeypatch, tmp_path: Path
- ) -> None:
+ def test_model_override(self, monkeypatch, tmp_path: Path) -> None:
_patch_ask(monkeypatch, tmp_path)
- result = CliRunner().invoke(
- cli, ["ask", "-m", "custom-model", "Hello"]
- )
+ result = CliRunner().invoke(cli, ["ask", "-m", "custom-model", "Hello"])
assert result.exit_code == 0
- def test_json_output(
- self, monkeypatch, tmp_path: Path
- ) -> None:
+ def test_json_output(self, monkeypatch, tmp_path: Path) -> None:
_patch_ask(monkeypatch, tmp_path)
- result = CliRunner().invoke(
- cli, ["ask", "--json", "Hello"]
- )
+ result = CliRunner().invoke(cli, ["ask", "--json", "Hello"])
assert result.exit_code == 0
data = json.loads(result.output)
assert "content" in data
- def test_telemetry_recorded(
- self, monkeypatch, tmp_path: Path
- ) -> None:
+ def test_telemetry_recorded(self, monkeypatch, tmp_path: Path) -> None:
_patch_ask(monkeypatch, tmp_path)
CliRunner().invoke(cli, ["ask", "Hello"])
db_path = tmp_path / "telemetry.db"
diff --git a/tests/cli/test_ask_router.py b/tests/cli/test_ask_router.py
index 22f5b2e4..39758979 100644
--- a/tests/cli/test_ask_router.py
+++ b/tests/cli/test_ask_router.py
@@ -31,15 +31,18 @@ def _patch_engine(engine):
"""Return context managers that patch engine discovery to use our mock."""
return (
mock.patch.object(
- _ask_mod, "get_engine",
+ _ask_mod,
+ "get_engine",
return_value=("mock", engine),
),
mock.patch.object(
- _ask_mod, "discover_engines",
+ _ask_mod,
+ "discover_engines",
return_value={"mock": engine},
),
mock.patch.object(
- _ask_mod, "discover_models",
+ _ask_mod,
+ "discover_models",
return_value={"mock": ["test-model"]},
),
mock.patch.object(_ask_mod, "register_builtin_models"),
@@ -64,7 +67,8 @@ class TestAskModelResolution:
patches = _patch_engine(engine)
with patches[0], patches[1], patches[2], patches[3], patches[4], patches[5]:
result = CliRunner().invoke(
- cli, ["ask", "-m", "test-model", "Hello"],
+ cli,
+ ["ask", "-m", "test-model", "Hello"],
)
assert result.exit_code == 0
assert "Hello!" in result.output
@@ -74,9 +78,15 @@ class TestAskModelResolution:
engine = _mock_engine()
patches = _patch_engine(engine)
with (
- patches[0], patches[1], patches[2], patches[3], patches[4], patches[5],
+ patches[0],
+ patches[1],
+ patches[2],
+ patches[3],
+ patches[4],
+ patches[5],
mock.patch.object(
- _ask_mod, "load_config",
+ _ask_mod,
+ "load_config",
) as mock_config,
):
cfg = mock_config.return_value
@@ -93,14 +103,19 @@ class TestAskModelResolution:
patches = _patch_engine(engine)
# Override discover_models to return empty list
with (
- patches[0], patches[1],
+ patches[0],
+ patches[1],
mock.patch.object(
- _ask_mod, "discover_models",
+ _ask_mod,
+ "discover_models",
return_value={"mock": []},
),
- patches[3], patches[4], patches[5],
+ patches[3],
+ patches[4],
+ patches[5],
mock.patch.object(
- _ask_mod, "load_config",
+ _ask_mod,
+ "load_config",
) as mock_config,
):
cfg = mock_config.return_value
diff --git a/tests/cli/test_bench_cmd.py b/tests/cli/test_bench_cmd.py
index 0eed437c..c92d8600 100644
--- a/tests/cli/test_bench_cmd.py
+++ b/tests/cli/test_bench_cmd.py
@@ -56,7 +56,8 @@ class TestBenchCLI:
return_value=("mock", engine),
):
result = CliRunner().invoke(
- cli, ["bench", "run", "-n", "2", "--json"],
+ cli,
+ ["bench", "run", "-n", "2", "--json"],
)
assert result.exit_code == 0
assert "benchmark_count" in result.output
@@ -76,7 +77,8 @@ class TestBenchCLI:
return_value=("mock", engine),
):
result = CliRunner().invoke(
- cli, ["bench", "run", "-n", "2", "-o", str(out_file)],
+ cli,
+ ["bench", "run", "-n", "2", "-o", str(out_file)],
)
assert result.exit_code == 0
assert out_file.exists()
diff --git a/tests/cli/test_channel_cmd.py b/tests/cli/test_channel_cmd.py
index 62ef5f97..f1ac6197 100644
--- a/tests/cli/test_channel_cmd.py
+++ b/tests/cli/test_channel_cmd.py
@@ -25,7 +25,8 @@ def _patch_channel(
bridge_instance.status.return_value = status_return
config_patch = mock.patch(
- "openjarvis.core.config.load_config", return_value=cfg,
+ "openjarvis.core.config.load_config",
+ return_value=cfg,
)
get_channel_patch = mock.patch(
"openjarvis.cli.channel_cmd._get_channel",
@@ -75,7 +76,8 @@ class TestChannelSend:
config_p, getch_p, _ = _patch_channel(send_return=True)
with config_p, getch_p:
result = CliRunner().invoke(
- cli, ["channel", "send", "slack", "Hello!"],
+ cli,
+ ["channel", "send", "slack", "Hello!"],
)
assert result.exit_code == 0
assert "Message sent" in result.output
@@ -84,7 +86,8 @@ class TestChannelSend:
config_p, getch_p, _ = _patch_channel(send_return=False)
with config_p, getch_p:
result = CliRunner().invoke(
- cli, ["channel", "send", "slack", "Hello!"],
+ cli,
+ ["channel", "send", "slack", "Hello!"],
)
assert result.exit_code == 0
assert "Failed to send" in result.output
diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py
index d283cfb3..ce9e7f56 100644
--- a/tests/cli/test_cli.py
+++ b/tests/cli/test_cli.py
@@ -31,11 +31,7 @@ class TestCLI:
# Either exits with error (no engine) or succeeds (deps missing)
# Both are valid states for testing
out = result.output.lower()
- assert (
- result.exit_code != 0
- or "not installed" in out
- or "no inference" in out
- )
+ assert result.exit_code != 0 or "not installed" in out or "no inference" in out
def test_model_subcommands_exist(self) -> None:
result = CliRunner().invoke(cli, ["model", "--help"])
@@ -82,12 +78,8 @@ class TestCLI:
config_dir = tmp_path / ".openjarvis"
config_path = config_dir / "config.toml"
with (
- mock.patch(
- "openjarvis.cli.init_cmd.DEFAULT_CONFIG_DIR", config_dir
- ),
- mock.patch(
- "openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path
- ),
+ mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_DIR", config_dir),
+ mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path),
):
result = CliRunner().invoke(cli, ["init", "--engine", "ollama"])
assert result.exit_code == 0
diff --git a/tests/cli/test_daemon_cmd.py b/tests/cli/test_daemon_cmd.py
index fc2cf567..4df3b393 100644
--- a/tests/cli/test_daemon_cmd.py
+++ b/tests/cli/test_daemon_cmd.py
@@ -23,18 +23,14 @@ class TestDaemonCommands:
def test_stop_no_server(self) -> None:
"""``jarvis stop`` when no PID file shows 'not running'."""
- with patch(
- "openjarvis.cli.daemon_cmd._read_pid", return_value=None
- ):
+ with patch("openjarvis.cli.daemon_cmd._read_pid", return_value=None):
result = CliRunner().invoke(cli, ["stop"])
assert result.exit_code != 0
assert "No running server" in result.output
def test_status_no_server(self) -> None:
"""``jarvis status`` when no PID file shows 'not running'."""
- with patch(
- "openjarvis.cli.daemon_cmd._read_pid", return_value=None
- ):
+ with patch("openjarvis.cli.daemon_cmd._read_pid", return_value=None):
result = CliRunner().invoke(cli, ["status"])
assert result.exit_code == 0
assert "not running" in result.output
@@ -52,9 +48,7 @@ class TestDaemonCommands:
pid_file = tmp_path / "server.pid"
with (
patch("openjarvis.cli.daemon_cmd._PID_FILE", pid_file),
- patch(
- "openjarvis.cli.daemon_cmd.DEFAULT_CONFIG_DIR", tmp_path
- ),
+ patch("openjarvis.cli.daemon_cmd.DEFAULT_CONFIG_DIR", tmp_path),
patch("os.kill", return_value=None),
):
_write_pid(12345)
@@ -68,9 +62,7 @@ class TestDaemonCommands:
mock_config.server.port = 8000
with (
- patch(
- "openjarvis.cli.daemon_cmd._read_pid", return_value=9999
- ),
+ patch("openjarvis.cli.daemon_cmd._read_pid", return_value=9999),
patch(
"openjarvis.cli.daemon_cmd.load_config",
return_value=mock_config,
@@ -83,9 +75,7 @@ class TestDaemonCommands:
def test_start_already_running(self) -> None:
"""``jarvis start`` exits with error when a server is already running."""
- with patch(
- "openjarvis.cli.daemon_cmd._read_pid", return_value=42
- ):
+ with patch("openjarvis.cli.daemon_cmd._read_pid", return_value=42):
result = CliRunner().invoke(cli, ["start"])
assert result.exit_code != 0
assert "already running" in result.output
diff --git a/tests/cli/test_doctor_cmd.py b/tests/cli/test_doctor_cmd.py
index f6fe32d2..2c06c4f3 100644
--- a/tests/cli/test_doctor_cmd.py
+++ b/tests/cli/test_doctor_cmd.py
@@ -32,19 +32,13 @@ class TestDoctorRuns:
mock_config.intelligence.default_model = ""
with (
- patch(
- "openjarvis.cli.doctor_cmd.load_config", return_value=mock_config
- ),
+ patch("openjarvis.cli.doctor_cmd.load_config", return_value=mock_config),
patch(
"openjarvis.cli.doctor_cmd.DEFAULT_CONFIG_PATH",
Path("/tmp/nonexistent/config.toml"),
),
- patch(
- "openjarvis.cli.doctor_cmd._check_engines", return_value=[]
- ),
- patch(
- "openjarvis.cli.doctor_cmd._check_models", return_value=[]
- ),
+ patch("openjarvis.cli.doctor_cmd._check_engines", return_value=[]),
+ patch("openjarvis.cli.doctor_cmd._check_models", return_value=[]),
):
result = CliRunner().invoke(cli, ["doctor"])
assert result.exit_code == 0
@@ -58,19 +52,13 @@ class TestDoctorJsonOutput:
mock_config.intelligence.default_model = ""
with (
- patch(
- "openjarvis.cli.doctor_cmd.load_config", return_value=mock_config
- ),
+ patch("openjarvis.cli.doctor_cmd.load_config", return_value=mock_config),
patch(
"openjarvis.cli.doctor_cmd.DEFAULT_CONFIG_PATH",
Path("/tmp/nonexistent/config.toml"),
),
- patch(
- "openjarvis.cli.doctor_cmd._check_engines", return_value=[]
- ),
- patch(
- "openjarvis.cli.doctor_cmd._check_models", return_value=[]
- ),
+ patch("openjarvis.cli.doctor_cmd._check_engines", return_value=[]),
+ patch("openjarvis.cli.doctor_cmd._check_models", return_value=[]),
):
result = CliRunner().invoke(cli, ["doctor", "--json"])
assert result.exit_code == 0
@@ -129,13 +117,9 @@ class TestCheckEngineProbing:
for key in sorted(keys):
engine = mock_make_engine(key, mock_config)
if engine.health():
- results.append(
- CheckResult(f"Engine: {key}", "ok", "Reachable")
- )
+ results.append(CheckResult(f"Engine: {key}", "ok", "Reachable"))
else:
- results.append(
- CheckResult(f"Engine: {key}", "warn", "Unreachable")
- )
+ results.append(CheckResult(f"Engine: {key}", "warn", "Unreachable"))
names = [r.name for r in results]
assert "Engine: ollama" in names
diff --git a/tests/cli/test_doctor_labels.py b/tests/cli/test_doctor_labels.py
index 3c0c0c94..0b570c80 100644
--- a/tests/cli/test_doctor_labels.py
+++ b/tests/cli/test_doctor_labels.py
@@ -15,10 +15,12 @@ _real_import = builtins.__import__
def _selective_import_blocker(*blocked: str):
"""Return an __import__ replacement that blocks specific packages."""
+
def _import(name, *args, **kwargs):
if name in blocked:
raise ImportError(f"mocked: {name} not installed")
return _real_import(name, *args, **kwargs)
+
return _import
@@ -45,8 +47,7 @@ class TestDoctorOptionalLabels:
result = runner.invoke(cli, ["doctor", "--json"])
data = json.loads(result.output)
apple_checks = [
- c for c in data
- if c["name"] == "Optional: Apple Silicon energy monitoring"
+ c for c in data if c["name"] == "Optional: Apple Silicon energy monitoring"
]
assert len(apple_checks) == 1
assert "Not installed (openjarvis[energy-apple])" == apple_checks[0]["message"]
@@ -59,8 +60,7 @@ class TestDoctorOptionalLabels:
result = runner.invoke(cli, ["doctor", "--json"])
data = json.loads(result.output)
nvidia_checks = [
- c for c in data
- if c["name"] == "Optional: NVIDIA energy monitoring"
+ c for c in data if c["name"] == "Optional: NVIDIA energy monitoring"
]
assert len(nvidia_checks) == 1
assert "Not installed (openjarvis[gpu-metrics])" == nvidia_checks[0]["message"]
diff --git a/tests/cli/test_init_guidance.py b/tests/cli/test_init_guidance.py
index 21ddbe9f..263d786d 100644
--- a/tests/cli/test_init_guidance.py
+++ b/tests/cli/test_init_guidance.py
@@ -17,12 +17,8 @@ class TestInitShowsNextSteps:
config_dir = tmp_path / ".openjarvis"
config_path = config_dir / "config.toml"
with (
- mock.patch(
- "openjarvis.cli.init_cmd.DEFAULT_CONFIG_DIR", config_dir
- ),
- mock.patch(
- "openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path
- ),
+ mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_DIR", config_dir),
+ mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path),
):
result = CliRunner().invoke(cli, ["init", "--engine", "llamacpp"])
assert result.exit_code == 0
@@ -35,12 +31,8 @@ class TestInitShowsNextSteps:
config_dir = tmp_path / ".openjarvis"
config_path = config_dir / "config.toml"
with (
- mock.patch(
- "openjarvis.cli.init_cmd.DEFAULT_CONFIG_DIR", config_dir
- ),
- mock.patch(
- "openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path
- ),
+ mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_DIR", config_dir),
+ mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path),
):
result = CliRunner().invoke(cli, ["init", "--engine", "llamacpp"])
assert result.exit_code == 0
@@ -98,12 +90,8 @@ class TestMinimalConfig:
config_dir = tmp_path / ".openjarvis"
config_path = config_dir / "config.toml"
with (
- mock.patch(
- "openjarvis.cli.init_cmd.DEFAULT_CONFIG_DIR", config_dir
- ),
- mock.patch(
- "openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path
- ),
+ mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_DIR", config_dir),
+ mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path),
):
result = CliRunner().invoke(cli, ["init", "--engine", "ollama"])
assert result.exit_code == 0
@@ -119,12 +107,8 @@ class TestMinimalConfig:
config_dir = tmp_path / ".openjarvis"
config_path = config_dir / "config.toml"
with (
- mock.patch(
- "openjarvis.cli.init_cmd.DEFAULT_CONFIG_DIR", config_dir
- ),
- mock.patch(
- "openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path
- ),
+ mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_DIR", config_dir),
+ mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path),
):
result = CliRunner().invoke(cli, ["init", "--full", "--engine", "ollama"])
assert result.exit_code == 0
diff --git a/tests/cli/test_log_config.py b/tests/cli/test_log_config.py
index 7bacfef0..3d554a2c 100644
--- a/tests/cli/test_log_config.py
+++ b/tests/cli/test_log_config.py
@@ -29,10 +29,7 @@ class TestSetupLogging:
log_file = tmp_path / "cli.log"
logger = setup_logging(verbose=True, quiet=False, log_file=log_file)
# Should have at least one file handler
- file_handlers = [
- h for h in logger.handlers
- if hasattr(h, "baseFilename")
- ]
+ file_handlers = [h for h in logger.handlers if hasattr(h, "baseFilename")]
assert len(file_handlers) >= 1
# Clean up
for h in logger.handlers[:]:
diff --git a/tests/cli/test_memory_cmd.py b/tests/cli/test_memory_cmd.py
index 9de534b2..dd86adaa 100644
--- a/tests/cli/test_memory_cmd.py
+++ b/tests/cli/test_memory_cmd.py
@@ -29,13 +29,12 @@ def test_memory_index_file(tmp_path: Path, monkeypatch):
mod = importlib.import_module("openjarvis.cli.memory_cmd")
monkeypatch.setattr(
- mod, "_get_backend",
+ mod,
+ "_get_backend",
lambda b=None: SQLiteMemory(db_path=db_path),
)
- result = CliRunner().invoke(
- cli, ["memory", "index", str(doc)]
- )
+ result = CliRunner().invoke(cli, ["memory", "index", str(doc)])
assert result.exit_code == 0
assert "Indexed" in result.output or "chunk" in result.output
@@ -43,9 +42,7 @@ def test_memory_index_file(tmp_path: Path, monkeypatch):
def test_memory_index_nonexistent(tmp_path: Path):
"""Indexing a nonexistent path should fail."""
_register_sqlite()
- result = CliRunner().invoke(
- cli, ["memory", "index", str(tmp_path / "nope")]
- )
+ result = CliRunner().invoke(cli, ["memory", "index", str(tmp_path / "nope")])
assert result.exit_code != 0
@@ -61,13 +58,12 @@ def test_memory_search_returns_results(tmp_path: Path, monkeypatch):
mod = importlib.import_module("openjarvis.cli.memory_cmd")
monkeypatch.setattr(
- mod, "_get_backend",
+ mod,
+ "_get_backend",
lambda b=None: SQLiteMemory(db_path=db_path),
)
- result = CliRunner().invoke(
- cli, ["memory", "search", "Python"]
- )
+ result = CliRunner().invoke(cli, ["memory", "search", "Python"])
assert result.exit_code == 0
assert "Python" in result.output
backend.close()
@@ -82,13 +78,12 @@ def test_memory_search_no_results(tmp_path: Path, monkeypatch):
mod = importlib.import_module("openjarvis.cli.memory_cmd")
monkeypatch.setattr(
- mod, "_get_backend",
+ mod,
+ "_get_backend",
lambda b=None: SQLiteMemory(db_path=db_path),
)
- result = CliRunner().invoke(
- cli, ["memory", "search", "quantum supercollider"]
- )
+ result = CliRunner().invoke(cli, ["memory", "search", "quantum supercollider"])
assert result.exit_code == 0
assert "No results" in result.output
backend.close()
@@ -104,7 +99,8 @@ def test_memory_stats_shows_count(tmp_path: Path, monkeypatch):
mod = importlib.import_module("openjarvis.cli.memory_cmd")
monkeypatch.setattr(
- mod, "_get_backend",
+ mod,
+ "_get_backend",
lambda b=None: SQLiteMemory(db_path=db_path),
)
diff --git a/tests/cli/test_model_cmd.py b/tests/cli/test_model_cmd.py
index 19c9e241..1cf93a9e 100644
--- a/tests/cli/test_model_cmd.py
+++ b/tests/cli/test_model_cmd.py
@@ -29,11 +29,13 @@ class TestModelList:
monkeypatch.setattr(_model_mod, "load_config", lambda: cfg)
fake = _mock_engine()
monkeypatch.setattr(
- _model_mod, "discover_engines",
+ _model_mod,
+ "discover_engines",
lambda c: [("mock", fake)],
)
monkeypatch.setattr(
- _model_mod, "discover_models",
+ _model_mod,
+ "discover_models",
lambda e: {"mock": ["model-a", "model-b"]},
)
result = CliRunner().invoke(cli, ["model", "list"])
@@ -43,9 +45,7 @@ class TestModelList:
def test_no_engines_message(self, monkeypatch) -> None:
cfg = JarvisConfig()
monkeypatch.setattr(_model_mod, "load_config", lambda: cfg)
- monkeypatch.setattr(
- _model_mod, "discover_engines", lambda c: []
- )
+ monkeypatch.setattr(_model_mod, "discover_engines", lambda c: [])
result = CliRunner().invoke(cli, ["model", "list"])
assert result.exit_code == 0
assert "No inference engines" in result.output
@@ -55,28 +55,16 @@ class TestModelInfo:
def test_info_known_model(self, monkeypatch) -> None:
cfg = JarvisConfig()
monkeypatch.setattr(_model_mod, "load_config", lambda: cfg)
- monkeypatch.setattr(
- _model_mod, "discover_engines", lambda c: []
- )
- monkeypatch.setattr(
- _model_mod, "discover_models", lambda e: {}
- )
- result = CliRunner().invoke(
- cli, ["model", "info", "qwen3:8b"]
- )
+ monkeypatch.setattr(_model_mod, "discover_engines", lambda c: [])
+ monkeypatch.setattr(_model_mod, "discover_models", lambda e: {})
+ result = CliRunner().invoke(cli, ["model", "info", "qwen3:8b"])
assert result.exit_code == 0
assert "Qwen3 8B" in result.output
def test_unknown_model_not_found(self, monkeypatch) -> None:
cfg = JarvisConfig()
monkeypatch.setattr(_model_mod, "load_config", lambda: cfg)
- monkeypatch.setattr(
- _model_mod, "discover_engines", lambda c: []
- )
- monkeypatch.setattr(
- _model_mod, "discover_models", lambda e: {}
- )
- result = CliRunner().invoke(
- cli, ["model", "info", "nonexistent-model"]
- )
+ monkeypatch.setattr(_model_mod, "discover_engines", lambda c: [])
+ monkeypatch.setattr(_model_mod, "discover_models", lambda e: {})
+ result = CliRunner().invoke(cli, ["model", "info", "nonexistent-model"])
assert result.exit_code != 0
diff --git a/tests/cli/test_quickstart.py b/tests/cli/test_quickstart.py
index 19acf906..c80a1357 100644
--- a/tests/cli/test_quickstart.py
+++ b/tests/cli/test_quickstart.py
@@ -32,28 +32,23 @@ class TestQuickstartCommand:
patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_PATH", config_path),
patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_DIR", tmp_path),
patch(
- "openjarvis.cli.quickstart_cmd"
- ".generate_default_toml",
+ "openjarvis.cli.quickstart_cmd.generate_default_toml",
return_value="[engine]\n",
),
patch(
- "openjarvis.cli.quickstart_cmd"
- ".recommend_engine",
+ "openjarvis.cli.quickstart_cmd.recommend_engine",
return_value="ollama",
),
patch(
- "openjarvis.cli.quickstart_cmd"
- "._check_engine_health",
+ "openjarvis.cli.quickstart_cmd._check_engine_health",
return_value=True,
),
patch(
- "openjarvis.cli.quickstart_cmd"
- "._check_model_available",
+ "openjarvis.cli.quickstart_cmd._check_model_available",
return_value=True,
),
patch(
- "openjarvis.cli.quickstart_cmd"
- "._test_query",
+ "openjarvis.cli.quickstart_cmd._test_query",
return_value="Hello!",
),
):
@@ -79,28 +74,23 @@ class TestQuickstartCommand:
patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_PATH", config_path),
patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_DIR", tmp_path),
patch(
- "openjarvis.cli.quickstart_cmd"
- ".generate_default_toml",
+ "openjarvis.cli.quickstart_cmd.generate_default_toml",
return_value="[engine]\n",
),
patch(
- "openjarvis.cli.quickstart_cmd"
- ".recommend_engine",
+ "openjarvis.cli.quickstart_cmd.recommend_engine",
return_value="ollama",
),
patch(
- "openjarvis.cli.quickstart_cmd"
- "._check_engine_health",
+ "openjarvis.cli.quickstart_cmd._check_engine_health",
return_value=True,
),
patch(
- "openjarvis.cli.quickstart_cmd"
- "._check_model_available",
+ "openjarvis.cli.quickstart_cmd._check_model_available",
return_value=True,
),
patch(
- "openjarvis.cli.quickstart_cmd"
- "._test_query",
+ "openjarvis.cli.quickstart_cmd._test_query",
return_value="Hello!",
),
):
@@ -128,23 +118,19 @@ class TestQuickstartCommand:
patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_PATH", config_path),
patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_DIR", tmp_path),
patch(
- "openjarvis.cli.quickstart_cmd"
- ".generate_default_toml",
+ "openjarvis.cli.quickstart_cmd.generate_default_toml",
return_value="[engine]\nnew = true\n",
),
patch(
- "openjarvis.cli.quickstart_cmd"
- ".recommend_engine",
+ "openjarvis.cli.quickstart_cmd.recommend_engine",
return_value="ollama",
),
patch(
- "openjarvis.cli.quickstart_cmd"
- "._check_engine_health",
+ "openjarvis.cli.quickstart_cmd._check_engine_health",
return_value=True,
),
patch(
- "openjarvis.cli.quickstart_cmd"
- "._check_model_available",
+ "openjarvis.cli.quickstart_cmd._check_model_available",
return_value=True,
),
patch("openjarvis.cli.quickstart_cmd._test_query", return_value="Hello!"),
@@ -169,18 +155,15 @@ class TestQuickstartCommand:
patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_PATH", config_path),
patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_DIR", tmp_path),
patch(
- "openjarvis.cli.quickstart_cmd"
- ".generate_default_toml",
+ "openjarvis.cli.quickstart_cmd.generate_default_toml",
return_value="[engine]\n",
),
patch(
- "openjarvis.cli.quickstart_cmd"
- ".recommend_engine",
+ "openjarvis.cli.quickstart_cmd.recommend_engine",
return_value="ollama",
),
patch(
- "openjarvis.cli.quickstart_cmd"
- "._check_engine_health",
+ "openjarvis.cli.quickstart_cmd._check_engine_health",
return_value=False,
),
):
@@ -207,33 +190,27 @@ class TestQuickstartCommand:
patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_PATH", config_path),
patch("openjarvis.cli.quickstart_cmd.DEFAULT_CONFIG_DIR", tmp_path),
patch(
- "openjarvis.cli.quickstart_cmd"
- ".generate_default_toml",
- return_value="[engine]\ndefault = \"mlx\"\n",
+ "openjarvis.cli.quickstart_cmd.generate_default_toml",
+ return_value='[engine]\ndefault = "mlx"\n',
),
patch(
- "openjarvis.cli.quickstart_cmd"
- ".recommend_engine",
+ "openjarvis.cli.quickstart_cmd.recommend_engine",
return_value="mlx",
),
patch(
- "openjarvis.cli.quickstart_cmd"
- "._check_engine_health",
+ "openjarvis.cli.quickstart_cmd._check_engine_health",
return_value=False,
),
patch(
- "openjarvis.cli.quickstart_cmd"
- "._discover_healthy_engines",
+ "openjarvis.cli.quickstart_cmd._discover_healthy_engines",
return_value=["ollama"],
),
patch(
- "openjarvis.cli.quickstart_cmd"
- "._check_model_available",
+ "openjarvis.cli.quickstart_cmd._check_model_available",
return_value=True,
),
patch(
- "openjarvis.cli.quickstart_cmd"
- "._test_query",
+ "openjarvis.cli.quickstart_cmd._test_query",
return_value="Hello!",
),
):
diff --git a/tests/cli/test_serve_channel_wiring.py b/tests/cli/test_serve_channel_wiring.py
new file mode 100644
index 00000000..81feb254
--- /dev/null
+++ b/tests/cli/test_serve_channel_wiring.py
@@ -0,0 +1,249 @@
+"""Tests for JarvisSystem.wire_channel() — channel → agent routing.
+
+These tests exercise wire_channel() on JarvisSystem directly. The serve.py
+entrypoint now delegates all channel-wiring logic there.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+from openjarvis.channels._stubs import ChannelMessage
+from openjarvis.core.config import JarvisConfig
+from openjarvis.core.events import EventBus
+from openjarvis.sessions.session import SessionStore
+from openjarvis.system import JarvisSystem
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _make_channel_message(
+ channel: str = "telegram",
+ sender: str = "42",
+ content: str = "hello",
+ conversation_id: str = "42",
+) -> ChannelMessage:
+ return ChannelMessage(
+ channel=channel,
+ sender=sender,
+ content=content,
+ message_id="1",
+ conversation_id=conversation_id,
+ )
+
+
+def _make_system(engine=None, agent_name="", tmp_path=None) -> JarvisSystem:
+ """Build a minimal JarvisSystem with mock engine for testing wire_channel."""
+ config = JarvisConfig()
+ if tmp_path is not None:
+ config.sessions.db_path = str(tmp_path / "sessions.db")
+ mock_engine = engine or MagicMock()
+ return JarvisSystem(
+ config=config,
+ bus=EventBus(record_history=False),
+ engine=mock_engine,
+ engine_key="mock",
+ model="test-model",
+ agent_name=agent_name,
+ )
+
+
+def _fire(channel_mock, cm: ChannelMessage) -> None:
+ """Invoke all registered on_message handlers as if the channel received cm."""
+ for handler in channel_mock.on_message.call_args_list:
+ handler[0][0](cm)
+
+
+# ---------------------------------------------------------------------------
+# Tests via JarvisSystem.wire_channel()
+# ---------------------------------------------------------------------------
+
+
+class TestWireChannelWithAgent:
+ """wire_channel routes incoming messages through the agent and replies."""
+
+ def test_ask_called_and_reply_sent(self, tmp_path):
+ system = _make_system(agent_name="simple", tmp_path=tmp_path)
+ # Patch ask() so we don't need a real engine/agent
+ system.ask = MagicMock(return_value={"content": "pong"})
+
+ mock_channel = MagicMock()
+ system.wire_channel(mock_channel)
+
+ # Simulate an incoming message
+ cm = _make_channel_message(content="ping")
+ handler = mock_channel.on_message.call_args[0][0]
+ handler(cm)
+
+ system.ask.assert_called_once()
+ assert system.ask.call_args[0][0] == "ping"
+ mock_channel.send.assert_called_once_with(
+ "telegram",
+ "pong",
+ conversation_id="42",
+ )
+
+ def test_session_store_created_lazily(self, tmp_path):
+ system = _make_system(tmp_path=tmp_path)
+ assert system.session_store is None
+
+ mock_channel = MagicMock()
+ system.ask = MagicMock(return_value={"content": "ok"})
+ system.wire_channel(mock_channel)
+
+ handler = mock_channel.on_message.call_args[0][0]
+ handler(_make_channel_message())
+
+ assert system.session_store is not None
+
+ def test_existing_session_store_reused(self, tmp_path):
+ system = _make_system(tmp_path=tmp_path)
+ existing_store = SessionStore(db_path=tmp_path / "sessions.db")
+ system.session_store = existing_store
+ system.ask = MagicMock(return_value={"content": "ok"})
+
+ mock_channel = MagicMock()
+ system.wire_channel(mock_channel)
+
+ handler = mock_channel.on_message.call_args[0][0]
+ handler(_make_channel_message())
+
+ assert system.session_store is existing_store
+
+
+class TestWireChannelWithEngine:
+ """wire_channel falls back to engine when no agent is set."""
+
+ def test_engine_path_used_when_no_agent(self, tmp_path):
+ system = _make_system(agent_name="", tmp_path=tmp_path)
+ system.ask = MagicMock(return_value={"content": "raw reply"})
+
+ mock_channel = MagicMock()
+ system.wire_channel(mock_channel)
+
+ handler = mock_channel.on_message.call_args[0][0]
+ handler(_make_channel_message(content="hi"))
+
+ mock_channel.send.assert_called_once_with(
+ "telegram",
+ "raw reply",
+ conversation_id="42",
+ )
+
+
+class TestWireChannelSessionIsolation:
+ """Separate conversation_ids get independent sessions."""
+
+ def test_two_chats_isolated(self, tmp_path):
+ system = _make_system(tmp_path=tmp_path)
+ replies = {"111": "reply-A", "222": "reply-B"}
+ system.ask = MagicMock(
+ side_effect=lambda q, **kw: {"content": replies.get(q, "")}
+ )
+
+ mock_channel = MagicMock()
+ system.wire_channel(mock_channel)
+ handler = mock_channel.on_message.call_args[0][0]
+
+ handler(_make_channel_message(content="111", conversation_id="111"))
+ handler(_make_channel_message(content="222", conversation_id="222"))
+
+ # Reload sessions — each chat must have only its own messages
+ s1 = system.session_store.get_or_create("telegram:111")
+ s2 = system.session_store.get_or_create("telegram:222")
+ c1 = {m.content for m in s1.messages}
+ c2 = {m.content for m in s2.messages}
+
+ assert "111" in c1 and "222" not in c1
+ assert "222" in c2 and "111" not in c2
+
+ def test_same_chat_accumulates_history(self, tmp_path):
+ system = _make_system(tmp_path=tmp_path)
+ system.ask = MagicMock(return_value={"content": "reply"})
+
+ mock_channel = MagicMock()
+ system.wire_channel(mock_channel)
+ handler = mock_channel.on_message.call_args[0][0]
+
+ handler(_make_channel_message(content="first"))
+ handler(_make_channel_message(content="second"))
+
+ session = system.session_store.get_or_create("telegram:42")
+ contents = [m.content for m in session.messages]
+ # user + assistant alternating for two turns
+ assert contents.count("first") == 1
+ assert contents.count("second") == 1
+
+
+class TestWireChannelErrorHandling:
+ """Handler sends a user-visible error message when ask() raises."""
+
+ def test_error_reply_sent(self, tmp_path):
+ system = _make_system(tmp_path=tmp_path)
+ system.ask = MagicMock(side_effect=RuntimeError("boom"))
+
+ mock_channel = MagicMock()
+ system.wire_channel(mock_channel)
+ handler = mock_channel.on_message.call_args[0][0]
+ handler(_make_channel_message())
+
+ mock_channel.send.assert_called_once()
+ sent_content = mock_channel.send.call_args[0][1]
+ assert "error" in sent_content.lower()
+
+
+class TestPerChatSessionIsolation:
+ """Direct SessionStore isolation tests (not via wire_channel)."""
+
+ def test_two_chats_have_separate_sessions(self, tmp_path):
+ store = SessionStore(db_path=tmp_path / "sessions.db")
+
+ session_a = store.get_or_create(
+ "telegram:111",
+ channel="telegram",
+ channel_user_id="111",
+ )
+ store.save_message(session_a.session_id, "user", "msg from A")
+
+ session_b = store.get_or_create(
+ "telegram:222",
+ channel="telegram",
+ channel_user_id="222",
+ )
+ store.save_message(session_b.session_id, "user", "msg from B")
+
+ reloaded_a = store.get_or_create(
+ "telegram:111",
+ channel="telegram",
+ channel_user_id="111",
+ )
+ reloaded_b = store.get_or_create(
+ "telegram:222",
+ channel="telegram",
+ channel_user_id="222",
+ )
+
+ contents_a = {m.content for m in reloaded_a.messages}
+ contents_b = {m.content for m in reloaded_b.messages}
+
+ assert "msg from A" in contents_a and "msg from B" not in contents_a
+ assert "msg from B" in contents_b and "msg from A" not in contents_b
+
+ def test_same_chat_accumulates_history(self, tmp_path):
+ store = SessionStore(db_path=tmp_path / "sessions.db")
+ session = store.get_or_create(
+ "telegram:42",
+ channel="telegram",
+ channel_user_id="42",
+ )
+ store.save_message(session.session_id, "user", "first")
+ store.save_message(session.session_id, "assistant", "reply")
+
+ reloaded = store.get_or_create(
+ "telegram:42",
+ channel="telegram",
+ channel_user_id="42",
+ )
+ assert [m.content for m in reloaded.messages] == ["first", "reply"]
diff --git a/tests/cli/test_telemetry_cmd.py b/tests/cli/test_telemetry_cmd.py
index d51d1405..b6b3ee10 100644
--- a/tests/cli/test_telemetry_cmd.py
+++ b/tests/cli/test_telemetry_cmd.py
@@ -18,16 +18,18 @@ def _populate_db(db_path: Path, n: int = 3) -> None:
"""Create a telemetry DB with *n* records."""
store = TelemetryStore(db_path)
for i in range(n):
- store.record(TelemetryRecord(
- timestamp=time.time() - (n - i),
- model_id=f"model-{i % 2}",
- engine="ollama",
- prompt_tokens=10 * (i + 1),
- completion_tokens=5 * (i + 1),
- total_tokens=15 * (i + 1),
- latency_seconds=0.5 * (i + 1),
- cost_usd=0.001 * (i + 1),
- ))
+ store.record(
+ TelemetryRecord(
+ timestamp=time.time() - (n - i),
+ model_id=f"model-{i % 2}",
+ engine="ollama",
+ prompt_tokens=10 * (i + 1),
+ completion_tokens=5 * (i + 1),
+ total_tokens=15 * (i + 1),
+ latency_seconds=0.5 * (i + 1),
+ cost_usd=0.001 * (i + 1),
+ )
+ )
store.close()
@@ -37,7 +39,8 @@ def _patch_config(tmp_path: Path):
cfg = mock.MagicMock()
cfg.telemetry.db_path = str(db_path)
return mock.patch(
- "openjarvis.cli.telemetry_cmd.load_config", return_value=cfg,
+ "openjarvis.cli.telemetry_cmd.load_config",
+ return_value=cfg,
), db_path
@@ -115,7 +118,8 @@ class TestTelemetryExport:
out_file = tmp_path / "export.json"
with patch:
result = CliRunner().invoke(
- cli, ["telemetry", "export", "-o", str(out_file)],
+ cli,
+ ["telemetry", "export", "-o", str(out_file)],
)
assert result.exit_code == 0
assert out_file.exists()
diff --git a/tests/cli/test_vault_cmd.py b/tests/cli/test_vault_cmd.py
index aaa01d2d..c839b25f 100644
--- a/tests/cli/test_vault_cmd.py
+++ b/tests/cli/test_vault_cmd.py
@@ -47,7 +47,8 @@ class TestVaultCmd:
mock.patch("openjarvis.cli.vault_cmd._VAULT_FILE", vault_file),
mock.patch("openjarvis.cli.vault_cmd._VAULT_KEY_FILE", key_file),
mock.patch(
- "openjarvis.cli.vault_cmd.DEFAULT_CONFIG_DIR", tmp_path,
+ "openjarvis.cli.vault_cmd.DEFAULT_CONFIG_DIR",
+ tmp_path,
),
):
runner = CliRunner()
diff --git a/tests/conftest.py b/tests/conftest.py
index af05a657..cd709c1e 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -54,8 +54,10 @@ def nvidia_gpu() -> GpuInfo:
def nvidia_consumer_gpu() -> GpuInfo:
"""NVIDIA consumer GPU fixture."""
return GpuInfo(
- vendor="nvidia", name="NVIDIA GeForce RTX 4090",
- vram_gb=24.0, count=1,
+ vendor="nvidia",
+ name="NVIDIA GeForce RTX 4090",
+ vram_gb=24.0,
+ count=1,
)
@@ -147,6 +149,7 @@ def has_ollama() -> bool:
"""Check if Ollama is running locally."""
try:
import httpx
+
resp = httpx.get("http://localhost:11434/api/tags", timeout=2.0)
return resp.status_code == 200
except Exception:
@@ -158,6 +161,7 @@ def has_vllm() -> bool:
"""Check if vLLM is running locally."""
try:
import httpx
+
resp = httpx.get("http://localhost:8000/v1/models", timeout=2.0)
return resp.status_code == 200
except Exception:
@@ -169,6 +173,7 @@ def has_llamacpp() -> bool:
"""Check if llama.cpp server is running locally."""
try:
import httpx
+
resp = httpx.get("http://localhost:8080/v1/models", timeout=2.0)
return resp.status_code == 200
except Exception:
diff --git a/tests/core/test_config.py b/tests/core/test_config.py
index 28407d26..69c1ee14 100644
--- a/tests/core/test_config.py
+++ b/tests/core/test_config.py
@@ -200,9 +200,10 @@ class TestAgentConfigNew:
def test_no_temperature_or_max_tokens(self) -> None:
ac = AgentConfig()
- assert not hasattr(ac.__class__, "temperature") or isinstance(
- getattr(ac.__class__, "temperature", None), property
- ) is False
+ assert (
+ not hasattr(ac.__class__, "temperature")
+ or isinstance(getattr(ac.__class__, "temperature", None), property) is False
+ )
class TestNestedEngineConfig:
@@ -288,9 +289,9 @@ class TestNestedLearningConfig:
def test_loads_nested_toml(self, tmp_path: Path) -> None:
toml_file = tmp_path / "config.toml"
toml_file.write_text(
- '[learning]\nenabled = true\nupdate_interval = 50\n\n'
+ "[learning]\nenabled = true\nupdate_interval = 50\n\n"
'[learning.routing]\npolicy = "learned"\n\n'
- '[learning.metrics]\nlatency_weight = 0.5\n'
+ "[learning.metrics]\nlatency_weight = 0.5\n"
)
cfg = load_config(toml_file)
assert cfg.learning.enabled is True
@@ -315,16 +316,14 @@ class TestNestedLearningConfig:
class TestBackwardCompatMigration:
def test_agent_temperature_migrates_to_intelligence(self, tmp_path: Path) -> None:
toml_file = tmp_path / "config.toml"
- toml_file.write_text(
- '[agent]\ntemperature = 0.3\nmax_tokens = 512\n'
- )
+ toml_file.write_text("[agent]\ntemperature = 0.3\nmax_tokens = 512\n")
cfg = load_config(toml_file)
assert cfg.intelligence.temperature == 0.3
assert cfg.intelligence.max_tokens == 512
def test_memory_context_injection_migrates_to_agent(self, tmp_path: Path) -> None:
toml_file = tmp_path / "config.toml"
- toml_file.write_text('[memory]\ncontext_injection = false\n')
+ toml_file.write_text("[memory]\ncontext_injection = false\n")
cfg = load_config(toml_file)
assert cfg.agent.context_from_memory is False
@@ -400,8 +399,7 @@ class TestSandboxConfig:
def test_loads_from_toml(self, tmp_path: Path) -> None:
toml_file = tmp_path / "config.toml"
toml_file.write_text(
- '[sandbox]\nenabled = true\ntimeout = 600\n'
- 'runtime = "podman"\n'
+ '[sandbox]\nenabled = true\ntimeout = 600\nruntime = "podman"\n'
)
cfg = load_config(toml_file)
assert cfg.sandbox.enabled is True
@@ -435,7 +433,7 @@ class TestSchedulerConfig:
def test_loads_from_toml(self, tmp_path: Path) -> None:
toml_file = tmp_path / "config.toml"
toml_file.write_text(
- '[scheduler]\nenabled = true\npoll_interval = 30\n'
+ "[scheduler]\nenabled = true\npoll_interval = 30\n"
'db_path = "/tmp/sched.db"\n'
)
cfg = load_config(toml_file)
diff --git a/tests/core/test_config_phase4.py b/tests/core/test_config_phase4.py
index c69c266f..4b2b4acf 100644
--- a/tests/core/test_config_phase4.py
+++ b/tests/core/test_config_phase4.py
@@ -45,8 +45,7 @@ class TestLearningConfig:
def test_toml_loading_with_learning(self, tmp_path: Path) -> None:
toml_file = tmp_path / "config.toml"
toml_file.write_text(
- '[learning]\ndefault_policy = "grpo"\n'
- 'reward_weights = "latency=0.5"\n'
+ '[learning]\ndefault_policy = "grpo"\nreward_weights = "latency=0.5"\n'
)
cfg = load_config(toml_file)
assert cfg.learning.routing.policy == "grpo"
@@ -55,9 +54,9 @@ class TestLearningConfig:
def test_toml_loading_nested(self, tmp_path: Path) -> None:
toml_file = tmp_path / "config.toml"
toml_file.write_text(
- '[learning]\nenabled = true\n\n'
+ "[learning]\nenabled = true\n\n"
'[learning.routing]\npolicy = "learned"\n\n'
- '[learning.metrics]\nlatency_weight = 0.5\n'
+ "[learning.metrics]\nlatency_weight = 0.5\n"
)
cfg = load_config(toml_file)
assert cfg.learning.enabled is True
diff --git a/tests/core/test_credentials.py b/tests/core/test_credentials.py
index e5679d47..2d8993fb 100644
--- a/tests/core/test_credentials.py
+++ b/tests/core/test_credentials.py
@@ -1,4 +1,5 @@
"""Tests for credential persistence module."""
+
import os
import pytest
diff --git a/tests/core/test_events.py b/tests/core/test_events.py
index 3f846472..471b6270 100644
--- a/tests/core/test_events.py
+++ b/tests/core/test_events.py
@@ -95,12 +95,14 @@ class TestEventBus:
class TestAgentEventTypes:
def test_agent_tick_events_exist(self):
from openjarvis.core.events import EventType
+
assert EventType.AGENT_TICK_START
assert EventType.AGENT_TICK_END
assert EventType.AGENT_TICK_ERROR
def test_agent_operational_events_exist(self):
from openjarvis.core.events import EventType
+
assert EventType.AGENT_BUDGET_EXCEEDED
assert EventType.AGENT_STALL_DETECTED
assert EventType.AGENT_MESSAGE_RECEIVED
diff --git a/tests/core/test_rust_bridge.py b/tests/core/test_rust_bridge.py
index 47ac6812..893250de 100644
--- a/tests/core/test_rust_bridge.py
+++ b/tests/core/test_rust_bridge.py
@@ -24,12 +24,14 @@ class TestScanResultFromJson:
def test_empty_findings(self):
from openjarvis._rust_bridge import scan_result_from_json
+
result = scan_result_from_json('{"findings": []}')
assert result.clean
assert result.findings == []
def test_with_findings(self):
from openjarvis._rust_bridge import scan_result_from_json
+
data = {
"findings": [
{
@@ -54,6 +56,7 @@ class TestInjectionResultFromJson:
def test_clean(self):
from openjarvis._rust_bridge import injection_result_from_json
+
data = {"is_clean": True, "findings": [], "threat_level": "low"}
result = injection_result_from_json(json.dumps(data))
assert result.is_clean
@@ -61,6 +64,7 @@ class TestInjectionResultFromJson:
def test_with_findings(self):
from openjarvis._rust_bridge import injection_result_from_json
+
data = {
"is_clean": False,
"findings": [
@@ -86,11 +90,13 @@ class TestRetrievalResultsFromJson:
def test_empty(self):
from openjarvis._rust_bridge import retrieval_results_from_json
+
results = retrieval_results_from_json("[]")
assert results == []
def test_with_items(self):
from openjarvis._rust_bridge import retrieval_results_from_json
+
data = [
{
"content": "hello world",
@@ -108,6 +114,7 @@ class TestRetrievalResultsFromJson:
def test_metadata_as_string(self):
from openjarvis._rust_bridge import retrieval_results_from_json
+
data = [
{
"content": "test",
diff --git a/tests/engine/test_cloud_extended.py b/tests/engine/test_cloud_extended.py
index e63c8dda..db946a80 100644
--- a/tests/engine/test_cloud_extended.py
+++ b/tests/engine/test_cloud_extended.py
@@ -324,8 +324,10 @@ class TestCloudAnthropic:
model="claude-opus-4-6",
)
call_kwargs = fake_client.messages.create.call_args
- assert call_kwargs.kwargs.get("system") == "You are helpful" or \
- call_kwargs[1].get("system") == "You are helpful"
+ assert (
+ call_kwargs.kwargs.get("system") == "You are helpful"
+ or call_kwargs[1].get("system") == "You are helpful"
+ )
# ---------------------------------------------------------------------------
@@ -339,10 +341,13 @@ class TestCloudGemini:
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
fake_genai = mock.MagicMock()
- with mock.patch.dict("sys.modules", {
- "google": mock.MagicMock(),
- "google.genai": fake_genai,
- }):
+ with mock.patch.dict(
+ "sys.modules",
+ {
+ "google": mock.MagicMock(),
+ "google.genai": fake_genai,
+ },
+ ):
if not EngineRegistry.contains("cloud"):
EngineRegistry.register_value("cloud", CloudEngine)
engine = CloudEngine()
@@ -360,11 +365,14 @@ class TestCloudGemini:
fake_config = mock.MagicMock()
fake_types = mock.MagicMock()
fake_types.GenerateContentConfig.return_value = fake_config
- with mock.patch.dict("sys.modules", {
- "google": mock.MagicMock(),
- "google.genai": mock.MagicMock(),
- "google.genai.types": fake_types,
- }):
+ with mock.patch.dict(
+ "sys.modules",
+ {
+ "google": mock.MagicMock(),
+ "google.genai": mock.MagicMock(),
+ "google.genai.types": fake_types,
+ },
+ ):
result = engine.generate(
[Message(role=Role.USER, content="Hi")], model="gemini-2.5-pro"
)
@@ -383,11 +391,14 @@ class TestCloudGemini:
fake_types = mock.MagicMock()
fake_types.GenerateContentConfig.return_value = mock.MagicMock()
- with mock.patch.dict("sys.modules", {
- "google": mock.MagicMock(),
- "google.genai": mock.MagicMock(),
- "google.genai.types": fake_types,
- }):
+ with mock.patch.dict(
+ "sys.modules",
+ {
+ "google": mock.MagicMock(),
+ "google.genai": mock.MagicMock(),
+ "google.genai.types": fake_types,
+ },
+ ):
result = engine.generate(
[Message(role=Role.USER, content="Hi")], model="gemini-2.5-flash"
)
@@ -403,11 +414,14 @@ class TestCloudGemini:
fake_types = mock.MagicMock()
fake_types.GenerateContentConfig.return_value = mock.MagicMock()
- with mock.patch.dict("sys.modules", {
- "google": mock.MagicMock(),
- "google.genai": mock.MagicMock(),
- "google.genai.types": fake_types,
- }):
+ with mock.patch.dict(
+ "sys.modules",
+ {
+ "google": mock.MagicMock(),
+ "google.genai": mock.MagicMock(),
+ "google.genai.types": fake_types,
+ },
+ ):
result = engine.generate(
[Message(role=Role.USER, content="Hi")], model="gemini-3-pro"
)
@@ -423,11 +437,14 @@ class TestCloudGemini:
fake_types = mock.MagicMock()
fake_types.GenerateContentConfig.return_value = mock.MagicMock()
- with mock.patch.dict("sys.modules", {
- "google": mock.MagicMock(),
- "google.genai": mock.MagicMock(),
- "google.genai.types": fake_types,
- }):
+ with mock.patch.dict(
+ "sys.modules",
+ {
+ "google": mock.MagicMock(),
+ "google.genai": mock.MagicMock(),
+ "google.genai.types": fake_types,
+ },
+ ):
result = engine.generate(
[Message(role=Role.USER, content="Hi")], model="gemini-3-flash"
)
@@ -441,16 +458,16 @@ class TestCloudGemini:
fake_client = mock.MagicMock()
# Build a response with a function_call part
- text_part = SimpleNamespace(
- text="Let me calculate.", function_call=None
- )
+ text_part = SimpleNamespace(text="Let me calculate.", function_call=None)
fc = SimpleNamespace(name="calculator", args={"expression": "2+2"})
fc_part = SimpleNamespace(text=None, function_call=fc)
content_obj = SimpleNamespace(parts=[text_part, fc_part])
candidate = SimpleNamespace(content=content_obj)
usage = SimpleNamespace(prompt_token_count=10, candidates_token_count=8)
fake_resp = SimpleNamespace(
- candidates=[candidate], usage_metadata=usage, text=None,
+ candidates=[candidate],
+ usage_metadata=usage,
+ text=None,
)
fake_client.models.generate_content.return_value = fake_resp
engine._google_client = fake_client
@@ -458,11 +475,14 @@ class TestCloudGemini:
fake_types = mock.MagicMock()
fake_config = mock.MagicMock()
fake_types.GenerateContentConfig.return_value = fake_config
- with mock.patch.dict("sys.modules", {
- "google": mock.MagicMock(),
- "google.genai": mock.MagicMock(),
- "google.genai.types": fake_types,
- }):
+ with mock.patch.dict(
+ "sys.modules",
+ {
+ "google": mock.MagicMock(),
+ "google.genai": mock.MagicMock(),
+ "google.genai.types": fake_types,
+ },
+ ):
result = engine.generate(
[Message(role=Role.USER, content="What is 2+2?")],
model="gemini-3-pro",
@@ -497,11 +517,14 @@ class TestCloudGemini:
fake_types = mock.MagicMock()
fake_types.GenerateContentConfig.return_value = mock.MagicMock()
- with mock.patch.dict("sys.modules", {
- "google": mock.MagicMock(),
- "google.genai": mock.MagicMock(),
- "google.genai.types": fake_types,
- }):
+ with mock.patch.dict(
+ "sys.modules",
+ {
+ "google": mock.MagicMock(),
+ "google.genai": mock.MagicMock(),
+ "google.genai.types": fake_types,
+ },
+ ):
result = engine.generate(
[Message(role=Role.USER, content="Hi")], model="gemini-3-pro"
)
@@ -538,11 +561,14 @@ class TestCloudGemini:
fake_types = mock.MagicMock()
fake_types.GenerateContentConfig.return_value = mock.MagicMock()
- with mock.patch.dict("sys.modules", {
- "google": mock.MagicMock(),
- "google.genai": mock.MagicMock(),
- "google.genai.types": fake_types,
- }):
+ with mock.patch.dict(
+ "sys.modules",
+ {
+ "google": mock.MagicMock(),
+ "google.genai": mock.MagicMock(),
+ "google.genai.types": fake_types,
+ },
+ ):
with pytest.raises(
EngineConnectionError,
match="Google client not available",
@@ -606,8 +632,13 @@ class TestPricingTable:
def test_all_new_models_in_pricing(self) -> None:
expected = [
"gpt-5-mini",
- "claude-opus-4-6", "claude-sonnet-4-6", "claude-haiku-4-5",
- "gemini-2.5-pro", "gemini-2.5-flash", "gemini-3-pro", "gemini-3-flash",
+ "claude-opus-4-6",
+ "claude-sonnet-4-6",
+ "claude-haiku-4-5",
+ "gemini-2.5-pro",
+ "gemini-2.5-flash",
+ "gemini-3-pro",
+ "gemini-3-flash",
]
for model_id in expected:
assert model_id in PRICING, f"{model_id} missing from PRICING dict"
diff --git a/tests/engine/test_discovery.py b/tests/engine/test_discovery.py
index fe3cb888..d111c169 100644
--- a/tests/engine/test_discovery.py
+++ b/tests/engine/test_discovery.py
@@ -54,9 +54,7 @@ class TestDiscoverEngines:
cfg = JarvisConfig()
with mock.patch(
"openjarvis.engine._discovery._make_engine",
- side_effect=lambda k, c: _FakeEngine(
- healthy=(k == "healthy")
- ),
+ side_effect=lambda k, c: _FakeEngine(healthy=(k == "healthy")),
):
result = discover_engines(cfg)
assert len(result) == 1
diff --git a/tests/engine/test_engine_model_matrix.py b/tests/engine/test_engine_model_matrix.py
index 8e0cc36a..a8ad1554 100644
--- a/tests/engine/test_engine_model_matrix.py
+++ b/tests/engine/test_engine_model_matrix.py
@@ -40,15 +40,18 @@ def _api_prefix(engine_key: str) -> str:
return ""
return "/v1"
-ENGINES_AND_HOSTS = [
- (key, host) for key, host, _ in _OPENAI_COMPAT_ENGINES
-] + [
+
+ENGINES_AND_HOSTS = [(key, host) for key, host, _ in _OPENAI_COMPAT_ENGINES] + [
("ollama", "http://testhost:11434"),
]
MODELS = [
- "gpt-oss:120b", "qwen3:8b", "glm-4.7-flash", "trinity-mini",
- "qwen3.5:35b-a3b", "LiquidAI/LFM2.5-1.2B-Instruct-GGUF",
+ "gpt-oss:120b",
+ "qwen3:8b",
+ "glm-4.7-flash",
+ "trinity-mini",
+ "qwen3.5:35b-a3b",
+ "LiquidAI/LFM2.5-1.2B-Instruct-GGUF",
]
_ENGINE_CLASSES = {key: cls for key, _, cls in _OPENAI_COMPAT_ENGINES}
@@ -69,26 +72,34 @@ def _mock_simple_chat(respx_mock, engine_key: str, host: str, model: str):
"""Set up mock for a simple chat response."""
if engine_key == "ollama":
respx_mock.post(f"{host}/api/chat").mock(
- return_value=httpx.Response(200, json={
- "message": {"role": "assistant", "content": "Hello!"},
- "model": model,
- "prompt_eval_count": 10,
- "eval_count": 5,
- "done": True,
- })
+ return_value=httpx.Response(
+ 200,
+ json={
+ "message": {"role": "assistant", "content": "Hello!"},
+ "model": model,
+ "prompt_eval_count": 10,
+ "eval_count": 5,
+ "done": True,
+ },
+ )
)
else: # All OpenAI-compatible engines
prefix = _api_prefix(engine_key)
respx_mock.post(f"{host}{prefix}/chat/completions").mock(
- return_value=httpx.Response(200, json={
- "choices": [
- {"message": {"content": "Hello!"}, "finish_reason": "stop"},
- ],
- "usage": {
- "prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15,
+ return_value=httpx.Response(
+ 200,
+ json={
+ "choices": [
+ {"message": {"content": "Hello!"}, "finish_reason": "stop"},
+ ],
+ "usage": {
+ "prompt_tokens": 10,
+ "completion_tokens": 5,
+ "total_tokens": 15,
+ },
+ "model": model,
},
- "model": model,
- })
+ )
)
@@ -96,39 +107,59 @@ def _mock_tool_call(respx_mock, engine_key: str, host: str, model: str):
"""Set up mock for a tool-call response."""
if engine_key == "ollama":
respx_mock.post(f"{host}/api/chat").mock(
- return_value=httpx.Response(200, json={
- "message": {
- "content": "",
- "tool_calls": [{
- "function": {"name": "calculator", "arguments": '{"x":1}'},
- }],
+ return_value=httpx.Response(
+ 200,
+ json={
+ "message": {
+ "content": "",
+ "tool_calls": [
+ {
+ "function": {
+ "name": "calculator",
+ "arguments": '{"x":1}',
+ },
+ }
+ ],
+ },
+ "model": model,
+ "prompt_eval_count": 10,
+ "eval_count": 8,
+ "done": True,
},
- "model": model,
- "prompt_eval_count": 10,
- "eval_count": 8,
- "done": True,
- })
+ )
)
else: # All OpenAI-compatible engines
prefix = _api_prefix(engine_key)
respx_mock.post(f"{host}{prefix}/chat/completions").mock(
- return_value=httpx.Response(200, json={
- "choices": [{
- "message": {
- "content": "",
- "tool_calls": [{
- "id": "call_1",
- "type": "function",
- "function": {"name": "calculator", "arguments": '{"x":1}'},
- }],
+ return_value=httpx.Response(
+ 200,
+ json={
+ "choices": [
+ {
+ "message": {
+ "content": "",
+ "tool_calls": [
+ {
+ "id": "call_1",
+ "type": "function",
+ "function": {
+ "name": "calculator",
+ "arguments": '{"x":1}',
+ },
+ }
+ ],
+ },
+ "finish_reason": "tool_calls",
+ }
+ ],
+ "usage": {
+ "prompt_tokens": 10,
+ "completion_tokens": 8,
+ "total_tokens": 18,
},
- "finish_reason": "tool_calls",
- }],
- "usage": {
- "prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18,
+ "model": model,
},
- "model": model,
- })
+ )
)
@@ -176,9 +207,7 @@ class TestEngineScenarios:
engine = _create_engine(engine_key, host)
_mock_error(respx_mock, engine_key, host)
with pytest.raises(EngineConnectionError):
- engine.generate(
- [Message(role=Role.USER, content="Hi")], model="qwen3:8b"
- )
+ engine.generate([Message(role=Role.USER, content="Hi")], model="qwen3:8b")
# ---------------------------------------------------------------------------
@@ -190,7 +219,11 @@ class TestEngineScenarios:
@pytest.mark.parametrize("model_id", MODELS)
class TestEngineModelMatrix:
def test_generate_with_model(
- self, respx_mock, engine_key: str, host: str, model_id: str,
+ self,
+ respx_mock,
+ engine_key: str,
+ host: str,
+ model_id: str,
) -> None:
engine = _create_engine(engine_key, host)
_mock_simple_chat(respx_mock, engine_key, host, model_id)
diff --git a/tests/engine/test_llamacpp_models.py b/tests/engine/test_llamacpp_models.py
index 4f740c82..03aa9b79 100644
--- a/tests/engine/test_llamacpp_models.py
+++ b/tests/engine/test_llamacpp_models.py
@@ -85,7 +85,9 @@ class TestLlamaCppGenerate:
return_value=httpx.Response(
200,
json=_openai_response(
- content="", model=model_id, tool_calls=tool_calls,
+ content="",
+ model=model_id,
+ tool_calls=tool_calls,
),
)
)
@@ -143,6 +145,7 @@ class TestLlamaCppGenerate:
return tokens
import asyncio
+
tokens = asyncio.run(collect())
assert tokens == ["Hi", " there"]
diff --git a/tests/engine/test_mlx.py b/tests/engine/test_mlx.py
index 2bd0e891..fb3c976f 100644
--- a/tests/engine/test_mlx.py
+++ b/tests/engine/test_mlx.py
@@ -51,9 +51,7 @@ class TestMLXGenerate:
side_effect=httpx.ConnectError("refused")
)
with pytest.raises(EngineConnectionError):
- engine.generate(
- [Message(role=Role.USER, content="Hi")], model="m"
- )
+ engine.generate([Message(role=Role.USER, content="Hi")], model="m")
class TestMLXHealth:
diff --git a/tests/engine/test_ollama_models.py b/tests/engine/test_ollama_models.py
index 66a6214a..11cf54da 100644
--- a/tests/engine/test_ollama_models.py
+++ b/tests/engine/test_ollama_models.py
@@ -81,7 +81,9 @@ class TestOllamaGenerate:
return_value=httpx.Response(
200,
json=_ollama_response(
- content="", model=model_id, tool_calls=tool_calls,
+ content="",
+ model=model_id,
+ tool_calls=tool_calls,
),
)
)
@@ -104,7 +106,8 @@ class TestOllamaGenerate:
return_value=httpx.Response(
200,
json=_ollama_response(
- content="", model=model_id,
+ content="",
+ model=model_id,
tool_calls=tool_calls,
),
)
@@ -136,6 +139,7 @@ class TestOllamaGenerate:
return tokens
import asyncio
+
tokens = asyncio.run(collect())
assert "Hello" in tokens
@@ -222,9 +226,7 @@ class TestOllamaErrors:
def capture(request):
captured["body"] = json.loads(request.content)
- return httpx.Response(
- 200, json=_ollama_response(content="ok")
- )
+ return httpx.Response(200, json=_ollama_response(content="ok"))
respx_mock.post(f"{OLLAMA_HOST}/api/chat").mock(side_effect=capture)
engine.generate(
@@ -241,12 +243,8 @@ class TestOllamaErrors:
def capture(request):
captured["body"] = json.loads(request.content)
- return httpx.Response(
- 200, json=_ollama_response(content="ok")
- )
+ return httpx.Response(200, json=_ollama_response(content="ok"))
respx_mock.post(f"{OLLAMA_HOST}/api/chat").mock(side_effect=capture)
- engine.generate(
- [Message(role=Role.USER, content="Hello")], model="qwen3:8b"
- )
+ engine.generate([Message(role=Role.USER, content="Hello")], model="qwen3:8b")
assert "tools" not in captured["body"]
diff --git a/tests/engine/test_openai_compat.py b/tests/engine/test_openai_compat.py
index 148594fd..31cc2af8 100644
--- a/tests/engine/test_openai_compat.py
+++ b/tests/engine/test_openai_compat.py
@@ -47,12 +47,11 @@ class TestOpenAICompatGenerate:
assert result["usage"]["total_tokens"] == 9
def test_empty_choices_returns_graceful_fallback(
- self, engine: VLLMEngine,
+ self,
+ engine: VLLMEngine,
) -> None:
with respx.mock:
- respx.post(
- "http://testhost:8000/v1/chat/completions"
- ).mock(
+ respx.post("http://testhost:8000/v1/chat/completions").mock(
return_value=httpx.Response(
200,
json={
diff --git a/tests/engine/test_openai_compat_tools.py b/tests/engine/test_openai_compat_tools.py
index 18857748..3214d088 100644
--- a/tests/engine/test_openai_compat_tools.py
+++ b/tests/engine/test_openai_compat_tools.py
@@ -24,17 +24,20 @@ class TestOpenAICompatToolCalls:
def test_no_tool_calls(self, respx_mock):
"""When no tool_calls in response, result has no tool_calls key."""
respx_mock.post("http://localhost:9999/v1/chat/completions").mock(
- return_value=httpx.Response(200, json={
- "choices": [
- {"message": {"content": "Hi"}, "finish_reason": "stop"},
- ],
- "usage": {
- "prompt_tokens": 5,
- "completion_tokens": 2,
- "total_tokens": 7,
+ return_value=httpx.Response(
+ 200,
+ json={
+ "choices": [
+ {"message": {"content": "Hi"}, "finish_reason": "stop"},
+ ],
+ "usage": {
+ "prompt_tokens": 5,
+ "completion_tokens": 2,
+ "total_tokens": 7,
+ },
+ "model": "test",
},
- "model": "test",
- })
+ )
)
engine = _TestEngine()
result = engine.generate(
@@ -47,28 +50,35 @@ class TestOpenAICompatToolCalls:
def test_with_tool_calls(self, respx_mock):
"""Extract tool_calls from OpenAI-format response."""
respx_mock.post("http://localhost:9999/v1/chat/completions").mock(
- return_value=httpx.Response(200, json={
- "choices": [{
- "message": {
- "content": None,
- "tool_calls": [{
- "id": "call_abc",
- "type": "function",
- "function": {
- "name": "calculator",
- "arguments": '{"expression":"2+2"}',
+ return_value=httpx.Response(
+ 200,
+ json={
+ "choices": [
+ {
+ "message": {
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_abc",
+ "type": "function",
+ "function": {
+ "name": "calculator",
+ "arguments": '{"expression":"2+2"}',
+ },
+ }
+ ],
},
- }],
+ "finish_reason": "tool_calls",
+ }
+ ],
+ "usage": {
+ "prompt_tokens": 5,
+ "completion_tokens": 10,
+ "total_tokens": 15,
},
- "finish_reason": "tool_calls",
- }],
- "usage": {
- "prompt_tokens": 5,
- "completion_tokens": 10,
- "total_tokens": 15,
+ "model": "test",
},
- "model": "test",
- })
+ )
)
engine = _TestEngine()
result = engine.generate(
@@ -89,11 +99,16 @@ class TestOpenAICompatToolCalls:
def capture(request):
captured["body"] = json.loads(request.content)
- return httpx.Response(200, json={
- "choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}],
- "usage": {},
- "model": "test",
- })
+ return httpx.Response(
+ 200,
+ json={
+ "choices": [
+ {"message": {"content": "ok"}, "finish_reason": "stop"}
+ ],
+ "usage": {},
+ "model": "test",
+ },
+ )
respx_mock.post("http://localhost:9999/v1/chat/completions").mock(
side_effect=capture,
@@ -108,26 +123,33 @@ class TestOpenAICompatToolCalls:
def test_multiple_tool_calls(self, respx_mock):
respx_mock.post("http://localhost:9999/v1/chat/completions").mock(
- return_value=httpx.Response(200, json={
- "choices": [{
- "message": {
- "content": "",
- "tool_calls": [
- {
- "id": "c1", "type": "function",
- "function": {"name": "a", "arguments": "{}"},
+ return_value=httpx.Response(
+ 200,
+ json={
+ "choices": [
+ {
+ "message": {
+ "content": "",
+ "tool_calls": [
+ {
+ "id": "c1",
+ "type": "function",
+ "function": {"name": "a", "arguments": "{}"},
+ },
+ {
+ "id": "c2",
+ "type": "function",
+ "function": {"name": "b", "arguments": "{}"},
+ },
+ ],
},
- {
- "id": "c2", "type": "function",
- "function": {"name": "b", "arguments": "{}"},
- },
- ],
- },
- "finish_reason": "tool_calls",
- }],
- "usage": {},
- "model": "test",
- })
+ "finish_reason": "tool_calls",
+ }
+ ],
+ "usage": {},
+ "model": "test",
+ },
+ )
)
engine = _TestEngine()
result = engine.generate(
@@ -145,12 +167,15 @@ class TestOpenAICompatToolCalls:
class TestOllamaToolCalls:
def test_no_tool_calls(self, respx_mock):
respx_mock.post("http://localhost:11434/api/chat").mock(
- return_value=httpx.Response(200, json={
- "message": {"content": "Hi"},
- "model": "test",
- "prompt_eval_count": 5,
- "eval_count": 2,
- })
+ return_value=httpx.Response(
+ 200,
+ json={
+ "message": {"content": "Hi"},
+ "model": "test",
+ "prompt_eval_count": 5,
+ "eval_count": 2,
+ },
+ )
)
engine = OllamaEngine()
result = engine.generate(
@@ -161,20 +186,25 @@ class TestOllamaToolCalls:
def test_with_tool_calls(self, respx_mock):
respx_mock.post("http://localhost:11434/api/chat").mock(
- return_value=httpx.Response(200, json={
- "message": {
- "content": "",
- "tool_calls": [{
- "function": {
- "name": "calculator",
- "arguments": '{"expression":"3*3"}',
- },
- }],
+ return_value=httpx.Response(
+ 200,
+ json={
+ "message": {
+ "content": "",
+ "tool_calls": [
+ {
+ "function": {
+ "name": "calculator",
+ "arguments": '{"expression":"3*3"}',
+ },
+ }
+ ],
+ },
+ "model": "test",
+ "prompt_eval_count": 5,
+ "eval_count": 3,
},
- "model": "test",
- "prompt_eval_count": 5,
- "eval_count": 3,
- })
+ )
)
engine = OllamaEngine()
result = engine.generate(
@@ -189,10 +219,13 @@ class TestOllamaToolCalls:
def capture(request):
captured["body"] = json.loads(request.content)
- return httpx.Response(200, json={
- "message": {"content": "ok"},
- "model": "test",
- })
+ return httpx.Response(
+ 200,
+ json={
+ "message": {"content": "ok"},
+ "model": "test",
+ },
+ )
respx_mock.post("http://localhost:11434/api/chat").mock(side_effect=capture)
engine = OllamaEngine()
@@ -205,23 +238,26 @@ class TestOllamaToolCalls:
def test_dict_arguments_serialized_to_json(self, respx_mock):
"""Ollama returns arguments as dict — engine must serialize."""
- respx_mock.post(
- "http://localhost:11434/api/chat"
- ).mock(
- return_value=httpx.Response(200, json={
- "message": {
- "content": "",
- "tool_calls": [{
- "function": {
- "name": "calculator",
- "arguments": {"expression": "3*3"},
- },
- }],
+ respx_mock.post("http://localhost:11434/api/chat").mock(
+ return_value=httpx.Response(
+ 200,
+ json={
+ "message": {
+ "content": "",
+ "tool_calls": [
+ {
+ "function": {
+ "name": "calculator",
+ "arguments": {"expression": "3*3"},
+ },
+ }
+ ],
+ },
+ "model": "test",
+ "prompt_eval_count": 5,
+ "eval_count": 3,
},
- "model": "test",
- "prompt_eval_count": 5,
- "eval_count": 3,
- })
+ )
)
engine = OllamaEngine()
result = engine.generate(
@@ -240,10 +276,13 @@ class TestOllamaToolCalls:
def capture(request):
captured["body"] = json.loads(request.content)
- return httpx.Response(200, json={
- "message": {"content": "ok"},
- "model": "test",
- })
+ return httpx.Response(
+ 200,
+ json={
+ "message": {"content": "ok"},
+ "model": "test",
+ },
+ )
respx_mock.post("http://localhost:11434/api/chat").mock(side_effect=capture)
engine = OllamaEngine()
diff --git a/tests/engine/test_structured_output.py b/tests/engine/test_structured_output.py
index 1ae964da..4ba8cd0c 100644
--- a/tests/engine/test_structured_output.py
+++ b/tests/engine/test_structured_output.py
@@ -180,9 +180,7 @@ class TestAnthropicStructuredOutput:
json_tool = [t for t in call_kwargs["tools"] if t["name"] == "json_output"][0]
assert json_tool["input_schema"] == schema
- def test_appends_to_existing_tools(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
+ def test_appends_to_existing_tools(self, monkeypatch: pytest.MonkeyPatch) -> None:
engine, fake_client = self._make_engine(monkeypatch)
rf = ResponseFormat()
existing_tools = [
@@ -228,12 +226,8 @@ class TestGoogleStructuredOutput:
text='{"answer": 42}',
function_call=None,
)
- fake_candidate = SimpleNamespace(
- content=SimpleNamespace(parts=[fake_part])
- )
- fake_um = SimpleNamespace(
- prompt_token_count=10, candidates_token_count=5
- )
+ fake_candidate = SimpleNamespace(content=SimpleNamespace(parts=[fake_part]))
+ fake_um = SimpleNamespace(prompt_token_count=10, candidates_token_count=5)
fake_resp = SimpleNamespace(
candidates=[fake_candidate],
usage_metadata=fake_um,
@@ -242,9 +236,7 @@ class TestGoogleStructuredOutput:
fake_client.models.generate_content.return_value = fake_resp
return engine, fake_client
- def test_json_mode_sets_mime_type(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
+ def test_json_mode_sets_mime_type(self, monkeypatch: pytest.MonkeyPatch) -> None:
engine, fake_client = self._make_engine(monkeypatch)
rf = ResponseFormat()
@@ -361,9 +353,7 @@ class TestOllamaStructuredOutput:
sent_payload = json.loads(route.calls[0].request.content)
assert sent_payload["format"] == "json"
- def test_no_format_without_response_format(
- self, engine: OllamaEngine
- ) -> None:
+ def test_no_format_without_response_format(self, engine: OllamaEngine) -> None:
with respx.mock:
route = respx.post("http://testhost:11434/api/chat").mock(
return_value=httpx.Response(
diff --git a/tests/engine/test_vllm_models.py b/tests/engine/test_vllm_models.py
index fd11a7ec..3ac27b43 100644
--- a/tests/engine/test_vllm_models.py
+++ b/tests/engine/test_vllm_models.py
@@ -84,7 +84,9 @@ class TestVLLMGenerate:
return_value=httpx.Response(
200,
json=_openai_response(
- content="", model=model_id, tool_calls=tool_calls,
+ content="",
+ model=model_id,
+ tool_calls=tool_calls,
),
)
)
@@ -112,9 +114,7 @@ class TestVLLMGenerate:
200, json=_openai_response(content="Fallback reply", model=model_id)
)
- respx_mock.post(f"{VLLM_HOST}/v1/chat/completions").mock(
- side_effect=handler
- )
+ respx_mock.post(f"{VLLM_HOST}/v1/chat/completions").mock(side_effect=handler)
result = engine.generate(
[Message(role=Role.USER, content="Hello")],
model=model_id,
@@ -144,6 +144,7 @@ class TestVLLMGenerate:
return tokens
import asyncio
+
tokens = asyncio.run(collect())
assert tokens == ["Hello", " world"]
diff --git a/tests/evals/scorers/test_browser_assistant.py b/tests/evals/scorers/test_browser_assistant.py
index dfccb90a..49ecb11a 100644
--- a/tests/evals/scorers/test_browser_assistant.py
+++ b/tests/evals/scorers/test_browser_assistant.py
@@ -11,9 +11,7 @@ def _make_record(exact_facts=None, semantic_facts=None):
if exact_facts:
all_facts += [{"fact": f, "type": "exact"} for f in exact_facts]
if semantic_facts:
- all_facts += [
- {"fact": f, "type": "semantic"} for f in semantic_facts
- ]
+ all_facts += [{"fact": f, "type": "semantic"} for f in semantic_facts]
return EvalRecord(
record_id="test-ba-1",
problem="Research this topic.",
@@ -76,9 +74,7 @@ def test_sources_with_url():
record = _make_record(exact_facts=["5432"])
scorer = BrowserAssistantScorer()
- answer = (
- "Port 5432. See https://www.postgresql.org/docs/"
- )
+ answer = "Port 5432. See https://www.postgresql.org/docs/"
is_correct, meta = scorer.score(record, answer)
assert meta["sources_cited"] is True
diff --git a/tests/evals/scorers/test_checklist.py b/tests/evals/scorers/test_checklist.py
index 00fae49b..51870ee4 100644
--- a/tests/evals/scorers/test_checklist.py
+++ b/tests/evals/scorers/test_checklist.py
@@ -64,10 +64,7 @@ def test_checklist_scorer_all_pass():
def test_checklist_scorer_partial():
- response = (
- "1. yes — Redis is mentioned\n"
- "2. no — Port number not found\n"
- )
+ response = "1. yes — Redis is mentioned\n2. no — Port number not found\n"
backend = FakeJudgeBackend(response)
scorer = ChecklistScorer(backend, "test-model")
score, details = scorer.score_checklist(
diff --git a/tests/evals/scorers/test_coding_assistant.py b/tests/evals/scorers/test_coding_assistant.py
index 5e4d4465..cb2ba861 100644
--- a/tests/evals/scorers/test_coding_assistant.py
+++ b/tests/evals/scorers/test_coding_assistant.py
@@ -28,7 +28,8 @@ def test_all_tests_fixed():
"def test_neg(): assert add(-1, 1) == 0\n"
)
record = _make_record(
- buggy, tests,
+ buggy,
+ tests,
["test_basic", "test_neg"],
["test_zero"],
)
@@ -49,7 +50,8 @@ def test_partial_fix():
"def test_zero(): assert div(0, 1) == 0.0\n"
)
record = _make_record(
- buggy, tests,
+ buggy,
+ tests,
["test_zero_div"], # not actually testable here
["test_normal", "test_zero"],
)
diff --git a/tests/evals/scorers/test_doc_qa.py b/tests/evals/scorers/test_doc_qa.py
index 2f1c2b7d..aa290145 100644
--- a/tests/evals/scorers/test_doc_qa.py
+++ b/tests/evals/scorers/test_doc_qa.py
@@ -23,10 +23,8 @@ def _make_record(required_facts):
def test_all_facts_with_citations():
facts = [
- {"fact": "reclaims storage from dead tuples",
- "source_doc_index": 0},
- {"fact": "must run periodically",
- "source_doc_index": 0},
+ {"fact": "reclaims storage from dead tuples", "source_doc_index": 0},
+ {"fact": "must run periodically", "source_doc_index": 0},
]
record = _make_record(facts)
scorer = DocQAScorer()
@@ -45,8 +43,7 @@ def test_all_facts_with_citations():
def test_facts_without_citations():
facts = [
- {"fact": "reclaims storage from dead tuples",
- "source_doc_index": 0},
+ {"fact": "reclaims storage from dead tuples", "source_doc_index": 0},
]
record = _make_record(facts)
scorer = DocQAScorer()
@@ -70,10 +67,7 @@ def test_partial_facts():
record = _make_record(facts)
scorer = DocQAScorer()
- answer = (
- "VACUUM reclaims storage [Doc 1]. "
- "It also prevents table bloat [Doc 1]."
- )
+ answer = "VACUUM reclaims storage [Doc 1]. It also prevents table bloat [Doc 1]."
is_correct, meta = scorer.score(record, answer)
assert meta["facts_found"] == 2
assert 0.6 <= meta["fact_score"] <= 0.7
diff --git a/tests/evals/scorers/test_security_scanner.py b/tests/evals/scorers/test_security_scanner.py
index ee406aa3..e81ac39b 100644
--- a/tests/evals/scorers/test_security_scanner.py
+++ b/tests/evals/scorers/test_security_scanner.py
@@ -62,8 +62,7 @@ def test_partial_detection():
scorer = SecurityScannerScorer()
answer = (
- "Found SQL injection in app.py. "
- "Severity: critical. Use parameterized queries."
+ "Found SQL injection in app.py. Severity: critical. Use parameterized queries."
)
is_correct, meta = scorer.score(record, answer)
assert meta["vulns_found"] == 1
@@ -107,10 +106,7 @@ def test_vuln_type_aliases():
record = _make_record(vulns)
scorer = SecurityScannerScorer()
- answer = (
- "app.py has a cross-site scripting "
- "vulnerability. Severity: high."
- )
+ answer = "app.py has a cross-site scripting vulnerability. Severity: high."
is_correct, meta = scorer.score(record, answer)
assert meta["vulns_found"] == 1
assert meta["detection_rate"] == 1.0
@@ -120,7 +116,8 @@ def test_no_vulnerabilities():
record = _make_record([])
scorer = SecurityScannerScorer()
is_correct, meta = scorer.score(
- record, "Everything looks clean.",
+ record,
+ "Everything looks clean.",
)
assert is_correct is None
assert meta["reason"] == "no_vulnerabilities_in_manifest"
diff --git a/tests/evals/test_agentic_runner.py b/tests/evals/test_agentic_runner.py
index 188b2387..122d3fe8 100644
--- a/tests/evals/test_agentic_runner.py
+++ b/tests/evals/test_agentic_runner.py
@@ -121,9 +121,7 @@ class TestAgenticRunner:
def test_artifacts_saved(self, tmp_path):
records = [MockRecord(record_id="r1", problem="test")]
dataset = MockDataset(records)
- runner = AgenticRunner(
- agent=MockAgent(), dataset=dataset, run_dir=tmp_path
- )
+ runner = AgenticRunner(agent=MockAgent(), dataset=dataset, run_dir=tmp_path)
self._run_async(runner.run())
arts = tmp_path / "artifacts"
@@ -137,9 +135,7 @@ class TestAgenticRunner:
"""Verify timeout is stored and runner accepts the parameter."""
records = [MockRecord(record_id="r1", problem="test")]
dataset = MockDataset(records)
- runner = AgenticRunner(
- agent=MockAgent(), dataset=dataset, query_timeout=30.0
- )
+ runner = AgenticRunner(agent=MockAgent(), dataset=dataset, query_timeout=30.0)
assert runner._query_timeout == 30.0
diff --git a/tests/evals/test_ama_bench.py b/tests/evals/test_ama_bench.py
index 51b05282..8c9affbf 100644
--- a/tests/evals/test_ama_bench.py
+++ b/tests/evals/test_ama_bench.py
@@ -91,6 +91,7 @@ class TestAMABenchDataset:
def test_question_types_mapped(self) -> None:
from openjarvis.evals.datasets.ama_bench import _QUESTION_TYPE_TO_SUBJECT
+
assert _QUESTION_TYPE_TO_SUBJECT["A"] == "recall"
assert _QUESTION_TYPE_TO_SUBJECT["B"] == "causal_inference"
assert _QUESTION_TYPE_TO_SUBJECT["C"] == "state_updating"
@@ -111,8 +112,10 @@ class TestAMABenchScorer:
def test_empty_response(self) -> None:
s = AMABenchScorer(_mock_backend(), "test-model")
record = EvalRecord(
- record_id="test-1", problem="question",
- reference="answer", category="agentic",
+ record_id="test-1",
+ problem="question",
+ reference="answer",
+ category="agentic",
)
is_correct, meta = s.score(record, "")
assert is_correct is False
@@ -175,14 +178,10 @@ class TestJudgeParsing:
assert _parse_judge_label("yes.") == "yes"
def test_with_thinking_tags(self) -> None:
- assert _parse_judge_label(
- "<think>Let me check...</think>yes"
- ) == "yes"
+ assert _parse_judge_label("<think>Let me check...</think>yes") == "yes"
def test_with_thinking_tags_no(self) -> None:
- assert _parse_judge_label(
- "<think>The answer doesn't match</think>\nno"
- ) == "no"
+ assert _parse_judge_label("<think>The answer doesn't match</think>\nno") == "no"
def test_multiline(self) -> None:
assert _parse_judge_label(" \n yes\n") == "yes"
@@ -216,14 +215,17 @@ class TestTokenF1:
class TestAMABenchCLI:
def test_in_benchmarks_dict(self) -> None:
from openjarvis.evals.cli import BENCHMARKS
+
assert "ama-bench" in BENCHMARKS
def test_build_dataset(self) -> None:
from openjarvis.evals.cli import _build_dataset
+
ds = _build_dataset("ama-bench")
assert ds.dataset_id == "ama-bench"
def test_build_scorer(self) -> None:
from openjarvis.evals.cli import _build_scorer
+
s = _build_scorer("ama-bench", _mock_backend(), "test-model")
assert s.scorer_id == "ama-bench"
diff --git a/tests/evals/test_benchmark_datasets.py b/tests/evals/test_benchmark_datasets.py
index 62753e97..ee5e73a3 100644
--- a/tests/evals/test_benchmark_datasets.py
+++ b/tests/evals/test_benchmark_datasets.py
@@ -24,84 +24,98 @@ class TestDatasetInstantiation:
def test_supergpqa(self) -> None:
from openjarvis.evals.datasets.supergpqa import SuperGPQADataset
+
ds = SuperGPQADataset()
assert ds.dataset_id == "supergpqa"
assert ds.dataset_name == "SuperGPQA"
def test_gpqa(self) -> None:
from openjarvis.evals.datasets.gpqa import GPQADataset
+
ds = GPQADataset()
assert ds.dataset_id == "gpqa"
assert ds.dataset_name == "GPQA"
def test_mmlu_pro(self) -> None:
from openjarvis.evals.datasets.mmlu_pro import MMLUProDataset
+
ds = MMLUProDataset()
assert ds.dataset_id == "mmlu-pro"
assert ds.dataset_name == "MMLU-Pro"
def test_math500(self) -> None:
from openjarvis.evals.datasets.math500 import MATH500Dataset
+
ds = MATH500Dataset()
assert ds.dataset_id == "math500"
assert ds.dataset_name == "MATH-500"
def test_natural_reasoning(self) -> None:
from openjarvis.evals.datasets.natural_reasoning import NaturalReasoningDataset
+
ds = NaturalReasoningDataset()
assert ds.dataset_id == "natural-reasoning"
assert ds.dataset_name == "Natural Reasoning"
def test_hle(self) -> None:
from openjarvis.evals.datasets.hle import HLEDataset
+
ds = HLEDataset()
assert ds.dataset_id == "hle"
assert ds.dataset_name == "HLE"
def test_simpleqa(self) -> None:
from openjarvis.evals.datasets.simpleqa import SimpleQADataset
+
ds = SimpleQADataset()
assert ds.dataset_id == "simpleqa"
assert ds.dataset_name == "SimpleQA"
def test_wildchat(self) -> None:
from openjarvis.evals.datasets.wildchat import WildChatDataset
+
ds = WildChatDataset()
assert ds.dataset_id == "wildchat"
assert ds.dataset_name == "WildChat"
def test_ipw(self) -> None:
from openjarvis.evals.datasets.ipw_mixed import IPWDataset
+
ds = IPWDataset()
assert ds.dataset_id == "ipw"
assert ds.dataset_name == "IPW"
def test_gaia(self) -> None:
from openjarvis.evals.datasets.gaia import GAIADataset
+
ds = GAIADataset()
assert ds.dataset_id == "gaia"
assert ds.dataset_name == "GAIA"
def test_frames(self) -> None:
from openjarvis.evals.datasets.frames import FRAMESDataset
+
ds = FRAMESDataset()
assert ds.dataset_id == "frames"
assert ds.dataset_name == "FRAMES"
def test_swebench(self) -> None:
from openjarvis.evals.datasets.swebench import SWEBenchDataset
+
ds = SWEBenchDataset()
assert ds.dataset_id == "swebench"
assert ds.dataset_name == "SWE-bench"
def test_swefficiency(self) -> None:
from openjarvis.evals.datasets.swefficiency import SWEfficiencyDataset
+
ds = SWEfficiencyDataset()
assert ds.dataset_id == "swefficiency"
assert ds.dataset_name == "SWEfficiency"
def test_terminalbench(self) -> None:
from openjarvis.evals.datasets.terminalbench import TerminalBenchDataset
+
ds = TerminalBenchDataset()
assert ds.dataset_id == "terminalbench"
assert ds.dataset_name == "TerminalBench"
@@ -110,6 +124,7 @@ class TestDatasetInstantiation:
from openjarvis.evals.datasets.terminalbench_native import (
TerminalBenchNativeDataset,
)
+
ds = TerminalBenchNativeDataset()
assert ds.dataset_id == "terminalbench-native"
assert ds.dataset_name == "TerminalBench Native"
@@ -132,56 +147,67 @@ class TestScorerInstantiation:
def test_supergpqa_scorer(self) -> None:
from openjarvis.evals.scorers.supergpqa_mcq import SuperGPQAScorer
+
s = SuperGPQAScorer(_mock_backend(), "test-model")
assert s.scorer_id == "supergpqa"
def test_gpqa_scorer(self) -> None:
from openjarvis.evals.scorers.gpqa_mcq import GPQAScorer
+
s = GPQAScorer(_mock_backend(), "test-model")
assert s.scorer_id == "gpqa"
def test_mmlu_pro_scorer(self) -> None:
from openjarvis.evals.scorers.mmlu_pro_mcq import MMLUProScorer
+
s = MMLUProScorer(_mock_backend(), "test-model")
assert s.scorer_id == "mmlu-pro"
def test_reasoning_judge_scorer(self) -> None:
from openjarvis.evals.scorers.reasoning_judge import ReasoningJudgeScorer
+
s = ReasoningJudgeScorer(_mock_backend(), "test-model")
assert s.scorer_id == "reasoning_judge"
def test_hle_scorer(self) -> None:
from openjarvis.evals.scorers.hle_judge import HLEScorer
+
s = HLEScorer(_mock_backend(), "test-model")
assert s.scorer_id == "hle"
def test_simpleqa_scorer(self) -> None:
from openjarvis.evals.scorers.simpleqa_judge import SimpleQAScorer
+
s = SimpleQAScorer(_mock_backend(), "test-model")
assert s.scorer_id == "simpleqa"
def test_wildchat_scorer(self) -> None:
from openjarvis.evals.scorers.wildchat_judge import WildChatScorer
+
s = WildChatScorer(_mock_backend(), "test-model")
assert s.scorer_id == "wildchat"
def test_ipw_mixed_scorer(self) -> None:
from openjarvis.evals.scorers.ipw_mixed import IPWMixedScorer
+
s = IPWMixedScorer(_mock_backend(), "test-model")
assert s.scorer_id == "ipw"
def test_gaia_scorer(self) -> None:
from openjarvis.evals.scorers.gaia_exact import GAIAScorer
+
s = GAIAScorer(_mock_backend(), "test-model")
assert s.scorer_id == "gaia"
def test_frames_scorer(self) -> None:
from openjarvis.evals.scorers.frames_judge import FRAMESScorer
+
s = FRAMESScorer(_mock_backend(), "test-model")
assert s.scorer_id == "frames"
def test_swebench_scorer(self) -> None:
from openjarvis.evals.scorers.swebench_structural import SWEBenchScorer
+
s = SWEBenchScorer(_mock_backend(), "test-model")
assert s.scorer_id == "swebench"
@@ -189,11 +215,13 @@ class TestScorerInstantiation:
from openjarvis.evals.scorers.swefficiency_structural import (
SWEfficiencyScorer,
)
+
s = SWEfficiencyScorer(_mock_backend(), "test-model")
assert s.scorer_id == "swefficiency"
def test_terminalbench_scorer(self) -> None:
from openjarvis.evals.scorers.terminalbench_judge import TerminalBenchScorer
+
s = TerminalBenchScorer(_mock_backend(), "test-model")
assert s.scorer_id == "terminalbench"
@@ -201,6 +229,7 @@ class TestScorerInstantiation:
from openjarvis.evals.scorers.terminalbench_native_structural import (
TerminalBenchNativeScorer,
)
+
s = TerminalBenchNativeScorer(_mock_backend(), "test-model")
assert s.scorer_id == "terminalbench-native"
@@ -211,9 +240,21 @@ class TestScorerInstantiation:
ALL_BENCHMARKS = [
- "supergpqa", "gpqa", "mmlu-pro", "math500", "natural-reasoning",
- "hle", "simpleqa", "wildchat", "ipw", "gaia", "frames",
- "swebench", "swefficiency", "terminalbench", "terminalbench-native",
+ "supergpqa",
+ "gpqa",
+ "mmlu-pro",
+ "math500",
+ "natural-reasoning",
+ "hle",
+ "simpleqa",
+ "wildchat",
+ "ipw",
+ "gaia",
+ "frames",
+ "swebench",
+ "swefficiency",
+ "terminalbench",
+ "terminalbench-native",
]
@@ -223,6 +264,7 @@ class TestCLIFactories:
@pytest.mark.parametrize("benchmark", ALL_BENCHMARKS)
def test_build_dataset(self, benchmark: str) -> None:
from openjarvis.evals.cli import _build_dataset
+
ds = _build_dataset(benchmark)
assert ds is not None
assert hasattr(ds, "load")
@@ -232,6 +274,7 @@ class TestCLIFactories:
@pytest.mark.parametrize("benchmark", ALL_BENCHMARKS)
def test_build_scorer(self, benchmark: str) -> None:
from openjarvis.evals.cli import _build_scorer
+
scorer = _build_scorer(benchmark, _mock_backend(), "test-model")
assert scorer is not None
assert hasattr(scorer, "score")
@@ -240,6 +283,7 @@ class TestCLIFactories:
import click
from openjarvis.evals.cli import _build_dataset
+
with pytest.raises(click.ClickException, match="Unknown benchmark"):
_build_dataset("nonexistent")
@@ -247,6 +291,7 @@ class TestCLIFactories:
import click
from openjarvis.evals.cli import _build_scorer
+
with pytest.raises(click.ClickException, match="Unknown benchmark"):
_build_scorer("nonexistent", _mock_backend(), "test-model")
@@ -261,11 +306,13 @@ class TestConfigBenchmarks:
def test_all_benchmarks_known(self) -> None:
from openjarvis.evals.core.config import KNOWN_BENCHMARKS
+
for b in ALL_BENCHMARKS:
assert b in KNOWN_BENCHMARKS, f"{b} missing from KNOWN_BENCHMARKS"
def test_benchmarks_count(self) -> None:
from openjarvis.evals.core.config import KNOWN_BENCHMARKS
+
assert len(KNOWN_BENCHMARKS) == 25
@@ -280,9 +327,12 @@ class TestStructuralScorers:
def test_swebench_empty_response(self) -> None:
from openjarvis.evals.core.types import EvalRecord
from openjarvis.evals.scorers.swebench_structural import SWEBenchScorer
+
scorer = SWEBenchScorer(_mock_backend(), "test-model")
record = EvalRecord(
- record_id="swe-1", problem="Fix bug", reference="patch",
+ record_id="swe-1",
+ problem="Fix bug",
+ reference="patch",
category="agentic",
)
is_correct, meta = scorer.score(record, "")
@@ -292,9 +342,12 @@ class TestStructuralScorers:
def test_swebench_with_diff(self) -> None:
from openjarvis.evals.core.types import EvalRecord
from openjarvis.evals.scorers.swebench_structural import SWEBenchScorer
+
scorer = SWEBenchScorer(_mock_backend(), "test-model")
record = EvalRecord(
- record_id="swe-2", problem="Fix bug", reference="patch",
+ record_id="swe-2",
+ problem="Fix bug",
+ reference="patch",
category="agentic",
)
answer = "--- a/file.py\n+++ b/file.py\n@@ -1 +1 @@\n-old\n+new"
@@ -308,10 +361,13 @@ class TestStructuralScorers:
from openjarvis.evals.scorers.terminalbench_native_structural import (
TerminalBenchNativeScorer,
)
+
scorer = TerminalBenchNativeScorer(_mock_backend(), "test-model")
record = EvalRecord(
- record_id="tb-1", problem="Run command",
- reference="", category="agentic",
+ record_id="tb-1",
+ problem="Run command",
+ reference="",
+ category="agentic",
)
is_correct, meta = scorer.score(record, "some output")
assert is_correct is None
@@ -322,10 +378,13 @@ class TestStructuralScorers:
from openjarvis.evals.scorers.terminalbench_native_structural import (
TerminalBenchNativeScorer,
)
+
scorer = TerminalBenchNativeScorer(_mock_backend(), "test-model")
record = EvalRecord(
- record_id="tb-2", problem="Run command",
- reference="", category="agentic",
+ record_id="tb-2",
+ problem="Run command",
+ reference="",
+ category="agentic",
metadata={"is_resolved": True},
)
is_correct, meta = scorer.score(record, "output")
diff --git a/tests/evals/test_deepplanning.py b/tests/evals/test_deepplanning.py
index d4db7903..f0b9482c 100644
--- a/tests/evals/test_deepplanning.py
+++ b/tests/evals/test_deepplanning.py
@@ -63,8 +63,10 @@ class TestDeepPlanningScorer:
def test_empty_response(self) -> None:
s = DeepPlanningScorer(_mock_backend(), "test-model")
record = EvalRecord(
- record_id="dp-3", problem="task",
- reference="answer", category="agentic",
+ record_id="dp-3",
+ problem="task",
+ reference="answer",
+ category="agentic",
)
is_correct, meta = s.score(record, "")
assert is_correct is False
@@ -74,14 +76,17 @@ class TestDeepPlanningScorer:
class TestDeepPlanningCLI:
def test_in_benchmarks(self) -> None:
from openjarvis.evals.cli import BENCHMARKS
+
assert "deepplanning" in BENCHMARKS
def test_build_dataset(self) -> None:
from openjarvis.evals.cli import _build_dataset
+
ds = _build_dataset("deepplanning")
assert ds.dataset_id == "deepplanning"
def test_build_scorer(self) -> None:
from openjarvis.evals.cli import _build_scorer
+
s = _build_scorer("deepplanning", _mock_backend(), "test-model")
assert s.scorer_id == "deepplanning"
diff --git a/tests/evals/test_display.py b/tests/evals/test_display.py
index 320f9660..1d307582 100644
--- a/tests/evals/test_display.py
+++ b/tests/evals/test_display.py
@@ -45,8 +45,14 @@ def _make_summary(**overrides) -> RunSummary:
def _make_metric_stats(**kw) -> MetricStats:
defaults = dict(
- mean=1.0, median=0.9, min=0.1, max=2.5,
- std=0.3, p90=2.0, p95=2.2, p99=2.4,
+ mean=1.0,
+ median=0.9,
+ min=0.1,
+ max=2.5,
+ std=0.3,
+ p90=2.0,
+ p95=2.2,
+ p99=2.4,
)
defaults.update(kw)
return MetricStats(**defaults)
@@ -189,7 +195,8 @@ class TestPrintCompletion:
summary = _make_summary()
console, buf = _make_console()
print_completion(
- console, summary,
+ console,
+ summary,
output_path=Path("results/test.jsonl"),
traces_dir=Path("results/traces/supergpqa_qwen3-8b"),
)
diff --git a/tests/evals/test_episode_mode.py b/tests/evals/test_episode_mode.py
index 4a139267..c4d1214d 100644
--- a/tests/evals/test_episode_mode.py
+++ b/tests/evals/test_episode_mode.py
@@ -37,13 +37,17 @@ class TestDatasetProviderEpisodes:
class TestRunConfigEpisodeMode:
def test_episode_mode_field(self) -> None:
from openjarvis.evals.core.types import RunConfig
+
cfg = RunConfig(
- benchmark="test", backend="test", model="test",
+ benchmark="test",
+ backend="test",
+ model="test",
episode_mode=True,
)
assert cfg.episode_mode is True
def test_episode_mode_default_false(self) -> None:
from openjarvis.evals.core.types import RunConfig
+
cfg = RunConfig(benchmark="test", backend="test", model="test")
assert cfg.episode_mode is False
diff --git a/tests/evals/test_export.py b/tests/evals/test_export.py
index 40605236..81df1e8e 100644
--- a/tests/evals/test_export.py
+++ b/tests/evals/test_export.py
@@ -17,25 +17,27 @@ from openjarvis.evals.core.trace import QueryTrace, TurnTrace
def _make_traces(n=3):
traces = []
for i in range(n):
- traces.append(QueryTrace(
- query_id=f"q{i:04d}",
- workload_type="test",
- query_text=f"Question {i}",
- response_text=f"Answer {i}",
- turns=[
- TurnTrace(
- turn_index=0,
- input_tokens=100 + i * 10,
- output_tokens=50 + i * 5,
- wall_clock_s=1.0 + i * 0.5,
- gpu_energy_joules=5.0 + i,
- cost_usd=0.01,
- ),
- ],
- total_wall_clock_s=1.0 + i * 0.5,
- completed=True,
- is_resolved=i % 2 == 0,
- ))
+ traces.append(
+ QueryTrace(
+ query_id=f"q{i:04d}",
+ workload_type="test",
+ query_text=f"Question {i}",
+ response_text=f"Answer {i}",
+ turns=[
+ TurnTrace(
+ turn_index=0,
+ input_tokens=100 + i * 10,
+ output_tokens=50 + i * 5,
+ wall_clock_s=1.0 + i * 0.5,
+ gpu_energy_joules=5.0 + i,
+ cost_usd=0.01,
+ ),
+ ],
+ total_wall_clock_s=1.0 + i * 0.5,
+ completed=True,
+ is_resolved=i % 2 == 0,
+ )
+ )
return traces
@@ -89,11 +91,20 @@ class TestExportSummaryJson:
summary = json.loads(path.read_text())
stats = summary["statistics"]
expected_stat_keys = {
- "wall_clock_s", "gpu_energy_joules", "cpu_energy_joules",
- "gpu_power_watts", "cpu_power_watts",
- "input_tokens", "output_tokens", "total_tokens",
- "throughput_tokens_per_sec", "energy_per_token_joules",
- "cost_usd", "turns", "tool_calls", "mbu_avg_pct",
+ "wall_clock_s",
+ "gpu_energy_joules",
+ "cpu_energy_joules",
+ "gpu_power_watts",
+ "cpu_power_watts",
+ "input_tokens",
+ "output_tokens",
+ "total_tokens",
+ "throughput_tokens_per_sec",
+ "energy_per_token_joules",
+ "cost_usd",
+ "turns",
+ "tool_calls",
+ "mbu_avg_pct",
}
assert set(stats.keys()) == expected_stat_keys
@@ -152,8 +163,10 @@ class TestComputeEfficiency:
def test_no_scored_traces(self):
traces = [
QueryTrace(
- query_id="q0", workload_type="test",
- completed=True, is_resolved=None,
+ query_id="q0",
+ workload_type="test",
+ completed=True,
+ is_resolved=None,
),
]
result = _compute_efficiency(traces, 5.0, 1.0)
@@ -169,8 +182,10 @@ class TestComputeEfficiency:
def test_with_gpu_power(self):
traces = [
QueryTrace(
- query_id="q0", workload_type="test",
- completed=True, is_resolved=True,
+ query_id="q0",
+ workload_type="test",
+ completed=True,
+ is_resolved=True,
query_gpu_power_avg_watts=100.0,
),
]
@@ -351,35 +366,37 @@ class TestActionEnergyBreakdown:
def test_action_energy_summary_in_export(self, tmp_path):
traces = []
for i in range(2):
- traces.append(QueryTrace(
- query_id=f"q{i:04d}",
- workload_type="test",
- turns=[
- TurnTrace(
- turn_index=0,
- input_tokens=100,
- output_tokens=50,
- wall_clock_s=2.0,
- gpu_energy_joules=5.0,
- action_energy_breakdown=[
- {
- "action_type": "lm_inference",
- "duration_s": 1.5,
- "gpu_energy_joules": 4.0,
- "cpu_energy_joules": 0.3,
- },
- {
- "action_type": "tool_call:search",
- "duration_s": 0.5,
- "gpu_energy_joules": 1.0,
- "cpu_energy_joules": 0.1,
- },
- ],
- ),
- ],
- total_wall_clock_s=2.0,
- completed=True,
- ))
+ traces.append(
+ QueryTrace(
+ query_id=f"q{i:04d}",
+ workload_type="test",
+ turns=[
+ TurnTrace(
+ turn_index=0,
+ input_tokens=100,
+ output_tokens=50,
+ wall_clock_s=2.0,
+ gpu_energy_joules=5.0,
+ action_energy_breakdown=[
+ {
+ "action_type": "lm_inference",
+ "duration_s": 1.5,
+ "gpu_energy_joules": 4.0,
+ "cpu_energy_joules": 0.3,
+ },
+ {
+ "action_type": "tool_call:search",
+ "duration_s": 0.5,
+ "gpu_energy_joules": 1.0,
+ "cpu_energy_joules": 0.1,
+ },
+ ],
+ ),
+ ],
+ total_wall_clock_s=2.0,
+ completed=True,
+ )
+ )
path = tmp_path / "summary.json"
export_summary_json(traces, {}, path)
summary = json.loads(path.read_text())
diff --git a/tests/evals/test_lifelong_agent.py b/tests/evals/test_lifelong_agent.py
index c26dbc99..bf10f964 100644
--- a/tests/evals/test_lifelong_agent.py
+++ b/tests/evals/test_lifelong_agent.py
@@ -55,13 +55,18 @@ def _db_record(direct=None, md5=None, sql="SELECT * FROM users", skills=None):
answer_info = {"direct": direct, "md5": md5, "sql": sql}
answer_type = "md5" if md5 else "direct"
return EvalRecord(
- record_id="test-1", problem="task",
+ record_id="test-1",
+ problem="task",
reference=json.dumps(answer_info),
- category="agentic", subject=f"db_{answer_type}",
+ category="agentic",
+ subject=f"db_{answer_type}",
metadata={
- "answer_info": answer_info, "answer_type": answer_type,
- "skills": skills or [], "table_info": _TABLE_INFO,
- "table_name": "users", "subset": "db_bench",
+ "answer_info": answer_info,
+ "answer_type": answer_type,
+ "skills": skills or [],
+ "table_info": _TABLE_INFO,
+ "table_name": "users",
+ "subset": "db_bench",
"sample_index": 0,
},
)
@@ -69,9 +74,11 @@ def _db_record(direct=None, md5=None, sql="SELECT * FROM users", skills=None):
def _kg_record(answer_list=None, skills=None, action_list=None):
return EvalRecord(
- record_id="test-kg-1", problem="question",
+ record_id="test-kg-1",
+ problem="question",
reference=json.dumps(answer_list or []),
- category="agentic", subject="knowledge_graph",
+ category="agentic",
+ subject="knowledge_graph",
metadata={
"subset": "knowledge_graph",
"question": "What is the answer?",
@@ -86,9 +93,11 @@ def _kg_record(answer_list=None, skills=None, action_list=None):
def _os_record():
return EvalRecord(
- record_id="test-os-1", problem="task",
+ record_id="test-os-1",
+ problem="task",
reference="{}",
- category="agentic", subject="os_interaction",
+ category="agentic",
+ subject="os_interaction",
metadata={
"subset": "os_interaction",
"instruction": "Create a file",
@@ -113,6 +122,7 @@ def _os_record():
# DB building
# ---------------------------------------------------------------------------
+
class TestBuildDB:
def test_creates_table_with_rows(self) -> None:
conn = build_db(_TABLE_INFO)
@@ -174,6 +184,7 @@ class TestTableStateComparison:
# SQL extraction (original's Action: Operation format)
# ---------------------------------------------------------------------------
+
class TestExtractSQL:
def test_action_operation_format(self) -> None:
text = "Action: Operation\n```sql\nSELECT * FROM users;\n```"
@@ -211,6 +222,7 @@ class TestExtractSQL:
# Text answer parsing (DirectTypeAnswerValidator format)
# ---------------------------------------------------------------------------
+
class TestTextAnswerParsing:
def test_tuple_list(self) -> None:
text = "Final Answer: [(1, 'Alice', 95.5), (2, 'Bob', 87.0)]"
@@ -249,6 +261,7 @@ class TestTextAnswerParsing:
# DB scorer: direct (SELECT)
# ---------------------------------------------------------------------------
+
class TestScorerDBDirect:
def test_correct_sql(self) -> None:
s = LifelongAgentScorer()
@@ -265,7 +278,8 @@ class TestScorerDBDirect:
direct=[[1, "Alice", 95.5], [2, "Bob", 87.0], [3, "Carol", 92.3]],
)
ok, meta = s.score(
- r, "Action: Operation\n```sql\nSELECT * FROM users\n```",
+ r,
+ "Action: Operation\n```sql\nSELECT * FROM users\n```",
)
assert ok is True
@@ -296,7 +310,8 @@ class TestScorerDBDirect:
def test_no_sql_in_response(self) -> None:
ok, meta = LifelongAgentScorer().score(
- _db_record(direct=[[1]]), "I don't know",
+ _db_record(direct=[[1]]),
+ "I don't know",
)
assert ok is False
@@ -304,7 +319,8 @@ class TestScorerDBDirect:
s = LifelongAgentScorer()
r = _db_record(direct=[[1, "Alice", 95.5]])
ok, meta = s.score(
- r, "Action: Answer\nFinal Answer: [(1, 'Alice', 95.5)]",
+ r,
+ "Action: Answer\nFinal Answer: [(1, 'Alice', 95.5)]",
)
assert ok is True
assert meta["strategy"] == "text_answer_parsing"
@@ -322,6 +338,7 @@ class TestScorerDBDirect:
# DB scorer: md5 (INSERT/UPDATE/DELETE)
# ---------------------------------------------------------------------------
+
class TestScorerDBMD5:
def test_correct_insert(self) -> None:
s = LifelongAgentScorer()
@@ -363,6 +380,7 @@ class TestScorerDBMD5:
# KG scorer
# ---------------------------------------------------------------------------
+
class TestScorerKG:
def test_single_shot_unscorable(self) -> None:
"""KG tasks should be unscorable in single-shot mode."""
@@ -399,6 +417,7 @@ class TestScorerKG:
# OS scorer
# ---------------------------------------------------------------------------
+
class TestScorerOS:
def test_returns_scorable_status(self) -> None:
s = LifelongAgentScorer()
@@ -413,6 +432,7 @@ class TestScorerOS:
# KG answer extraction
# ---------------------------------------------------------------------------
+
class TestExtractKGAnswers:
def test_entity_id(self) -> None:
assert extract_kg_answers("Final Answer: m.02h8b9t") == ["m.02h8b9t"]
@@ -441,6 +461,7 @@ class TestExtractKGAnswers:
# Bash command extraction
# ---------------------------------------------------------------------------
+
class TestExtractBashCommands:
def test_act_format(self) -> None:
text = "Act: ```bash\nls -la /tmp\n```"
@@ -469,6 +490,7 @@ class TestExtractBashCommands:
# Value comparison
# ---------------------------------------------------------------------------
+
class TestValueComparison:
def test_int(self) -> None:
assert values_match(42, 42)
@@ -512,6 +534,7 @@ class TestTupleComparison:
# Episode grouping
# ---------------------------------------------------------------------------
+
class TestEpisodeGrouping:
def test_single_subset_episode(self) -> None:
ds = LifelongAgentDataset(subset="db_bench")
@@ -536,14 +559,18 @@ class TestEpisodeGrouping:
ds._records = [
EvalRecord(
record_id="lifelong-db-0",
- problem="task", reference="{}",
- category="agentic", subject="db_direct",
+ problem="task",
+ reference="{}",
+ category="agentic",
+ subject="db_direct",
metadata={"subset": "db_bench", "sample_index": 0},
),
EvalRecord(
record_id="lifelong-kg-0",
- problem="task", reference="{}",
- category="agentic", subject="knowledge_graph",
+ problem="task",
+ reference="{}",
+ category="agentic",
+ subject="knowledge_graph",
metadata={"subset": "knowledge_graph", "sample_index": 0},
),
]
@@ -555,8 +582,10 @@ class TestEpisodeGrouping:
ds._records = [
EvalRecord(
record_id=f"lifelong-db-{i}",
- problem="task", reference="{}",
- category="agentic", subject="db_direct",
+ problem="task",
+ reference="{}",
+ category="agentic",
+ subject="db_direct",
metadata={"subset": "db_bench", "sample_index": i},
)
for i in range(5)
@@ -571,6 +600,7 @@ class TestEpisodeGrouping:
# Dataset
# ---------------------------------------------------------------------------
+
class TestDataset:
def test_instantiation_default(self) -> None:
ds = LifelongAgentDataset()
@@ -603,6 +633,7 @@ class TestDataset:
# Multi-turn environments
# ---------------------------------------------------------------------------
+
class TestDBEnvironment:
def test_multi_turn_select(self) -> None:
"""DB environment should handle multi-turn SQL interaction."""
@@ -643,15 +674,15 @@ class TestDBEnvironment:
from openjarvis.evals.environments.lifelong_agent_env import DBEnvironment
record = _db_record(
- md5="x", sql="INSERT INTO users VALUES (4, 'Dave', 88.0)",
+ md5="x",
+ sql="INSERT INTO users VALUES (4, 'Dave', 88.0)",
)
env = DBEnvironment(use_mysql=False)
env.reset(record)
# Agent executes the correct INSERT
obs, done = env.step(
- "Action: Operation\n```sql\n"
- "INSERT INTO users VALUES (4, 'Dave', 88.0)\n```"
+ "Action: Operation\n```sql\nINSERT INTO users VALUES (4, 'Dave', 88.0)\n```"
)
assert not done
assert "successfully" in obs.lower() or "Result" in obs
@@ -814,20 +845,24 @@ class TestOSEnvironment:
# CLI wiring
# ---------------------------------------------------------------------------
+
class TestCLI:
def test_in_benchmarks(self) -> None:
from openjarvis.evals.cli import BENCHMARKS
+
assert "lifelong-agent" in BENCHMARKS
assert BENCHMARKS["lifelong-agent"]["category"] == "agentic"
def test_build_dataset(self) -> None:
from openjarvis.evals.cli import _build_dataset
+
ds = _build_dataset("lifelong-agent")
assert ds.dataset_id == "lifelong-agent"
assert hasattr(ds, "create_task_env")
def test_build_scorer(self) -> None:
from openjarvis.evals.cli import _build_scorer
+
s = _build_scorer("lifelong-agent", None, "test-model")
assert s.scorer_id == "lifelong-agent"
@@ -836,9 +871,11 @@ class TestCLI:
# Runner episode_mode integration
# ---------------------------------------------------------------------------
+
class TestRunnerEpisodeMode:
def test_episode_mode_field_exists(self) -> None:
from openjarvis.evals.core.types import RunConfig
+
config = RunConfig(
benchmark="lifelong-agent",
backend="jarvis-direct",
@@ -849,6 +886,7 @@ class TestRunnerEpisodeMode:
def test_runner_has_episode_mode_method(self) -> None:
from openjarvis.evals.core.runner import EvalRunner
+
assert hasattr(EvalRunner, "_run_episode_mode")
assert hasattr(EvalRunner, "_process_interactive")
assert hasattr(EvalRunner, "_inject_examples")
@@ -858,8 +896,11 @@ class TestRunnerEpisodeMode:
from openjarvis.evals.core.runner import EvalRunner
record = EvalRecord(
- record_id="test", problem="What is 2+2?",
- reference="4", category="reasoning", subject="math",
+ record_id="test",
+ problem="What is 2+2?",
+ reference="4",
+ category="reasoning",
+ subject="math",
metadata={},
)
# Call the static-ish method
@@ -872,8 +913,11 @@ class TestRunnerEpisodeMode:
from openjarvis.evals.core.runner import EvalRunner
record = EvalRecord(
- record_id="test", problem="What is 2+2?",
- reference="4", category="reasoning", subject="math",
+ record_id="test",
+ problem="What is 2+2?",
+ reference="4",
+ category="reasoning",
+ subject="math",
metadata={},
)
examples = [{"problem": "What is 1+1?", "answer": "2"}]
@@ -888,20 +932,25 @@ class TestRunnerEpisodeMode:
from openjarvis.evals.core.runner import EvalRunner
record = EvalRecord(
- record_id="test", problem="What is 3+3?",
- reference="6", category="reasoning", subject="math",
+ record_id="test",
+ problem="What is 3+3?",
+ reference="6",
+ category="reasoning",
+ subject="math",
metadata={},
)
- examples = [{
- "problem": "What is 1+1?",
- "answer": "2",
- "interaction_history": [
- {"role": "user", "content": "What is 1+1?"},
- {"role": "assistant", "content": "Action: compute(1+1)"},
- {"role": "user", "content": "Result: 2"},
- {"role": "assistant", "content": "Final Answer: 2"},
- ],
- }]
+ examples = [
+ {
+ "problem": "What is 1+1?",
+ "answer": "2",
+ "interaction_history": [
+ {"role": "user", "content": "What is 1+1?"},
+ {"role": "assistant", "content": "Action: compute(1+1)"},
+ {"role": "user", "content": "Result: 2"},
+ {"role": "assistant", "content": "Final Answer: 2"},
+ ],
+ }
+ ]
runner = EvalRunner.__new__(EvalRunner)
result = runner._inject_examples(record, examples)
assert "Previously Completed Tasks" in result.problem
@@ -912,6 +961,7 @@ class TestRunnerEpisodeMode:
def test_max_prior_examples_constant(self) -> None:
"""Runner should have a FIFO buffer size matching original default."""
from openjarvis.evals.core.runner import EvalRunner
+
assert EvalRunner._MAX_PRIOR_EXAMPLES == 3
@@ -919,14 +969,13 @@ class TestRunnerEpisodeMode:
# KG variable reference resolution
# ---------------------------------------------------------------------------
+
class TestKGVariableReference:
def test_variable_ref_in_scorer(self) -> None:
"""Scorer should handle Final Answer: #N format."""
# When a variable ref is given and entity IDs are elsewhere in text
result = extract_kg_answers(
- "I found the answer.\n"
- "The entity m.02h8b9t matches.\n"
- "Final Answer: #2"
+ "I found the answer.\nThe entity m.02h8b9t matches.\nFinal Answer: #2"
)
assert "m.02h8b9t" in result
@@ -962,8 +1011,7 @@ class TestKGVariableReference:
def test_variable_ref_with_var_keyword(self) -> None:
"""Should handle 'Final Answer: Variable #2' format."""
result = extract_kg_answers(
- "Based on my analysis, m.001 is the answer.\n"
- "Final Answer: Variable #2"
+ "Based on my analysis, m.001 is the answer.\nFinal Answer: Variable #2"
)
assert "m.001" in result
@@ -972,12 +1020,11 @@ class TestKGVariableReference:
# OS action format
# ---------------------------------------------------------------------------
+
class TestOSActionFormat:
def test_original_format_act_bash(self) -> None:
"""Should parse original format: Act: bash\\n```bash\\n...\\n```"""
- cmds = _extract_bash_commands(
- "Act: bash\n```bash\nls -la /tmp\n```"
- )
+ cmds = _extract_bash_commands("Act: bash\n```bash\nls -la /tmp\n```")
assert len(cmds) == 1
assert "ls -la" in cmds[0]
@@ -995,12 +1042,14 @@ class TestOSActionFormat:
# Per-subset max turns
# ---------------------------------------------------------------------------
+
class TestMaxTurns:
def test_db_max_turns(self) -> None:
from openjarvis.evals.environments.lifelong_agent_env import (
MAX_TURNS_DB,
DBEnvironment,
)
+
env = DBEnvironment(use_mysql=False)
assert env.max_turns == MAX_TURNS_DB
assert env.max_turns == 3
@@ -1010,6 +1059,7 @@ class TestMaxTurns:
MAX_TURNS_KG,
KGEnvironment,
)
+
env = KGEnvironment()
assert env.max_turns == MAX_TURNS_KG
assert env.max_turns == 15
@@ -1019,12 +1069,14 @@ class TestMaxTurns:
MAX_TURNS_OS,
OSEnvironment,
)
+
env = OSEnvironment()
assert env.max_turns == MAX_TURNS_OS
assert env.max_turns == 5
def test_base_default(self) -> None:
from openjarvis.evals.environments.base import TaskEnvironment
+
# Can't instantiate ABC, but verify the property exists
assert hasattr(TaskEnvironment, "max_turns")
@@ -1033,6 +1085,7 @@ class TestMaxTurns:
# Numeric tolerance
# ---------------------------------------------------------------------------
+
class TestNumericTolerance:
def test_abs_tol_near_zero(self) -> None:
"""abs_tol=1e-6 should make very small values match zero."""
diff --git a/tests/evals/test_loghub.py b/tests/evals/test_loghub.py
index 06812f30..4b18e891 100644
--- a/tests/evals/test_loghub.py
+++ b/tests/evals/test_loghub.py
@@ -101,8 +101,10 @@ class TestLogHubScorer:
def test_exact_match_anomaly(self) -> None:
s = LogHubScorer(_mock_backend(), "test-model")
record = EvalRecord(
- record_id="test-1", problem="analyze logs",
- reference="anomaly", category="agentic",
+ record_id="test-1",
+ problem="analyze logs",
+ reference="anomaly",
+ category="agentic",
)
is_correct, meta = s.score(record, "ANOMALY\nThe logs show errors.")
assert is_correct is True
@@ -111,8 +113,10 @@ class TestLogHubScorer:
def test_exact_match_normal(self) -> None:
s = LogHubScorer(_mock_backend(), "test-model")
record = EvalRecord(
- record_id="test-2", problem="analyze logs",
- reference="normal", category="agentic",
+ record_id="test-2",
+ problem="analyze logs",
+ reference="normal",
+ category="agentic",
)
is_correct, meta = s.score(record, "NORMAL - no issues detected")
assert is_correct is True
@@ -120,8 +124,10 @@ class TestLogHubScorer:
def test_empty_response(self) -> None:
s = LogHubScorer(_mock_backend(), "test-model")
record = EvalRecord(
- record_id="test-3", problem="analyze logs",
- reference="anomaly", category="agentic",
+ record_id="test-3",
+ problem="analyze logs",
+ reference="anomaly",
+ category="agentic",
)
is_correct, meta = s.score(record, "")
assert is_correct is False
@@ -130,8 +136,10 @@ class TestLogHubScorer:
def test_wrong_classification(self) -> None:
s = LogHubScorer(_mock_backend(), "test-model")
record = EvalRecord(
- record_id="test-4", problem="analyze logs",
- reference="anomaly", category="agentic",
+ record_id="test-4",
+ problem="analyze logs",
+ reference="anomaly",
+ category="agentic",
)
is_correct, meta = s.score(record, "NORMAL - everything looks fine")
assert is_correct is False
@@ -140,17 +148,20 @@ class TestLogHubScorer:
class TestLogHubCLI:
def test_in_benchmarks_dict(self) -> None:
from openjarvis.evals.cli import BENCHMARKS
+
assert "loghub" in BENCHMARKS
assert BENCHMARKS["loghub"]["category"] == "agentic"
def test_build_dataset(self) -> None:
from openjarvis.evals.cli import _build_dataset
+
ds = _build_dataset("loghub")
assert ds is not None
assert ds.dataset_id == "loghub"
def test_build_scorer(self) -> None:
from openjarvis.evals.cli import _build_scorer
+
s = _build_scorer("loghub", _mock_backend(), "test-model")
assert s is not None
assert s.scorer_id == "loghub"
diff --git a/tests/evals/test_paperarena.py b/tests/evals/test_paperarena.py
index 022aeb9c..2339c41e 100644
--- a/tests/evals/test_paperarena.py
+++ b/tests/evals/test_paperarena.py
@@ -91,8 +91,10 @@ class TestPaperArenaScorer:
def test_empty_response(self) -> None:
s = PaperArenaScorer(_mock_backend(), "test-model")
record = EvalRecord(
- record_id="pa-5", problem="q",
- reference="a", category="agentic",
+ record_id="pa-5",
+ problem="q",
+ reference="a",
+ category="agentic",
)
is_correct, meta = s.score(record, "")
assert is_correct is False
@@ -102,14 +104,17 @@ class TestPaperArenaScorer:
class TestPaperArenaCLI:
def test_in_benchmarks(self) -> None:
from openjarvis.evals.cli import BENCHMARKS
+
assert "paperarena" in BENCHMARKS
def test_build_dataset(self) -> None:
from openjarvis.evals.cli import _build_dataset
+
ds = _build_dataset("paperarena")
assert ds.dataset_id == "paperarena"
def test_build_scorer(self) -> None:
from openjarvis.evals.cli import _build_scorer
+
s = _build_scorer("paperarena", _mock_backend(), "test-model")
assert s.scorer_id == "paperarena"
diff --git a/tests/evals/test_trackers.py b/tests/evals/test_trackers.py
index 1654efa2..e8645873 100644
--- a/tests/evals/test_trackers.py
+++ b/tests/evals/test_trackers.py
@@ -15,6 +15,7 @@ from openjarvis.evals.core.types import EvalResult, RunConfig, RunSummary
# Test double
# ---------------------------------------------------------------------------
+
class RecordingTracker(ResultTracker):
"""Records all lifecycle calls for testing."""
@@ -58,6 +59,7 @@ class CrashingTracker(ResultTracker):
# Helpers
# ---------------------------------------------------------------------------
+
def _make_config(**overrides) -> RunConfig:
defaults = dict(benchmark="test", backend="jarvis-direct", model="test-model")
defaults.update(overrides)
@@ -92,6 +94,7 @@ def _make_result(**overrides) -> EvalResult:
# RecordingTracker through EvalRunner lifecycle
# ---------------------------------------------------------------------------
+
class TestRecordingTrackerIntegration:
"""Test that trackers receive all lifecycle calls through EvalRunner."""
@@ -111,11 +114,13 @@ class TestRecordingTrackerIntegration:
dataset.iter_records = MagicMock(return_value=[record])
backend = MagicMock()
- backend.generate_full = MagicMock(return_value={
- "content": "2",
- "usage": {"prompt_tokens": 10, "completion_tokens": 5},
- "latency_seconds": 0.5,
- })
+ backend.generate_full = MagicMock(
+ return_value={
+ "content": "2",
+ "usage": {"prompt_tokens": 10, "completion_tokens": 5},
+ "latency_seconds": 0.5,
+ }
+ )
scorer = MagicMock()
scorer.score = MagicMock(return_value=(True, {}))
@@ -152,11 +157,13 @@ class TestRecordingTrackerIntegration:
dataset.iter_records = MagicMock(return_value=[record])
backend = MagicMock()
- backend.generate_full = MagicMock(return_value={
- "content": "yes",
- "usage": {},
- "latency_seconds": 0.1,
- })
+ backend.generate_full = MagicMock(
+ return_value={
+ "content": "yes",
+ "usage": {},
+ "latency_seconds": 0.1,
+ }
+ )
scorer = MagicMock()
scorer.score = MagicMock(return_value=(True, {}))
@@ -178,6 +185,7 @@ class TestRecordingTrackerIntegration:
# WandbTracker unit tests
# ---------------------------------------------------------------------------
+
class TestWandbTracker:
"""Unit tests for WandbTracker (mocked wandb module)."""
@@ -185,6 +193,7 @@ class TestWandbTracker:
"""WandbTracker raises ImportError when wandb is not installed."""
with patch.dict(sys.modules, {"wandb": None}):
import openjarvis.evals.trackers.wandb_tracker as wt_mod
+
original = wt_mod.wandb
wt_mod.wandb = None
try:
@@ -277,12 +286,14 @@ class TestWandbTracker:
# SheetsTracker unit tests
# ---------------------------------------------------------------------------
+
class TestSheetsTracker:
"""Unit tests for SheetsTracker."""
def test_import_error_when_gspread_missing(self):
"""SheetsTracker raises ImportError when gspread not installed."""
import openjarvis.evals.trackers.sheets_tracker as st_mod
+
original = st_mod.gspread
st_mod.gspread = None
try:
diff --git a/tests/evals/test_use_case_benchmarks.py b/tests/evals/test_use_case_benchmarks.py
index f50ed24f..ec3ebbe3 100644
--- a/tests/evals/test_use_case_benchmarks.py
+++ b/tests/evals/test_use_case_benchmarks.py
@@ -25,12 +25,14 @@ import pytest
class TestEmailTriageDataset:
def test_instantiation(self) -> None:
from openjarvis.evals.datasets.email_triage import EmailTriageDataset
+
ds = EmailTriageDataset()
assert ds.dataset_id == "email_triage"
assert ds.dataset_name == "Email Triage"
def test_load(self) -> None:
from openjarvis.evals.datasets.email_triage import EmailTriageDataset
+
ds = EmailTriageDataset()
ds.load(max_samples=5, seed=42)
assert ds.size() == 5
@@ -44,6 +46,7 @@ class TestEmailTriageDataset:
def test_load_all(self) -> None:
from openjarvis.evals.datasets.email_triage import EmailTriageDataset
+
ds = EmailTriageDataset()
ds.load()
assert ds.size() == 30
@@ -52,12 +55,14 @@ class TestEmailTriageDataset:
class TestMorningBriefDataset:
def test_instantiation(self) -> None:
from openjarvis.evals.datasets.morning_brief import MorningBriefDataset
+
ds = MorningBriefDataset()
assert ds.dataset_id == "morning_brief"
assert ds.dataset_name == "Morning Brief"
def test_load(self) -> None:
from openjarvis.evals.datasets.morning_brief import MorningBriefDataset
+
ds = MorningBriefDataset()
ds.load(max_samples=5, seed=42)
assert ds.size() == 5
@@ -68,6 +73,7 @@ class TestMorningBriefDataset:
def test_load_all(self) -> None:
from openjarvis.evals.datasets.morning_brief import MorningBriefDataset
+
ds = MorningBriefDataset()
ds.load()
assert ds.size() == 15
@@ -76,12 +82,14 @@ class TestMorningBriefDataset:
class TestResearchMiningDataset:
def test_instantiation(self) -> None:
from openjarvis.evals.datasets.research_mining import ResearchMiningDataset
+
ds = ResearchMiningDataset()
assert ds.dataset_id == "research_mining"
assert ds.dataset_name == "Research Mining"
def test_load(self) -> None:
from openjarvis.evals.datasets.research_mining import ResearchMiningDataset
+
ds = ResearchMiningDataset()
ds.load(max_samples=5, seed=42)
assert ds.size() == 5
@@ -92,6 +100,7 @@ class TestResearchMiningDataset:
def test_load_all(self) -> None:
from openjarvis.evals.datasets.research_mining import ResearchMiningDataset
+
ds = ResearchMiningDataset()
ds.load()
assert ds.size() == 31
@@ -100,12 +109,14 @@ class TestResearchMiningDataset:
class TestKnowledgeBaseDataset:
def test_instantiation(self) -> None:
from openjarvis.evals.datasets.knowledge_base import KnowledgeBaseDataset
+
ds = KnowledgeBaseDataset()
assert ds.dataset_id == "knowledge_base"
assert ds.dataset_name == "Knowledge Base"
def test_load(self) -> None:
from openjarvis.evals.datasets.knowledge_base import KnowledgeBaseDataset
+
ds = KnowledgeBaseDataset()
ds.load(max_samples=5, seed=42)
assert ds.size() == 5
@@ -116,6 +127,7 @@ class TestKnowledgeBaseDataset:
def test_load_all(self) -> None:
from openjarvis.evals.datasets.knowledge_base import KnowledgeBaseDataset
+
ds = KnowledgeBaseDataset()
ds.load()
assert ds.size() == 30
@@ -124,12 +136,14 @@ class TestKnowledgeBaseDataset:
class TestCodingTaskDataset:
def test_instantiation(self) -> None:
from openjarvis.evals.datasets.coding_task import CodingTaskDataset
+
ds = CodingTaskDataset()
assert ds.dataset_id == "coding_task"
assert ds.dataset_name == "Coding Task"
def test_load(self) -> None:
from openjarvis.evals.datasets.coding_task import CodingTaskDataset
+
ds = CodingTaskDataset()
ds.load(max_samples=5, seed=42)
assert ds.size() == 5
@@ -141,6 +155,7 @@ class TestCodingTaskDataset:
def test_load_all(self) -> None:
from openjarvis.evals.datasets.coding_task import CodingTaskDataset
+
ds = CodingTaskDataset()
ds.load()
assert ds.size() == 29
@@ -157,26 +172,31 @@ class TestScorerInstantiation:
def test_email_triage_scorer(self) -> None:
from openjarvis.evals.scorers.email_triage import EmailTriageScorer
+
scorer = EmailTriageScorer(self._mock_backend(), "gpt-5-mini")
assert scorer.scorer_id == "email_triage"
def test_morning_brief_scorer(self) -> None:
from openjarvis.evals.scorers.morning_brief import MorningBriefScorer
+
scorer = MorningBriefScorer(self._mock_backend(), "gpt-5-mini")
assert scorer.scorer_id == "morning_brief"
def test_research_mining_scorer(self) -> None:
from openjarvis.evals.scorers.research_mining import ResearchMiningScorer
+
scorer = ResearchMiningScorer(self._mock_backend(), "gpt-5-mini")
assert scorer.scorer_id == "research_mining"
def test_knowledge_base_scorer(self) -> None:
from openjarvis.evals.scorers.knowledge_base import KnowledgeBaseScorer
+
scorer = KnowledgeBaseScorer(self._mock_backend(), "gpt-5-mini")
assert scorer.scorer_id == "knowledge_base"
def test_coding_task_scorer(self) -> None:
from openjarvis.evals.scorers.coding_task import CodingTaskScorer
+
scorer = CodingTaskScorer(self._mock_backend(), "gpt-5-mini")
assert scorer.scorer_id == "coding_task"
@@ -208,10 +228,7 @@ class TestCodingTaskScoring:
),
},
)
- answer = (
- "def is_palindrome(s):\n"
- " return s == s[::-1]"
- )
+ answer = "def is_palindrome(s):\n return s == s[::-1]"
is_correct, meta = scorer.score(record, answer)
assert is_correct is True
assert meta["tests_passed"] == 3
@@ -228,10 +245,7 @@ class TestCodingTaskScoring:
reference="",
category="use-case",
metadata={
- "test_cases": (
- "assert add(1, 2) == 3\n"
- "assert add(0, 0) == 0"
- ),
+ "test_cases": ("assert add(1, 2) == 3\nassert add(0, 0) == 0"),
},
)
# Wrong implementation
@@ -285,27 +299,35 @@ class TestEmailTriageScoring:
class TestCLIFactories:
"""Test that _build_dataset and _build_scorer work for new benchmarks."""
- @pytest.mark.parametrize("benchmark", [
- "email_triage",
- "morning_brief",
- "research_mining",
- "knowledge_base",
- "coding_task",
- ])
+ @pytest.mark.parametrize(
+ "benchmark",
+ [
+ "email_triage",
+ "morning_brief",
+ "research_mining",
+ "knowledge_base",
+ "coding_task",
+ ],
+ )
def test_build_dataset(self, benchmark: str) -> None:
from openjarvis.evals.cli import _build_dataset
+
ds = _build_dataset(benchmark)
assert ds.dataset_id == benchmark
- @pytest.mark.parametrize("benchmark", [
- "email_triage",
- "morning_brief",
- "research_mining",
- "knowledge_base",
- "coding_task",
- ])
+ @pytest.mark.parametrize(
+ "benchmark",
+ [
+ "email_triage",
+ "morning_brief",
+ "research_mining",
+ "knowledge_base",
+ "coding_task",
+ ],
+ )
def test_build_scorer(self, benchmark: str) -> None:
from openjarvis.evals.cli import _build_scorer
+
scorer = _build_scorer(benchmark, MagicMock(), "gpt-5-mini")
assert scorer.scorer_id == benchmark
@@ -320,6 +342,7 @@ class TestCostCalculator:
def test_estimate_monthly_cost(self) -> None:
from openjarvis.server.cost_calculator import estimate_monthly_cost
+
est = estimate_monthly_cost(
calls_per_month=1000,
avg_input_tokens=500,
@@ -332,6 +355,7 @@ class TestCostCalculator:
def test_estimate_scenario(self) -> None:
from openjarvis.server.cost_calculator import estimate_scenario
+
estimates = estimate_scenario("daily_briefing")
assert len(estimates) == 3 # 3 cloud providers
for est in estimates:
@@ -339,16 +363,19 @@ class TestCostCalculator:
def test_estimate_all_scenarios(self) -> None:
from openjarvis.server.cost_calculator import estimate_all_scenarios
+
all_est = estimate_all_scenarios()
assert len(all_est) == 5 # 5 scenarios
def test_unknown_provider(self) -> None:
from openjarvis.server.cost_calculator import estimate_monthly_cost
+
with pytest.raises(ValueError, match="Unknown provider"):
estimate_monthly_cost(100, 100, 100, "nonexistent")
def test_unknown_scenario(self) -> None:
from openjarvis.server.cost_calculator import estimate_scenario
+
with pytest.raises(ValueError, match="Unknown scenario"):
estimate_scenario("nonexistent")
@@ -363,6 +390,7 @@ class TestSavings:
def test_compute_savings_basic(self) -> None:
from openjarvis.server.savings import compute_savings
+
summary = compute_savings(1000, 500, total_calls=10)
assert summary.total_calls == 10
assert summary.total_tokens == 1500
@@ -375,15 +403,20 @@ class TestSavings:
import time
from openjarvis.server.savings import compute_savings
+
start = time.time() - 3600 # 1 hour ago
summary = compute_savings(
- 100000, 50000, total_calls=100, session_start=start,
+ 100000,
+ 50000,
+ total_calls=100,
+ session_start=start,
)
assert summary.session_duration_hours > 0
assert summary.monthly_projection # not empty
def test_savings_to_dict(self) -> None:
from openjarvis.server.savings import compute_savings, savings_to_dict
+
summary = compute_savings(1000, 500, total_calls=5)
d = savings_to_dict(summary)
assert isinstance(d, dict)
diff --git a/tests/hardware/test_amd.py b/tests/hardware/test_amd.py
index 87cfad9a..3eecdae8 100644
--- a/tests/hardware/test_amd.py
+++ b/tests/hardware/test_amd.py
@@ -28,9 +28,9 @@ class TestAMDDetection:
@patch(
"openjarvis.core.config._run_cmd",
side_effect=[
- "AMD Instinct MI300X", # --showproductname
+ "AMD Instinct MI300X", # --showproductname
"GPU[0] : vram Total Memory (B): 206158430208", # --showmeminfo vram
- "GPU[0] : Some info", # --showallinfo
+ "GPU[0] : Some info", # --showallinfo
],
)
def test_rocm_smi_parsing(self, mock_run, mock_which):
@@ -48,8 +48,8 @@ class TestAMDDetection:
"openjarvis.core.config._run_cmd",
side_effect=[
"AMD Instinct MI250X\nAMD Instinct MI250X", # --showproductname
- "", # --showmeminfo vram (empty)
- "", # --showallinfo (empty)
+ "", # --showmeminfo vram (empty)
+ "", # --showallinfo (empty)
],
)
def test_amd_gpu_model(self, mock_run, mock_which):
@@ -140,8 +140,10 @@ class TestAMDEngineRecommendation:
cpu_count=96,
ram_gb=768.0,
gpu=GpuInfo(
- vendor="amd", name="AMD Instinct MI300X",
- vram_gb=192.0, count=1,
+ vendor="amd",
+ name="AMD Instinct MI300X",
+ vram_gb=192.0,
+ count=1,
),
)
assert recommend_engine(hw) == "vllm"
@@ -163,8 +165,10 @@ class TestAMDEngineRecommendation:
cpu_count=128,
ram_gb=1024.0,
gpu=GpuInfo(
- vendor="amd", name="AMD Instinct MI300X",
- vram_gb=192.0, count=4,
+ vendor="amd",
+ name="AMD Instinct MI300X",
+ vram_gb=192.0,
+ count=4,
),
)
assert recommend_engine(hw) == "vllm"
diff --git a/tests/hardware/test_apple.py b/tests/hardware/test_apple.py
index f662ea00..e289d338 100644
--- a/tests/hardware/test_apple.py
+++ b/tests/hardware/test_apple.py
@@ -87,11 +87,7 @@ class TestAppleDetection:
@patch("openjarvis.core.config.platform.system", return_value="Darwin")
@patch(
"openjarvis.core.config._run_cmd",
- return_value=(
- "Graphics/Displays:\n"
- " Apple Silicon\n"
- " Type: GPU\n"
- ),
+ return_value=("Graphics/Displays:\n Apple Silicon\n Type: GPU\n"),
)
def test_apple_no_chipset_line_falls_back(self, mock_run, mock_system):
"""When no 'Chipset Model' line exists, falls back to 'Apple Silicon'."""
diff --git a/tests/hardware/test_hardware_profiles.py b/tests/hardware/test_hardware_profiles.py
index 8556e98a..d1686931 100644
--- a/tests/hardware/test_hardware_profiles.py
+++ b/tests/hardware/test_hardware_profiles.py
@@ -38,9 +38,9 @@ class TestDetectHardware:
@patch(
"openjarvis.core.config._run_cmd",
side_effect=[
- "AMD Instinct MI300X", # --showproductname
+ "AMD Instinct MI300X", # --showproductname
"GPU[0] : vram Total Memory (B): 206158430208", # --showmeminfo vram
- "GPU[0] : Some info", # --showallinfo
+ "GPU[0] : Some info", # --showallinfo
],
)
def test_detect_amd_gpu(self, mock_run, mock_which):
diff --git a/tests/hardware/test_nvidia.py b/tests/hardware/test_nvidia.py
index c8c94adb..d0a68e44 100644
--- a/tests/hardware/test_nvidia.py
+++ b/tests/hardware/test_nvidia.py
@@ -104,7 +104,8 @@ class TestNVIDIAEngineRecommendation:
gpu=GpuInfo(
vendor="nvidia",
name="NVIDIA A100-SXM4-80GB",
- vram_gb=80.0, count=1,
+ vram_gb=80.0,
+ count=1,
),
)
assert recommend_engine(hw) == "vllm"
@@ -118,7 +119,8 @@ class TestNVIDIAEngineRecommendation:
gpu=GpuInfo(
vendor="nvidia",
name="NVIDIA H100 80GB HBM3",
- vram_gb=80.0, count=1,
+ vram_gb=80.0,
+ count=1,
),
)
assert recommend_engine(hw) == "vllm"
@@ -132,7 +134,8 @@ class TestNVIDIAEngineRecommendation:
gpu=GpuInfo(
vendor="nvidia",
name="NVIDIA Tesla V100-SXM2-32GB",
- vram_gb=32.0, count=1,
+ vram_gb=32.0,
+ count=1,
),
)
assert recommend_engine(hw) == "ollama"
@@ -146,7 +149,8 @@ class TestNVIDIAEngineRecommendation:
gpu=GpuInfo(
vendor="nvidia",
name="NVIDIA GeForce RTX 4090",
- vram_gb=24.0, count=1,
+ vram_gb=24.0,
+ count=1,
),
)
assert recommend_engine(hw) == "ollama"
diff --git a/tests/integration/test_agent_manager_e2e.py b/tests/integration/test_agent_manager_e2e.py
index b098c51f..c9b5d3a4 100644
--- a/tests/integration/test_agent_manager_e2e.py
+++ b/tests/integration/test_agent_manager_e2e.py
@@ -25,9 +25,7 @@ class TestResearchMonitorE2E:
templates = manager.list_templates()
assert any(t["id"] == "research_monitor" for t in templates)
- agent = manager.create_from_template(
- "research_monitor", "My Researcher"
- )
+ agent = manager.create_from_template("research_monitor", "My Researcher")
assert agent["name"] == "My Researcher"
assert agent["agent_type"] == "monitor_operative"
assert agent["status"] == "idle"
@@ -64,7 +62,9 @@ class TestResearchMonitorE2E:
# Complete task
manager.update_task(
- t1["id"], status="completed", findings=["Paper A", "Paper B"],
+ t1["id"],
+ status="completed",
+ findings=["Paper A", "Paper B"],
)
task = manager._get_task(t1["id"])
assert task["status"] == "completed"
diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py
index ced76ecc..babbbb11 100644
--- a/tests/integration/test_integration.py
+++ b/tests/integration/test_integration.py
@@ -138,7 +138,8 @@ class TestOrchestratorWithCalculator:
bus = EventBus(record_history=True)
agent_cls = AgentRegistry.get("orchestrator")
agent = agent_cls(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[CalculatorTool()],
bus=bus,
)
@@ -250,8 +251,7 @@ class TestTelemetryThroughAgent:
agent.run("Hello")
telem_events = [
- e for e in bus.history
- if e.event_type == EventType.TELEMETRY_RECORD
+ e for e in bus.history if e.event_type == EventType.TELEMETRY_RECORD
]
assert len(telem_events) == 1
rec = telem_events[0].data["record"]
@@ -286,14 +286,8 @@ class TestToolExecutorIntegration:
assert think_result.content == "Step 1: solve"
# Verify events
- starts = [
- e for e in bus.history
- if e.event_type == EventType.TOOL_CALL_START
- ]
- ends = [
- e for e in bus.history
- if e.event_type == EventType.TOOL_CALL_END
- ]
+ starts = [e for e in bus.history if e.event_type == EventType.TOOL_CALL_START]
+ ends = [e for e in bus.history if e.event_type == EventType.TOOL_CALL_END]
assert len(starts) == 2
assert len(ends) == 2
@@ -321,7 +315,9 @@ class TestHeuristicRewardWithTelemetry:
)
rf = HeuristicRewardFunction()
score = rf.compute(
- RoutingContext(query="test"), rec.model_id, "response",
+ RoutingContext(query="test"),
+ rec.model_id,
+ "response",
latency_seconds=rec.latency_seconds,
cost_usd=rec.cost_usd,
prompt_tokens=rec.prompt_tokens,
@@ -352,16 +348,30 @@ class TestTelemetryPipeline:
db = tmp_path / "telemetry.db"
store = TelemetryStore(db)
- store.record(TelemetryRecord(
- timestamp=time.time(), model_id="m1", engine="ollama",
- prompt_tokens=10, completion_tokens=5, total_tokens=15,
- latency_seconds=1.0, cost_usd=0.001,
- ))
- store.record(TelemetryRecord(
- timestamp=time.time(), model_id="m2", engine="vllm",
- prompt_tokens=20, completion_tokens=10, total_tokens=30,
- latency_seconds=0.5, cost_usd=0.002,
- ))
+ store.record(
+ TelemetryRecord(
+ timestamp=time.time(),
+ model_id="m1",
+ engine="ollama",
+ prompt_tokens=10,
+ completion_tokens=5,
+ total_tokens=15,
+ latency_seconds=1.0,
+ cost_usd=0.001,
+ )
+ )
+ store.record(
+ TelemetryRecord(
+ timestamp=time.time(),
+ model_id="m2",
+ engine="vllm",
+ prompt_tokens=20,
+ completion_tokens=10,
+ total_tokens=30,
+ latency_seconds=0.5,
+ cost_usd=0.002,
+ )
+ )
store.close()
agg = TelemetryAggregator(db)
@@ -387,8 +397,12 @@ class TestEventBusTelemetryAggregator:
# Publish a telemetry event
rec = TelemetryRecord(
- timestamp=1000.0, model_id="event-model", engine="test",
- prompt_tokens=5, completion_tokens=3, total_tokens=8,
+ timestamp=1000.0,
+ model_id="event-model",
+ engine="test",
+ prompt_tokens=5,
+ completion_tokens=3,
+ total_tokens=8,
latency_seconds=0.1,
)
bus.publish(EventType.TELEMETRY_RECORD, {"record": rec})
@@ -435,11 +449,18 @@ class TestRewardTelemetryIntegration:
db = tmp_path / "telemetry.db"
store = TelemetryStore(db)
- store.record(TelemetryRecord(
- timestamp=time.time(), model_id="scored-model", engine="test",
- prompt_tokens=50, completion_tokens=25, total_tokens=75,
- latency_seconds=3.0, cost_usd=0.003,
- ))
+ store.record(
+ TelemetryRecord(
+ timestamp=time.time(),
+ model_id="scored-model",
+ engine="test",
+ prompt_tokens=50,
+ completion_tokens=25,
+ total_tokens=75,
+ latency_seconds=3.0,
+ cost_usd=0.003,
+ )
+ )
store.close()
agg = TelemetryAggregator(db)
@@ -448,7 +469,9 @@ class TestRewardTelemetryIntegration:
rf = HeuristicRewardFunction()
score = rf.compute(
- RoutingContext(query="test"), ms.model_id, "response",
+ RoutingContext(query="test"),
+ ms.model_id,
+ "response",
latency_seconds=ms.avg_latency,
cost_usd=ms.total_cost,
prompt_tokens=ms.prompt_tokens,
@@ -581,7 +604,8 @@ class TestFullPipeline:
def run(self, input, context=None, **kwargs):
result = self.engine.generate(
- [], model=self.model,
+ [],
+ model=self.model,
)
return AgentResult(content=result["content"], turns=1)
diff --git a/tests/integration/test_integration_extended.py b/tests/integration/test_integration_extended.py
index 9687a70f..bec9b97d 100644
--- a/tests/integration/test_integration_extended.py
+++ b/tests/integration/test_integration_extended.py
@@ -87,16 +87,15 @@ class TestReActPipeline:
"Action: calculator\n"
'Action Input: {"expression":"2+2"}'
),
- _simple_response(
- "Thought: The result is 4.\n"
- "Final Answer: 2+2 equals 4."
- ),
+ _simple_response("Thought: The result is 4.\nFinal Answer: 2+2 equals 4."),
]
engine = _make_engine(responses)
bus = EventBus(record_history=True)
agent = NativeReActAgent(
- engine, "test-model",
- tools=[CalculatorTool()], bus=bus,
+ engine,
+ "test-model",
+ tools=[CalculatorTool()],
+ bus=bus,
)
result = agent.run("What is 2+2?")
@@ -118,13 +117,14 @@ class TestReActPipeline:
'Action Input: {"thought":"Step 1: analyze"}'
),
_simple_response(
- "Thought: I have my analysis.\n"
- "Final Answer: The answer is clear."
+ "Thought: I have my analysis.\nFinal Answer: The answer is clear."
),
]
engine = _make_engine(responses)
agent = NativeReActAgent(
- engine, "test-model", tools=[ThinkTool()],
+ engine,
+ "test-model",
+ tools=[ThinkTool()],
)
result = agent.run("Analyze this.")
assert result.turns == 2
@@ -136,10 +136,7 @@ class TestReActPipeline:
from openjarvis.agents.native_react import NativeReActAgent
engine = _make_engine(
- _simple_response(
- "Thought: This is simple.\n"
- "Final Answer: Hello!"
- )
+ _simple_response("Thought: This is simple.\nFinal Answer: Hello!")
)
agent = NativeReActAgent(engine, "test-model")
result = agent.run("Say hello")
@@ -155,14 +152,12 @@ class TestReActPipeline:
_register_all()
from openjarvis.agents.native_react import NativeReActAgent
- engine = _make_engine(
- _simple_response(
- "Thought: done.\nFinal Answer: ok"
- )
- )
+ engine = _make_engine(_simple_response("Thought: done.\nFinal Answer: ok"))
bus = EventBus(record_history=True)
agent = NativeReActAgent(
- engine, "test-model", bus=bus,
+ engine,
+ "test-model",
+ bus=bus,
)
agent.run("Test")
@@ -188,19 +183,18 @@ class TestOpenHandsPipeline:
if not ToolRegistry.contains("code_interpreter"):
ToolRegistry.register_value(
- "code_interpreter", CodeInterpreterTool,
+ "code_interpreter",
+ CodeInterpreterTool,
)
responses = [
- _simple_response(
- "I'll calculate this:\n"
- "```python\nprint(2 + 2)\n```"
- ),
+ _simple_response("I'll calculate this:\n```python\nprint(2 + 2)\n```"),
_simple_response("The result is 4."),
]
engine = _make_engine(responses)
agent = NativeOpenHandsAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[CodeInterpreterTool()],
)
result = agent.run("What is 2+2?")
@@ -216,9 +210,7 @@ class TestOpenHandsPipeline:
_register_all()
from openjarvis.agents.native_openhands import NativeOpenHandsAgent
- engine = _make_engine(
- _simple_response("Hello! How can I help?")
- )
+ engine = _make_engine(_simple_response("Hello! How can I help?"))
agent = NativeOpenHandsAgent(engine, "test-model")
result = agent.run("Say hello")
assert result.content == "Hello! How can I help?"
@@ -229,12 +221,12 @@ class TestOpenHandsPipeline:
_register_all()
from openjarvis.agents.native_openhands import NativeOpenHandsAgent
- engine = _make_engine(
- _simple_response("Direct answer.")
- )
+ engine = _make_engine(_simple_response("Direct answer."))
bus = EventBus(record_history=True)
agent = NativeOpenHandsAgent(
- engine, "test-model", bus=bus,
+ engine,
+ "test-model",
+ bus=bus,
)
agent.run("Test")
@@ -275,14 +267,16 @@ class TestMCPIntegration:
# Call calculator
result = client.call_tool(
- "calculator", {"expression": "10*5"},
+ "calculator",
+ {"expression": "10*5"},
)
assert result["content"][0]["text"] == "50.0"
assert result["isError"] is False
# Call think
result = client.call_tool(
- "think", {"thought": "reasoning step"},
+ "think",
+ {"thought": "reasoning step"},
)
assert result["content"][0]["text"] == "reasoning step"
@@ -325,7 +319,8 @@ class TestMCPIntegration:
# 3. Call tool
result = client.call_tool(
- "calculator", {"expression": "7+3"},
+ "calculator",
+ {"expression": "7+3"},
)
assert result["content"][0]["text"] == "10.0"
@@ -372,15 +367,13 @@ class TestCrossEngineConsistency:
"Action: calculator\n"
'Action Input: {"expression":"3*3"}'
),
- _simple_response(
- "Thought: got 9.\n"
- "Final Answer: 9"
- ),
+ _simple_response("Thought: got 9.\nFinal Answer: 9"),
]
engine = _make_engine(responses)
engine.engine_id = engine_name
agent = NativeReActAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[CalculatorTool()],
)
result = agent.run("What is 3*3?")
@@ -453,13 +446,9 @@ class TestModelCatalogIntegration:
BUILTIN_MODELS,
)
- local = [
- s for s in BUILTIN_MODELS if not s.requires_api_key
- ]
+ local = [s for s in BUILTIN_MODELS if not s.requires_api_key]
for spec in local:
- assert len(spec.supported_engines) >= 1, (
- f"{spec.model_id} has no engines"
- )
+ assert len(spec.supported_engines) >= 1, f"{spec.model_id} has no engines"
def test_cloud_models_require_api_key(self):
"""All cloud models require an API key."""
@@ -478,9 +467,7 @@ class TestModelCatalogIntegration:
"gemini-3-flash",
]
for mid in cloud_ids:
- matches = [
- s for s in BUILTIN_MODELS if s.model_id == mid
- ]
+ matches = [s for s in BUILTIN_MODELS if s.model_id == mid]
assert len(matches) == 1, f"Missing {mid}"
assert matches[0].requires_api_key is True
@@ -494,7 +481,8 @@ class TestAgentRoutingMatrix:
"""Agents run consistently across different configurations."""
@pytest.mark.parametrize(
- "agent_key", ["native_react", "native_openhands"],
+ "agent_key",
+ ["native_react", "native_openhands"],
)
def test_agent_returns_valid_result(self, agent_key):
_register_all()
@@ -515,7 +503,8 @@ class TestAgentRoutingMatrix:
assert len(result.content) > 0
@pytest.mark.parametrize(
- "agent_key", ["native_react", "native_openhands"],
+ "agent_key",
+ ["native_react", "native_openhands"],
)
def test_agent_emits_events(self, agent_key):
_register_all()
@@ -543,15 +532,16 @@ class TestAgentRoutingMatrix:
engine = _make_engine(
_simple_response(
- "Thought: I see the system message.\n"
- "Final Answer: Got context."
+ "Thought: I see the system message.\nFinal Answer: Got context."
)
)
conv = Conversation()
- conv.add(Message(
- role=Role.SYSTEM,
- content="You are helpful.",
- ))
+ conv.add(
+ Message(
+ role=Role.SYSTEM,
+ content="You are helpful.",
+ )
+ )
ctx = AgentContext(conversation=conv)
agent = NativeReActAgent(engine, "test-model")
result = agent.run("Hello", context=ctx)
@@ -561,6 +551,4 @@ class TestAgentRoutingMatrix:
call_args = engine.generate.call_args
msgs = call_args[0][0]
# First message is ReAct system prompt, then context
- assert any(
- m.content == "You are helpful." for m in msgs
- )
+ assert any(m.content == "You are helpful." for m in msgs)
diff --git a/tests/intelligence/test_intelligence_stubs.py b/tests/intelligence/test_intelligence_stubs.py
index d483c81a..7f46cd2a 100644
--- a/tests/intelligence/test_intelligence_stubs.py
+++ b/tests/intelligence/test_intelligence_stubs.py
@@ -39,7 +39,8 @@ class TestBackwardCompatShims:
ctx = build_routing_context("hello")
assert ctx.query == "hello"
router = HeuristicRouter(
- available_models=[], default_model="m",
+ available_models=[],
+ default_model="m",
)
assert router.select_model(ctx) == "m"
diff --git a/tests/intelligence/test_model_catalog_extended.py b/tests/intelligence/test_model_catalog_extended.py
index 3e6e5582..bb5e8d38 100644
--- a/tests/intelligence/test_model_catalog_extended.py
+++ b/tests/intelligence/test_model_catalog_extended.py
@@ -16,6 +16,7 @@ from openjarvis.intelligence.model_catalog import (
# Helpers
# ---------------------------------------------------------------------------
+
def _get_spec(model_id: str) -> ModelSpec:
"""Lookup a spec from the BUILTIN_MODELS list (not registry)."""
for spec in BUILTIN_MODELS:
@@ -273,10 +274,19 @@ class TestModelDiscovery:
def test_cloud_models_require_api_key(self) -> None:
"""All cloud models have requires_api_key=True."""
cloud_ids = {
- "gpt-4o", "gpt-4o-mini", "gpt-5-mini", "gpt-5-mini-2025-08-07",
- "claude-sonnet-4-20250514", "claude-opus-4-20250514",
- "claude-opus-4-6", "claude-sonnet-4-6", "claude-haiku-4-5",
- "gemini-2.5-pro", "gemini-2.5-flash", "gemini-3-pro", "gemini-3-flash",
+ "gpt-4o",
+ "gpt-4o-mini",
+ "gpt-5-mini",
+ "gpt-5-mini-2025-08-07",
+ "claude-sonnet-4-20250514",
+ "claude-opus-4-20250514",
+ "claude-opus-4-6",
+ "claude-sonnet-4-6",
+ "claude-haiku-4-5",
+ "gemini-2.5-pro",
+ "gemini-2.5-flash",
+ "gemini-3-pro",
+ "gemini-3-flash",
}
for spec in BUILTIN_MODELS:
if spec.model_id in cloud_ids:
@@ -287,9 +297,15 @@ class TestModelDiscovery:
def test_moe_models_have_active_params(self) -> None:
"""MoE models have active_parameter_count_b set."""
moe_ids = {
- "gpt-oss:120b", "glm-4.7-flash", "trinity-mini",
- "qwen3.5:3b", "qwen3.5:8b", "qwen3.5:14b",
- "qwen3.5:35b", "qwen3.5:122b", "qwen3.5:397b",
+ "gpt-oss:120b",
+ "glm-4.7-flash",
+ "trinity-mini",
+ "qwen3.5:3b",
+ "qwen3.5:8b",
+ "qwen3.5:14b",
+ "qwen3.5:35b",
+ "qwen3.5:122b",
+ "qwen3.5:397b",
"granite4.0-h-small",
}
for spec in BUILTIN_MODELS:
@@ -308,8 +324,12 @@ class TestModelDiscovery:
"""merge_discovered_models works for all new model IDs."""
register_builtin_models()
new_ids = [
- "gpt-oss:120b", "glm-4.7-flash", "trinity-mini",
- "gpt-5-mini", "claude-opus-4-6", "gemini-3-pro",
+ "gpt-oss:120b",
+ "glm-4.7-flash",
+ "trinity-mini",
+ "gpt-5-mini",
+ "claude-opus-4-6",
+ "gemini-3-pro",
]
# Merging known IDs should not raise
merge_discovered_models("vllm", new_ids)
diff --git a/tests/intelligence/test_router.py b/tests/intelligence/test_router.py
index b6ea9319..83c5f4d2 100644
--- a/tests/intelligence/test_router.py
+++ b/tests/intelligence/test_router.py
@@ -19,15 +19,19 @@ def _register_models() -> None:
ModelRegistry.register_value(
"small",
ModelSpec(
- model_id="small", name="Small",
- parameter_count_b=3.0, context_length=4096,
+ model_id="small",
+ name="Small",
+ parameter_count_b=3.0,
+ context_length=4096,
),
)
ModelRegistry.register_value(
"large",
ModelSpec(
- model_id="large", name="Large",
- parameter_count_b=70.0, context_length=131072,
+ model_id="large",
+ name="Large",
+ parameter_count_b=70.0,
+ context_length=131072,
),
)
diff --git a/tests/learning/agents/test_dspy_optimizer.py b/tests/learning/agents/test_dspy_optimizer.py
index e83d45f8..3904b75d 100644
--- a/tests/learning/agents/test_dspy_optimizer.py
+++ b/tests/learning/agents/test_dspy_optimizer.py
@@ -48,23 +48,27 @@ class TestDSPyOptimizerTraceConversion:
now = time.time()
traces = []
for i in range(5):
- traces.append(Trace(
- query=f"test query {i}",
- agent="native_react",
- model="qwen3:8b",
- result=f"result {i}",
- outcome="success",
- feedback=0.9,
- started_at=now,
- ended_at=now + 1,
- total_tokens=100,
- total_latency_seconds=1.0,
- steps=[TraceStep(
- step_type=StepType.GENERATE,
- timestamp=now,
- duration_seconds=0.5,
- )],
- ))
+ traces.append(
+ Trace(
+ query=f"test query {i}",
+ agent="native_react",
+ model="qwen3:8b",
+ result=f"result {i}",
+ outcome="success",
+ feedback=0.9,
+ started_at=now,
+ ended_at=now + 1,
+ total_tokens=100,
+ total_latency_seconds=1.0,
+ steps=[
+ TraceStep(
+ step_type=StepType.GENERATE,
+ timestamp=now,
+ duration_seconds=0.5,
+ )
+ ],
+ )
+ )
mock_store = MagicMock()
mock_store.list_traces.return_value = traces
@@ -73,10 +77,14 @@ class TestDSPyOptimizerTraceConversion:
import openjarvis.learning.agents.dspy_optimizer as mod
with patch.object(mod, "HAS_DSPY", True):
- with patch.object(optimizer, "_run_dspy_optimization", return_value={
- "system_prompt": "You are a helpful assistant.",
- "few_shot_examples": [{"input": "hi", "output": "hello"}],
- }):
+ with patch.object(
+ optimizer,
+ "_run_dspy_optimization",
+ return_value={
+ "system_prompt": "You are a helpful assistant.",
+ "few_shot_examples": [{"input": "hi", "output": "hello"}],
+ },
+ ):
result = optimizer.optimize(mock_store)
assert result["status"] == "completed"
assert "config_updates" in result
diff --git a/tests/learning/agents/test_gepa_optimizer.py b/tests/learning/agents/test_gepa_optimizer.py
index acb59198..db738b60 100644
--- a/tests/learning/agents/test_gepa_optimizer.py
+++ b/tests/learning/agents/test_gepa_optimizer.py
@@ -45,17 +45,27 @@ class TestGEPAOptimizerOptimize:
optimizer = GEPAAgentOptimizer(cfg)
now = time.time()
- traces = [Trace(
- query="test", agent="native_react", model="qwen3:8b",
- result="result", outcome="success", feedback=0.9,
- started_at=now, ended_at=now + 1,
- total_tokens=100, total_latency_seconds=1.0,
- steps=[TraceStep(
- step_type=StepType.GENERATE,
- timestamp=now,
- duration_seconds=0.5,
- )],
- )]
+ traces = [
+ Trace(
+ query="test",
+ agent="native_react",
+ model="qwen3:8b",
+ result="result",
+ outcome="success",
+ feedback=0.9,
+ started_at=now,
+ ended_at=now + 1,
+ total_tokens=100,
+ total_latency_seconds=1.0,
+ steps=[
+ TraceStep(
+ step_type=StepType.GENERATE,
+ timestamp=now,
+ duration_seconds=0.5,
+ )
+ ],
+ )
+ ]
mock_store = MagicMock()
mock_store.list_traces.return_value = traces
@@ -80,6 +90,8 @@ class TestOpenJarvisGEPAAdapter:
mock_store = MagicMock()
adapter = OpenJarvisGEPAAdapter(
- mock_store, "native_react", GEPAOptimizerConfig(),
+ mock_store,
+ "native_react",
+ GEPAOptimizerConfig(),
)
assert adapter.agent_name == "native_react"
diff --git a/tests/learning/routing/test_learned_router.py b/tests/learning/routing/test_learned_router.py
index 20dc4d71..574d2c5c 100644
--- a/tests/learning/routing/test_learned_router.py
+++ b/tests/learning/routing/test_learned_router.py
@@ -46,6 +46,7 @@ class TestLearnedRouterPolicy:
def test_registered_as_learned(self) -> None:
from openjarvis.core.registry import RouterPolicyRegistry
from openjarvis.learning.routing.learned_router import ensure_registered
+
ensure_registered()
assert RouterPolicyRegistry.contains("learned")
@@ -66,19 +67,23 @@ class TestLearnedRouterPolicy:
def test_update_from_traces(self, tmp_path: Path) -> None:
store = TraceStore(tmp_path / "test.db")
for _ in range(6):
- store.save(_make_trace(
- query="def foo(): pass",
- model="codestral",
- outcome="success",
- feedback=0.9,
- ))
+ store.save(
+ _make_trace(
+ query="def foo(): pass",
+ model="codestral",
+ outcome="success",
+ feedback=0.9,
+ )
+ )
for _ in range(6):
- store.save(_make_trace(
- query="def bar(): return 1",
- model="qwen3:8b",
- outcome="failure",
- feedback=0.3,
- ))
+ store.save(
+ _make_trace(
+ query="def bar(): return 1",
+ model="qwen3:8b",
+ outcome="failure",
+ feedback=0.3,
+ )
+ )
analyzer = TraceAnalyzer(store)
policy = LearnedRouterPolicy(
@@ -96,10 +101,13 @@ class TestLearnedRouterPolicy:
def test_policy_map_readable(self, tmp_path: Path) -> None:
store = TraceStore(tmp_path / "test.db")
for _ in range(5):
- store.save(_make_trace(
- query="hello", model="small-model",
- outcome="success",
- ))
+ store.save(
+ _make_trace(
+ query="hello",
+ model="small-model",
+ outcome="success",
+ )
+ )
analyzer = TraceAnalyzer(store)
policy = LearnedRouterPolicy(analyzer=analyzer, default_model="default")
@@ -126,24 +134,34 @@ class TestLearnedRouterPolicy:
mock_store = MagicMock()
mock_store.list_traces.return_value = [
_make_trace(
- query="def foo(): pass", model="code-model",
- outcome="success", feedback=0.9,
+ query="def foo(): pass",
+ model="code-model",
+ outcome="success",
+ feedback=0.9,
),
_make_trace(
- query="def bar(): pass", model="code-model",
- outcome="success", feedback=0.85,
+ query="def bar(): pass",
+ model="code-model",
+ outcome="success",
+ feedback=0.85,
),
_make_trace(
- query="def baz(): pass", model="code-model",
- outcome="success", feedback=0.88,
+ query="def baz(): pass",
+ model="code-model",
+ outcome="success",
+ feedback=0.88,
),
_make_trace(
- query="def qux(): pass", model="code-model",
- outcome="success", feedback=0.92,
+ query="def qux(): pass",
+ model="code-model",
+ outcome="success",
+ feedback=0.92,
),
_make_trace(
- query="def quux(): pass", model="code-model",
- outcome="success", feedback=0.87,
+ query="def quux(): pass",
+ model="code-model",
+ outcome="success",
+ feedback=0.87,
),
]
result = policy.update(mock_store)
diff --git a/tests/learning/test_feedback_collector.py b/tests/learning/test_feedback_collector.py
index 93b2d50b..976c2fd7 100644
--- a/tests/learning/test_feedback_collector.py
+++ b/tests/learning/test_feedback_collector.py
@@ -222,9 +222,9 @@ class TestStats:
def test_distribution_mixed(self) -> None:
fc = FeedbackCollector()
- fc.record_explicit("a", 0.1) # low
- fc.record_explicit("b", 0.5) # medium
- fc.record_explicit("c", 0.9) # high
+ fc.record_explicit("a", 0.1) # low
+ fc.record_explicit("b", 0.5) # medium
+ fc.record_explicit("c", 0.9) # high
s = fc.stats()
assert s["distribution"]["low"] == 1
assert s["distribution"]["medium"] == 1
diff --git a/tests/learning/test_heuristic_reward.py b/tests/learning/test_heuristic_reward.py
index e4534eac..2217267a 100644
--- a/tests/learning/test_heuristic_reward.py
+++ b/tests/learning/test_heuristic_reward.py
@@ -12,9 +12,13 @@ class TestHeuristicRewardFunction:
def test_perfect_score(self) -> None:
rf = HeuristicRewardFunction()
score = rf.compute(
- RoutingContext(), "model-a", "response",
- latency_seconds=0.0, cost_usd=0.0,
- prompt_tokens=10, completion_tokens=10,
+ RoutingContext(),
+ "model-a",
+ "response",
+ latency_seconds=0.0,
+ cost_usd=0.0,
+ prompt_tokens=10,
+ completion_tokens=10,
)
# latency=1.0, cost=1.0, efficiency=0.5 → 0.4*1 + 0.3*1 + 0.3*0.5 = 0.85
assert score == pytest.approx(0.85)
@@ -22,45 +26,67 @@ class TestHeuristicRewardFunction:
def test_worst_score(self) -> None:
rf = HeuristicRewardFunction()
score = rf.compute(
- RoutingContext(), "model-a", "response",
- latency_seconds=60.0, cost_usd=0.1,
- prompt_tokens=100, completion_tokens=0,
+ RoutingContext(),
+ "model-a",
+ "response",
+ latency_seconds=60.0,
+ cost_usd=0.1,
+ prompt_tokens=100,
+ completion_tokens=0,
)
# latency clamped to 0, cost clamped to 0, efficiency=0/100=0
assert score == pytest.approx(0.0)
def test_latency_only_weight(self) -> None:
rf = HeuristicRewardFunction(
- weight_latency=1.0, weight_cost=0.0, weight_efficiency=0.0,
+ weight_latency=1.0,
+ weight_cost=0.0,
+ weight_efficiency=0.0,
)
score = rf.compute(
- RoutingContext(), "m", "",
- latency_seconds=15.0, cost_usd=0.0,
- prompt_tokens=10, completion_tokens=10,
+ RoutingContext(),
+ "m",
+ "",
+ latency_seconds=15.0,
+ cost_usd=0.0,
+ prompt_tokens=10,
+ completion_tokens=10,
)
# 1 - 15/30 = 0.5
assert score == pytest.approx(0.5)
def test_cost_only_weight(self) -> None:
rf = HeuristicRewardFunction(
- weight_latency=0.0, weight_cost=1.0, weight_efficiency=0.0,
+ weight_latency=0.0,
+ weight_cost=1.0,
+ weight_efficiency=0.0,
)
score = rf.compute(
- RoutingContext(), "m", "",
- latency_seconds=0.0, cost_usd=0.005,
- prompt_tokens=10, completion_tokens=10,
+ RoutingContext(),
+ "m",
+ "",
+ latency_seconds=0.0,
+ cost_usd=0.005,
+ prompt_tokens=10,
+ completion_tokens=10,
)
# 1 - 0.005/0.01 = 0.5
assert score == pytest.approx(0.5)
def test_efficiency_only_weight(self) -> None:
rf = HeuristicRewardFunction(
- weight_latency=0.0, weight_cost=0.0, weight_efficiency=1.0,
+ weight_latency=0.0,
+ weight_cost=0.0,
+ weight_efficiency=1.0,
)
score = rf.compute(
- RoutingContext(), "m", "",
- latency_seconds=0.0, cost_usd=0.0,
- prompt_tokens=25, completion_tokens=75,
+ RoutingContext(),
+ "m",
+ "",
+ latency_seconds=0.0,
+ cost_usd=0.0,
+ prompt_tokens=25,
+ completion_tokens=75,
)
# 75/100 = 0.75
assert score == pytest.approx(0.75)
@@ -69,18 +95,26 @@ class TestHeuristicRewardFunction:
rf = HeuristicRewardFunction()
# Even with extreme values, score should be in [0, 1]
score = rf.compute(
- RoutingContext(), "m", "",
- latency_seconds=-10.0, cost_usd=-0.1,
- prompt_tokens=0, completion_tokens=100,
+ RoutingContext(),
+ "m",
+ "",
+ latency_seconds=-10.0,
+ cost_usd=-0.1,
+ prompt_tokens=0,
+ completion_tokens=100,
)
assert 0.0 <= score <= 1.0
def test_custom_max_values(self) -> None:
rf = HeuristicRewardFunction(max_latency=10.0, max_cost=1.0)
score = rf.compute(
- RoutingContext(), "m", "",
- latency_seconds=5.0, cost_usd=0.5,
- prompt_tokens=50, completion_tokens=50,
+ RoutingContext(),
+ "m",
+ "",
+ latency_seconds=5.0,
+ cost_usd=0.5,
+ prompt_tokens=50,
+ completion_tokens=50,
)
# latency: 1-5/10=0.5, cost: 1-0.5/1.0=0.5, efficiency: 50/100=0.5
# 0.4*0.5 + 0.3*0.5 + 0.3*0.5 = 0.5
@@ -89,9 +123,13 @@ class TestHeuristicRewardFunction:
def test_zero_total_tokens_default(self) -> None:
rf = HeuristicRewardFunction()
score = rf.compute(
- RoutingContext(), "m", "",
- latency_seconds=0.0, cost_usd=0.0,
- prompt_tokens=0, completion_tokens=0,
+ RoutingContext(),
+ "m",
+ "",
+ latency_seconds=0.0,
+ cost_usd=0.0,
+ prompt_tokens=0,
+ completion_tokens=0,
)
# latency=1, cost=1, efficiency default=0.5
# 0.4*1 + 0.3*1 + 0.3*0.5 = 0.85
@@ -101,9 +139,13 @@ class TestHeuristicRewardFunction:
rf = HeuristicRewardFunction()
# Should not raise even with extra kwargs
score = rf.compute(
- RoutingContext(), "m", "",
- latency_seconds=1.0, cost_usd=0.001,
- prompt_tokens=10, completion_tokens=10,
+ RoutingContext(),
+ "m",
+ "",
+ latency_seconds=1.0,
+ cost_usd=0.001,
+ prompt_tokens=10,
+ completion_tokens=10,
random_extra="ignored",
)
assert 0.0 <= score <= 1.0
diff --git a/tests/learning/test_llm_optimizer.py b/tests/learning/test_llm_optimizer.py
index 8e114c13..33375c3f 100644
--- a/tests/learning/test_llm_optimizer.py
+++ b/tests/learning/test_llm_optimizer.py
@@ -183,9 +183,7 @@ class TestInit:
def test_stores_optimizer_backend(self) -> None:
space = _make_search_space()
backend = _make_mock_backend("")
- opt = LLMOptimizer(
- search_space=space, optimizer_backend=backend
- )
+ opt = LLMOptimizer(search_space=space, optimizer_backend=backend)
assert opt.optimizer_backend is backend
def test_default_backend_is_none(self) -> None:
@@ -203,13 +201,15 @@ class TestProposeInitial:
"""Tests for LLMOptimizer.propose_initial."""
def test_returns_trial_config(self) -> None:
- response = json.dumps({
- "params": {
- "agent.type": "native_react",
- "intelligence.temperature": 0.3,
- },
- "reasoning": "Balanced starting point",
- })
+ response = json.dumps(
+ {
+ "params": {
+ "agent.type": "native_react",
+ "intelligence.temperature": 0.3,
+ },
+ "reasoning": "Balanced starting point",
+ }
+ )
response = f"```json\n{response}\n```"
backend = _make_mock_backend(response)
opt = LLMOptimizer(
@@ -265,14 +265,16 @@ class TestProposeNext:
"""Tests for LLMOptimizer.propose_next."""
def test_returns_trial_config_with_history(self) -> None:
- response = json.dumps({
- "params": {
- "agent.type": "native_react",
- "intelligence.temperature": 0.2,
- "agent.max_turns": 15,
- },
- "reasoning": "Lower temp for better accuracy",
- })
+ response = json.dumps(
+ {
+ "params": {
+ "agent.type": "native_react",
+ "intelligence.temperature": 0.2,
+ "agent.max_turns": 15,
+ },
+ "reasoning": "Lower temp for better accuracy",
+ }
+ )
response = f"```json\n{response}\n```"
backend = _make_mock_backend(response)
opt = LLMOptimizer(
@@ -507,8 +509,7 @@ class TestParseConfigResponse:
def test_raw_json(self) -> None:
opt = self._make_optimizer()
response = (
- 'I suggest: {"params": {"agent.max_turns": 10}, '
- '"reasoning": "More turns"}'
+ 'I suggest: {"params": {"agent.max_turns": 10}, "reasoning": "More turns"}'
)
config = opt._parse_config_response(response)
assert config.params["agent.max_turns"] == 10
@@ -666,10 +667,7 @@ class TestFormatTraces:
def test_limits_to_last_10(self) -> None:
opt = LLMOptimizer(search_space=_make_search_space())
- traces = [
- _make_trace(trace_id=f"trace-{i:03d}")
- for i in range(20)
- ]
+ traces = [_make_trace(trace_id=f"trace-{i:03d}") for i in range(20)]
result = opt._format_traces(traces)
# Should only include the last 10 (indices 10-19)
assert "trace-010" in result
@@ -740,26 +738,24 @@ class TestIntegration:
"""Simulate a two-step optimization loop."""
space = _make_search_space()
initial_response = (
- '```json\n'
+ "```json\n"
'{"params": {"agent.type": "orchestrator", '
'"intelligence.temperature": 0.5, "agent.max_turns": 10}, '
'"reasoning": "Balanced start"}\n'
- '```'
+ "```"
)
next_response = (
- '```json\n'
+ "```json\n"
'{"params": {"agent.type": "native_react", '
'"intelligence.temperature": 0.2, "agent.max_turns": 15}, '
'"reasoning": "Switch to ReAct for better tool use"}\n'
- '```'
+ "```"
)
backend = MagicMock(spec=InferenceBackend)
backend.backend_id = "mock"
backend.generate.side_effect = [initial_response, next_response]
- opt = LLMOptimizer(
- search_space=space, optimizer_backend=backend
- )
+ opt = LLMOptimizer(search_space=space, optimizer_backend=backend)
# Step 1: initial proposal
config1 = opt.propose_initial()
@@ -781,10 +777,10 @@ class TestIntegration:
"""Simulate propose -> evaluate -> analyze."""
space = _make_search_space()
propose_response = (
- '```json\n'
+ "```json\n"
'{"params": {"agent.type": "orchestrator"}, '
'"reasoning": "Start with orchestrator"}\n'
- '```'
+ "```"
)
analysis_response = (
"The orchestrator agent achieved moderate accuracy. "
@@ -798,9 +794,7 @@ class TestIntegration:
analysis_response,
]
- opt = LLMOptimizer(
- search_space=space, optimizer_backend=backend
- )
+ opt = LLMOptimizer(search_space=space, optimizer_backend=backend)
config = opt.propose_initial()
summary = _make_run_summary()
@@ -833,13 +827,15 @@ class TestAnalyzeTrialStructured:
def test_analyze_trial_returns_trial_feedback(self) -> None:
"""Mock backend returns structured JSON -> TrialFeedback."""
- feedback_json = json.dumps({
- "summary_text": "Good accuracy but high latency",
- "failure_patterns": ["timeout on complex queries"],
- "primitive_ratings": {"agent": "high", "intelligence": "medium"},
- "suggested_changes": ["reduce max_turns"],
- "target_primitive": "agent",
- })
+ feedback_json = json.dumps(
+ {
+ "summary_text": "Good accuracy but high latency",
+ "failure_patterns": ["timeout on complex queries"],
+ "primitive_ratings": {"agent": "high", "intelligence": "medium"},
+ "suggested_changes": ["reduce max_turns"],
+ "target_primitive": "agent",
+ }
+ )
response = f"```json\n{feedback_json}\n```"
backend = _make_mock_backend(response)
opt = LLMOptimizer(
@@ -875,13 +871,15 @@ class TestAnalyzeTrialStructured:
def test_analyze_trial_with_sample_scores(self) -> None:
"""Verify sample_scores are included in the prompt."""
- feedback_json = json.dumps({
- "summary_text": "Analysis with scores",
- "failure_patterns": [],
- "primitive_ratings": {},
- "suggested_changes": [],
- "target_primitive": "",
- })
+ feedback_json = json.dumps(
+ {
+ "summary_text": "Analysis with scores",
+ "failure_patterns": [],
+ "primitive_ratings": {},
+ "suggested_changes": [],
+ "target_primitive": "",
+ }
+ )
response = f"```json\n{feedback_json}\n```"
backend = _make_mock_backend(response)
opt = LLMOptimizer(
@@ -909,14 +907,16 @@ class TestProposeTargeted:
"""Tests for LLMOptimizer.propose_targeted."""
def test_preserves_non_target_params(self) -> None:
- response = json.dumps({
- "params": {
- "agent.type": "native_react",
- "agent.max_turns": 20,
- "intelligence.temperature": 0.9,
- },
- "reasoning": "Change agent only",
- })
+ response = json.dumps(
+ {
+ "params": {
+ "agent.type": "native_react",
+ "agent.max_turns": 20,
+ "intelligence.temperature": 0.9,
+ },
+ "reasoning": "Change agent only",
+ }
+ )
response = f"```json\n{response}\n```"
backend = _make_mock_backend(response)
opt = LLMOptimizer(
diff --git a/tests/learning/test_multi_bench_runner.py b/tests/learning/test_multi_bench_runner.py
index aceef32a..8e6b21bc 100644
--- a/tests/learning/test_multi_bench_runner.py
+++ b/tests/learning/test_multi_bench_runner.py
@@ -74,7 +74,9 @@ class TestBenchmarkSpec:
class TestBenchmarkScore:
def test_creation(self):
score = BenchmarkScore(
- benchmark="hle", accuracy=0.75, weight=0.2,
+ benchmark="hle",
+ accuracy=0.75,
+ weight=0.2,
)
assert score.benchmark == "hle"
assert score.accuracy == 0.75
diff --git a/tests/learning/test_optimizer_engine.py b/tests/learning/test_optimizer_engine.py
index 55694044..1634dee6 100644
--- a/tests/learning/test_optimizer_engine.py
+++ b/tests/learning/test_optimizer_engine.py
@@ -152,7 +152,8 @@ class TestOptimizationEngineRun:
optimizer.optimizer_model = "test-model"
runner.run_trial.return_value = _sample_trial_result(
- "init", accuracy=0.8,
+ "init",
+ accuracy=0.8,
)
runner.benchmark = "supergpqa"
@@ -212,7 +213,8 @@ class TestOptimizationEngineRun:
runner = MagicMock()
optimizer.propose_initial.return_value = TrialConfig(
- trial_id="t1", params={},
+ trial_id="t1",
+ params={},
)
optimizer.analyze_trial.return_value = TrialFeedback(
summary_text="detailed analysis",
@@ -237,7 +239,8 @@ class TestOptimizationEngineRun:
runner = MagicMock()
optimizer.propose_initial.return_value = TrialConfig(
- trial_id="t1", params={},
+ trial_id="t1",
+ params={},
)
optimizer.optimizer_model = "m"
@@ -276,11 +279,11 @@ class TestEarlyStopping:
runner = MagicMock()
optimizer.propose_initial.return_value = TrialConfig(
- trial_id="t0", params={},
+ trial_id="t0",
+ params={},
)
optimizer.propose_next.side_effect = [
- TrialConfig(trial_id=f"t{i}", params={})
- for i in range(1, 20)
+ TrialConfig(trial_id=f"t{i}", params={}) for i in range(1, 20)
]
optimizer.analyze_trial.return_value = TrialFeedback(summary_text="ok")
optimizer.optimizer_model = "m"
@@ -311,11 +314,11 @@ class TestEarlyStopping:
runner = MagicMock()
optimizer.propose_initial.return_value = TrialConfig(
- trial_id="t0", params={},
+ trial_id="t0",
+ params={},
)
optimizer.propose_next.side_effect = [
- TrialConfig(trial_id=f"t{i}", params={})
- for i in range(1, 5)
+ TrialConfig(trial_id=f"t{i}", params={}) for i in range(1, 5)
]
optimizer.analyze_trial.return_value = TrialFeedback(summary_text="ok")
optimizer.optimizer_model = "m"
@@ -357,19 +360,18 @@ class TestProgressCallback:
runner = MagicMock()
optimizer.propose_initial.return_value = TrialConfig(
- trial_id="t0", params={},
+ trial_id="t0",
+ params={},
)
optimizer.propose_next.side_effect = [
- TrialConfig(trial_id=f"t{i}", params={})
- for i in range(1, 3)
+ TrialConfig(trial_id=f"t{i}", params={}) for i in range(1, 3)
]
optimizer.analyze_trial.return_value = TrialFeedback(summary_text="ok")
optimizer.optimizer_model = "m"
runner.benchmark = "b"
results = [
- _sample_trial_result(f"t{i}", accuracy=0.5 + i * 0.1)
- for i in range(3)
+ _sample_trial_result(f"t{i}", accuracy=0.5 + i * 0.1) for i in range(3)
]
runner.run_trial.side_effect = results
@@ -394,7 +396,8 @@ class TestProgressCallback:
runner = MagicMock()
optimizer.propose_initial.return_value = TrialConfig(
- trial_id="t0", params={},
+ trial_id="t0",
+ params={},
)
optimizer.analyze_trial.return_value = TrialFeedback(summary_text="ok")
optimizer.optimizer_model = "m"
@@ -513,7 +516,9 @@ class TestExportBestRecipe:
config = TrialConfig(trial_id="min", params={})
best = TrialResult(
- trial_id="min", config=config, accuracy=0.5,
+ trial_id="min",
+ config=config,
+ accuracy=0.5,
)
run = OptimizationRun(
run_id="run-min",
@@ -544,10 +549,12 @@ class TestRunWithStore:
runner = MagicMock()
optimizer.propose_initial.return_value = TrialConfig(
- trial_id="t0", params={},
+ trial_id="t0",
+ params={},
)
optimizer.propose_next.return_value = TrialConfig(
- trial_id="t1", params={},
+ trial_id="t1",
+ params={},
)
optimizer.analyze_trial.return_value = TrialFeedback(summary_text="analysis")
optimizer.optimizer_model = "m"
@@ -584,7 +591,8 @@ class TestRunWithStore:
runner = MagicMock()
optimizer.propose_initial.return_value = TrialConfig(
- trial_id="t0", params={},
+ trial_id="t0",
+ params={},
)
optimizer.analyze_trial.return_value = TrialFeedback(summary_text="ok")
optimizer.optimizer_model = "m"
@@ -648,7 +656,7 @@ engine = "ollama"
from openjarvis.optimize.config import load_optimize_config
path = tmp_path / "test.toml"
- path.write_bytes(b'[optimize]\nmax_trials = 5\n')
+ path.write_bytes(b"[optimize]\nmax_trials = 5\n")
config = load_optimize_config(str(path))
assert config["optimize"]["max_trials"] == 5
@@ -662,10 +670,12 @@ class TestParetoFrontier:
runner = MagicMock()
optimizer.propose_initial.return_value = TrialConfig(
- trial_id="t0", params={},
+ trial_id="t0",
+ params={},
)
optimizer.propose_next.return_value = TrialConfig(
- trial_id="t1", params={},
+ trial_id="t1",
+ params={},
)
# analyze_trial returns TrialFeedback
optimizer.analyze_trial.return_value = TrialFeedback(
@@ -700,10 +710,7 @@ class TestTargetedAndMerge:
optimizer = MagicMock()
runner = MagicMock()
- configs = [
- TrialConfig(trial_id=f"t{i}", params={})
- for i in range(5)
- ]
+ configs = [TrialConfig(trial_id=f"t{i}", params={}) for i in range(5)]
optimizer.propose_initial.return_value = configs[0]
optimizer.propose_next.side_effect = configs[1:]
optimizer.propose_targeted.return_value = configs[3]
@@ -718,15 +725,14 @@ class TestTargetedAndMerge:
)
optimizer.analyze_trial.side_effect = [
- fb_normal, # trial 0
- fb_normal, # trial 1
- fb_targeted, # trial 2 -> will trigger targeted on next
- fb_normal, # trial 3
+ fb_normal, # trial 0
+ fb_normal, # trial 1
+ fb_targeted, # trial 2 -> will trigger targeted on next
+ fb_normal, # trial 3
]
results = [
- _sample_trial_result(f"t{i}", accuracy=0.5 + i * 0.05)
- for i in range(4)
+ _sample_trial_result(f"t{i}", accuracy=0.5 + i * 0.05) for i in range(4)
]
runner.run_trial.side_effect = results
@@ -745,10 +751,7 @@ class TestTargetedAndMerge:
optimizer = MagicMock()
runner = MagicMock()
- configs = [
- TrialConfig(trial_id=f"t{i}", params={})
- for i in range(7)
- ]
+ configs = [TrialConfig(trial_id=f"t{i}", params={}) for i in range(7)]
optimizer.propose_initial.return_value = configs[0]
optimizer.propose_next.side_effect = configs[1:]
optimizer.propose_merge.return_value = configs[5]
@@ -761,9 +764,9 @@ class TestTargetedAndMerge:
# Create diverse results with genuine tradeoffs so
# frontier has >= 2 members (high acc/high lat vs low acc/low lat)
tradeoffs = [
- (0.9, 3.0, 0.05), # t0: high accuracy, high latency, high cost
- (0.5, 0.5, 0.01), # t1: low accuracy, low latency, low cost
- (0.7, 1.5, 0.03), # t2: dominated by t0+t1 combo
+ (0.9, 3.0, 0.05), # t0: high accuracy, high latency, high cost
+ (0.5, 0.5, 0.01), # t1: low accuracy, low latency, low cost
+ (0.7, 1.5, 0.03), # t2: dominated by t0+t1 combo
(0.85, 2.5, 0.04), # t3: close to t0 tradeoff
(0.6, 0.8, 0.015), # t4: close to t1 tradeoff
(0.75, 1.0, 0.02), # t5: merged result
diff --git a/tests/learning/test_pareto.py b/tests/learning/test_pareto.py
index c0542daf..47fa9979 100644
--- a/tests/learning/test_pareto.py
+++ b/tests/learning/test_pareto.py
@@ -16,6 +16,7 @@ from openjarvis.optimize.types import (
# Helpers
# ---------------------------------------------------------------------------
+
def _make_trial(
trial_id: str,
accuracy: float = 0.0,
@@ -92,6 +93,7 @@ DEFAULT_OBJECTIVES = [
# Tests
# ---------------------------------------------------------------------------
+
class TestComputeParetoFrontier:
"""Tests for :func:`compute_pareto_frontier`."""
@@ -185,7 +187,8 @@ class TestComputeParetoFrontier:
)
frontier = compute_pareto_frontier(
- [trial_a, trial_b, trial_c], objectives,
+ [trial_a, trial_b, trial_c],
+ objectives,
)
ids = {t.trial_id for t in frontier}
@@ -210,7 +213,8 @@ class TestComputeParetoFrontier:
trial_c = _make_trial("C", accuracy=0.55, energy=120.0)
frontier = compute_pareto_frontier(
- [trial_a, trial_b, trial_c], objectives,
+ [trial_a, trial_b, trial_c],
+ objectives,
)
ids = {t.trial_id for t in frontier}
diff --git a/tests/learning/test_personal_synthesizer.py b/tests/learning/test_personal_synthesizer.py
index 85dd3406..02bdc7ea 100644
--- a/tests/learning/test_personal_synthesizer.py
+++ b/tests/learning/test_personal_synthesizer.py
@@ -60,25 +60,33 @@ def trace_store(tmp_path: Path) -> TraceStore:
class TestPersonalBenchmarkSampleDefaults:
def test_default_category(self) -> None:
sample = PersonalBenchmarkSample(
- trace_id="t1", query="hello", reference_answer="world",
+ trace_id="t1",
+ query="hello",
+ reference_answer="world",
)
assert sample.category == "chat"
def test_default_agent_empty(self) -> None:
sample = PersonalBenchmarkSample(
- trace_id="t1", query="q", reference_answer="a",
+ trace_id="t1",
+ query="q",
+ reference_answer="a",
)
assert sample.agent == ""
def test_default_feedback_score_zero(self) -> None:
sample = PersonalBenchmarkSample(
- trace_id="t1", query="q", reference_answer="a",
+ trace_id="t1",
+ query="q",
+ reference_answer="a",
)
assert sample.feedback_score == 0.0
def test_default_metadata_empty(self) -> None:
sample = PersonalBenchmarkSample(
- trace_id="t1", query="q", reference_answer="a",
+ trace_id="t1",
+ query="q",
+ reference_answer="a",
)
assert sample.metadata == {}
@@ -90,7 +98,8 @@ class TestPersonalBenchmarkSampleDefaults:
class TestPersonalBenchmarkSynthesizer:
def test_synthesize_creates_benchmark_from_traces(
- self, trace_store: TraceStore,
+ self,
+ trace_store: TraceStore,
) -> None:
trace_store.save(_make_trace(trace_id="t1", feedback=0.9))
trace_store.save(_make_trace(trace_id="t2", query="What is 3+3?", feedback=0.8))
@@ -119,38 +128,46 @@ class TestPersonalBenchmarkSynthesizer:
def test_grouping_by_query_class(self, trace_store: TraceStore) -> None:
"""Same agent + same query prefix -> same group, so only one sample."""
- trace_store.save(_make_trace(
- trace_id="t1",
- query="What is 2+2?",
- agent="simple",
- feedback=0.8,
- ))
- trace_store.save(_make_trace(
- trace_id="t2",
- query="What is 2+2?",
- agent="simple",
- feedback=0.95,
- ))
+ trace_store.save(
+ _make_trace(
+ trace_id="t1",
+ query="What is 2+2?",
+ agent="simple",
+ feedback=0.8,
+ )
+ )
+ trace_store.save(
+ _make_trace(
+ trace_id="t2",
+ query="What is 2+2?",
+ agent="simple",
+ feedback=0.95,
+ )
+ )
synth = PersonalBenchmarkSynthesizer(trace_store)
bm = synth.synthesize()
# Should collapse into one sample (same group)
assert len(bm.samples) == 1
def test_picks_highest_feedback_per_group(self, trace_store: TraceStore) -> None:
- trace_store.save(_make_trace(
- trace_id="t1",
- query="Tell me a joke",
- agent="simple",
- feedback=0.7,
- result="bad joke",
- ))
- trace_store.save(_make_trace(
- trace_id="t2",
- query="Tell me a joke",
- agent="simple",
- feedback=0.99,
- result="great joke",
- ))
+ trace_store.save(
+ _make_trace(
+ trace_id="t1",
+ query="Tell me a joke",
+ agent="simple",
+ feedback=0.7,
+ result="bad joke",
+ )
+ )
+ trace_store.save(
+ _make_trace(
+ trace_id="t2",
+ query="Tell me a joke",
+ agent="simple",
+ feedback=0.99,
+ result="great joke",
+ )
+ )
synth = PersonalBenchmarkSynthesizer(trace_store)
bm = synth.synthesize()
assert len(bm.samples) == 1
@@ -162,12 +179,14 @@ class TestPersonalBenchmarkSynthesizer:
trace_store.save(
_make_trace(trace_id="t1", query="Hello", agent="simple", feedback=0.9),
)
- trace_store.save(_make_trace(
- trace_id="t2",
- query="Hello",
- agent="orchestrator",
- feedback=0.8,
- ))
+ trace_store.save(
+ _make_trace(
+ trace_id="t2",
+ query="Hello",
+ agent="orchestrator",
+ feedback=0.8,
+ )
+ )
synth = PersonalBenchmarkSynthesizer(trace_store)
bm = synth.synthesize()
assert len(bm.samples) == 2
@@ -186,7 +205,8 @@ class TestPersonalBenchmarkSynthesizer:
assert len(bm.samples) == 3
def test_empty_traces_returns_empty_benchmark(
- self, trace_store: TraceStore,
+ self,
+ trace_store: TraceStore,
) -> None:
synth = PersonalBenchmarkSynthesizer(trace_store)
bm = synth.synthesize()
diff --git a/tests/learning/test_router.py b/tests/learning/test_router.py
index 5e242153..bf80b4ea 100644
--- a/tests/learning/test_router.py
+++ b/tests/learning/test_router.py
@@ -16,22 +16,28 @@ def _register_models() -> None:
ModelRegistry.register_value(
"small",
ModelSpec(
- model_id="small", name="Small",
- parameter_count_b=3.0, context_length=4096,
+ model_id="small",
+ name="Small",
+ parameter_count_b=3.0,
+ context_length=4096,
),
)
ModelRegistry.register_value(
"large",
ModelSpec(
- model_id="large", name="Large",
- parameter_count_b=70.0, context_length=131072,
+ model_id="large",
+ name="Large",
+ parameter_count_b=70.0,
+ context_length=131072,
),
)
ModelRegistry.register_value(
"coder",
ModelSpec(
- model_id="coder", name="DeepSeek Coder",
- parameter_count_b=16.0, context_length=131072,
+ model_id="coder",
+ name="DeepSeek Coder",
+ parameter_count_b=16.0,
+ context_length=131072,
),
)
@@ -71,7 +77,9 @@ class TestHeuristicRouter:
available_models=["small", "large", "coder"],
)
ctx = RoutingContext(
- query="def foo():", query_length=10, has_code=True,
+ query="def foo():",
+ query_length=10,
+ has_code=True,
)
assert router.select_model(ctx) == "coder"
@@ -81,7 +89,9 @@ class TestHeuristicRouter:
available_models=["small", "large", "coder"],
)
ctx = RoutingContext(
- query="solve x", query_length=7, has_math=True,
+ query="solve x",
+ query_length=7,
+ has_math=True,
)
assert router.select_model(ctx) == "large"
@@ -99,7 +109,9 @@ class TestHeuristicRouter:
available_models=["small", "large", "coder"],
)
ctx = RoutingContext(
- query="x" * 501, query_length=501, urgency=0.9,
+ query="x" * 501,
+ query_length=501,
+ urgency=0.9,
)
assert router.select_model(ctx) == "small"
@@ -112,13 +124,15 @@ class TestHeuristicRouter:
)
# Medium-length, no code/math, no reasoning → falls to default
ctx = RoutingContext(
- query="Tell me about cats", query_length=60,
+ query="Tell me about cats",
+ query_length=60,
)
assert router.select_model(ctx) == "large"
def test_no_available_models(self) -> None:
router = HeuristicRouter(
- available_models=[], default_model="fallback-model",
+ available_models=[],
+ default_model="fallback-model",
)
ctx = RoutingContext(query="test", query_length=4)
assert router.select_model(ctx) == "fallback-model"
@@ -139,6 +153,8 @@ class TestHeuristicRouter:
available_models=["small", "large"],
)
ctx = RoutingContext(
- query="def foo():", query_length=10, has_code=True,
+ query="def foo():",
+ query_length=10,
+ has_code=True,
)
assert router.select_model(ctx) == "large"
diff --git a/tests/learning/test_router_stubs.py b/tests/learning/test_router_stubs.py
index 91ef44d6..41fd6c49 100644
--- a/tests/learning/test_router_stubs.py
+++ b/tests/learning/test_router_stubs.py
@@ -16,10 +16,13 @@ class _DummyRouter(RouterPolicy):
class _DummyAnalyzer(QueryAnalyzer):
def analyze(
- self, query: str, **kwargs: object,
+ self,
+ query: str,
+ **kwargs: object,
) -> RoutingContext:
return RoutingContext(
- query=query, query_length=len(query),
+ query=query,
+ query_length=len(query),
)
diff --git a/tests/learning/test_routing_models.py b/tests/learning/test_routing_models.py
index 309004d0..febbd22c 100644
--- a/tests/learning/test_routing_models.py
+++ b/tests/learning/test_routing_models.py
@@ -13,10 +13,10 @@ from openjarvis.learning.routing.router import (
# New local model keys for testing
NEW_LOCAL_MODELS = [
- "gpt-oss:120b", # 117B total, 5.1B active, MoE
- "qwen3:8b", # 8.2B, dense
+ "gpt-oss:120b", # 117B total, 5.1B active, MoE
+ "qwen3:8b", # 8.2B, dense
"glm-4.7-flash", # 30B total, 3.0B active, MoE
- "trinity-mini", # 26B total, 3.0B active, MoE
+ "trinity-mini", # 26B total, 3.0B active, MoE
]
# Cloud model keys
@@ -114,8 +114,7 @@ class TestRouterWithNewModels:
available_models=NEW_LOCAL_MODELS,
)
ctx = build_routing_context(
- "Please explain step by step how neural networks"
- " learn"
+ "Please explain step by step how neural networks learn"
)
selected = router.select_model(ctx)
assert selected == "gpt-oss:120b"
@@ -148,7 +147,9 @@ class TestRouterCloudFallback:
_setup_models()
router = HeuristicRouter(available_models=CLOUD_MODELS)
ctx = RoutingContext(
- query="solve x", query_length=7, has_math=True,
+ query="solve x",
+ query_length=7,
+ has_math=True,
)
selected = router.select_model(ctx)
assert selected in CLOUD_MODELS
@@ -192,18 +193,21 @@ class TestRouterParameterized:
@pytest.mark.parametrize("model_id", NEW_LOCAL_MODELS)
def test_single_model_always_returns_it(
- self, model_id: str,
+ self,
+ model_id: str,
) -> None:
_setup_models()
router = HeuristicRouter(available_models=[model_id])
ctx = RoutingContext(
- query="hello world", query_length=11,
+ query="hello world",
+ query_length=11,
)
assert router.select_model(ctx) == model_id
@pytest.mark.parametrize("urgency", [0.85, 0.9, 1.0])
def test_high_urgency_always_smallest(
- self, urgency: float,
+ self,
+ urgency: float,
) -> None:
_setup_models()
router = HeuristicRouter(
diff --git a/tests/learning/test_search_space.py b/tests/learning/test_search_space.py
index e220ff2c..4d28b4ba 100644
--- a/tests/learning/test_search_space.py
+++ b/tests/learning/test_search_space.py
@@ -36,8 +36,7 @@ class TestBuildSearchSpace:
},
"constraints": {
"rules": [
- "SimpleAgent should only have "
- "max_turns = 1",
+ "SimpleAgent should only have max_turns = 1",
],
},
},
@@ -350,9 +349,16 @@ class TestDefaultSearchSpace:
dim = _find_dim("engine.backend")
assert dim.dim_type == "categorical"
expected = {
- "ollama", "vllm", "sglang",
- "llamacpp", "mlx", "lmstudio",
- "exo", "nexa", "uzu", "apple_fm",
+ "ollama",
+ "vllm",
+ "sglang",
+ "llamacpp",
+ "mlx",
+ "lmstudio",
+ "exo",
+ "nexa",
+ "uzu",
+ "apple_fm",
}
assert set(dim.values) == expected
@@ -360,8 +366,10 @@ class TestDefaultSearchSpace:
dim = _find_dim("agent.type")
assert dim.dim_type == "categorical"
expected = {
- "simple", "orchestrator",
- "native_react", "native_openhands",
+ "simple",
+ "orchestrator",
+ "native_react",
+ "native_openhands",
}
assert set(dim.values) == expected
@@ -388,15 +396,11 @@ class TestDefaultSearchSpace:
def test_all_dimensions_have_descriptions(self) -> None:
for dim in _DIMS:
- assert dim.description != "", (
- f"Dimension {dim.name} has no description"
- )
+ assert dim.description != "", f"Dimension {dim.name} has no description"
def test_all_dimensions_have_primitives(self) -> None:
for dim in _DIMS:
- assert dim.primitive != "", (
- f"Dimension {dim.name} has no primitive"
- )
+ assert dim.primitive != "", f"Dimension {dim.name} has no primitive"
# ---------------------------------------------------------------------------
@@ -415,15 +419,16 @@ class TestToPromptDescription:
def test_all_dimensions_appear_in_description(self) -> None:
desc = DEFAULT_SEARCH_SPACE.to_prompt_description()
for dim in _DIMS:
- assert dim.name in desc, (
- f"Dimension {dim.name} not in description"
- )
+ assert dim.name in desc, f"Dimension {dim.name} not in description"
def test_all_primitive_headers_in_description(self) -> None:
desc = DEFAULT_SEARCH_SPACE.to_prompt_description()
for primitive in (
- "Intelligence", "Engine", "Agent",
- "Tools", "Learning",
+ "Intelligence",
+ "Engine",
+ "Agent",
+ "Tools",
+ "Learning",
):
assert f"## {primitive}" in desc, (
f"Primitive header {primitive} not in description"
diff --git a/tests/learning/test_skill_discovery.py b/tests/learning/test_skill_discovery.py
index 7873fc4c..0b9d61e4 100644
--- a/tests/learning/test_skill_discovery.py
+++ b/tests/learning/test_skill_discovery.py
@@ -11,6 +11,7 @@ from openjarvis.learning.agents.skill_discovery import SkillDiscovery
# Helpers
# ---------------------------------------------------------------------------
+
@dataclass
class _Step:
step_type: str = "tool_call"
@@ -93,10 +94,7 @@ class TestSkillDiscovery:
def test_outcome_threshold(self):
"""Low-outcome sequences should be filtered out."""
sd = SkillDiscovery(min_frequency=3, min_outcome=0.8)
- traces = [
- _make_trace(["a", "b"], outcome=0.3)
- for _ in range(5)
- ]
+ traces = [_make_trace(["a", "b"], outcome=0.3) for _ in range(5)]
result = sd.analyze_traces(traces)
assert result == []
@@ -104,7 +102,9 @@ class TestSkillDiscovery:
"""Sequences shorter than min or longer than max should be excluded."""
# Only length-1 tools (below min_sequence_length=2)
sd = SkillDiscovery(
- min_frequency=2, min_sequence_length=2, max_sequence_length=3,
+ min_frequency=2,
+ min_sequence_length=2,
+ max_sequence_length=3,
)
short_traces = [_make_trace(["a"], outcome=1.0) for _ in range(5)]
result = sd.analyze_traces(short_traces)
@@ -113,12 +113,11 @@ class TestSkillDiscovery:
# Long sequence: with max_sequence_length=2, a 4-tool sequence
# should only produce subsequences of length 2
sd2 = SkillDiscovery(
- min_frequency=3, min_sequence_length=2, max_sequence_length=2,
+ min_frequency=3,
+ min_sequence_length=2,
+ max_sequence_length=2,
)
- long_traces = [
- _make_trace(["a", "b", "c", "d"], outcome=1.0)
- for _ in range(3)
- ]
+ long_traces = [_make_trace(["a", "b", "c", "d"], outcome=1.0) for _ in range(3)]
result2 = sd2.analyze_traces(long_traces)
# All discovered skills should have exactly 2 tools
for skill in result2:
@@ -127,10 +126,7 @@ class TestSkillDiscovery:
def test_to_skill_manifests(self):
"""Verify manifest dict format has expected keys."""
sd = SkillDiscovery(min_frequency=2, min_outcome=0.0)
- traces = [
- _make_trace(["calc", "save"], outcome=0.9)
- for _ in range(3)
- ]
+ traces = [_make_trace(["calc", "save"], outcome=0.9) for _ in range(3)]
sd.analyze_traces(traces)
manifests = sd.to_skill_manifests()
assert len(manifests) >= 1
@@ -150,15 +146,9 @@ class TestSkillDiscovery:
"""Higher frequency*outcome skills should come first."""
sd = SkillDiscovery(min_frequency=2, min_outcome=0.0)
# Group A: high freq, high outcome
- traces_a = [
- _make_trace(["alpha", "beta"], outcome=1.0)
- for _ in range(10)
- ]
+ traces_a = [_make_trace(["alpha", "beta"], outcome=1.0) for _ in range(10)]
# Group B: low freq, low outcome
- traces_b = [
- _make_trace(["gamma", "delta"], outcome=0.3)
- for _ in range(2)
- ]
+ traces_b = [_make_trace(["gamma", "delta"], outcome=0.3) for _ in range(2)]
result = sd.analyze_traces(traces_a + traces_b)
assert len(result) >= 2
# First should be the higher quality one
@@ -172,7 +162,9 @@ class TestSkillDiscovery:
sd = SkillDiscovery(min_frequency=3, min_outcome=0.5)
traces = [
_make_dict_trace(
- ["read", "compute", "write"], outcome=0.8, query=f"task {i}",
+ ["read", "compute", "write"],
+ outcome=0.8,
+ query=f"task {i}",
)
for i in range(4)
]
diff --git a/tests/learning/test_trial_runner.py b/tests/learning/test_trial_runner.py
index c65c6d29..fd0fc042 100644
--- a/tests/learning/test_trial_runner.py
+++ b/tests/learning/test_trial_runner.py
@@ -185,8 +185,12 @@ class TestRunTrial:
@patch("openjarvis.evals.cli._build_backend")
@patch("openjarvis.evals.core.runner.EvalRunner")
def test_run_trial_returns_trial_result(
- self, mock_runner_cls, mock_build_backend, mock_build_dataset,
- mock_build_judge, mock_build_scorer,
+ self,
+ mock_runner_cls,
+ mock_build_backend,
+ mock_build_dataset,
+ mock_build_judge,
+ mock_build_scorer,
) -> None:
summary = self._make_summary()
mock_runner_instance = MagicMock()
@@ -218,8 +222,12 @@ class TestRunTrial:
@patch("openjarvis.evals.cli._build_backend")
@patch("openjarvis.evals.core.runner.EvalRunner")
def test_run_trial_accuracy_from_summary(
- self, mock_runner_cls, mock_build_backend, mock_build_dataset,
- mock_build_judge, mock_build_scorer,
+ self,
+ mock_runner_cls,
+ mock_build_backend,
+ mock_build_dataset,
+ mock_build_judge,
+ mock_build_scorer,
) -> None:
summary = self._make_summary(accuracy=0.92)
mock_runner_cls.return_value.run.return_value = summary
@@ -238,8 +246,12 @@ class TestRunTrial:
@patch("openjarvis.evals.cli._build_backend")
@patch("openjarvis.evals.core.runner.EvalRunner")
def test_run_trial_tokens_summed(
- self, mock_runner_cls, mock_build_backend, mock_build_dataset,
- mock_build_judge, mock_build_scorer,
+ self,
+ mock_runner_cls,
+ mock_build_backend,
+ mock_build_dataset,
+ mock_build_judge,
+ mock_build_scorer,
) -> None:
summary = self._make_summary(
total_input_tokens=3000,
@@ -261,8 +273,12 @@ class TestRunTrial:
@patch("openjarvis.evals.cli._build_backend")
@patch("openjarvis.evals.core.runner.EvalRunner")
def test_run_trial_summary_attached(
- self, mock_runner_cls, mock_build_backend, mock_build_dataset,
- mock_build_judge, mock_build_scorer,
+ self,
+ mock_runner_cls,
+ mock_build_backend,
+ mock_build_dataset,
+ mock_build_judge,
+ mock_build_scorer,
) -> None:
summary = self._make_summary()
mock_runner_cls.return_value.run.return_value = summary
@@ -281,8 +297,12 @@ class TestRunTrial:
@patch("openjarvis.evals.cli._build_backend")
@patch("openjarvis.evals.core.runner.EvalRunner")
def test_run_trial_failure_modes_on_errors(
- self, mock_runner_cls, mock_build_backend, mock_build_dataset,
- mock_build_judge, mock_build_scorer,
+ self,
+ mock_runner_cls,
+ mock_build_backend,
+ mock_build_dataset,
+ mock_build_judge,
+ mock_build_scorer,
) -> None:
summary = self._make_summary(errors=5)
mock_runner_cls.return_value.run.return_value = summary
@@ -302,8 +322,12 @@ class TestRunTrial:
@patch("openjarvis.evals.cli._build_backend")
@patch("openjarvis.evals.core.runner.EvalRunner")
def test_run_trial_no_failure_modes_when_clean(
- self, mock_runner_cls, mock_build_backend, mock_build_dataset,
- mock_build_judge, mock_build_scorer,
+ self,
+ mock_runner_cls,
+ mock_build_backend,
+ mock_build_dataset,
+ mock_build_judge,
+ mock_build_scorer,
) -> None:
summary = self._make_summary(errors=0)
mock_runner_cls.return_value.run.return_value = summary
@@ -322,8 +346,12 @@ class TestRunTrial:
@patch("openjarvis.evals.cli._build_backend")
@patch("openjarvis.evals.core.runner.EvalRunner")
def test_run_trial_closes_backends(
- self, mock_runner_cls, mock_build_backend, mock_build_dataset,
- mock_build_judge, mock_build_scorer,
+ self,
+ mock_runner_cls,
+ mock_build_backend,
+ mock_build_dataset,
+ mock_build_judge,
+ mock_build_scorer,
) -> None:
summary = self._make_summary()
mock_runner_cls.return_value.run.return_value = summary
@@ -345,8 +373,12 @@ class TestRunTrial:
@patch("openjarvis.evals.cli._build_backend")
@patch("openjarvis.evals.core.runner.EvalRunner")
def test_run_trial_populates_sample_scores(
- self, mock_runner_cls, mock_build_backend, mock_build_dataset,
- mock_build_judge, mock_build_scorer,
+ self,
+ mock_runner_cls,
+ mock_build_backend,
+ mock_build_dataset,
+ mock_build_judge,
+ mock_build_scorer,
) -> None:
from openjarvis.evals.core.types import EvalResult
diff --git a/tests/learning/training/test_data.py b/tests/learning/training/test_data.py
index f5dc8725..086a4763 100644
--- a/tests/learning/training/test_data.py
+++ b/tests/learning/training/test_data.py
@@ -78,7 +78,7 @@ class FakeTraceStore:
self._traces = traces or []
def list_traces(self, *, limit: int = 10000, **kwargs: Any) -> list[Trace]:
- return self._traces[: limit]
+ return self._traces[:limit]
# ---------------------------------------------------------------------------
diff --git a/tests/mcp/test_protocol.py b/tests/mcp/test_protocol.py
index f397cc0a..b6d3b727 100644
--- a/tests/mcp/test_protocol.py
+++ b/tests/mcp/test_protocol.py
@@ -91,7 +91,10 @@ class TestMCPResponse:
def test_error_response_with_data(self):
resp = MCPResponse.error_response(
- 1, INTERNAL_ERROR, "Oops", data={"detail": "stack"},
+ 1,
+ INTERNAL_ERROR,
+ "Oops",
+ data={"detail": "stack"},
)
assert resp.error["data"] == {"detail": "stack"}
diff --git a/tests/mcp/test_transport.py b/tests/mcp/test_transport.py
index e43e0ec3..f1cb545e 100644
--- a/tests/mcp/test_transport.py
+++ b/tests/mcp/test_transport.py
@@ -71,7 +71,8 @@ class TestStdioTransport:
def test_send_receive(self, tmp_path):
"""Use a simple Python echo script as the subprocess."""
script = tmp_path / "echo_server.py"
- script.write_text(textwrap.dedent("""\
+ script.write_text(
+ textwrap.dedent("""\
import sys
import json
for line in sys.stdin:
@@ -86,7 +87,8 @@ class TestStdioTransport:
}
sys.stdout.write(json.dumps(resp) + "\\n")
sys.stdout.flush()
- """))
+ """)
+ )
transport = StdioTransport([sys.executable, str(script)])
try:
@@ -101,7 +103,8 @@ class TestStdioTransport:
def test_multiple_requests(self, tmp_path):
"""Send multiple requests to the subprocess."""
script = tmp_path / "echo_server.py"
- script.write_text(textwrap.dedent("""\
+ script.write_text(
+ textwrap.dedent("""\
import sys
import json
for line in sys.stdin:
@@ -116,7 +119,8 @@ class TestStdioTransport:
}
sys.stdout.write(json.dumps(resp) + "\\n")
sys.stdout.flush()
- """))
+ """)
+ )
transport = StdioTransport([sys.executable, str(script)])
try:
@@ -129,11 +133,13 @@ class TestStdioTransport:
def test_close_terminates_process(self, tmp_path):
script = tmp_path / "sleep_server.py"
- script.write_text(textwrap.dedent("""\
+ script.write_text(
+ textwrap.dedent("""\
import sys
import time
time.sleep(300)
- """))
+ """)
+ )
transport = StdioTransport([sys.executable, str(script)])
proc = transport._process
diff --git a/tests/memory/test_bm25.py b/tests/memory/test_bm25.py
index 6209104c..6aae1269 100644
--- a/tests/memory/test_bm25.py
+++ b/tests/memory/test_bm25.py
@@ -130,10 +130,7 @@ def test_event_bus_store():
mod.get_event_bus = lambda: bus
try:
backend.store("test event emission")
- events = [
- e for e in bus.history
- if e.event_type == EventType.MEMORY_STORE
- ]
+ events = [e for e in bus.history if e.event_type == EventType.MEMORY_STORE]
assert len(events) == 1
assert events[0].data["backend"] == "bm25"
assert "doc_id" in events[0].data
@@ -151,10 +148,7 @@ def test_event_bus_retrieve():
mod.get_event_bus = lambda: bus
try:
backend.retrieve("searchable")
- events = [
- e for e in bus.history
- if e.event_type == EventType.MEMORY_RETRIEVE
- ]
+ events = [e for e in bus.history if e.event_type == EventType.MEMORY_RETRIEVE]
assert len(events) == 1
assert events[0].data["backend"] == "bm25"
assert events[0].data["num_results"] >= 1
diff --git a/tests/memory/test_colbert.py b/tests/memory/test_colbert.py
index 598ca52c..1053fa87 100644
--- a/tests/memory/test_colbert.py
+++ b/tests/memory/test_colbert.py
@@ -64,16 +64,20 @@ def test_maxsim_scoring():
"""Test the _maxsim function directly with synthetic tensors."""
backend = _make_backend()
# 2 query tokens, dim 4
- q = torch.tensor([
- [1.0, 0.0, 0.0, 0.0],
- [0.0, 1.0, 0.0, 0.0],
- ])
+ q = torch.tensor(
+ [
+ [1.0, 0.0, 0.0, 0.0],
+ [0.0, 1.0, 0.0, 0.0],
+ ]
+ )
# 3 doc tokens, dim 4
- d = torch.tensor([
- [1.0, 0.0, 0.0, 0.0],
- [0.0, 0.0, 1.0, 0.0],
- [0.0, 1.0, 0.0, 0.0],
- ])
+ d = torch.tensor(
+ [
+ [1.0, 0.0, 0.0, 0.0],
+ [0.0, 0.0, 1.0, 0.0],
+ [0.0, 1.0, 0.0, 0.0],
+ ]
+ )
score = backend._maxsim(q, d)
# query[0] max sim: max(1.0, 0.0, 0.0) = 1.0
# query[1] max sim: max(0.0, 0.0, 1.0) = 1.0
@@ -134,10 +138,7 @@ def test_event_bus_store():
mod.get_event_bus = lambda: bus
try:
backend.store("test event emission")
- events = [
- e for e in bus.history
- if e.event_type == EventType.MEMORY_STORE
- ]
+ events = [e for e in bus.history if e.event_type == EventType.MEMORY_STORE]
assert len(events) == 1
assert events[0].data["backend"] == "colbert"
assert "doc_id" in events[0].data
@@ -155,10 +156,7 @@ def test_event_bus_retrieve():
mod.get_event_bus = lambda: bus
try:
backend.retrieve("searchable")
- events = [
- e for e in bus.history
- if e.event_type == EventType.MEMORY_RETRIEVE
- ]
+ events = [e for e in bus.history if e.event_type == EventType.MEMORY_RETRIEVE]
assert len(events) == 1
assert events[0].data["backend"] == "colbert"
assert events[0].data["num_results"] >= 0
diff --git a/tests/memory/test_context.py b/tests/memory/test_context.py
index 5f7f0a15..8aa036ad 100644
--- a/tests/memory/test_context.py
+++ b/tests/memory/test_context.py
@@ -17,6 +17,7 @@ from openjarvis.tools.storage.context import (
# -- Fake backend for testing ------------------------------------------------
+
class _FakeMemory(MemoryBackend):
"""In-memory backend that returns pre-set results."""
@@ -113,7 +114,10 @@ def test_inject_context_filters_low_score():
messages = [Message(role=Role.USER, content="hello")]
cfg = ContextConfig(min_score=0.1)
augmented = inject_context(
- "query", messages, backend, config=cfg,
+ "query",
+ messages,
+ backend,
+ config=cfg,
)
# Low score filtered out — no context added
assert len(augmented) == 1
@@ -130,7 +134,10 @@ def test_inject_context_respects_max_tokens():
messages = [Message(role=Role.USER, content="test")]
cfg = ContextConfig(max_context_tokens=150)
augmented = inject_context(
- "query", messages, backend, config=cfg,
+ "query",
+ messages,
+ backend,
+ config=cfg,
)
assert len(augmented) == 2 # system + user
# Only one source should be cited
@@ -145,7 +152,10 @@ def test_inject_context_disabled():
messages = [Message(role=Role.USER, content="hello")]
cfg = ContextConfig(enabled=False)
augmented = inject_context(
- "query", messages, backend, config=cfg,
+ "query",
+ messages,
+ backend,
+ config=cfg,
)
assert len(augmented) == 1
@@ -166,14 +176,12 @@ def test_inject_context_publishes_event():
messages = [Message(role=Role.USER, content="hello")]
import openjarvis.tools.storage.context as mod
+
original = mod.get_event_bus
mod.get_event_bus = lambda: bus
try:
inject_context("query", messages, backend)
- events = [
- e for e in bus.history
- if e.event_type == EventType.MEMORY_RETRIEVE
- ]
+ events = [e for e in bus.history if e.event_type == EventType.MEMORY_RETRIEVE]
assert len(events) == 1
assert events[0].data["context_injection"] is True
finally:
diff --git a/tests/memory/test_faiss.py b/tests/memory/test_faiss.py
index a7100ec1..6d0e37a9 100644
--- a/tests/memory/test_faiss.py
+++ b/tests/memory/test_faiss.py
@@ -30,14 +30,10 @@ class _FakeEmbedder(Embedder):
def embed(self, texts: list[str]) -> np.ndarray: # type: ignore[override]
results = []
for text in texts:
- rng = np.random.RandomState(
- hash(text) % 2**31
- )
+ rng = np.random.RandomState(hash(text) % 2**31)
vec = rng.randn(64).astype(np.float32)
results.append(vec)
- return np.array(results) if results else np.empty(
- (0, 64), dtype=np.float32
- )
+ return np.array(results) if results else np.empty((0, 64), dtype=np.float32)
def dim(self) -> int:
return 64
@@ -174,11 +170,7 @@ def test_event_bus_store():
mod.get_event_bus = lambda: bus
try:
backend.store("event test document")
- events = [
- e
- for e in bus.history
- if e.event_type == EventType.MEMORY_STORE
- ]
+ events = [e for e in bus.history if e.event_type == EventType.MEMORY_STORE]
assert len(events) == 1
assert events[0].data["backend"] == "faiss"
assert "doc_id" in events[0].data
@@ -198,11 +190,7 @@ def test_event_bus_retrieve():
mod.get_event_bus = lambda: bus
try:
backend.retrieve("searchable")
- events = [
- e
- for e in bus.history
- if e.event_type == EventType.MEMORY_RETRIEVE
- ]
+ events = [e for e in bus.history if e.event_type == EventType.MEMORY_RETRIEVE]
assert len(events) == 1
assert events[0].data["backend"] == "faiss"
assert events[0].data["num_results"] >= 0
diff --git a/tests/memory/test_hybrid.py b/tests/memory/test_hybrid.py
index dc2f5223..70326d78 100644
--- a/tests/memory/test_hybrid.py
+++ b/tests/memory/test_hybrid.py
@@ -46,12 +46,14 @@ class _FakeBackend(MemoryBackend):
results = []
for doc_id, (content, source, meta) in self._docs.items():
if query.lower() in content.lower():
- results.append(RetrievalResult(
- content=content,
- score=1.0,
- source=source,
- metadata=meta,
- ))
+ results.append(
+ RetrievalResult(
+ content=content,
+ score=1.0,
+ source=source,
+ metadata=meta,
+ )
+ )
return results[:top_k]
def delete(self, doc_id: str) -> bool:
@@ -127,7 +129,9 @@ def test_rrf_custom_weights():
list1 = [RetrievalResult(content="A", score=1.0)]
list2 = [RetrievalResult(content="B", score=1.0)]
fused = reciprocal_rank_fusion(
- [list1, list2], k=60, weights=[2.0, 1.0],
+ [list1, list2],
+ k=60,
+ weights=[2.0, 1.0],
)
scores = {r.content: r.score for r in fused}
# A has weight 2, B has weight 1, both at rank 0
@@ -191,19 +195,14 @@ def test_event_bus_store():
bus = EventBus(record_history=True)
hybrid = _make_hybrid()
import openjarvis.tools.storage.hybrid as mod
+
original = mod.get_event_bus
mod.get_event_bus = lambda: bus
try:
hybrid.store("event test content")
- events = [
- e for e in bus.history
- if e.event_type == EventType.MEMORY_STORE
- ]
+ events = [e for e in bus.history if e.event_type == EventType.MEMORY_STORE]
assert len(events) >= 1
- assert any(
- e.data.get("backend") == "hybrid"
- for e in events
- )
+ assert any(e.data.get("backend") == "hybrid" for e in events)
finally:
mod.get_event_bus = original
@@ -213,18 +212,13 @@ def test_event_bus_retrieve():
hybrid = _make_hybrid()
hybrid.store("retrievable content here")
import openjarvis.tools.storage.hybrid as mod
+
original = mod.get_event_bus
mod.get_event_bus = lambda: bus
try:
hybrid.retrieve("retrievable")
- events = [
- e for e in bus.history
- if e.event_type == EventType.MEMORY_RETRIEVE
- ]
+ events = [e for e in bus.history if e.event_type == EventType.MEMORY_RETRIEVE]
assert len(events) >= 1
- assert any(
- e.data.get("backend") == "hybrid"
- for e in events
- )
+ assert any(e.data.get("backend") == "hybrid" for e in events)
finally:
mod.get_event_bus = original
diff --git a/tests/memory/test_retrieval_quality.py b/tests/memory/test_retrieval_quality.py
index d520c378..d49ce7f2 100644
--- a/tests/memory/test_retrieval_quality.py
+++ b/tests/memory/test_retrieval_quality.py
@@ -39,7 +39,8 @@ def _make_sqlite(tmp_path):
def _make_bm25():
bm25_mod = pytest.importorskip(
- "openjarvis.tools.storage.bm25", exc_type=ImportError,
+ "openjarvis.tools.storage.bm25",
+ exc_type=ImportError,
)
BM25Memory = bm25_mod.BM25Memory
if not MemoryRegistry.contains("bm25"):
diff --git a/tests/memory/test_sqlite.py b/tests/memory/test_sqlite.py
index bbbf2dc7..987c6cb4 100644
--- a/tests/memory/test_sqlite.py
+++ b/tests/memory/test_sqlite.py
@@ -134,14 +134,12 @@ def test_event_bus_integration_store(tmp_path: Path):
backend = _make_backend(tmp_path)
# Monkey-patch the global bus for this test
import openjarvis.tools.storage.sqlite as mod
+
original = mod.get_event_bus
mod.get_event_bus = lambda: bus
try:
backend.store("test event emission")
- events = [
- e for e in bus.history
- if e.event_type == EventType.MEMORY_STORE
- ]
+ events = [e for e in bus.history if e.event_type == EventType.MEMORY_STORE]
assert len(events) == 1
assert events[0].data["backend"] == "sqlite"
finally:
@@ -154,14 +152,12 @@ def test_event_bus_integration_retrieve(tmp_path: Path):
backend = _make_backend(tmp_path)
backend.store("searchable content for events")
import openjarvis.tools.storage.sqlite as mod
+
original = mod.get_event_bus
mod.get_event_bus = lambda: bus
try:
backend.retrieve("searchable")
- events = [
- e for e in bus.history
- if e.event_type == EventType.MEMORY_RETRIEVE
- ]
+ events = [e for e in bus.history if e.event_type == EventType.MEMORY_RETRIEVE]
assert len(events) == 1
assert events[0].data["backend"] == "sqlite"
finally:
diff --git a/tests/memory/test_storage_suite.py b/tests/memory/test_storage_suite.py
index ae5bbbcf..9adbee6f 100644
--- a/tests/memory/test_storage_suite.py
+++ b/tests/memory/test_storage_suite.py
@@ -20,7 +20,8 @@ def _make_sqlite(tmp_path):
def _make_bm25():
bm25_mod = pytest.importorskip(
- "openjarvis.tools.storage.bm25", exc_type=ImportError,
+ "openjarvis.tools.storage.bm25",
+ exc_type=ImportError,
)
BM25Memory = bm25_mod.BM25Memory
if not MemoryRegistry.contains("bm25"):
@@ -171,9 +172,7 @@ class TestStorageSuiteOptional:
def test_delete_document(self, backend_key, tmp_path):
if backend_key == "hybrid":
- pytest.skip(
- "HybridMemory sub-backend BM25 lacks delete()"
- )
+ pytest.skip("HybridMemory sub-backend BM25 lacks delete()")
backend = _make_backend(backend_key, tmp_path)
doc_id = backend.store("content to delete")
assert backend.delete(doc_id) is True
@@ -181,9 +180,7 @@ class TestStorageSuiteOptional:
def test_clear_all(self, backend_key, tmp_path):
if backend_key == "hybrid":
- pytest.skip(
- "HybridMemory sub-backend BM25 lacks clear()"
- )
+ pytest.skip("HybridMemory sub-backend BM25 lacks clear()")
backend = _make_backend(backend_key, tmp_path)
backend.store("first document")
backend.store("second document")
diff --git a/tests/operators/test_operators.py b/tests/operators/test_operators.py
index b67bffe5..065dc596 100644
--- a/tests/operators/test_operators.py
+++ b/tests/operators/test_operators.py
@@ -314,8 +314,11 @@ name = "Discovered"
mgr = OperatorManager(system)
m = OperatorManifest(
- id="test_op", name="Test",
- tools=["think"], schedule_type="interval", schedule_value="60",
+ id="test_op",
+ name="Test",
+ tools=["think"],
+ schedule_type="interval",
+ schedule_value="60",
)
mgr.register(m)
task_id = mgr.activate("test_op")
@@ -352,8 +355,10 @@ name = "Discovered"
mgr = OperatorManager(system)
m = OperatorManifest(
- id="meta_test", name="Meta",
- system_prompt="Do stuff", temperature=0.5,
+ id="meta_test",
+ name="Meta",
+ system_prompt="Do stuff",
+ temperature=0.5,
)
mgr.register(m)
mgr.activate("meta_test")
@@ -452,8 +457,10 @@ name = "Discovered"
mgr = OperatorManager(system)
m = OperatorManifest(
- id="run_test", name="Run",
- system_prompt="Test prompt", tools=["think"],
+ id="run_test",
+ name="Run",
+ system_prompt="Test prompt",
+ tools=["think"],
)
mgr.register(m)
@@ -494,7 +501,8 @@ class TestOperativeAgent:
engine = FakeEngine([{"content": "Response with prompt."}])
agent = OperativeAgent(
- engine, "test-model",
+ engine,
+ "test-model",
system_prompt="You are a test operator.",
)
result = agent.run("Execute tick")
@@ -504,8 +512,7 @@ class TestOperativeAgent:
call = engine.calls[0]
messages = call["messages"]
assert any(
- m.role.value == "system" and "test operator" in m.content
- for m in messages
+ m.role.value == "system" and "test operator" in m.content for m in messages
)
def test_run_loads_session(self):
@@ -521,7 +528,8 @@ class TestOperativeAgent:
engine = FakeEngine([{"content": "New tick done."}])
agent = OperativeAgent(
- engine, "test-model",
+ engine,
+ "test-model",
operator_id="test_op",
session_store=session_store,
)
@@ -535,7 +543,8 @@ class TestOperativeAgent:
engine = FakeEngine([{"content": "Tick response."}])
agent = OperativeAgent(
- engine, "test-model",
+ engine,
+ "test-model",
operator_id="save_test",
session_store=session_store,
)
@@ -555,7 +564,8 @@ class TestOperativeAgent:
engine = FakeEngine([{"content": "State recalled."}])
agent = OperativeAgent(
- engine, "test-model",
+ engine,
+ "test-model",
operator_id="recall_test",
memory_backend=memory,
)
@@ -583,22 +593,25 @@ class TestOperativeAgent:
tool.spec.taint_labels = []
tool.run = MagicMock(return_value="Thought result")
- engine = FakeEngine([
- {
- "content": "",
- "tool_calls": [
- {
- "id": "call_1",
- "name": "think",
- "arguments": '{"thought": "test"}',
- },
- ],
- },
- {"content": "Final answer after tool use."},
- ])
+ engine = FakeEngine(
+ [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "id": "call_1",
+ "name": "think",
+ "arguments": '{"thought": "test"}',
+ },
+ ],
+ },
+ {"content": "Final answer after tool use."},
+ ]
+ )
agent = OperativeAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[tool],
)
result = agent.run("Do something")
@@ -620,7 +633,8 @@ class TestOperativeAgent:
engine = FakeEngine([{"content": "Tick complete."}])
agent = OperativeAgent(
- engine, "test-model",
+ engine,
+ "test-model",
operator_id="persist_test",
memory_backend=memory,
)
@@ -660,7 +674,8 @@ class TestOperativeAgent:
tool.run = MagicMock(return_value="thought")
agent = OperativeAgent(
- engine, "test-model",
+ engine,
+ "test-model",
tools=[tool],
max_turns=3,
)
@@ -685,15 +700,26 @@ class TestSystemAskPassthrough:
# Patch _run_agent to capture kwargs
captured = {}
- def patched_run_agent(self, query, messages, agent_name, tool_names,
- temperature, max_tokens, **kwargs):
+ def patched_run_agent(
+ self,
+ query,
+ messages,
+ agent_name,
+ tool_names,
+ temperature,
+ max_tokens,
+ **kwargs,
+ ):
captured.update(kwargs)
return {"content": "OK"}
with patch.object(JarvisSystem, "_run_agent", patched_run_agent):
real_system = JarvisSystem(
- config=system.config, bus=system.bus,
- engine=engine, engine_key="test", model="test-model",
+ config=system.config,
+ bus=system.bus,
+ engine=engine,
+ engine_key="test",
+ model="test-model",
agent_name="operative",
)
real_system.ask("test", system_prompt="Custom prompt")
@@ -709,15 +735,26 @@ class TestSystemAskPassthrough:
captured = {}
- def patched_run_agent(self, query, messages, agent_name, tool_names,
- temperature, max_tokens, **kwargs):
+ def patched_run_agent(
+ self,
+ query,
+ messages,
+ agent_name,
+ tool_names,
+ temperature,
+ max_tokens,
+ **kwargs,
+ ):
captured.update(kwargs)
return {"content": "OK"}
with patch.object(JarvisSystem, "_run_agent", patched_run_agent):
real_system = JarvisSystem(
- config=system.config, bus=system.bus,
- engine=engine, engine_key="test", model="test-model",
+ config=system.config,
+ bus=system.bus,
+ engine=engine,
+ engine_key="test",
+ model="test-model",
agent_name="operative",
)
real_system.ask("test", operator_id="my_op")
@@ -762,8 +799,10 @@ class TestSchedulerOperatorExecution:
mock_system.ask.assert_called_once()
call_kwargs = mock_system.ask.call_args
# The scheduler passes agent and tools
- assert call_kwargs.kwargs.get("agent") == "operative" or \
- call_kwargs[1].get("agent") == "operative"
+ assert (
+ call_kwargs.kwargs.get("agent") == "operative"
+ or call_kwargs[1].get("agent") == "operative"
+ )
# ---------------------------------------------------------------------------
@@ -815,6 +854,7 @@ class TestAgentRegistration:
# Re-register if cleared by another test
if not AgentRegistry.contains("operative"):
from openjarvis.agents.operative import OperativeAgent
+
AgentRegistry.register_value("operative", OperativeAgent)
assert AgentRegistry.contains("operative")
diff --git a/tests/prompt/test_builder.py b/tests/prompt/test_builder.py
index 13cc20ac..14b1d0f9 100644
--- a/tests/prompt/test_builder.py
+++ b/tests/prompt/test_builder.py
@@ -20,6 +20,7 @@ def memory_dir(tmp_path: Path) -> Path:
def test_build_frozen_prefix(memory_dir: Path):
from openjarvis.prompt.builder import SystemPromptBuilder
+
builder = SystemPromptBuilder(
agent_template="You are Jarvis.",
memory_files_config=MemoryFilesConfig(
@@ -38,6 +39,7 @@ def test_build_frozen_prefix(memory_dir: Path):
def test_frozen_prefix_stability(memory_dir: Path):
from openjarvis.prompt.builder import SystemPromptBuilder
+
builder = SystemPromptBuilder(
agent_template="You are Jarvis.",
memory_files_config=MemoryFilesConfig(
@@ -55,6 +57,7 @@ def test_frozen_prefix_stability(memory_dir: Path):
def test_char_limit_truncation(memory_dir: Path):
from openjarvis.prompt.builder import SystemPromptBuilder
+
(memory_dir / "SOUL.md").write_text("x" * 10000)
builder = SystemPromptBuilder(
agent_template="You are Jarvis.",
@@ -72,6 +75,7 @@ def test_char_limit_truncation(memory_dir: Path):
def test_skill_index_in_prompt(memory_dir: Path):
from openjarvis.prompt.builder import SystemPromptBuilder
+
skills = [("api_health_check", "Check API health across all endpoints")]
builder = SystemPromptBuilder(
agent_template="You are Jarvis.",
@@ -90,6 +94,7 @@ def test_skill_index_in_prompt(memory_dir: Path):
def test_dynamic_section_appended(memory_dir: Path):
from openjarvis.prompt.builder import SystemPromptBuilder
+
builder = SystemPromptBuilder(
agent_template="You are Jarvis.",
memory_files_config=MemoryFilesConfig(
@@ -106,6 +111,7 @@ def test_dynamic_section_appended(memory_dir: Path):
def test_missing_files_handled(tmp_path: Path):
from openjarvis.prompt.builder import SystemPromptBuilder
+
builder = SystemPromptBuilder(
agent_template="You are Jarvis.",
memory_files_config=MemoryFilesConfig(
diff --git a/tests/recipes/test_loader.py b/tests/recipes/test_loader.py
index 02fa024b..6a04d700 100644
--- a/tests/recipes/test_loader.py
+++ b/tests/recipes/test_loader.py
@@ -74,7 +74,7 @@ class TestLoadRecipe:
def test_load_recipe_defaults(self, tmp_path: Path) -> None:
"""Minimal TOML should yield sensible defaults."""
toml_file = tmp_path / "minimal.toml"
- toml_file.write_text("[recipe]\nname = \"minimal\"\n")
+ toml_file.write_text('[recipe]\nname = "minimal"\n')
recipe = load_recipe(toml_file)
@@ -87,7 +87,7 @@ class TestLoadRecipe:
def test_load_recipe_name_from_filename(self, tmp_path: Path) -> None:
"""When [recipe] has no name, use the file stem."""
toml_file = tmp_path / "my_recipe.toml"
- toml_file.write_text("[recipe]\ndescription = \"no name\"\n")
+ toml_file.write_text('[recipe]\ndescription = "no name"\n')
recipe = load_recipe(toml_file)
assert recipe.name == "my_recipe"
@@ -104,9 +104,7 @@ class TestDiscoverRecipes:
def test_discover_extra_dirs(self, tmp_path: Path) -> None:
toml_file = tmp_path / "custom.toml"
- toml_file.write_text(
- '[recipe]\nname = "custom"\ndescription = "extra"\n'
- )
+ toml_file.write_text('[recipe]\nname = "custom"\ndescription = "extra"\n')
recipes = discover_recipes(extra_dirs=[tmp_path])
names = {r.name for r in recipes}
assert "custom" in names
diff --git a/tests/sandbox/test_runner.py b/tests/sandbox/test_runner.py
index 308dd84d..308ab272 100644
--- a/tests/sandbox/test_runner.py
+++ b/tests/sandbox/test_runner.py
@@ -70,7 +70,9 @@ class TestBuildDockerArgs:
runner = ContainerRunner()
with patch("shutil.which", return_value="/usr/bin/docker"):
args = runner._build_docker_args(
- "test-container", [], None,
+ "test-container",
+ [],
+ None,
)
assert "/usr/bin/docker" in args
assert "run" in args
@@ -85,7 +87,9 @@ class TestBuildDockerArgs:
runner = ContainerRunner()
with patch("shutil.which", return_value="/usr/bin/docker"):
args = runner._build_docker_args(
- "test-container", ["/data/project"], None,
+ "test-container",
+ ["/data/project"],
+ None,
)
assert "-v" in args
idx = args.index("-v")
@@ -95,7 +99,9 @@ class TestBuildDockerArgs:
runner = ContainerRunner()
with patch("shutil.which", return_value="/usr/bin/docker"):
args = runner._build_docker_args(
- "test-container", [], {"FOO": "bar"},
+ "test-container",
+ [],
+ {"FOO": "bar"},
)
assert "-e" in args
idx = args.index("-e")
@@ -106,40 +112,62 @@ class TestContainerRunnerRun:
def test_successful_run(self):
runner = ContainerRunner()
output = _wrap_output({"content": "Hello!"})
- with patch("shutil.which", return_value="/usr/bin/docker"), \
- patch("subprocess.run", return_value=_mock_proc(
- stdout=output,
- )):
+ with (
+ patch("shutil.which", return_value="/usr/bin/docker"),
+ patch(
+ "subprocess.run",
+ return_value=_mock_proc(
+ stdout=output,
+ ),
+ ),
+ ):
result = runner.run({"prompt": "test"})
assert result["content"] == "Hello!"
def test_timeout(self):
runner = ContainerRunner(timeout=5)
- with patch("shutil.which", return_value="/usr/bin/docker"), \
- patch("subprocess.run", side_effect=subprocess.TimeoutExpired(
- cmd=["docker"], timeout=5,
- )), \
- patch.object(runner, "stop"):
+ with (
+ patch("shutil.which", return_value="/usr/bin/docker"),
+ patch(
+ "subprocess.run",
+ side_effect=subprocess.TimeoutExpired(
+ cmd=["docker"],
+ timeout=5,
+ ),
+ ),
+ patch.object(runner, "stop"),
+ ):
result = runner.run({"prompt": "test"})
assert result["error"] is True
assert result["error_type"] == "timeout"
def test_nonzero_exit(self):
runner = ContainerRunner()
- with patch("shutil.which", return_value="/usr/bin/docker"), \
- patch("subprocess.run", return_value=_mock_proc(
- returncode=1, stderr="OOM killed",
- )):
+ with (
+ patch("shutil.which", return_value="/usr/bin/docker"),
+ patch(
+ "subprocess.run",
+ return_value=_mock_proc(
+ returncode=1,
+ stderr="OOM killed",
+ ),
+ ),
+ ):
result = runner.run({"prompt": "test"})
assert result["error"] is True
assert "OOM killed" in result["content"]
def test_no_sentinel_output(self):
runner = ContainerRunner()
- with patch("shutil.which", return_value="/usr/bin/docker"), \
- patch("subprocess.run", return_value=_mock_proc(
- stdout="plain text output",
- )):
+ with (
+ patch("shutil.which", return_value="/usr/bin/docker"),
+ patch(
+ "subprocess.run",
+ return_value=_mock_proc(
+ stdout="plain text output",
+ ),
+ ),
+ ):
result = runner.run({"prompt": "test"})
assert result["content"] == "plain text output"
@@ -155,8 +183,10 @@ class TestContainerRunnerRuntimeCheck:
class TestCleanupOrphans:
def test_cleanup_orphans(self):
runner = ContainerRunner()
- with patch("shutil.which", return_value="/usr/bin/docker"), \
- patch("subprocess.run") as mock_run:
+ with (
+ patch("shutil.which", return_value="/usr/bin/docker"),
+ patch("subprocess.run") as mock_run,
+ ):
mock_run.return_value = _mock_proc(stdout="abc123\ndef456")
runner.cleanup_orphans()
assert mock_run.call_count == 2 # ps + rm
@@ -198,8 +228,10 @@ class TestSandboxedAgentInit:
runner = MagicMock(spec=ContainerRunner)
agent = SandboxedAgent(
- inner, runner,
- engine=inner._engine, model="test-model",
+ inner,
+ runner,
+ engine=inner._engine,
+ model="test-model",
)
assert agent._wrapped_agent is inner
assert agent._runner is runner
@@ -219,7 +251,10 @@ class TestSandboxedAgentRun:
engine = MagicMock()
agent = SandboxedAgent(
- inner, runner, engine=engine, model="test-model",
+ inner,
+ runner,
+ engine=engine,
+ model="test-model",
)
result = agent.run("hello")
@@ -246,7 +281,10 @@ class TestSandboxedAgentRun:
engine = MagicMock()
agent = SandboxedAgent(
- inner, runner, engine=engine, model="m",
+ inner,
+ runner,
+ engine=engine,
+ model="m",
)
result = agent.run("compute")
@@ -264,7 +302,11 @@ class TestSandboxedAgentRun:
bus = EventBus(record_history=True)
engine = MagicMock()
agent = SandboxedAgent(
- inner, runner, engine=engine, model="m", bus=bus,
+ inner,
+ runner,
+ engine=engine,
+ model="m",
+ bus=bus,
)
agent.run("test")
diff --git a/tests/scheduler/test_scheduler.py b/tests/scheduler/test_scheduler.py
index abdddeda..0643bc15 100644
--- a/tests/scheduler/test_scheduler.py
+++ b/tests/scheduler/test_scheduler.py
@@ -164,15 +164,20 @@ class TestComputeNextRun:
def test_once_not_yet_run(self, scheduler):
target = "2099-06-15T12:00:00+00:00"
task = ScheduledTask(
- id="t", prompt="p", schedule_type="once",
- schedule_value=target, last_run=None,
+ id="t",
+ prompt="p",
+ schedule_type="once",
+ schedule_value=target,
+ last_run=None,
)
next_run = scheduler._compute_next_run(task)
assert next_run == target
def test_once_already_run(self, scheduler):
task = ScheduledTask(
- id="t", prompt="p", schedule_type="once",
+ id="t",
+ prompt="p",
+ schedule_type="once",
schedule_value="2099-06-15T12:00:00+00:00",
last_run="2099-06-15T12:01:00+00:00",
)
@@ -181,7 +186,9 @@ class TestComputeNextRun:
def test_cron_fallback(self, scheduler):
task = ScheduledTask(
- id="t", prompt="p", schedule_type="cron",
+ id="t",
+ prompt="p",
+ schedule_type="cron",
schedule_value="30 2 * * *",
)
next_run = scheduler._compute_next_run(task)
@@ -266,7 +273,9 @@ class TestExecuteTask:
sched = TaskScheduler(store, system=mock_system, poll_interval=1)
task = sched.create_task(
- "what is 2+2?", "once", "2026-01-01T00:00:00+00:00",
+ "what is 2+2?",
+ "once",
+ "2026-01-01T00:00:00+00:00",
tools="calculator,think",
)
sched._execute_task(task)
diff --git a/tests/scheduler/test_store.py b/tests/scheduler/test_store.py
index 00fc8f46..cb4098be 100644
--- a/tests/scheduler/test_store.py
+++ b/tests/scheduler/test_store.py
@@ -152,8 +152,8 @@ class TestRunLogs:
for i in range(20):
store.log_run(
task_id="t1",
- started_at=f"2026-01-{i+1:02d}T00:00:00+00:00",
- finished_at=f"2026-01-{i+1:02d}T00:01:00+00:00",
+ started_at=f"2026-01-{i + 1:02d}T00:00:00+00:00",
+ finished_at=f"2026-01-{i + 1:02d}T00:01:00+00:00",
success=True,
)
logs = store.get_run_logs("t1", limit=5)
diff --git a/tests/security/test_audit.py b/tests/security/test_audit.py
index b358e53e..6e86e8d5 100644
--- a/tests/security/test_audit.py
+++ b/tests/security/test_audit.py
@@ -61,18 +61,24 @@ class TestAuditLogger:
def test_query_by_type(self, tmp_path: Path) -> None:
logger = AuditLogger(db_path=tmp_path / "audit.db")
- logger.log(SecurityEvent(
- event_type=SecurityEventType.SECRET_DETECTED,
- timestamp=time.time(),
- ))
- logger.log(SecurityEvent(
- event_type=SecurityEventType.PII_DETECTED,
- timestamp=time.time(),
- ))
- logger.log(SecurityEvent(
- event_type=SecurityEventType.SECRET_DETECTED,
- timestamp=time.time(),
- ))
+ logger.log(
+ SecurityEvent(
+ event_type=SecurityEventType.SECRET_DETECTED,
+ timestamp=time.time(),
+ )
+ )
+ logger.log(
+ SecurityEvent(
+ event_type=SecurityEventType.PII_DETECTED,
+ timestamp=time.time(),
+ )
+ )
+ logger.log(
+ SecurityEvent(
+ event_type=SecurityEventType.SECRET_DETECTED,
+ timestamp=time.time(),
+ )
+ )
secrets = logger.query(event_type="secret_detected")
assert len(secrets) == 2
@@ -87,14 +93,18 @@ class TestAuditLogger:
t1 = time.time() - 100
t2 = time.time()
- logger.log(SecurityEvent(
- event_type=SecurityEventType.SECRET_DETECTED,
- timestamp=t1,
- ))
- logger.log(SecurityEvent(
- event_type=SecurityEventType.SECRET_DETECTED,
- timestamp=t2,
- ))
+ logger.log(
+ SecurityEvent(
+ event_type=SecurityEventType.SECRET_DETECTED,
+ timestamp=t1,
+ )
+ )
+ logger.log(
+ SecurityEvent(
+ event_type=SecurityEventType.SECRET_DETECTED,
+ timestamp=t2,
+ )
+ )
recent = logger.query(since=t2 - 1)
assert len(recent) == 1
@@ -146,10 +156,12 @@ class TestAuditLogger:
def test_close_and_reopen(self, tmp_path: Path) -> None:
db_path = tmp_path / "audit.db"
logger = AuditLogger(db_path=db_path)
- logger.log(SecurityEvent(
- event_type=SecurityEventType.SECRET_DETECTED,
- timestamp=time.time(),
- ))
+ logger.log(
+ SecurityEvent(
+ event_type=SecurityEventType.SECRET_DETECTED,
+ timestamp=time.time(),
+ )
+ )
logger.close()
logger2 = AuditLogger(db_path=db_path)
diff --git a/tests/security/test_capabilities.py b/tests/security/test_capabilities.py
index 3b1a592e..958f2328 100644
--- a/tests/security/test_capabilities.py
+++ b/tests/security/test_capabilities.py
@@ -18,9 +18,16 @@ class TestCapability:
def test_all_capabilities_exist(self):
expected = {
- "file:read", "file:write", "network:fetch", "code:execute",
- "memory:read", "memory:write", "channel:send", "tool:invoke",
- "schedule:create", "system:admin",
+ "file:read",
+ "file:write",
+ "network:fetch",
+ "code:execute",
+ "memory:read",
+ "memory:write",
+ "channel:send",
+ "tool:invoke",
+ "schedule:create",
+ "system:admin",
}
actual = {c.value for c in Capability}
assert expected == actual
diff --git a/tests/security/test_credential_stripper.py b/tests/security/test_credential_stripper.py
index 01f7f781..b7a65d2f 100644
--- a/tests/security/test_credential_stripper.py
+++ b/tests/security/test_credential_stripper.py
@@ -3,6 +3,7 @@ from __future__ import annotations
def test_strips_openai_key():
from openjarvis.security.credential_stripper import CredentialStripper
+
stripper = CredentialStripper()
text = (
"Error: auth failed with key "
@@ -15,6 +16,7 @@ def test_strips_openai_key():
def test_strips_aws_key():
from openjarvis.security.credential_stripper import CredentialStripper
+
stripper = CredentialStripper()
text = "Using credentials AKIAIOSFODNN7EXAMPLE for access"
result = stripper.strip(text)
@@ -24,6 +26,7 @@ def test_strips_aws_key():
def test_strips_github_token():
from openjarvis.security.credential_stripper import CredentialStripper
+
stripper = CredentialStripper()
text = "Token: ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij"
result = stripper.strip(text)
@@ -32,6 +35,7 @@ def test_strips_github_token():
def test_preserves_normal_text():
from openjarvis.security.credential_stripper import CredentialStripper
+
stripper = CredentialStripper()
text = "The function returned 42 results."
result = stripper.strip(text)
@@ -40,6 +44,7 @@ def test_preserves_normal_text():
def test_tool_output_wrapping():
from openjarvis.security.credential_stripper import wrap_tool_output
+
content = "Search results: found 3 items"
wrapped = wrap_tool_output("web_search", content, success=True)
assert '<tool_result name="web_search" status="success">' in wrapped
diff --git a/tests/security/test_guardrails.py b/tests/security/test_guardrails.py
index 5eee6dc8..67fa72e0 100644
--- a/tests/security/test_guardrails.py
+++ b/tests/security/test_guardrails.py
@@ -111,7 +111,10 @@ class TestGuardrailsEngineInputScanning:
bus = EventBus(record_history=True)
mock = _make_mock_engine("OK")
ge = GuardrailsEngine(
- mock, mode=RedactionMode.WARN, scan_input=True, bus=bus,
+ mock,
+ mode=RedactionMode.WARN,
+ scan_input=True,
+ bus=bus,
)
secret = "my key sk-abc123def456ghi789jkl012"
@@ -129,7 +132,9 @@ class TestGuardrailsEngineInputScanning:
"""REDACT mode on input must send redacted messages."""
mock = _make_mock_engine("OK")
ge = GuardrailsEngine(
- mock, mode=RedactionMode.REDACT, scan_input=True,
+ mock,
+ mode=RedactionMode.REDACT,
+ scan_input=True,
)
secret = "my key sk-abc123def456ghi789jkl012"
@@ -147,7 +152,9 @@ class TestGuardrailsEngineInputScanning:
bus = EventBus(record_history=True)
mock = _make_mock_engine("OK")
ge = GuardrailsEngine(
- mock, mode=RedactionMode.WARN, scan_input=False,
+ mock,
+ mode=RedactionMode.WARN,
+ scan_input=False,
bus=bus,
)
diff --git a/tests/security/test_security_wiring.py b/tests/security/test_security_wiring.py
index 9651bd4f..eca55808 100644
--- a/tests/security/test_security_wiring.py
+++ b/tests/security/test_security_wiring.py
@@ -1,4 +1,5 @@
"""Verify security wiring reaches agents and ToolExecutor."""
+
from __future__ import annotations
from unittest.mock import MagicMock
diff --git a/tests/security/test_setup_security.py b/tests/security/test_setup_security.py
index 754dc73f..2ad1c181 100644
--- a/tests/security/test_setup_security.py
+++ b/tests/security/test_setup_security.py
@@ -1,4 +1,5 @@
"""Tests for setup_security() helper."""
+
from __future__ import annotations
from unittest.mock import MagicMock
@@ -34,6 +35,7 @@ def _make_config(*, enabled: bool = True, caps_enabled: bool = False) -> JarvisC
def _has_rust() -> bool:
try:
import openjarvis_rust # noqa: F401
+
return True
except ImportError:
return False
diff --git a/tests/security/test_severity_policy.py b/tests/security/test_severity_policy.py
index 907ca826..37c34c00 100644
--- a/tests/security/test_severity_policy.py
+++ b/tests/security/test_severity_policy.py
@@ -4,6 +4,7 @@ from __future__ import annotations
def test_severity_policy_block():
from openjarvis.security.severity_policy import SeverityPolicy
from openjarvis.security.types import ThreatLevel
+
policy = SeverityPolicy()
assert policy.action_for(ThreatLevel.CRITICAL) == "block"
@@ -11,6 +12,7 @@ def test_severity_policy_block():
def test_severity_policy_warn():
from openjarvis.security.severity_policy import SeverityPolicy
from openjarvis.security.types import ThreatLevel
+
policy = SeverityPolicy()
assert policy.action_for(ThreatLevel.HIGH) == "warn"
@@ -18,6 +20,7 @@ def test_severity_policy_warn():
def test_severity_policy_sanitize():
from openjarvis.security.severity_policy import SeverityPolicy
from openjarvis.security.types import ThreatLevel
+
policy = SeverityPolicy()
assert policy.action_for(ThreatLevel.MEDIUM) == "sanitize"
@@ -25,5 +28,6 @@ def test_severity_policy_sanitize():
def test_severity_policy_log():
from openjarvis.security.severity_policy import SeverityPolicy
from openjarvis.security.types import ThreatLevel
+
policy = SeverityPolicy()
assert policy.action_for(ThreatLevel.LOW) == "log"
diff --git a/tests/security/test_signing.py b/tests/security/test_signing.py
index 2562c92e..5562448f 100644
--- a/tests/security/test_signing.py
+++ b/tests/security/test_signing.py
@@ -16,6 +16,7 @@ class TestSigning:
def test_generate_keypair(self):
_skip_if_no_cryptography()
from openjarvis.security.signing import generate_keypair
+
kp = generate_keypair()
assert len(kp.private_key) == 32
assert len(kp.public_key) == 32
@@ -23,6 +24,7 @@ class TestSigning:
def test_sign_and_verify(self):
_skip_if_no_cryptography()
from openjarvis.security.signing import generate_keypair, sign, verify
+
kp = generate_keypair()
data = b"hello world"
sig = sign(data, kp.private_key)
@@ -32,6 +34,7 @@ class TestSigning:
def test_verify_wrong_data(self):
_skip_if_no_cryptography()
from openjarvis.security.signing import generate_keypair, sign, verify
+
kp = generate_keypair()
sig = sign(b"hello", kp.private_key)
assert not verify(b"wrong", sig, kp.public_key)
@@ -39,6 +42,7 @@ class TestSigning:
def test_verify_wrong_key(self):
_skip_if_no_cryptography()
from openjarvis.security.signing import generate_keypair, sign, verify
+
kp1 = generate_keypair()
kp2 = generate_keypair()
sig = sign(b"data", kp1.private_key)
@@ -47,6 +51,7 @@ class TestSigning:
def test_sign_b64(self):
_skip_if_no_cryptography()
from openjarvis.security.signing import generate_keypair, sign_b64, verify_b64
+
kp = generate_keypair()
data = b"test data"
sig_b64 = sign_b64(data, kp.private_key)
@@ -56,6 +61,7 @@ class TestSigning:
def test_verify_b64_invalid(self):
_skip_if_no_cryptography()
from openjarvis.security.signing import generate_keypair, verify_b64
+
kp = generate_keypair()
assert not verify_b64(b"data", "invalid-base64!!!", kp.public_key)
@@ -68,6 +74,7 @@ class TestSigning:
sign,
verify,
)
+
kp = generate_keypair()
priv_path = str(tmp_path / "private.key")
pub_path = str(tmp_path / "public.key")
diff --git a/tests/security/test_subprocess_sandbox.py b/tests/security/test_subprocess_sandbox.py
index 1bca470c..92fe2d95 100644
--- a/tests/security/test_subprocess_sandbox.py
+++ b/tests/security/test_subprocess_sandbox.py
@@ -21,8 +21,16 @@ class TestBuildSafeEnv:
env = build_safe_env()
# All keys should be from the safe set
safe_keys = {
- "PATH", "HOME", "USER", "LANG", "TERM", "SHELL",
- "LC_ALL", "LC_CTYPE", "TMPDIR", "TZ",
+ "PATH",
+ "HOME",
+ "USER",
+ "LANG",
+ "TERM",
+ "SHELL",
+ "LC_ALL",
+ "LC_CTYPE",
+ "TMPDIR",
+ "TZ",
}
for key in env:
assert key in safe_keys
@@ -78,7 +86,8 @@ class TestRunSandboxed:
os.environ["TEST_SECRET"] = "super_secret_value"
try:
result = run_sandboxed(
- 'echo "val=$TEST_SECRET"', timeout=10.0,
+ 'echo "val=$TEST_SECRET"',
+ timeout=10.0,
)
assert result.returncode == 0
assert "super_secret_value" not in result.stdout
diff --git a/tests/server/test_agent_manager_routes.py b/tests/server/test_agent_manager_routes.py
index b694edd5..498bc167 100644
--- a/tests/server/test_agent_manager_routes.py
+++ b/tests/server/test_agent_manager_routes.py
@@ -49,10 +49,13 @@ class TestAgentManagerRoutes:
assert resp.json()["agents"] == []
def test_create_agent(self, client):
- resp = client.post("/v1/managed-agents", json={
- "name": "researcher",
- "agent_type": "monitor_operative",
- })
+ resp = client.post(
+ "/v1/managed-agents",
+ json={
+ "name": "researcher",
+ "agent_type": "monitor_operative",
+ },
+ )
assert resp.status_code == 200
data = resp.json()
assert data["name"] == "researcher"
@@ -95,9 +98,12 @@ class TestAgentManagerRoutes:
def test_create_task(self, client):
create_resp = client.post("/v1/managed-agents", json={"name": "worker"})
agent_id = create_resp.json()["id"]
- resp = client.post(f"/v1/managed-agents/{agent_id}/tasks", json={
- "description": "Find papers on reasoning",
- })
+ resp = client.post(
+ f"/v1/managed-agents/{agent_id}/tasks",
+ json={
+ "description": "Find papers on reasoning",
+ },
+ )
assert resp.status_code == 200
assert resp.json()["description"] == "Find papers on reasoning"
@@ -113,10 +119,13 @@ class TestAgentManagerRoutes:
create_resp = client.post("/v1/managed-agents", json={"name": "slacker"})
agent_id = create_resp.json()["id"]
# Bind
- bind_resp = client.post(f"/v1/managed-agents/{agent_id}/channels", json={
- "channel_type": "slack",
- "config": {"channel": "#research"},
- })
+ bind_resp = client.post(
+ f"/v1/managed-agents/{agent_id}/channels",
+ json={
+ "channel_type": "slack",
+ "config": {"channel": "#research"},
+ },
+ )
assert bind_resp.status_code == 200
binding_id = bind_resp.json()["id"]
# List
diff --git a/tests/server/test_api_routes.py b/tests/server/test_api_routes.py
index d2856f21..d846c348 100644
--- a/tests/server/test_api_routes.py
+++ b/tests/server/test_api_routes.py
@@ -1,4 +1,5 @@
"""Tests for extended API routes."""
+
import pytest
fastapi = pytest.importorskip("fastapi")
diff --git a/tests/server/test_channel_routes.py b/tests/server/test_channel_routes.py
index c3da6ecd..eab9210b 100644
--- a/tests/server/test_channel_routes.py
+++ b/tests/server/test_channel_routes.py
@@ -47,7 +47,8 @@ def app_with_bridge(mock_engine, mock_bridge):
from openjarvis.server.app import create_app
return create_app(
- mock_engine, "test-model",
+ mock_engine,
+ "test-model",
channel_bridge=mock_bridge,
)
diff --git a/tests/server/test_middleware.py b/tests/server/test_middleware.py
index 07dac292..d2016082 100644
--- a/tests/server/test_middleware.py
+++ b/tests/server/test_middleware.py
@@ -48,6 +48,7 @@ class TestSecurityHeaders:
if middleware_cls is None:
# starlette not installed -- skip
import pytest
+
pytest.skip("starlette not available")
assert middleware_cls is not None
assert callable(middleware_cls)
@@ -55,6 +56,7 @@ class TestSecurityHeaders:
def test_middleware_adds_headers(self) -> None:
"""Middleware adds all security headers to responses."""
import pytest
+
fastapi = pytest.importorskip("fastapi")
from fastapi.testclient import TestClient
@@ -80,6 +82,7 @@ class TestSecurityHeaders:
def test_middleware_skips_options(self) -> None:
"""OPTIONS requests pass through without security headers."""
import pytest
+
fastapi = pytest.importorskip("fastapi")
from fastapi.middleware.cors import CORSMiddleware
from fastapi.testclient import TestClient
diff --git a/tests/server/test_model_management.py b/tests/server/test_model_management.py
index 7c5f99ef..7e300135 100644
--- a/tests/server/test_model_management.py
+++ b/tests/server/test_model_management.py
@@ -71,7 +71,6 @@ class TestModelPull:
engine = _make_ollama_engine()
client = TestClient(_app(engine, engine_name="ollama"))
-
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.raise_for_status = MagicMock()
@@ -157,11 +156,14 @@ class TestStreamingResilience:
app = create_app(engine, "test-model")
client = TestClient(app)
- resp = client.post("/v1/chat/completions", json={
- "model": "bad-model",
- "messages": [{"role": "user", "content": "Hello"}],
- "stream": True,
- })
+ resp = client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "bad-model",
+ "messages": [{"role": "user", "content": "Hello"}],
+ "stream": True,
+ },
+ )
assert resp.status_code == 200
# Should contain partial content + error message + [DONE]
@@ -176,11 +178,14 @@ class TestStreamingResilience:
app = create_app(engine, "test-model")
client = TestClient(app)
- resp = client.post("/v1/chat/completions", json={
- "model": "test-model",
- "messages": [{"role": "user", "content": "Hello"}],
- "stream": True,
- })
+ resp = client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "test-model",
+ "messages": [{"role": "user", "content": "Hello"}],
+ "stream": True,
+ },
+ )
assert resp.status_code == 200
# Collect tokens
@@ -203,18 +208,22 @@ class TestStreamingResilience:
agent = MagicMock()
agent.agent_id = "simple"
agent.run.return_value = AgentResult(
- content="agent response", turns=1,
+ content="agent response",
+ turns=1,
)
app = create_app(engine, "test-model", agent=agent)
client = TestClient(app)
- resp = client.post("/v1/chat/completions", json={
- "model": "test-model",
- "messages": [{"role": "user", "content": "Hello"}],
- "stream": True,
- # No tools — should use direct engine stream
- })
+ resp = client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "test-model",
+ "messages": [{"role": "user", "content": "Hello"}],
+ "stream": True,
+ # No tools — should use direct engine stream
+ },
+ )
assert resp.status_code == 200
# Should get engine tokens, not agent response
diff --git a/tests/server/test_models_pydantic.py b/tests/server/test_models_pydantic.py
index dc80bbe5..f3757fb0 100644
--- a/tests/server/test_models_pydantic.py
+++ b/tests/server/test_models_pydantic.py
@@ -83,10 +83,12 @@ class TestModelListResponse:
assert resp.data == []
def test_with_models(self):
- resp = ModelListResponse(data=[
- ModelObject(id="model-a"),
- ModelObject(id="model-b"),
- ])
+ resp = ModelListResponse(
+ data=[
+ ModelObject(id="model-a"),
+ ModelObject(id="model-b"),
+ ]
+ )
assert len(resp.data) == 2
assert resp.data[0].id == "model-a"
assert resp.data[0].object == "model"
diff --git a/tests/server/test_routes.py b/tests/server/test_routes.py
index 8880d4e2..f82d08d5 100644
--- a/tests/server/test_routes.py
+++ b/tests/server/test_routes.py
@@ -31,8 +31,12 @@ def _make_engine(content="Hello from server", models=None):
# Set up async stream
async def mock_stream(
- messages, *, model, temperature=0.7,
- max_tokens=1024, **kwargs,
+ messages,
+ *,
+ model,
+ temperature=0.7,
+ max_tokens=1024,
+ **kwargs,
):
for token in ["Hello", " ", "world"]:
yield token
@@ -43,6 +47,7 @@ def _make_engine(content="Hello from server", models=None):
def _make_agent(content="Hello from agent"):
from openjarvis.agents._stubs import AgentResult
+
agent = MagicMock()
agent.agent_id = "mock"
agent.run.return_value = AgentResult(content=content, turns=1)
@@ -71,47 +76,62 @@ def client_with_agent():
class TestChatCompletions:
def test_basic_completion(self, client):
- resp = client.post("/v1/chat/completions", json={
- "model": "test-model",
- "messages": [{"role": "user", "content": "Hello"}],
- })
+ resp = client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "test-model",
+ "messages": [{"role": "user", "content": "Hello"}],
+ },
+ )
assert resp.status_code == 200
data = resp.json()
assert data["object"] == "chat.completion"
assert data["choices"][0]["message"]["content"] == "Hello from server"
def test_completion_has_usage(self, client):
- resp = client.post("/v1/chat/completions", json={
- "model": "test-model",
- "messages": [{"role": "user", "content": "Hello"}],
- })
+ resp = client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "test-model",
+ "messages": [{"role": "user", "content": "Hello"}],
+ },
+ )
data = resp.json()
assert data["usage"]["total_tokens"] == 8
def test_completion_has_id(self, client):
- resp = client.post("/v1/chat/completions", json={
- "model": "test-model",
- "messages": [{"role": "user", "content": "Hello"}],
- })
+ resp = client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "test-model",
+ "messages": [{"role": "user", "content": "Hello"}],
+ },
+ )
data = resp.json()
assert data["id"].startswith("chatcmpl-")
def test_custom_temperature(self, client):
- resp = client.post("/v1/chat/completions", json={
- "model": "test-model",
- "messages": [{"role": "user", "content": "Hello"}],
- "temperature": 0.1,
- })
+ resp = client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "test-model",
+ "messages": [{"role": "user", "content": "Hello"}],
+ "temperature": 0.1,
+ },
+ )
assert resp.status_code == 200
def test_with_system_message(self, client):
- resp = client.post("/v1/chat/completions", json={
- "model": "test-model",
- "messages": [
- {"role": "system", "content": "Be helpful"},
- {"role": "user", "content": "Hello"},
- ],
- })
+ resp = client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "test-model",
+ "messages": [
+ {"role": "system", "content": "Be helpful"},
+ {"role": "user", "content": "Hello"},
+ ],
+ },
+ )
assert resp.status_code == 200
def test_with_tools(self):
@@ -127,40 +147,52 @@ class TestChatCompletions:
}
app = create_app(engine, "test-model")
client = TestClient(app)
- resp = client.post("/v1/chat/completions", json={
- "model": "test-model",
- "messages": [{"role": "user", "content": "Calc"}],
- "tools": [{"type": "function", "function": {"name": "calc"}}],
- })
+ resp = client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "test-model",
+ "messages": [{"role": "user", "content": "Calc"}],
+ "tools": [{"type": "function", "function": {"name": "calc"}}],
+ },
+ )
assert resp.status_code == 200
data = resp.json()
assert data["choices"][0]["message"]["tool_calls"] is not None
def test_agent_mode(self, client_with_agent):
- resp = client_with_agent.post("/v1/chat/completions", json={
- "model": "test-model",
- "messages": [{"role": "user", "content": "Hello"}],
- })
+ resp = client_with_agent.post(
+ "/v1/chat/completions",
+ json={
+ "model": "test-model",
+ "messages": [{"role": "user", "content": "Hello"}],
+ },
+ )
assert resp.status_code == 200
data = resp.json()
assert data["choices"][0]["message"]["content"] == "Hello from agent"
def test_agent_with_conversation(self, client_with_agent):
- resp = client_with_agent.post("/v1/chat/completions", json={
- "model": "test-model",
- "messages": [
- {"role": "system", "content": "Be helpful"},
- {"role": "user", "content": "Hello"},
- ],
- })
+ resp = client_with_agent.post(
+ "/v1/chat/completions",
+ json={
+ "model": "test-model",
+ "messages": [
+ {"role": "system", "content": "Be helpful"},
+ {"role": "user", "content": "Hello"},
+ ],
+ },
+ )
assert resp.status_code == 200
def test_streaming(self, client):
- resp = client.post("/v1/chat/completions", json={
- "model": "test-model",
- "messages": [{"role": "user", "content": "Hello"}],
- "stream": True,
- })
+ resp = client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "test-model",
+ "messages": [{"role": "user", "content": "Hello"}],
+ "stream": True,
+ },
+ )
assert resp.status_code == 200
assert "text/event-stream" in resp.headers.get("content-type", "")
# Parse SSE events
@@ -171,29 +203,40 @@ class TestChatCompletions:
assert data_lines[-1].strip() == "data: [DONE]"
def test_streaming_content(self, client):
- resp = client.post("/v1/chat/completions", json={
- "model": "test-model",
- "messages": [{"role": "user", "content": "Hello"}],
- "stream": True,
- })
+ resp = client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "test-model",
+ "messages": [{"role": "user", "content": "Hello"}],
+ "stream": True,
+ },
+ )
# Collect content tokens from stream
content = ""
for line in resp.text.strip().split("\n"):
if line.startswith("data:") and "[DONE]" not in line:
data = json.loads(line[5:].strip())
choices = data.get("choices", [{}])
- delta_content = choices[0].get(
- "delta", {},
- ).get("content")
+ delta_content = (
+ choices[0]
+ .get(
+ "delta",
+ {},
+ )
+ .get("content")
+ )
if delta_content:
content += delta_content
assert content == "Hello world"
def test_finish_reason_default(self, client):
- resp = client.post("/v1/chat/completions", json={
- "model": "test-model",
- "messages": [{"role": "user", "content": "Hello"}],
- })
+ resp = client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "test-model",
+ "messages": [{"role": "user", "content": "Hello"}],
+ },
+ )
data = resp.json()
assert data["choices"][0]["finish_reason"] == "stop"
diff --git a/tests/server/test_tools_endpoint.py b/tests/server/test_tools_endpoint.py
index 414b194d..4f061209 100644
--- a/tests/server/test_tools_endpoint.py
+++ b/tests/server/test_tools_endpoint.py
@@ -1,4 +1,5 @@
"""Tests for GET /v1/tools endpoint."""
+
import pytest
try:
@@ -14,6 +15,7 @@ pytestmark = pytest.mark.skipif(
def test_tools_endpoint_returns_list():
from openjarvis.server.agent_manager_routes import build_tools_list
+
tools = build_tools_list()
assert isinstance(tools, list)
assert len(tools) > 0
@@ -29,6 +31,7 @@ def test_tools_endpoint_returns_list():
def test_tools_includes_channels():
from openjarvis.server.agent_manager_routes import build_tools_list
+
tools = build_tools_list()
names = {t["name"] for t in tools}
channel_names = {"slack", "telegram", "discord", "email"}
@@ -37,6 +40,7 @@ def test_tools_includes_channels():
def test_browser_meta_group():
from openjarvis.server.agent_manager_routes import build_tools_list
+
tools = build_tools_list()
names = {t["name"] for t in tools}
assert "browser" in names
diff --git a/tests/server/test_ws_bridge.py b/tests/server/test_ws_bridge.py
index 9e714efe..1727c1e0 100644
--- a/tests/server/test_ws_bridge.py
+++ b/tests/server/test_ws_bridge.py
@@ -38,10 +38,13 @@ class TestWSBridge:
def test_websocket_receives_events(self, app, event_bus):
client = TestClient(app)
with client.websocket_connect("/v1/agents/events") as ws:
- event_bus.publish(EventType.AGENT_TICK_START, {
- "agent_id": "test-123",
- "agent_name": "test",
- })
+ event_bus.publish(
+ EventType.AGENT_TICK_START,
+ {
+ "agent_id": "test-123",
+ "agent_name": "test",
+ },
+ )
time.sleep(0.05) # Let call_soon_threadsafe deliver to queue
data = ws.receive_json()
assert data["type"] == "agent_tick_start"
diff --git a/tests/sessions/test_session.py b/tests/sessions/test_session.py
index 8368f3f1..ca6b7d53 100644
--- a/tests/sessions/test_session.py
+++ b/tests/sessions/test_session.py
@@ -29,7 +29,8 @@ class TestSession:
class TestSessionIdentity:
def test_create_identity(self):
identity = SessionIdentity(
- user_id="u1", display_name="Alice",
+ user_id="u1",
+ display_name="Alice",
channel_ids={"telegram": "t123"},
)
assert identity.user_id == "u1"
diff --git a/tests/skills/test_bundled_skills.py b/tests/skills/test_bundled_skills.py
index 44940e4a..56b645e5 100644
--- a/tests/skills/test_bundled_skills.py
+++ b/tests/skills/test_bundled_skills.py
@@ -10,11 +10,7 @@ from openjarvis.skills.loader import load_skill
# Resolve the skills/builtin/ directory relative to the project root.
BUILTIN_DIR = (
- Path(__file__).resolve().parents[2]
- / "src"
- / "openjarvis"
- / "skills"
- / "data"
+ Path(__file__).resolve().parents[2] / "src" / "openjarvis" / "skills" / "data"
)
# Collect all TOML files once so parametrized IDs are readable.
@@ -98,6 +94,4 @@ class TestStepsHaveToolNames:
def test_steps_have_tool_names(self, toml_path: Path):
manifest = load_skill(toml_path)
for i, step in enumerate(manifest.steps):
- assert step.tool_name, (
- f"{toml_path.name} step {i} has empty tool_name"
- )
+ assert step.tool_name, f"{toml_path.name} step {i} has empty tool_name"
diff --git a/tests/skills/test_skills.py b/tests/skills/test_skills.py
index ec2dace2..b151558e 100644
--- a/tests/skills/test_skills.py
+++ b/tests/skills/test_skills.py
@@ -11,6 +11,7 @@ from openjarvis.tools._stubs import BaseTool, ToolExecutor, ToolSpec
class EchoTool(BaseTool):
"""Simple tool that echoes input."""
+
tool_id = "echo"
@property
@@ -27,6 +28,7 @@ class EchoTool(BaseTool):
class UpperTool(BaseTool):
"""Simple tool that uppercases input."""
+
tool_id = "upper"
@property
@@ -68,11 +70,13 @@ class TestSkillExecutor:
executor = self._make_executor()
manifest = SkillManifest(
name="single",
- steps=[SkillStep(
- tool_name="echo",
- arguments_template='{"text": "hello"}',
- output_key="result",
- )],
+ steps=[
+ SkillStep(
+ tool_name="echo",
+ arguments_template='{"text": "hello"}',
+ output_key="result",
+ )
+ ],
)
result = executor.run(manifest)
assert result.success
@@ -116,11 +120,13 @@ class TestSkillExecutor:
executor = self._make_executor()
manifest = SkillManifest(
name="with_ctx",
- steps=[SkillStep(
- tool_name="echo",
- arguments_template='{"text": "{greeting}"}',
- output_key="result",
- )],
+ steps=[
+ SkillStep(
+ tool_name="echo",
+ arguments_template='{"text": "{greeting}"}',
+ output_key="result",
+ )
+ ],
)
result = executor.run(manifest, initial_context={"greeting": "hi"})
assert result.success
@@ -152,11 +158,13 @@ class TestSkillTool:
manifest = SkillManifest(
name="tool_skill",
description="A skill exposed as a tool",
- steps=[SkillStep(
- tool_name="echo",
- arguments_template='{"text": "{input}"}',
- output_key="result",
- )],
+ steps=[
+ SkillStep(
+ tool_name="echo",
+ arguments_template='{"text": "{input}"}',
+ output_key="result",
+ )
+ ],
)
skill_tool = SkillTool(manifest, executor)
assert skill_tool.spec.name == "skill_tool_skill"
diff --git a/tests/speech/test_discovery.py b/tests/speech/test_discovery.py
index 523b73cc..50f44220 100644
--- a/tests/speech/test_discovery.py
+++ b/tests/speech/test_discovery.py
@@ -13,10 +13,14 @@ def test_get_speech_backend_explicit():
config.speech.backend = "faster-whisper"
with patch("openjarvis.speech._discovery._create_backend") as mock_create:
- mock_backend = type("MockBackend", (), {
- "backend_id": "faster-whisper",
- "health": lambda self: True,
- })()
+ mock_backend = type(
+ "MockBackend",
+ (),
+ {
+ "backend_id": "faster-whisper",
+ "health": lambda self: True,
+ },
+ )()
mock_create.return_value = mock_backend
result = get_speech_backend(config)
diff --git a/tests/telemetry/test_aggregator.py b/tests/telemetry/test_aggregator.py
index d71d7e4c..ed3bb74a 100644
--- a/tests/telemetry/test_aggregator.py
+++ b/tests/telemetry/test_aggregator.py
@@ -71,11 +71,14 @@ class TestTelemetryAggregator:
agg.close()
def test_multiple_models_grouped(self, tmp_path: Path) -> None:
- agg = _setup(tmp_path, [
- _make_record(model_id="m1"),
- _make_record(model_id="m1"),
- _make_record(model_id="m2"),
- ])
+ agg = _setup(
+ tmp_path,
+ [
+ _make_record(model_id="m1"),
+ _make_record(model_id="m1"),
+ _make_record(model_id="m2"),
+ ],
+ )
stats = agg.per_model_stats()
assert len(stats) == 2
# Ordered by call_count DESC
@@ -86,11 +89,14 @@ class TestTelemetryAggregator:
agg.close()
def test_per_engine_stats(self, tmp_path: Path) -> None:
- agg = _setup(tmp_path, [
- _make_record(engine="ollama"),
- _make_record(engine="vllm"),
- _make_record(engine="vllm"),
- ])
+ agg = _setup(
+ tmp_path,
+ [
+ _make_record(engine="ollama"),
+ _make_record(engine="vllm"),
+ _make_record(engine="vllm"),
+ ],
+ )
stats = agg.per_engine_stats()
assert len(stats) == 2
assert stats[0].engine == "vllm"
@@ -105,22 +111,28 @@ class TestTelemetryAggregator:
agg.close()
def test_top_models_ordering(self, tmp_path: Path) -> None:
- agg = _setup(tmp_path, [
- _make_record(model_id="rare"),
- _make_record(model_id="popular"),
- _make_record(model_id="popular"),
- _make_record(model_id="popular"),
- ])
+ agg = _setup(
+ tmp_path,
+ [
+ _make_record(model_id="rare"),
+ _make_record(model_id="popular"),
+ _make_record(model_id="popular"),
+ _make_record(model_id="popular"),
+ ],
+ )
top = agg.top_models(n=2)
assert top[0].model_id == "popular"
assert top[0].call_count == 3
agg.close()
def test_summary_totals(self, tmp_path: Path) -> None:
- agg = _setup(tmp_path, [
- _make_record(prompt_tokens=10, completion_tokens=5, cost=0.001),
- _make_record(prompt_tokens=20, completion_tokens=10, cost=0.002),
- ])
+ agg = _setup(
+ tmp_path,
+ [
+ _make_record(prompt_tokens=10, completion_tokens=5, cost=0.001),
+ _make_record(prompt_tokens=20, completion_tokens=10, cost=0.002),
+ ],
+ )
s = agg.summary()
assert s.total_calls == 2
assert s.total_tokens == 45 # (10+5) + (20+10)
@@ -136,11 +148,14 @@ class TestTelemetryAggregator:
def test_time_range_since(self, tmp_path: Path) -> None:
now = time.time()
- agg = _setup(tmp_path, [
- _make_record(ts=now - 100),
- _make_record(ts=now - 10),
- _make_record(ts=now),
- ])
+ agg = _setup(
+ tmp_path,
+ [
+ _make_record(ts=now - 100),
+ _make_record(ts=now - 10),
+ _make_record(ts=now),
+ ],
+ )
stats = agg.per_model_stats(since=now - 50)
total = sum(s.call_count for s in stats)
assert total == 2
@@ -148,10 +163,13 @@ class TestTelemetryAggregator:
def test_time_range_until(self, tmp_path: Path) -> None:
now = time.time()
- agg = _setup(tmp_path, [
- _make_record(ts=now - 100),
- _make_record(ts=now),
- ])
+ agg = _setup(
+ tmp_path,
+ [
+ _make_record(ts=now - 100),
+ _make_record(ts=now),
+ ],
+ )
stats = agg.per_model_stats(until=now - 50)
total = sum(s.call_count for s in stats)
assert total == 1
@@ -159,11 +177,14 @@ class TestTelemetryAggregator:
def test_time_range_since_and_until(self, tmp_path: Path) -> None:
now = time.time()
- agg = _setup(tmp_path, [
- _make_record(ts=now - 200),
- _make_record(ts=now - 100),
- _make_record(ts=now),
- ])
+ agg = _setup(
+ tmp_path,
+ [
+ _make_record(ts=now - 200),
+ _make_record(ts=now - 100),
+ _make_record(ts=now),
+ ],
+ )
stats = agg.per_model_stats(since=now - 150, until=now - 50)
total = sum(s.call_count for s in stats)
assert total == 1
@@ -178,10 +199,13 @@ class TestTelemetryAggregator:
def test_export_records_filtered(self, tmp_path: Path) -> None:
now = time.time()
- agg = _setup(tmp_path, [
- _make_record(ts=now - 100),
- _make_record(ts=now),
- ])
+ agg = _setup(
+ tmp_path,
+ [
+ _make_record(ts=now - 100),
+ _make_record(ts=now),
+ ],
+ )
records = agg.export_records(since=now - 50)
assert len(records) == 1
agg.close()
diff --git a/tests/telemetry/test_derived_metrics.py b/tests/telemetry/test_derived_metrics.py
index c2f5ddb6..50e66a8a 100644
--- a/tests/telemetry/test_derived_metrics.py
+++ b/tests/telemetry/test_derived_metrics.py
@@ -176,13 +176,15 @@ class TestDerivedMetricsInStore:
def test_summary_weighted_averages(self, tmp_path):
store = TelemetryStore(tmp_path / "test.db")
for i in range(3):
- store.record(TelemetryRecord(
- timestamp=time.time() + i,
- model_id="m1",
- engine="e1",
- energy_per_output_token_joules=0.1 * (i + 1),
- throughput_per_watt=1.0 * (i + 1),
- ))
+ store.record(
+ TelemetryRecord(
+ timestamp=time.time() + i,
+ model_id="m1",
+ engine="e1",
+ energy_per_output_token_joules=0.1 * (i + 1),
+ throughput_per_watt=1.0 * (i + 1),
+ )
+ )
agg = TelemetryAggregator(tmp_path / "test.db")
summary = agg.summary()
assert summary.avg_energy_per_output_token_joules > 0
diff --git a/tests/telemetry/test_energy_apple.py b/tests/telemetry/test_energy_apple.py
index c9c48b4f..4b1c0268 100644
--- a/tests/telemetry/test_energy_apple.py
+++ b/tests/telemetry/test_energy_apple.py
@@ -50,8 +50,9 @@ class TestAvailable:
orig = mod._ZEUS_APPLE_AVAILABLE
mod._ZEUS_APPLE_AVAILABLE = False
try:
- with patch("platform.system", return_value="Darwin"), patch(
- "platform.machine", return_value="arm64"
+ with (
+ patch("platform.system", return_value="Darwin"),
+ patch("platform.machine", return_value="arm64"),
):
assert mod.AppleEnergyMonitor.available() is True
monitor = mod.AppleEnergyMonitor.__new__(mod.AppleEnergyMonitor)
diff --git a/tests/telemetry/test_energy_monitor.py b/tests/telemetry/test_energy_monitor.py
index 848b095b..07d753b6 100644
--- a/tests/telemetry/test_energy_monitor.py
+++ b/tests/telemetry/test_energy_monitor.py
@@ -77,40 +77,51 @@ class TestEnergyMonitorABC:
class TestCreateEnergyMonitor:
def test_returns_none_when_nothing_available(self):
- with patch(
- "openjarvis.telemetry.energy_nvidia.NvidiaEnergyMonitor.available",
- return_value=False,
- ), patch(
- "openjarvis.telemetry.energy_amd.AmdEnergyMonitor.available",
- return_value=False,
- ), patch(
- "openjarvis.telemetry.energy_apple.AppleEnergyMonitor.available",
- return_value=False,
- ), patch(
- "openjarvis.telemetry.energy_rapl.RaplEnergyMonitor.available",
- return_value=False,
+ with (
+ patch(
+ "openjarvis.telemetry.energy_nvidia.NvidiaEnergyMonitor.available",
+ return_value=False,
+ ),
+ patch(
+ "openjarvis.telemetry.energy_amd.AmdEnergyMonitor.available",
+ return_value=False,
+ ),
+ patch(
+ "openjarvis.telemetry.energy_apple.AppleEnergyMonitor.available",
+ return_value=False,
+ ),
+ patch(
+ "openjarvis.telemetry.energy_rapl.RaplEnergyMonitor.available",
+ return_value=False,
+ ),
):
result = create_energy_monitor()
assert result is None
def test_prefer_vendor_parameter(self):
"""When prefer_vendor is set, that vendor is tried first."""
- with patch(
- "openjarvis.telemetry.energy_nvidia.NvidiaEnergyMonitor.available",
- return_value=False,
- ), patch(
- "openjarvis.telemetry.energy_amd.AmdEnergyMonitor.available",
- return_value=False,
- ), patch(
- "openjarvis.telemetry.energy_apple.AppleEnergyMonitor.available",
- return_value=False,
- ), patch(
- "openjarvis.telemetry.energy_rapl.RaplEnergyMonitor.available",
- return_value=True,
- ), patch(
- "openjarvis.telemetry.energy_rapl.RaplEnergyMonitor.__init__",
- return_value=None,
- ) as mock_init:
+ with (
+ patch(
+ "openjarvis.telemetry.energy_nvidia.NvidiaEnergyMonitor.available",
+ return_value=False,
+ ),
+ patch(
+ "openjarvis.telemetry.energy_amd.AmdEnergyMonitor.available",
+ return_value=False,
+ ),
+ patch(
+ "openjarvis.telemetry.energy_apple.AppleEnergyMonitor.available",
+ return_value=False,
+ ),
+ patch(
+ "openjarvis.telemetry.energy_rapl.RaplEnergyMonitor.available",
+ return_value=True,
+ ),
+ patch(
+ "openjarvis.telemetry.energy_rapl.RaplEnergyMonitor.__init__",
+ return_value=None,
+ ) as mock_init,
+ ):
create_energy_monitor(prefer_vendor="cpu_rapl")
# RaplEnergyMonitor was available and preferred
mock_init.assert_called_once_with(poll_interval_ms=50)
@@ -127,21 +138,27 @@ class TestCreateEnergyMonitor:
call_order.append("amd")
return True
- with patch(
- "openjarvis.telemetry.energy_nvidia.NvidiaEnergyMonitor.available",
- side_effect=nvidia_available,
- ), patch(
- "openjarvis.telemetry.energy_amd.AmdEnergyMonitor.available",
- side_effect=amd_available,
- ), patch(
- "openjarvis.telemetry.energy_apple.AppleEnergyMonitor.available",
- return_value=False,
- ), patch(
- "openjarvis.telemetry.energy_rapl.RaplEnergyMonitor.available",
- return_value=False,
- ), patch(
- "openjarvis.telemetry.energy_nvidia.NvidiaEnergyMonitor.__init__",
- return_value=None,
+ with (
+ patch(
+ "openjarvis.telemetry.energy_nvidia.NvidiaEnergyMonitor.available",
+ side_effect=nvidia_available,
+ ),
+ patch(
+ "openjarvis.telemetry.energy_amd.AmdEnergyMonitor.available",
+ side_effect=amd_available,
+ ),
+ patch(
+ "openjarvis.telemetry.energy_apple.AppleEnergyMonitor.available",
+ return_value=False,
+ ),
+ patch(
+ "openjarvis.telemetry.energy_rapl.RaplEnergyMonitor.available",
+ return_value=False,
+ ),
+ patch(
+ "openjarvis.telemetry.energy_nvidia.NvidiaEnergyMonitor.__init__",
+ return_value=None,
+ ),
):
create_energy_monitor()
# NVIDIA was tried first and returned True
@@ -159,21 +176,27 @@ class TestCreateEnergyMonitor:
call_order.append("amd")
return True
- with patch(
- "openjarvis.telemetry.energy_nvidia.NvidiaEnergyMonitor.available",
- side_effect=nvidia_available,
- ), patch(
- "openjarvis.telemetry.energy_amd.AmdEnergyMonitor.available",
- side_effect=amd_available,
- ), patch(
- "openjarvis.telemetry.energy_apple.AppleEnergyMonitor.available",
- return_value=False,
- ), patch(
- "openjarvis.telemetry.energy_rapl.RaplEnergyMonitor.available",
- return_value=False,
- ), patch(
- "openjarvis.telemetry.energy_amd.AmdEnergyMonitor.__init__",
- return_value=None,
+ with (
+ patch(
+ "openjarvis.telemetry.energy_nvidia.NvidiaEnergyMonitor.available",
+ side_effect=nvidia_available,
+ ),
+ patch(
+ "openjarvis.telemetry.energy_amd.AmdEnergyMonitor.available",
+ side_effect=amd_available,
+ ),
+ patch(
+ "openjarvis.telemetry.energy_apple.AppleEnergyMonitor.available",
+ return_value=False,
+ ),
+ patch(
+ "openjarvis.telemetry.energy_rapl.RaplEnergyMonitor.available",
+ return_value=False,
+ ),
+ patch(
+ "openjarvis.telemetry.energy_amd.AmdEnergyMonitor.__init__",
+ return_value=None,
+ ),
):
create_energy_monitor()
assert call_order == ["nvidia", "amd"]
@@ -198,18 +221,23 @@ class TestCreateEnergyMonitor:
call_order.append("apple")
return False
- with patch(
- "openjarvis.telemetry.energy_nvidia.NvidiaEnergyMonitor.available",
- side_effect=nvidia_available,
- ), patch(
- "openjarvis.telemetry.energy_amd.AmdEnergyMonitor.available",
- side_effect=amd_available,
- ), patch(
- "openjarvis.telemetry.energy_apple.AppleEnergyMonitor.available",
- side_effect=apple_available,
- ), patch(
- "openjarvis.telemetry.energy_rapl.RaplEnergyMonitor.available",
- side_effect=rapl_available,
+ with (
+ patch(
+ "openjarvis.telemetry.energy_nvidia.NvidiaEnergyMonitor.available",
+ side_effect=nvidia_available,
+ ),
+ patch(
+ "openjarvis.telemetry.energy_amd.AmdEnergyMonitor.available",
+ side_effect=amd_available,
+ ),
+ patch(
+ "openjarvis.telemetry.energy_apple.AppleEnergyMonitor.available",
+ side_effect=apple_available,
+ ),
+ patch(
+ "openjarvis.telemetry.energy_rapl.RaplEnergyMonitor.available",
+ side_effect=rapl_available,
+ ),
):
result = create_energy_monitor(prefer_vendor="cpu_rapl")
assert result is None
diff --git a/tests/telemetry/test_energy_nvidia.py b/tests/telemetry/test_energy_nvidia.py
index 264748a1..f9f4264d 100644
--- a/tests/telemetry/test_energy_nvidia.py
+++ b/tests/telemetry/test_energy_nvidia.py
@@ -34,14 +34,10 @@ def _make_fake_pynvml(device_count: int = 1, power_mw: int = 300_000):
mod.nvmlInit = MagicMock()
mod.nvmlShutdown = MagicMock()
mod.nvmlDeviceGetCount = MagicMock(return_value=device_count)
- mod.nvmlDeviceGetHandleByIndex = MagicMock(
- side_effect=lambda i: f"handle-{i}"
- )
+ mod.nvmlDeviceGetHandleByIndex = MagicMock(side_effect=lambda i: f"handle-{i}")
mod.nvmlDeviceGetName = MagicMock(return_value="NVIDIA A100-SXM")
mod.nvmlDeviceGetPowerUsage = MagicMock(return_value=power_mw)
- mod.nvmlDeviceGetUtilizationRates = MagicMock(
- return_value=_FakeUtilization()
- )
+ mod.nvmlDeviceGetUtilizationRates = MagicMock(return_value=_FakeUtilization())
mod.nvmlDeviceGetMemoryInfo = MagicMock(return_value=_FakeMemInfo())
mod.nvmlDeviceGetTemperature = MagicMock(return_value=65)
mod.nvmlDeviceGetTotalEnergyConsumption = MagicMock(return_value=5000.0)
@@ -115,8 +111,8 @@ class TestHwCounterProbe:
def test_probe_fails_on_pre_volta(self):
"""nvmlDeviceGetTotalEnergyConsumption raises => polling fallback."""
fake_pynvml = _make_fake_pynvml(device_count=1)
- fake_pynvml.nvmlDeviceGetTotalEnergyConsumption.side_effect = (
- RuntimeError("Not supported")
+ fake_pynvml.nvmlDeviceGetTotalEnergyConsumption.side_effect = RuntimeError(
+ "Not supported"
)
with patch.dict(sys.modules, {"pynvml": fake_pynvml}):
@@ -155,8 +151,8 @@ class TestEnergyMethod:
def test_returns_polling_when_no_hw_counter(self):
fake_pynvml = _make_fake_pynvml(device_count=1)
- fake_pynvml.nvmlDeviceGetTotalEnergyConsumption.side_effect = (
- RuntimeError("Not supported")
+ fake_pynvml.nvmlDeviceGetTotalEnergyConsumption.side_effect = RuntimeError(
+ "Not supported"
)
with patch.dict(sys.modules, {"pynvml": fake_pynvml}):
@@ -238,8 +234,8 @@ class TestSamplePolling:
"""Fallback mode uses trapezoidal integration of power readings."""
fake_pynvml = _make_fake_pynvml(device_count=1, power_mw=300_000)
# Make hw counter probe fail => polling mode
- fake_pynvml.nvmlDeviceGetTotalEnergyConsumption.side_effect = (
- RuntimeError("Not supported")
+ fake_pynvml.nvmlDeviceGetTotalEnergyConsumption.side_effect = RuntimeError(
+ "Not supported"
)
with patch.dict(sys.modules, {"pynvml": fake_pynvml}):
@@ -276,16 +272,18 @@ class TestSampleMultiGpu:
# 2 devices: __init__ probe reads device 0 once.
# Then sample() reads start (dev0, dev1), end (dev0, dev1).
- readings = iter([
- 1000.0, # probe: device 0
- 2000.0, # sample start: device 0
- 3000.0, # sample start: device 1
- 5000.0, # sample end: device 0
- 7000.0, # sample end: device 1
- ])
-
- fake_pynvml.nvmlDeviceGetTotalEnergyConsumption.side_effect = (
- lambda h: next(readings)
+ readings = iter(
+ [
+ 1000.0, # probe: device 0
+ 2000.0, # sample start: device 0
+ 3000.0, # sample start: device 1
+ 5000.0, # sample end: device 0
+ 7000.0, # sample end: device 1
+ ]
+ )
+
+ fake_pynvml.nvmlDeviceGetTotalEnergyConsumption.side_effect = lambda h: next(
+ readings
)
with patch.dict(sys.modules, {"pynvml": fake_pynvml}):
diff --git a/tests/telemetry/test_energy_rapl.py b/tests/telemetry/test_energy_rapl.py
index 8695c779..2bb80b08 100644
--- a/tests/telemetry/test_energy_rapl.py
+++ b/tests/telemetry/test_energy_rapl.py
@@ -33,9 +33,7 @@ def _create_rapl_domain(
domain_dir.mkdir(parents=True, exist_ok=True)
(domain_dir / "name").write_text(name)
(domain_dir / "energy_uj").write_text(str(energy_uj))
- (domain_dir / "max_energy_range_uj").write_text(
- str(max_energy_uj)
- )
+ (domain_dir / "max_energy_range_uj").write_text(str(max_energy_uj))
return domain_dir
@@ -50,12 +48,18 @@ def _build_fake_sysfs(tmp_path: Path) -> Path:
rapl_base.mkdir()
_create_rapl_domain(
- rapl_base, "package-0", "intel-rapl:0",
- energy_uj=500000, max_energy_uj=262143328850,
+ rapl_base,
+ "package-0",
+ "intel-rapl:0",
+ energy_uj=500000,
+ max_energy_uj=262143328850,
)
_create_rapl_domain(
- rapl_base, "dram", "intel-rapl:0/intel-rapl:0:0",
- energy_uj=100000, max_energy_uj=65535999603,
+ rapl_base,
+ "dram",
+ "intel-rapl:0/intel-rapl:0:0",
+ energy_uj=100000,
+ max_energy_uj=65535999603,
)
return rapl_base
@@ -127,16 +131,15 @@ class TestSampleNormalDelta:
with patch(_PLAT, return_value="Linux"):
monitor = RaplEnergyMonitor(
- poll_interval_ms=50, rapl_base=rapl_base,
+ poll_interval_ms=50,
+ rapl_base=rapl_base,
)
assert monitor._initialized is True
assert len(monitor._domains) == 2
# Set start values
pkg_energy = rapl_base / "intel-rapl:0" / "energy_uj"
- dram_energy = (
- rapl_base / "intel-rapl:0" / "intel-rapl:0:0" / "energy_uj"
- )
+ dram_energy = rapl_base / "intel-rapl:0" / "intel-rapl:0:0" / "energy_uj"
pkg_energy.write_text("500000")
dram_energy.write_text("100000")
@@ -169,13 +172,17 @@ class TestSampleWrapAround:
max_energy = 1000000
_create_rapl_domain(
- rapl_base, "package-0", "intel-rapl:0",
- energy_uj=900000, max_energy_uj=max_energy,
+ rapl_base,
+ "package-0",
+ "intel-rapl:0",
+ energy_uj=900000,
+ max_energy_uj=max_energy,
)
with patch(_PLAT, return_value="Linux"):
monitor = RaplEnergyMonitor(
- poll_interval_ms=50, rapl_base=rapl_base,
+ poll_interval_ms=50,
+ rapl_base=rapl_base,
)
assert monitor._initialized is True
@@ -205,14 +212,13 @@ class TestSampleDomainCategorization:
with patch(_PLAT, return_value="Linux"):
monitor = RaplEnergyMonitor(
- poll_interval_ms=50, rapl_base=rapl_base,
+ poll_interval_ms=50,
+ rapl_base=rapl_base,
)
# Set start values
pkg_energy = rapl_base / "intel-rapl:0" / "energy_uj"
- dram_energy = (
- rapl_base / "intel-rapl:0" / "intel-rapl:0:0" / "energy_uj"
- )
+ dram_energy = rapl_base / "intel-rapl:0" / "intel-rapl:0:0" / "energy_uj"
pkg_energy.write_text("1000")
dram_energy.write_text("2000")
@@ -237,7 +243,8 @@ class TestClose:
with patch(_PLAT, return_value="Linux"):
monitor = RaplEnergyMonitor(
- poll_interval_ms=50, rapl_base=rapl_base,
+ poll_interval_ms=50,
+ rapl_base=rapl_base,
)
assert len(monitor._domains) == 2
assert monitor._initialized is True
diff --git a/tests/telemetry/test_energy_wiring.py b/tests/telemetry/test_energy_wiring.py
index 1ceb9b7f..399b717d 100644
--- a/tests/telemetry/test_energy_wiring.py
+++ b/tests/telemetry/test_energy_wiring.py
@@ -138,25 +138,32 @@ class TestCliAskWiring:
engine = _mock_engine()
monkeypatch.setattr(
- _ask_mod, "get_engine",
+ _ask_mod,
+ "get_engine",
lambda *a, **kw: ("mock", engine),
)
monkeypatch.setattr(
- _ask_mod, "discover_engines",
+ _ask_mod,
+ "discover_engines",
lambda c: [("mock", engine)],
)
monkeypatch.setattr(
- _ask_mod, "discover_models",
+ _ask_mod,
+ "discover_models",
lambda e: {"mock": ["test-model"]},
)
return cfg, engine
def test_engine_wrapped_with_instrumented(
- self, monkeypatch, tmp_path,
+ self,
+ monkeypatch,
+ tmp_path,
):
"""InstrumentedEngine wraps engine, not instrumented_generate."""
cfg, engine = self._patch_ask(
- monkeypatch, tmp_path, gpu_metrics=False,
+ monkeypatch,
+ tmp_path,
+ gpu_metrics=False,
)
result = CliRunner().invoke(cli, ["ask", "Hello"])
assert result.exit_code == 0
@@ -165,11 +172,15 @@ class TestCliAskWiring:
engine.generate.assert_called_once()
def test_energy_monitor_created_when_gpu_metrics_on(
- self, monkeypatch, tmp_path,
+ self,
+ monkeypatch,
+ tmp_path,
):
"""Energy monitor is created when gpu_metrics=True."""
cfg, engine = self._patch_ask(
- monkeypatch, tmp_path, gpu_metrics=True,
+ monkeypatch,
+ tmp_path,
+ gpu_metrics=True,
)
mock_monitor = _mock_energy_monitor()
with patch(
@@ -182,22 +193,30 @@ class TestCliAskWiring:
mock_monitor.close.assert_called_once()
def test_no_energy_monitor_when_gpu_metrics_off(
- self, monkeypatch, tmp_path,
+ self,
+ monkeypatch,
+ tmp_path,
):
"""No energy monitor when gpu_metrics=False."""
cfg, engine = self._patch_ask(
- monkeypatch, tmp_path, gpu_metrics=False,
+ monkeypatch,
+ tmp_path,
+ gpu_metrics=False,
)
# Should not attempt to import create_energy_monitor
result = CliRunner().invoke(cli, ["ask", "Hello"])
assert result.exit_code == 0
def test_telemetry_events_published(
- self, monkeypatch, tmp_path,
+ self,
+ monkeypatch,
+ tmp_path,
):
"""InstrumentedEngine publishes TELEMETRY_RECORD events."""
cfg, engine = self._patch_ask(
- monkeypatch, tmp_path, gpu_metrics=False,
+ monkeypatch,
+ tmp_path,
+ gpu_metrics=False,
)
result = CliRunner().invoke(cli, ["ask", "Hello"])
assert result.exit_code == 0
@@ -209,11 +228,15 @@ class TestCliAskWiring:
agg.close()
def test_energy_data_in_telemetry_record(
- self, monkeypatch, tmp_path,
+ self,
+ monkeypatch,
+ tmp_path,
):
"""Energy data flows into TelemetryRecord in SQLite."""
cfg, engine = self._patch_ask(
- monkeypatch, tmp_path, gpu_metrics=True,
+ monkeypatch,
+ tmp_path,
+ gpu_metrics=True,
)
mock_monitor = _mock_energy_monitor()
with patch(
@@ -236,11 +259,15 @@ class TestCliAskWiring:
agg.close()
def test_agent_mode_uses_instrumented_engine(
- self, monkeypatch, tmp_path,
+ self,
+ monkeypatch,
+ tmp_path,
):
"""Agent mode passes InstrumentedEngine to agent."""
cfg, engine = self._patch_ask(
- monkeypatch, tmp_path, gpu_metrics=False,
+ monkeypatch,
+ tmp_path,
+ gpu_metrics=False,
)
# Register a trivial agent that calls engine.generate
@@ -262,11 +289,13 @@ class TestCliAskWiring:
return AgentResult(content="Agent OK", turns=1)
AgentRegistry.register_value(
- "test-wiring-agent", _TestAgent,
+ "test-wiring-agent",
+ _TestAgent,
)
result = CliRunner().invoke(
- cli, ["ask", "--agent", "test-wiring-agent", "Hi"],
+ cli,
+ ["ask", "--agent", "test-wiring-agent", "Hi"],
)
assert result.exit_code == 0
assert "Agent OK" in result.output
@@ -309,12 +338,15 @@ class TestSdkWiring:
cfg = _energy_config(tmp_path, gpu_metrics=True)
mock_monitor = _mock_energy_monitor()
- with patch(
- "openjarvis.sdk.get_engine",
- return_value=("mock", engine),
- ), patch(
- "openjarvis.telemetry.energy_monitor.create_energy_monitor",
- return_value=mock_monitor,
+ with (
+ patch(
+ "openjarvis.sdk.get_engine",
+ return_value=("mock", engine),
+ ),
+ patch(
+ "openjarvis.telemetry.energy_monitor.create_energy_monitor",
+ return_value=mock_monitor,
+ ),
):
j = Jarvis(config=cfg, model="test-model")
j._ensure_engine()
@@ -347,12 +379,15 @@ class TestSdkWiring:
cfg = _energy_config(tmp_path, gpu_metrics=True)
mock_monitor = _mock_energy_monitor()
- with patch(
- "openjarvis.sdk.get_engine",
- return_value=("mock", engine),
- ), patch(
- "openjarvis.telemetry.energy_monitor.create_energy_monitor",
- return_value=mock_monitor,
+ with (
+ patch(
+ "openjarvis.sdk.get_engine",
+ return_value=("mock", engine),
+ ),
+ patch(
+ "openjarvis.telemetry.energy_monitor.create_energy_monitor",
+ return_value=mock_monitor,
+ ),
):
j = Jarvis(config=cfg, model="test-model")
result = j.ask_full("Hello")
@@ -376,12 +411,15 @@ class TestSdkWiring:
cfg.telemetry.gpu_metrics = True
mock_monitor = _mock_energy_monitor()
- with patch(
- "openjarvis.sdk.get_engine",
- return_value=("mock", engine),
- ), patch(
- "openjarvis.telemetry.energy_monitor.create_energy_monitor",
- return_value=mock_monitor,
+ with (
+ patch(
+ "openjarvis.sdk.get_engine",
+ return_value=("mock", engine),
+ ),
+ patch(
+ "openjarvis.telemetry.energy_monitor.create_energy_monitor",
+ return_value=mock_monitor,
+ ),
):
j = Jarvis(config=cfg, model="test-model")
j._ensure_engine()
@@ -398,12 +436,15 @@ class TestSdkWiring:
cfg.telemetry.gpu_metrics = True
mock_monitor = _mock_energy_monitor()
- with patch(
- "openjarvis.sdk.get_engine",
- return_value=("mock", engine),
- ), patch(
- "openjarvis.telemetry.energy_monitor.create_energy_monitor",
- return_value=mock_monitor,
+ with (
+ patch(
+ "openjarvis.sdk.get_engine",
+ return_value=("mock", engine),
+ ),
+ patch(
+ "openjarvis.telemetry.energy_monitor.create_energy_monitor",
+ return_value=mock_monitor,
+ ),
):
j = Jarvis(config=cfg, model="test-model")
j._ensure_engine()
@@ -426,7 +467,9 @@ class TestInstrumentedEngineEnergy:
monitor = _mock_energy_monitor()
ie = InstrumentedEngine(
- engine, bus, energy_monitor=monitor,
+ engine,
+ bus,
+ energy_monitor=monitor,
)
messages = [Message(role=Role.USER, content="Hi")]
result = ie.generate(messages, model="test")
@@ -434,8 +477,7 @@ class TestInstrumentedEngineEnergy:
assert result["content"] == "Test response"
# Verify energy data is in the telemetry record
tel_events = [
- e for e in bus.history
- if e.event_type == EventType.TELEMETRY_RECORD
+ e for e in bus.history if e.event_type == EventType.TELEMETRY_RECORD
]
assert len(tel_events) == 1
rec = tel_events[0].data["record"]
@@ -452,7 +494,9 @@ class TestInstrumentedEngineEnergy:
monitor = _mock_energy_monitor()
ie = InstrumentedEngine(
- engine, bus, energy_monitor=monitor,
+ engine,
+ bus,
+ energy_monitor=monitor,
)
messages = [Message(role=Role.USER, content="Hi")]
result = ie.generate(messages, model="test")
@@ -477,10 +521,7 @@ class TestInstrumentedEngineEnergy:
result = ie.generate(messages, model="test")
assert result["content"] == "Test response"
- tel = [
- e for e in bus.history
- if e.event_type == EventType.TELEMETRY_RECORD
- ]
+ tel = [e for e in bus.history if e.event_type == EventType.TELEMETRY_RECORD]
rec = tel[0].data["record"]
assert rec.energy_joules == 0.0
assert rec.energy_method == ""
@@ -499,7 +540,9 @@ class TestInstrumentedEngineEnergy:
monitor.sample = _broken_sample
ie = InstrumentedEngine(
- engine, bus, energy_monitor=monitor,
+ engine,
+ bus,
+ energy_monitor=monitor,
)
messages = [Message(role=Role.USER, content="Hi")]
# Should not crash — energy is best-effort.
@@ -537,18 +580,23 @@ class TestBenchWiring:
cfg.telemetry.gpu_metrics = True
mock_monitor = _mock_energy_monitor()
- with patch(
- "openjarvis.cli.bench_cmd.get_engine",
- return_value=("mock", engine),
- ), patch(
- "openjarvis.cli.bench_cmd.load_config",
- return_value=cfg,
- ), patch(
- "openjarvis.telemetry.energy_monitor.create_energy_monitor",
- return_value=mock_monitor,
- ) as mock_create:
+ with (
+ patch(
+ "openjarvis.cli.bench_cmd.get_engine",
+ return_value=("mock", engine),
+ ),
+ patch(
+ "openjarvis.cli.bench_cmd.load_config",
+ return_value=cfg,
+ ),
+ patch(
+ "openjarvis.telemetry.energy_monitor.create_energy_monitor",
+ return_value=mock_monitor,
+ ) as mock_create,
+ ):
result = CliRunner().invoke(
- cli, ["bench", "run", "-n", "2"],
+ cli,
+ ["bench", "run", "-n", "2"],
)
assert result.exit_code == 0
@@ -572,15 +620,19 @@ class TestBenchWiring:
cfg = JarvisConfig()
cfg.telemetry.gpu_metrics = False
- with patch(
- "openjarvis.cli.bench_cmd.get_engine",
- return_value=("mock", engine),
- ), patch(
- "openjarvis.cli.bench_cmd.load_config",
- return_value=cfg,
+ with (
+ patch(
+ "openjarvis.cli.bench_cmd.get_engine",
+ return_value=("mock", engine),
+ ),
+ patch(
+ "openjarvis.cli.bench_cmd.load_config",
+ return_value=cfg,
+ ),
):
result = CliRunner().invoke(
- cli, ["bench", "run", "-n", "2"],
+ cli,
+ ["bench", "run", "-n", "2"],
)
assert result.exit_code == 0
@@ -602,15 +654,19 @@ class TestBenchWiring:
cfg = JarvisConfig()
cfg.telemetry.gpu_metrics = False
- with patch(
- "openjarvis.cli.bench_cmd.get_engine",
- return_value=("mock", engine),
- ), patch(
- "openjarvis.cli.bench_cmd.load_config",
- return_value=cfg,
+ with (
+ patch(
+ "openjarvis.cli.bench_cmd.get_engine",
+ return_value=("mock", engine),
+ ),
+ patch(
+ "openjarvis.cli.bench_cmd.load_config",
+ return_value=cfg,
+ ),
):
result = CliRunner().invoke(
- cli, ["bench", "run", "-n", "2", "-w", "3"],
+ cli,
+ ["bench", "run", "-n", "2", "-w", "3"],
)
assert result.exit_code == 0
@@ -625,14 +681,16 @@ def _populate_energy_db(db_path: Path, n: int = 3) -> None:
"""Create a telemetry DB with energy-enriched records."""
store = TelemetryStore(db_path)
for i in range(n):
- store.record(_make_energy_record(
- model_id=f"model-{i % 2}",
- energy_joules=10.0 * (i + 1),
- throughput=100.0 + i * 10,
- gpu_util=70.0 + i * 5,
- power=200.0 + i * 25,
- ts=time.time() - (n - i),
- ))
+ store.record(
+ _make_energy_record(
+ model_id=f"model-{i % 2}",
+ energy_joules=10.0 * (i + 1),
+ throughput=100.0 + i * 10,
+ gpu_util=70.0 + i * 5,
+ power=200.0 + i * 25,
+ ts=time.time() - (n - i),
+ )
+ )
store.close()
@@ -656,7 +714,8 @@ class TestTelemetryStatsEnergy:
_populate_energy_db(db_path)
with p:
result = CliRunner().invoke(
- cli, ["telemetry", "stats"],
+ cli,
+ ["telemetry", "stats"],
)
assert result.exit_code == 0
assert "Total Energy (J)" in result.output
@@ -674,21 +733,24 @@ class TestTelemetryStatsEnergy:
# Populate with non-energy records
store = TelemetryStore(db_path)
for i in range(3):
- store.record(TelemetryRecord(
- timestamp=time.time(),
- model_id="model-0",
- engine="ollama",
- prompt_tokens=10,
- completion_tokens=5,
- total_tokens=15,
- latency_seconds=0.5,
- cost_usd=0.001,
- ))
+ store.record(
+ TelemetryRecord(
+ timestamp=time.time(),
+ model_id="model-0",
+ engine="ollama",
+ prompt_tokens=10,
+ completion_tokens=5,
+ total_tokens=15,
+ latency_seconds=0.5,
+ cost_usd=0.001,
+ )
+ )
store.close()
with p:
result = CliRunner().invoke(
- cli, ["telemetry", "stats"],
+ cli,
+ ["telemetry", "stats"],
)
assert result.exit_code == 0
assert "Total Calls" in result.output
@@ -702,7 +764,8 @@ class TestTelemetryStatsEnergy:
_populate_energy_db(db_path, n=1)
with p:
result = CliRunner().invoke(
- cli, ["telemetry", "export", "-f", "json"],
+ cli,
+ ["telemetry", "export", "-f", "json"],
)
assert result.exit_code == 0
data = json.loads(result.output)
@@ -726,7 +789,8 @@ class TestTelemetryStatsEnergy:
_populate_energy_db(db_path, n=1)
with p:
result = CliRunner().invoke(
- cli, ["telemetry", "export", "-f", "csv"],
+ cli,
+ ["telemetry", "export", "-f", "csv"],
)
assert result.exit_code == 0
header = result.output.strip().splitlines()[0]
@@ -755,24 +819,30 @@ class TestAggregatorEnergy:
"""summary() sums energy and computes weighted averages."""
db_path = tmp_path / "telemetry.db"
store = TelemetryStore(db_path)
- store.record(_make_energy_record(
- model_id="m1",
- energy_joules=10.0,
- throughput=100.0,
- gpu_util=80.0,
- ))
- store.record(_make_energy_record(
- model_id="m1",
- energy_joules=20.0,
- throughput=120.0,
- gpu_util=90.0,
- ))
- store.record(_make_energy_record(
- model_id="m2",
- energy_joules=30.0,
- throughput=80.0,
- gpu_util=60.0,
- ))
+ store.record(
+ _make_energy_record(
+ model_id="m1",
+ energy_joules=10.0,
+ throughput=100.0,
+ gpu_util=80.0,
+ )
+ )
+ store.record(
+ _make_energy_record(
+ model_id="m1",
+ energy_joules=20.0,
+ throughput=120.0,
+ gpu_util=90.0,
+ )
+ )
+ store.record(
+ _make_energy_record(
+ model_id="m2",
+ energy_joules=30.0,
+ throughput=80.0,
+ gpu_util=60.0,
+ )
+ )
store.close()
agg = TelemetryAggregator(db_path)
@@ -784,7 +854,8 @@ class TestAggregatorEnergy:
assert s.avg_throughput_tok_per_sec == pytest.approx(100.0)
# Weighted avg GPU util: (85*2 + 60*1) / 3 = 76.67
assert s.avg_gpu_utilization_pct == pytest.approx(
- 76.666, rel=0.01,
+ 76.666,
+ rel=0.01,
)
agg.close()
@@ -792,9 +863,12 @@ class TestAggregatorEnergy:
"""per_model_stats includes energy fields."""
db_path = tmp_path / "telemetry.db"
store = TelemetryStore(db_path)
- store.record(_make_energy_record(
- model_id="m1", energy_joules=50.0,
- ))
+ store.record(
+ _make_energy_record(
+ model_id="m1",
+ energy_joules=50.0,
+ )
+ )
store.close()
agg = TelemetryAggregator(db_path)
@@ -811,9 +885,12 @@ class TestAggregatorEnergy:
"""per_engine_stats includes energy fields."""
db_path = tmp_path / "telemetry.db"
store = TelemetryStore(db_path)
- store.record(_make_energy_record(
- engine="vllm", energy_joules=25.0,
- ))
+ store.record(
+ _make_energy_record(
+ engine="vllm",
+ energy_joules=25.0,
+ )
+ )
store.close()
agg = TelemetryAggregator(db_path)
@@ -845,7 +922,9 @@ class TestEndToEndPipeline:
"""Full pipeline: ask → InstrumentedEngine → energy → SQLite → stats."""
def test_ask_to_stats_with_energy(
- self, monkeypatch, tmp_path,
+ self,
+ monkeypatch,
+ tmp_path,
):
"""Full flow: ask records energy, stats displays it."""
cfg = _energy_config(tmp_path, gpu_metrics=True)
@@ -853,15 +932,18 @@ class TestEndToEndPipeline:
monkeypatch.setattr(_ask_mod, "load_config", lambda: cfg)
monkeypatch.setattr(
- _ask_mod, "get_engine",
+ _ask_mod,
+ "get_engine",
lambda *a, **kw: ("mock", engine),
)
monkeypatch.setattr(
- _ask_mod, "discover_engines",
+ _ask_mod,
+ "discover_engines",
lambda c: [("mock", engine)],
)
monkeypatch.setattr(
- _ask_mod, "discover_models",
+ _ask_mod,
+ "discover_models",
lambda e: {"mock": ["test-model"]},
)
@@ -880,7 +962,8 @@ class TestEndToEndPipeline:
return_value=telem_cfg,
):
result = CliRunner().invoke(
- cli, ["telemetry", "stats"],
+ cli,
+ ["telemetry", "stats"],
)
assert result.exit_code == 0
@@ -888,7 +971,9 @@ class TestEndToEndPipeline:
assert "42.50" in result.output # energy_joules value
def test_ask_to_export_with_energy(
- self, monkeypatch, tmp_path,
+ self,
+ monkeypatch,
+ tmp_path,
):
"""Full flow: ask records energy, export includes it."""
cfg = _energy_config(tmp_path, gpu_metrics=True)
@@ -896,15 +981,18 @@ class TestEndToEndPipeline:
monkeypatch.setattr(_ask_mod, "load_config", lambda: cfg)
monkeypatch.setattr(
- _ask_mod, "get_engine",
+ _ask_mod,
+ "get_engine",
lambda *a, **kw: ("mock", engine),
)
monkeypatch.setattr(
- _ask_mod, "discover_engines",
+ _ask_mod,
+ "discover_engines",
lambda c: [("mock", engine)],
)
monkeypatch.setattr(
- _ask_mod, "discover_models",
+ _ask_mod,
+ "discover_models",
lambda e: {"mock": ["test-model"]},
)
@@ -923,7 +1011,8 @@ class TestEndToEndPipeline:
return_value=telem_cfg,
):
result = CliRunner().invoke(
- cli, ["telemetry", "export", "-f", "json"],
+ cli,
+ ["telemetry", "export", "-f", "json"],
)
data = json.loads(result.output)
diff --git a/tests/telemetry/test_gpu_monitor.py b/tests/telemetry/test_gpu_monitor.py
index 2ff34b6b..e4b0f1dc 100644
--- a/tests/telemetry/test_gpu_monitor.py
+++ b/tests/telemetry/test_gpu_monitor.py
@@ -28,32 +28,22 @@ class _FakeMemInfo:
free: int = 12 * 1024**3
-def _make_fake_pynvml(
- device_count: int = 1, power_mw: int = 300_000
-):
+def _make_fake_pynvml(device_count: int = 1, power_mw: int = 300_000):
"""Return a fake pynvml module object."""
mod = types.ModuleType("pynvml")
mod.nvmlInit = MagicMock()
mod.nvmlShutdown = MagicMock()
mod.nvmlDeviceGetCount = MagicMock(return_value=device_count)
- mod.nvmlDeviceGetHandleByIndex = MagicMock(
- side_effect=lambda i: f"handle-{i}"
- )
+ mod.nvmlDeviceGetHandleByIndex = MagicMock(side_effect=lambda i: f"handle-{i}")
mod.nvmlDeviceGetPowerUsage = MagicMock(return_value=power_mw)
- mod.nvmlDeviceGetUtilizationRates = MagicMock(
- return_value=_FakeUtilization()
- )
- mod.nvmlDeviceGetMemoryInfo = MagicMock(
- return_value=_FakeMemInfo()
- )
+ mod.nvmlDeviceGetUtilizationRates = MagicMock(return_value=_FakeUtilization())
+ mod.nvmlDeviceGetMemoryInfo = MagicMock(return_value=_FakeMemInfo())
mod.nvmlDeviceGetTemperature = MagicMock(return_value=65)
mod.NVML_TEMPERATURE_GPU = 0
return mod
-def _snap(
- power=300, util=80, mem=12, temp=65, dev=0
-):
+def _snap(power=300, util=80, mem=12, temp=65, dev=0):
"""Shorthand to build a GpuSnapshot."""
from openjarvis.telemetry.gpu_monitor import GpuSnapshot
@@ -105,12 +95,18 @@ class TestGpuHardwareSpec:
expected = {
"B200-SXM",
- "A100-SXM", "A100-PCIE",
- "H100-SXM", "H100-PCIE",
- "L40S", "A10",
- "RTX 4090", "RTX 3090",
- "MI300X", "MI250X",
- "M4 Max", "M2 Ultra",
+ "A100-SXM",
+ "A100-PCIE",
+ "H100-SXM",
+ "H100-PCIE",
+ "L40S",
+ "A10",
+ "RTX 4090",
+ "RTX 3090",
+ "MI300X",
+ "MI250X",
+ "M4 Max",
+ "M2 Ultra",
}
assert set(GPU_SPECS.keys()) == expected
@@ -132,15 +128,10 @@ class TestEnergyIntegration:
"""Constant 300W for 10 seconds = 3000 J."""
from openjarvis.telemetry.gpu_monitor import GpuMonitor
- snapshots = [
- [_snap(power=300)]
- for _ in range(11)
- ]
+ snapshots = [[_snap(power=300)] for _ in range(11)]
timestamps = [float(i) for i in range(11)]
- sample = GpuMonitor._aggregate(
- snapshots, timestamps, wall_duration=10.0
- )
+ sample = GpuMonitor._aggregate(snapshots, timestamps, wall_duration=10.0)
assert sample.energy_joules == pytest.approx(3000.0, rel=1e-6)
assert sample.mean_power_watts == pytest.approx(300.0, rel=1e-6)
assert sample.peak_power_watts == pytest.approx(300.0, rel=1e-6)
@@ -152,14 +143,11 @@ class TestEnergyIntegration:
from openjarvis.telemetry.gpu_monitor import GpuMonitor
snapshots = [
- [_snap(power=100.0 * i, util=50, mem=8, temp=60)]
- for i in range(5)
+ [_snap(power=100.0 * i, util=50, mem=8, temp=60)] for i in range(5)
]
timestamps = [float(i) for i in range(5)]
- sample = GpuMonitor._aggregate(
- snapshots, timestamps, wall_duration=4.0
- )
+ sample = GpuMonitor._aggregate(snapshots, timestamps, wall_duration=4.0)
assert sample.energy_joules == pytest.approx(800.0, rel=1e-6)
assert sample.mean_power_watts == pytest.approx(200.0, rel=1e-6)
assert sample.peak_power_watts == pytest.approx(400.0)
@@ -177,14 +165,10 @@ class TestEnergyIntegration:
"""One snapshot: energy is zero (no interval)."""
from openjarvis.telemetry.gpu_monitor import GpuMonitor
- snapshots = [
- [_snap(power=250, util=90, mem=16, temp=70)]
- ]
+ snapshots = [[_snap(power=250, util=90, mem=16, temp=70)]]
timestamps = [0.0]
- sample = GpuMonitor._aggregate(
- snapshots, timestamps, wall_duration=0.05
- )
+ sample = GpuMonitor._aggregate(snapshots, timestamps, wall_duration=0.05)
assert sample.energy_joules == 0.0
assert sample.num_snapshots == 1
assert sample.mean_power_watts == pytest.approx(250.0)
@@ -206,9 +190,7 @@ class TestGpuSampleAggregation:
]
timestamps = [0.0, 1.0, 2.0]
- sample = GpuMonitor._aggregate(
- snapshots, timestamps, wall_duration=2.0
- )
+ sample = GpuMonitor._aggregate(snapshots, timestamps, wall_duration=2.0)
assert sample.peak_power_watts == pytest.approx(400.0)
assert sample.peak_utilization_pct == pytest.approx(95.0)
assert sample.peak_memory_used_gb == pytest.approx(20.0)
@@ -224,9 +206,7 @@ class TestGpuSampleAggregation:
]
timestamps = [0.0, 1.0, 2.0]
- sample = GpuMonitor._aggregate(
- snapshots, timestamps, wall_duration=2.0
- )
+ sample = GpuMonitor._aggregate(snapshots, timestamps, wall_duration=2.0)
assert sample.mean_power_watts == pytest.approx(200.0)
assert sample.mean_utilization_pct == pytest.approx(60.0)
assert sample.mean_memory_used_gb == pytest.approx(12.0)
@@ -250,9 +230,7 @@ class TestMultiGpu:
snapshots = [tick, tick]
timestamps = [0.0, 1.0]
- sample = GpuMonitor._aggregate(
- snapshots, timestamps, wall_duration=1.0
- )
+ sample = GpuMonitor._aggregate(snapshots, timestamps, wall_duration=1.0)
assert sample.energy_joules == pytest.approx(500.0)
assert sample.mean_power_watts == pytest.approx(500.0)
assert sample.mean_utilization_pct == pytest.approx(85.0)
@@ -275,9 +253,7 @@ class TestMultiGpu:
]
timestamps = [0.0, 2.0]
- sample = GpuMonitor._aggregate(
- snapshots, timestamps, wall_duration=2.0
- )
+ sample = GpuMonitor._aggregate(snapshots, timestamps, wall_duration=2.0)
# Tick powers: 200, 600 => 0.5*(200+600)*2 = 800
assert sample.energy_joules == pytest.approx(800.0)
assert sample.peak_power_watts == pytest.approx(600.0)
@@ -291,9 +267,7 @@ class TestMultiGpu:
class TestContextManager:
def test_sample_context_manager(self):
"""sample() starts/stops polling and populates result."""
- fake_pynvml = _make_fake_pynvml(
- device_count=1, power_mw=300_000
- )
+ fake_pynvml = _make_fake_pynvml(device_count=1, power_mw=300_000)
import openjarvis.telemetry.gpu_monitor as mod
diff --git a/tests/telemetry/test_instrumented_engine.py b/tests/telemetry/test_instrumented_engine.py
index 2795ad9e..50e23965 100644
--- a/tests/telemetry/test_instrumented_engine.py
+++ b/tests/telemetry/test_instrumented_engine.py
@@ -63,8 +63,7 @@ class TestInstrumentedEngine:
ie.generate(messages, model="test")
tel_events = [
- e for e in bus.history
- if e.event_type == EventType.TELEMETRY_RECORD
+ e for e in bus.history if e.event_type == EventType.TELEMETRY_RECORD
]
assert len(tel_events) == 1
record = tel_events[0].data["record"]
@@ -92,9 +91,8 @@ class TestInstrumentedEngine:
messages = [Message(role=Role.USER, content="Hi")]
ie.generate(messages, model="test", temperature=0.5, max_tokens=100)
call_kwargs = mock_engine.generate.call_args
- temp = (
- call_kwargs.kwargs.get("temperature")
- or call_kwargs[1].get("temperature")
+ temp = call_kwargs.kwargs.get("temperature") or call_kwargs[1].get(
+ "temperature"
)
assert temp == 0.5
@@ -130,8 +128,7 @@ class TestTokensPerJoule:
ie.generate(messages, model="test")
tel_events = [
- e for e in bus.history
- if e.event_type == EventType.TELEMETRY_RECORD
+ e for e in bus.history if e.event_type == EventType.TELEMETRY_RECORD
]
record = tel_events[0].data["record"]
assert record.tokens_per_joule == 0.0
diff --git a/tests/telemetry/test_itl_metrics.py b/tests/telemetry/test_itl_metrics.py
index ff067575..0bd8aeb8 100644
--- a/tests/telemetry/test_itl_metrics.py
+++ b/tests/telemetry/test_itl_metrics.py
@@ -330,18 +330,20 @@ class TestItlStorage:
def test_store_and_query_itl(self, tmp_path):
store = TelemetryStore(tmp_path / "test.db")
- store.record(TelemetryRecord(
- timestamp=time.time(),
- model_id="m1",
- engine="mock",
- mean_itl_ms=15.0,
- median_itl_ms=14.0,
- p90_itl_ms=20.0,
- p95_itl_ms=25.0,
- p99_itl_ms=30.0,
- std_itl_ms=5.0,
- is_streaming=True,
- ))
+ store.record(
+ TelemetryRecord(
+ timestamp=time.time(),
+ model_id="m1",
+ engine="mock",
+ mean_itl_ms=15.0,
+ median_itl_ms=14.0,
+ p90_itl_ms=20.0,
+ p95_itl_ms=25.0,
+ p99_itl_ms=30.0,
+ std_itl_ms=5.0,
+ is_streaming=True,
+ )
+ )
agg = TelemetryAggregator(tmp_path / "test.db")
stats = agg.per_model_stats()
diff --git a/tests/telemetry/test_itl_new.py b/tests/telemetry/test_itl_new.py
index 66f1ec00..b553b20d 100644
--- a/tests/telemetry/test_itl_new.py
+++ b/tests/telemetry/test_itl_new.py
@@ -57,7 +57,12 @@ class TestComputeItlStats:
def test_all_keys_present(self):
result = compute_itl_stats([0.0, 5.0, 10.0])
expected_keys = {
- "p50_ms", "p90_ms", "p95_ms", "p99_ms",
- "mean_ms", "min_ms", "max_ms",
+ "p50_ms",
+ "p90_ms",
+ "p95_ms",
+ "p99_ms",
+ "mean_ms",
+ "min_ms",
+ "max_ms",
}
assert set(result.keys()) == expected_keys
diff --git a/tests/telemetry/test_phase_energy.py b/tests/telemetry/test_phase_energy.py
index 2878e28c..40f539b0 100644
--- a/tests/telemetry/test_phase_energy.py
+++ b/tests/telemetry/test_phase_energy.py
@@ -85,9 +85,7 @@ class TestDecodeLatency:
ie.generate([Message(role=Role.USER, content="hi")], model="m")
rec = records[0]
# decode_latency = latency - ttft
- assert rec.decode_latency_seconds == pytest.approx(
- rec.latency_seconds - 0.1
- )
+ assert rec.decode_latency_seconds == pytest.approx(rec.latency_seconds - 0.1)
def test_decode_latency_zero_when_no_ttft(self):
bus = EventBus()
@@ -213,14 +211,16 @@ class TestPhaseEnergyStorage:
def test_store_and_aggregate(self, tmp_path):
store = TelemetryStore(tmp_path / "test.db")
- store.record(TelemetryRecord(
- timestamp=time.time(),
- model_id="m1",
- engine="mock",
- energy_joules=10.0,
- prefill_energy_joules=3.0,
- decode_energy_joules=7.0,
- ))
+ store.record(
+ TelemetryRecord(
+ timestamp=time.time(),
+ model_id="m1",
+ engine="mock",
+ energy_joules=10.0,
+ prefill_energy_joules=3.0,
+ decode_energy_joules=7.0,
+ )
+ )
agg = TelemetryAggregator(tmp_path / "test.db")
stats = agg.per_model_stats()
diff --git a/tests/telemetry/test_phase_metrics_new.py b/tests/telemetry/test_phase_metrics_new.py
index d4c5f23f..569b88a8 100644
--- a/tests/telemetry/test_phase_metrics_new.py
+++ b/tests/telemetry/test_phase_metrics_new.py
@@ -14,11 +14,13 @@ class TestComputePhaseMetrics:
session = TelemetrySession(monitor=None)
# Manually push samples into the buffer
for i in range(11):
- session._buffer.push(TelemetrySample(
- timestamp_ns=i * 100_000_000,
- gpu_power_w=100.0,
- cpu_power_w=50.0,
- ))
+ session._buffer.push(
+ TelemetrySample(
+ timestamp_ns=i * 100_000_000,
+ gpu_power_w=100.0,
+ cpu_power_w=50.0,
+ )
+ )
return session
def test_basic_metrics(self):
@@ -38,7 +40,7 @@ class TestComputePhaseMetrics:
prefill, decode = split_at_ttft(
session,
start_ns=0,
- ttft_ns=500_000_000, # 500ms
+ ttft_ns=500_000_000, # 500ms
end_ns=1_000_000_000, # 1s
input_tokens=50,
output_tokens=100,
diff --git a/tests/telemetry/test_session.py b/tests/telemetry/test_session.py
index 7226b27e..24b15801 100644
--- a/tests/telemetry/test_session.py
+++ b/tests/telemetry/test_session.py
@@ -40,14 +40,16 @@ class TestPythonRingBuffer:
buf = _PythonRingBuffer(capacity=100)
# 100W GPU, 50W CPU for 1 second (10 samples, 100ms apart)
for i in range(11):
- buf.push(TelemetrySample(
- timestamp_ns=i * 100_000_000, # 0ms, 100ms, ..., 1000ms
- gpu_power_w=100.0,
- cpu_power_w=50.0,
- ))
+ buf.push(
+ TelemetrySample(
+ timestamp_ns=i * 100_000_000, # 0ms, 100ms, ..., 1000ms
+ gpu_power_w=100.0,
+ cpu_power_w=50.0,
+ )
+ )
gpu_j, cpu_j = buf.compute_energy_delta(0, 1_000_000_000)
assert abs(gpu_j - 100.0) < 1.0 # 100W * 1s = 100J
- assert abs(cpu_j - 50.0) < 1.0 # 50W * 1s = 50J
+ assert abs(cpu_j - 50.0) < 1.0 # 50W * 1s = 50J
def test_energy_delta_insufficient_samples(self):
buf = _PythonRingBuffer(capacity=100)
@@ -59,10 +61,13 @@ class TestPythonRingBuffer:
def test_avg_power(self):
buf = _PythonRingBuffer(capacity=100)
buf.push(TelemetrySample(timestamp_ns=0, gpu_power_w=100.0, cpu_power_w=40.0))
- buf.push(TelemetrySample(
- timestamp_ns=500_000_000, gpu_power_w=200.0,
- cpu_power_w=60.0,
- ))
+ buf.push(
+ TelemetrySample(
+ timestamp_ns=500_000_000,
+ gpu_power_w=200.0,
+ cpu_power_w=60.0,
+ )
+ )
gpu_w, cpu_w = buf.compute_avg_power(0, 1_000_000_000)
assert gpu_w == 150.0
assert cpu_w == 50.0
diff --git a/tests/telemetry/test_steady_state.py b/tests/telemetry/test_steady_state.py
index 0420afc4..34eaa307 100644
--- a/tests/telemetry/test_steady_state.py
+++ b/tests/telemetry/test_steady_state.py
@@ -68,7 +68,10 @@ class TestSteadyStateDetector:
def test_erratic_values_no_steady_state(self):
"""Highly variable values should not reach steady state."""
cfg = SteadyStateConfig(
- warmup_samples=2, window_size=3, cv_threshold=0.01, min_steady_samples=3,
+ warmup_samples=2,
+ window_size=3,
+ cv_threshold=0.01,
+ min_steady_samples=3,
)
detector = SteadyStateDetector(cfg)
@@ -96,7 +99,10 @@ class TestSteadyStateDetector:
def test_cv_calculation_correctness(self):
"""Verify CV-based detection with known values."""
cfg = SteadyStateConfig(
- warmup_samples=2, window_size=3, cv_threshold=0.05, min_steady_samples=1,
+ warmup_samples=2,
+ window_size=3,
+ cv_threshold=0.05,
+ min_steady_samples=1,
)
detector = SteadyStateDetector(cfg)
diff --git a/tests/telemetry/test_vllm_metrics.py b/tests/telemetry/test_vllm_metrics.py
index eb651e58..89dcf4ef 100644
--- a/tests/telemetry/test_vllm_metrics.py
+++ b/tests/telemetry/test_vllm_metrics.py
@@ -166,6 +166,7 @@ class TestVLLMMetricsScraper:
class FakeResp:
status_code = 200
text = SAMPLE_METRICS
+
def raise_for_status(self):
pass
diff --git a/tests/telemetry/test_wrapper.py b/tests/telemetry/test_wrapper.py
index d280c661..63462214 100644
--- a/tests/telemetry/test_wrapper.py
+++ b/tests/telemetry/test_wrapper.py
@@ -73,8 +73,7 @@ class TestInstrumentedGenerate:
bus=bus,
)
telem_events = [
- e for e in bus.history
- if e.event_type == EventType.TELEMETRY_RECORD
+ e for e in bus.history if e.event_type == EventType.TELEMETRY_RECORD
]
assert len(telem_events) == 1
rec = telem_events[0].data["record"]
@@ -91,8 +90,7 @@ class TestInstrumentedGenerate:
bus=bus,
)
telem_events = [
- e for e in bus.history
- if e.event_type == EventType.TELEMETRY_RECORD
+ e for e in bus.history if e.event_type == EventType.TELEMETRY_RECORD
]
rec = telem_events[0].data["record"]
assert rec.latency_seconds >= 0
diff --git a/tests/templates/test_agent_templates.py b/tests/templates/test_agent_templates.py
index 5877af44..5f703393 100644
--- a/tests/templates/test_agent_templates.py
+++ b/tests/templates/test_agent_templates.py
@@ -13,11 +13,7 @@ from openjarvis.templates.agent_templates import (
)
TEMPLATES_DIR = (
- Path(__file__).resolve().parents[2]
- / "src"
- / "openjarvis"
- / "templates"
- / "data"
+ Path(__file__).resolve().parents[2] / "src" / "openjarvis" / "templates" / "data"
)
VALID_AGENT_TYPES = {"simple", "orchestrator", "native_react", "monitor"}
diff --git a/tests/test_orchestrator_learning/test_environment.py b/tests/test_orchestrator_learning/test_environment.py
index eee9b532..bcce6ac2 100644
--- a/tests/test_orchestrator_learning/test_environment.py
+++ b/tests/test_orchestrator_learning/test_environment.py
@@ -86,9 +86,7 @@ class TestOrchestratorEnvironment:
assert env.is_done(state) is True
def test_is_done_on_max_turns(self):
- env = OrchestratorEnvironment(
- tools=[_MockCalculator()], max_turns=2
- )
+ env = OrchestratorEnvironment(tools=[_MockCalculator()], max_turns=2)
state = env.reset("q")
for _ in range(2):
action = OrchestratorAction(
@@ -107,9 +105,7 @@ class TestOrchestratorEnvironment:
env.step(state, action)
def test_max_turns_exceeded_raises(self):
- env = OrchestratorEnvironment(
- tools=[_MockCalculator()], max_turns=1
- )
+ env = OrchestratorEnvironment(tools=[_MockCalculator()], max_turns=1)
state = env.reset("q")
action = OrchestratorAction(
thought="go", tool_name="calculator", tool_input="1+1"
diff --git a/tests/test_orchestrator_learning/test_grpo_trainer.py b/tests/test_orchestrator_learning/test_grpo_trainer.py
index 066e5c24..4f122640 100644
--- a/tests/test_orchestrator_learning/test_grpo_trainer.py
+++ b/tests/test_orchestrator_learning/test_grpo_trainer.py
@@ -99,4 +99,5 @@ class TestGRPORegistration:
def test_registered_in_learning_registry(self):
import openjarvis.learning.intelligence.orchestrator.grpo_trainer # noqa: F401
from openjarvis.core.registry import LearningRegistry
+
assert LearningRegistry.contains("orchestrator_grpo")
diff --git a/tests/test_orchestrator_learning/test_orchestrator_structured.py b/tests/test_orchestrator_learning/test_orchestrator_structured.py
index d96ef7bb..63643935 100644
--- a/tests/test_orchestrator_learning/test_orchestrator_structured.py
+++ b/tests/test_orchestrator_learning/test_orchestrator_structured.py
@@ -55,9 +55,7 @@ class _MockTool(BaseTool):
def execute(self, **params) -> ToolResult:
expr = params.get("expression", "")
- return ToolResult(
- tool_name="calculator", content=str(expr), success=True
- )
+ return ToolResult(tool_name="calculator", content=str(expr), success=True)
# -- Tests -------------------------------------------------------------------
@@ -66,10 +64,12 @@ class _MockTool(BaseTool):
class TestStructuredMode:
def test_thought_tool_input_then_final_answer(self):
"""Test full structured loop: TOOL call -> FINAL_ANSWER."""
- engine = _MockEngine([
- "THOUGHT: I need to calculate\nTOOL: calculator\nINPUT: 2+2",
- "THOUGHT: Got result\nFINAL_ANSWER: 4",
- ])
+ engine = _MockEngine(
+ [
+ "THOUGHT: I need to calculate\nTOOL: calculator\nINPUT: 2+2",
+ "THOUGHT: Got result\nFINAL_ANSWER: 4",
+ ]
+ )
agent = OrchestratorAgent(
engine=engine,
model="test",
@@ -83,9 +83,11 @@ class TestStructuredMode:
def test_direct_final_answer(self):
"""Test that FINAL_ANSWER on first turn works."""
- engine = _MockEngine([
- "THOUGHT: Easy\nFINAL_ANSWER: Paris",
- ])
+ engine = _MockEngine(
+ [
+ "THOUGHT: Easy\nFINAL_ANSWER: Paris",
+ ]
+ )
agent = OrchestratorAgent(
engine=engine,
model="test",
diff --git a/tests/test_orchestrator_learning/test_policy_model.py b/tests/test_orchestrator_learning/test_policy_model.py
index 59890251..09795353 100644
--- a/tests/test_orchestrator_learning/test_policy_model.py
+++ b/tests/test_orchestrator_learning/test_policy_model.py
@@ -22,11 +22,7 @@ class TestParseOutput:
def test_valid_thought_tool_input(self):
m = self._model()
- text = (
- "THOUGHT: I need to calculate 2+2\n"
- "TOOL: calculator\n"
- "INPUT: 2+2"
- )
+ text = "THOUGHT: I need to calculate 2+2\nTOOL: calculator\nINPUT: 2+2"
po = m._parse_output(text, ["calculator", "think"])
assert po.thought == "I need to calculate 2+2"
assert po.tool_name == "calculator"
@@ -35,10 +31,7 @@ class TestParseOutput:
def test_final_answer(self):
m = self._model()
- text = (
- "THOUGHT: I have the result\n"
- "FINAL_ANSWER: 42"
- )
+ text = "THOUGHT: I have the result\nFINAL_ANSWER: 42"
po = m._parse_output(text, ["calculator"])
assert po.is_final_answer is True
assert po.tool_input == "42"
diff --git a/tests/test_orchestrator_learning/test_prompt_registry.py b/tests/test_orchestrator_learning/test_prompt_registry.py
index 30028ebc..2d1c527b 100644
--- a/tests/test_orchestrator_learning/test_prompt_registry.py
+++ b/tests/test_orchestrator_learning/test_prompt_registry.py
@@ -60,9 +60,7 @@ class TestBuildSystemPrompt:
assert "custom_tool_xyz" in prompt
def test_with_memory_tools(self):
- prompt = build_system_prompt(
- ["calculator", "memory_search", "memory_store"]
- )
+ prompt = build_system_prompt(["calculator", "memory_search", "memory_store"])
assert "memory_search" in prompt
def test_with_llm_tool(self):
diff --git a/tests/test_orchestrator_learning/test_sft_trainer.py b/tests/test_orchestrator_learning/test_sft_trainer.py
index b9768eab..3e2dfbcf 100644
--- a/tests/test_orchestrator_learning/test_sft_trainer.py
+++ b/tests/test_orchestrator_learning/test_sft_trainer.py
@@ -97,15 +97,15 @@ class TestOrchestratorSFTDataset:
trace_file = tmp_path / "traces.jsonl"
traces = []
for i in range(5):
- traces.append({
- "conversations": [
- {"role": "user", "content": f"q{i}"},
- {"role": "assistant", "content": f"a{i}"},
- ]
- })
- trace_file.write_text(
- "\n".join(json.dumps(t) for t in traces) + "\n"
- )
+ traces.append(
+ {
+ "conversations": [
+ {"role": "user", "content": f"q{i}"},
+ {"role": "assistant", "content": f"a{i}"},
+ ]
+ }
+ )
+ trace_file.write_text("\n".join(json.dumps(t) for t in traces) + "\n")
tok = MagicMock()
tok.eos_token = ""
@@ -128,4 +128,5 @@ class TestSFTRegistration:
# Import to trigger registration
import openjarvis.learning.intelligence.orchestrator.sft_trainer # noqa: F401
from openjarvis.core.registry import LearningRegistry
+
assert LearningRegistry.contains("orchestrator_sft")
diff --git a/tests/test_orchestrator_learning/test_types.py b/tests/test_orchestrator_learning/test_types.py
index 63930f60..f330f7d2 100644
--- a/tests/test_orchestrator_learning/test_types.py
+++ b/tests/test_orchestrator_learning/test_types.py
@@ -147,9 +147,7 @@ class TestEpisodeState:
def test_add_turn(self):
state = EpisodeState(initial_prompt="q")
- action = OrchestratorAction(
- thought="t", tool_name="calc", tool_input="1+1"
- )
+ action = OrchestratorAction(thought="t", tool_name="calc", tool_input="1+1")
obs = OrchestratorObservation(content="2")
state.add_turn(action, obs)
assert state.num_turns() == 1
diff --git a/tests/tools/test_apply_patch.py b/tests/tools/test_apply_patch.py
index b606b66e..734c0b81 100644
--- a/tests/tools/test_apply_patch.py
+++ b/tests/tools/test_apply_patch.py
@@ -127,13 +127,7 @@ class TestApplyPatchTool:
def test_blocks_sensitive_files(self, tmp_path):
f = tmp_path / ".env"
f.write_text("SECRET=foo\n", encoding="utf-8")
- patch = (
- "--- a/.env\n"
- "+++ b/.env\n"
- "@@ -1 +1 @@\n"
- "-SECRET=foo\n"
- "+SECRET=bar\n"
- )
+ patch = "--- a/.env\n+++ b/.env\n@@ -1 +1 @@\n-SECRET=foo\n+SECRET=bar\n"
tool = ApplyPatchTool()
result = tool.execute(patch=patch, path=str(f))
assert result.success is False
@@ -143,13 +137,7 @@ class TestApplyPatchTool:
f = tmp_path / "auto.txt"
f.write_text("one\ntwo\nthree\n", encoding="utf-8")
patch = (
- "--- a/auto.txt\n"
- f"+++ b/{f}\n"
- "@@ -1,3 +1,3 @@\n"
- " one\n"
- "-two\n"
- "+TWO\n"
- " three\n"
+ f"--- a/auto.txt\n+++ b/{f}\n@@ -1,3 +1,3 @@\n one\n-two\n+TWO\n three\n"
)
tool = ApplyPatchTool()
# No explicit path — should auto-detect from +++ header
@@ -167,11 +155,7 @@ class TestApplyPatchTool:
def test_file_not_found(self):
tool = ApplyPatchTool()
patch = (
- "--- a/nonexistent.txt\n"
- "+++ b/nonexistent.txt\n"
- "@@ -1 +1 @@\n"
- "-old\n"
- "+new\n"
+ "--- a/nonexistent.txt\n+++ b/nonexistent.txt\n@@ -1 +1 @@\n-old\n+new\n"
)
result = tool.execute(patch=patch, path="/nonexistent/path/file.txt")
assert result.success is False
diff --git a/tests/tools/test_audio_tool.py b/tests/tools/test_audio_tool.py
index d9cee4cc..bef3ccba 100644
--- a/tests/tools/test_audio_tool.py
+++ b/tests/tools/test_audio_tool.py
@@ -169,9 +169,7 @@ class TestAudioTranscribeTool:
monkeypatch.setenv("OPENAI_API_KEY", "test-key")
mock_client = MagicMock()
- mock_client.audio.transcriptions.create.side_effect = RuntimeError(
- "API error"
- )
+ mock_client.audio.transcriptions.create.side_effect = RuntimeError("API error")
mock_openai = MagicMock()
mock_openai.OpenAI.return_value = mock_client
diff --git a/tests/tools/test_browser.py b/tests/tools/test_browser.py
index df810a9a..3f658924 100644
--- a/tests/tools/test_browser.py
+++ b/tests/tools/test_browser.py
@@ -48,8 +48,7 @@ def _make_import_error_session():
session = MagicMock()
type(session).page = PropertyMock(
side_effect=ImportError(
- "playwright not installed. Install with: "
- "uv sync --extra browser"
+ "playwright not installed. Install with: uv sync --extra browser"
)
)
return session
@@ -145,6 +144,7 @@ class TestBrowserNavigateTool:
# Make the ssrf import fail inside execute
import builtins
+
original_import = builtins.__import__
def _mock_import(name, *args, **kwargs):
@@ -202,9 +202,7 @@ class TestBrowserNavigateTool:
tool = BrowserNavigateTool()
tool.execute(url="https://example.com", wait_for="invalid")
- page.goto.assert_called_once_with(
- "https://example.com", wait_until="load"
- )
+ page.goto.assert_called_once_with("https://example.com", wait_until="load")
def test_execute_content_truncation(self):
from openjarvis.tools.browser import BrowserNavigateTool
@@ -840,6 +838,7 @@ class TestBrowserSession:
session = _BrowserSession()
import builtins
+
original_import = builtins.__import__
def _mock_import(name, *args, **kwargs):
diff --git a/tests/tools/test_channel_tools.py b/tests/tools/test_channel_tools.py
index 07ea94fb..2c9b1907 100644
--- a/tests/tools/test_channel_tools.py
+++ b/tests/tools/test_channel_tools.py
@@ -34,11 +34,13 @@ class _MockChannel(BaseChannel):
self._status = ChannelStatus.DISCONNECTED
def send(self, channel, content, *, conversation_id="", metadata=None) -> bool:
- self._sent.append({
- "channel": channel,
- "content": content,
- "conversation_id": conversation_id,
- })
+ self._sent.append(
+ {
+ "channel": channel,
+ "content": content,
+ "conversation_id": conversation_id,
+ }
+ )
return True
def status(self) -> ChannelStatus:
@@ -95,7 +97,9 @@ class TestChannelSendTool:
def test_send_with_conversation_id(self, channel):
tool = ChannelSendTool(channel)
result = tool.execute(
- channel="chat-123", content="Reply", conversation_id="conv-1",
+ channel="chat-123",
+ content="Reply",
+ conversation_id="conv-1",
)
assert result.success is True
assert channel._sent[0]["conversation_id"] == "conv-1"
diff --git a/tests/tools/test_file_write.py b/tests/tools/test_file_write.py
index 643f5035..002de3e7 100644
--- a/tests/tools/test_file_write.py
+++ b/tests/tools/test_file_write.py
@@ -45,7 +45,9 @@ class TestFileWriteTool:
f = tmp_path / "sub" / "deep" / "test.txt"
tool = FileWriteTool()
result = tool.execute(
- path=str(f), content="nested", create_dirs=True,
+ path=str(f),
+ content="nested",
+ create_dirs=True,
)
assert result.success is True
assert f.read_text(encoding="utf-8") == "nested"
@@ -68,7 +70,8 @@ class TestFileWriteTool:
f = tmp_path / "server.pem"
tool = FileWriteTool()
result = tool.execute(
- path=str(f), content="-----BEGIN CERTIFICATE-----",
+ path=str(f),
+ content="-----BEGIN CERTIFICATE-----",
)
assert result.success is False
assert "sensitive" in result.content.lower()
@@ -124,7 +127,9 @@ class TestFileWriteTool:
f = tmp_path / "test.txt"
tool = FileWriteTool()
result = tool.execute(
- path=str(f), content="data", mode="invalid",
+ path=str(f),
+ content="data",
+ mode="invalid",
)
assert result.success is False
assert "Invalid mode" in result.content
diff --git a/tests/tools/test_git_tool.py b/tests/tools/test_git_tool.py
index 79c72144..e3cc5b80 100644
--- a/tests/tools/test_git_tool.py
+++ b/tests/tools/test_git_tool.py
@@ -497,10 +497,7 @@ class TestGitLogTool:
assert result.success is True
# Rust ignores the count param (reads "n", gets "count") and
# defaults to 10, so all 6 commits are returned.
- lines = [
- line for line in result.content.strip().splitlines()
- if line.strip()
- ]
+ lines = [line for line in result.content.strip().splitlines() if line.strip()]
assert len(lines) == 6
def test_default_count_is_10(self):
diff --git a/tests/tools/test_http_request.py b/tests/tools/test_http_request.py
index 544e4289..04f68805 100644
--- a/tests/tools/test_http_request.py
+++ b/tests/tools/test_http_request.py
@@ -83,8 +83,7 @@ class TestHttpRequestTool:
tool = HttpRequestTool()
with patch("openjarvis.tools.http_request.check_ssrf") as mock_ssrf:
mock_ssrf.return_value = (
- "Blocked host: 169.254.169.254"
- " (cloud metadata endpoint)"
+ "Blocked host: 169.254.169.254 (cloud metadata endpoint)"
)
result = tool.execute(url="http://169.254.169.254/latest/meta-data/")
assert result.success is False
diff --git a/tests/tools/test_knowledge_graph.py b/tests/tools/test_knowledge_graph.py
index 79d97c6f..7b094e71 100644
--- a/tests/tools/test_knowledge_graph.py
+++ b/tests/tools/test_knowledge_graph.py
@@ -16,8 +16,10 @@ class TestKnowledgeGraph:
def test_add_and_get_entity(self, tmp_path):
kg = self._make_kg(tmp_path)
entity = Entity(
- entity_id="e1", entity_type="concept",
- name="Machine Learning", properties={"field": "AI"},
+ entity_id="e1",
+ entity_type="concept",
+ name="Machine Learning",
+ properties={"field": "AI"},
)
kg.add_entity(entity)
result = kg.get_entity("e1")
@@ -35,10 +37,13 @@ class TestKnowledgeGraph:
kg = self._make_kg(tmp_path)
kg.add_entity(Entity(entity_id="a", entity_type="concept", name="A"))
kg.add_entity(Entity(entity_id="b", entity_type="concept", name="B"))
- kg.add_relation(Relation(
- source_id="a", target_id="b",
- relation_type="depends_on",
- ))
+ kg.add_relation(
+ Relation(
+ source_id="a",
+ target_id="b",
+ relation_type="depends_on",
+ )
+ )
assert kg.relation_count() == 1
kg.close()
@@ -61,10 +66,13 @@ class TestKnowledgeGraph:
kg.add_entity(Entity(entity_id="b", entity_type="concept", name="B"))
kg.add_entity(Entity(entity_id="c", entity_type="concept", name="C"))
kg.add_relation(Relation(source_id="a", target_id="b", relation_type="uses"))
- kg.add_relation(Relation(
- source_id="a", target_id="c",
- relation_type="produces",
- ))
+ kg.add_relation(
+ Relation(
+ source_id="a",
+ target_id="c",
+ relation_type="produces",
+ )
+ )
neighbors = kg.neighbors("a", relation_type="uses", direction="out")
assert len(neighbors) == 1
assert neighbors[0].name == "B"
diff --git a/tests/tools/test_retrieval.py b/tests/tools/test_retrieval.py
index eaac1847..1219defa 100644
--- a/tests/tools/test_retrieval.py
+++ b/tests/tools/test_retrieval.py
@@ -17,13 +17,20 @@ class _FakeBackend(MemoryBackend):
self._results = results or []
def store(
- self, content: str, *, source: str = "",
+ self,
+ content: str,
+ *,
+ source: str = "",
metadata: Optional[Dict[str, Any]] = None,
) -> str:
return "fake-id"
def retrieve(
- self, query: str, *, top_k: int = 5, **kwargs: Any,
+ self,
+ query: str,
+ *,
+ top_k: int = 5,
+ **kwargs: Any,
) -> List[RetrievalResult]:
return self._results[:top_k]
@@ -36,7 +43,11 @@ class _FakeBackend(MemoryBackend):
class _ErrorBackend(_FakeBackend):
def retrieve(
- self, query: str, *, top_k: int = 5, **kwargs: Any,
+ self,
+ query: str,
+ *,
+ top_k: int = 5,
+ **kwargs: Any,
) -> List[RetrievalResult]:
raise RuntimeError("backend error")
diff --git a/tests/tools/test_storage_stubs.py b/tests/tools/test_storage_stubs.py
index ba63793f..96ad2470 100644
--- a/tests/tools/test_storage_stubs.py
+++ b/tests/tools/test_storage_stubs.py
@@ -24,11 +24,13 @@ class _DummyStorage(MemoryBackend):
results = []
for doc_id, doc in self._data.items():
if query.lower() in doc["content"].lower():
- results.append(RetrievalResult(
- content=doc["content"],
- score=1.0,
- source=doc["source"],
- ))
+ results.append(
+ RetrievalResult(
+ content=doc["content"],
+ score=1.0,
+ source=doc["source"],
+ )
+ )
return results[:top_k]
def delete(self, doc_id):
@@ -67,12 +69,14 @@ class TestStorageStubs:
"""Memory imports should still work via shim."""
from openjarvis.tools.storage._stubs import MemoryBackend as MB
from openjarvis.tools.storage._stubs import RetrievalResult as RR
+
assert MB is MemoryBackend
assert RR is RetrievalResult
def test_canonical_import(self) -> None:
"""Canonical import from tools.storage should work."""
from openjarvis.tools.storage._stubs import MemoryBackend as MB
+
assert MB is MemoryBackend
def test_sqlite_backward_compat(self) -> None:
diff --git a/tests/tools/test_storage_tools.py b/tests/tools/test_storage_tools.py
index 6d56d1df..558ac143 100644
--- a/tests/tools/test_storage_tools.py
+++ b/tests/tools/test_storage_tools.py
@@ -181,7 +181,9 @@ class TestMemoryIndexTool:
def test_index_file(self, backend):
with tempfile.NamedTemporaryFile(
- mode="w", suffix=".txt", delete=False,
+ mode="w",
+ suffix=".txt",
+ delete=False,
) as f:
f.write("This is test content for indexing.")
f.flush()
diff --git a/tests/tools/test_templates.py b/tests/tools/test_templates.py
index 2ccf6af5..ae504c99 100644
--- a/tests/tools/test_templates.py
+++ b/tests/tools/test_templates.py
@@ -7,107 +7,129 @@ from openjarvis.tools.templates.loader import ToolTemplate, discover_templates
class TestToolTemplate:
def test_create_template(self):
- template = ToolTemplate({
- "name": "test_tool",
- "description": "A test tool",
- "action": {"type": "transform", "transform": "upper"},
- })
+ template = ToolTemplate(
+ {
+ "name": "test_tool",
+ "description": "A test tool",
+ "action": {"type": "transform", "transform": "upper"},
+ }
+ )
assert template.spec.name == "test_tool"
def test_execute_upper_transform(self):
- template = ToolTemplate({
- "name": "upper",
- "description": "Uppercase",
- "action": {"type": "transform", "transform": "upper"},
- })
+ template = ToolTemplate(
+ {
+ "name": "upper",
+ "description": "Uppercase",
+ "action": {"type": "transform", "transform": "upper"},
+ }
+ )
result = template.execute(input="hello")
assert result.success
assert result.content == "HELLO"
def test_execute_lower_transform(self):
- template = ToolTemplate({
- "name": "lower",
- "description": "Lowercase",
- "action": {"type": "transform", "transform": "lower"},
- })
+ template = ToolTemplate(
+ {
+ "name": "lower",
+ "description": "Lowercase",
+ "action": {"type": "transform", "transform": "lower"},
+ }
+ )
result = template.execute(input="HELLO")
assert result.success
assert result.content == "hello"
def test_execute_reverse_transform(self):
- template = ToolTemplate({
- "name": "reverse",
- "description": "Reverse",
- "action": {"type": "transform", "transform": "reverse"},
- })
+ template = ToolTemplate(
+ {
+ "name": "reverse",
+ "description": "Reverse",
+ "action": {"type": "transform", "transform": "reverse"},
+ }
+ )
result = template.execute(input="hello")
assert result.success
assert result.content == "olleh"
def test_execute_length_transform(self):
- template = ToolTemplate({
- "name": "length",
- "description": "Length",
- "action": {"type": "transform", "transform": "length"},
- })
+ template = ToolTemplate(
+ {
+ "name": "length",
+ "description": "Length",
+ "action": {"type": "transform", "transform": "length"},
+ }
+ )
result = template.execute(input="hello")
assert result.success
assert result.content == "5"
def test_execute_json_pretty_transform(self):
- template = ToolTemplate({
- "name": "json",
- "description": "JSON pretty",
- "action": {"type": "transform", "transform": "json_pretty"},
- })
+ template = ToolTemplate(
+ {
+ "name": "json",
+ "description": "JSON pretty",
+ "action": {"type": "transform", "transform": "json_pretty"},
+ }
+ )
result = template.execute(input='{"a":1}')
assert result.success
assert '"a": 1' in result.content
def test_execute_json_pretty_invalid(self):
- template = ToolTemplate({
- "name": "json",
- "description": "JSON pretty",
- "action": {"type": "transform", "transform": "json_pretty"},
- })
+ template = ToolTemplate(
+ {
+ "name": "json",
+ "description": "JSON pretty",
+ "action": {"type": "transform", "transform": "json_pretty"},
+ }
+ )
result = template.execute(input="not json")
assert not result.success
def test_execute_python_action(self):
- template = ToolTemplate({
- "name": "py",
- "description": "Python",
- "action": {"type": "python", "expression": "str(len(input))"},
- })
+ template = ToolTemplate(
+ {
+ "name": "py",
+ "description": "Python",
+ "action": {"type": "python", "expression": "str(len(input))"},
+ }
+ )
result = template.execute(input="hello")
assert result.success
assert result.content == "5"
def test_execute_identity_transform(self):
- template = ToolTemplate({
- "name": "identity",
- "description": "Identity",
- "action": {"type": "transform", "transform": "identity"},
- })
+ template = ToolTemplate(
+ {
+ "name": "identity",
+ "description": "Identity",
+ "action": {"type": "transform", "transform": "identity"},
+ }
+ )
result = template.execute(input="unchanged")
assert result.success
assert result.content == "unchanged"
def test_unknown_action_type(self):
- template = ToolTemplate({
- "name": "bad",
- "description": "Bad",
- "action": {"type": "unknown"},
- })
+ template = ToolTemplate(
+ {
+ "name": "bad",
+ "description": "Bad",
+ "action": {"type": "unknown"},
+ }
+ )
result = template.execute()
assert not result.success
def test_template_metadata(self):
- template = ToolTemplate({
- "name": "test",
- "description": "test",
- "action": {"type": "transform"},
- })
+ template = ToolTemplate(
+ {
+ "name": "test",
+ "description": "test",
+ "action": {"type": "transform"},
+ }
+ )
assert template.spec.metadata.get("template") is True
diff --git a/tests/tools/test_tool_descriptions.py b/tests/tools/test_tool_descriptions.py
index 96bd8b2b..3c7ce965 100644
--- a/tests/tools/test_tool_descriptions.py
+++ b/tests/tools/test_tool_descriptions.py
@@ -28,8 +28,7 @@ class _CalcTool(BaseTool):
"expression": {
"type": "string",
"description": (
- "Math expression to evaluate"
- " (e.g. '2+3*4', 'sqrt(16)')"
+ "Math expression to evaluate (e.g. '2+3*4', 'sqrt(16)')"
),
},
},
diff --git a/tests/workflow/test_workflow.py b/tests/workflow/test_workflow.py
index 87a2ae29..841d31eb 100644
--- a/tests/workflow/test_workflow.py
+++ b/tests/workflow/test_workflow.py
@@ -138,22 +138,14 @@ class TestWorkflowEngine:
def test_run_transform_node(self):
bus = EventBus(record_history=True)
engine = WorkflowEngine(bus=bus)
- wf = (
- WorkflowBuilder("test")
- .add_transform("t", transform="concatenate")
- .build()
- )
+ wf = WorkflowBuilder("test").add_transform("t", transform="concatenate").build()
result = engine.run(wf, initial_input="hello")
assert result.success
def test_events_emitted(self):
bus = EventBus(record_history=True)
engine = WorkflowEngine(bus=bus)
- wf = (
- WorkflowBuilder("test")
- .add_transform("t", transform="concatenate")
- .build()
- )
+ wf = WorkflowBuilder("test").add_transform("t", transform="concatenate").build()
engine.run(wf, initial_input="hello")
event_types = {e.event_type for e in bus.history}
assert EventType.WORKFLOW_START in event_types
diff --git a/uv.lock b/uv.lock
index 5629a0a3..da90bd67 100644
--- a/uv.lock
+++ b/uv.lock
@@ -4679,6 +4679,7 @@ dependencies = [
{ name = "ddgs" },
{ name = "httpx" },
{ name = "openai" },
+ { name = "python-telegram-bot" },
{ name = "rich" },
{ name = "tomli", marker = "python_full_version < '3.11'" },
]
@@ -4903,6 +4904,7 @@ requires-dist = [
{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.24" },
{ name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5" },
{ name = "python-multipart", marker = "extra == 'server'", specifier = ">=0.0.9" },
+ { name = "python-telegram-bot", specifier = ">=22.6" },
{ name = "python-telegram-bot", marker = "extra == 'channel-telegram'", specifier = ">=21.0" },
{ name = "rank-bm25", marker = "extra == 'memory-bm25'", specifier = ">=0.2.2" },
{ name = "respx", marker = "extra == 'dev'", specifier = ">=0.22" },