Personal AI, On Personal Devices
Python
fix(agents): merge inline system messages so exactly one always leads (#823)
andrew-twiggc85ba54fb6ed
diff --git a/src/openjarvis/agents/_stubs.py b/src/openjarvis/agents/_stubs.pyindex 3c4ce9e4..c8354c69 100644--- a/src/openjarvis/agents/_stubs.py+++ b/src/openjarvis/agents/_stubs.py@@ -178,27 +178,32 @@ class BaseAgent(ABC):effective_system_prompt = cfg.agent.default_system_prompt or Noneexcept Exception:effective_system_prompt = None- if effective_system_prompt:- context_system_text = "\n\n".join(- message.text- for message in context_messages- if message.role == Role.SYSTEM- and message.metadata.get("memory_context")- and message.text+ # Fold ALL in-context system messages (both auto-captured memory+ # context and caller-supplied system messages) into one leading system+ # message. Do this even when there is no independently-built prompt:+ # Qwen-family chat templates reject a system message after the first+ # slot or more than one system message. Empty system messages must be+ # removed too, otherwise they can leave a second system entry behind.+ identity_already_applied = any(+ message.role == Role.SYSTEM+ and message.metadata.get("openjarvis_identity_prompt")+ for message in context_messages+ )+ system_parts = []+ if effective_system_prompt and not identity_already_applied:+ system_parts.append(effective_system_prompt)+ system_parts.extend(+ message.text+ for message in context_messages+ if message.role == Role.SYSTEM and message.text+ )+ context_messages = [+ message for message in context_messages if message.role != Role.SYSTEM+ ]+ if system_parts:+ messages.append(+ Message(role=Role.SYSTEM, content="\n\n".join(system_parts)))- if context_system_text:- effective_system_prompt = (- f"{effective_system_prompt}\n\n{context_system_text}"- )- context_messages = [- message- for message in context_messages- if not (- message.role == Role.SYSTEM- and message.metadata.get("memory_context")- )- ]- messages.append(Message(role=Role.SYSTEM, content=effective_system_prompt))if context_messages:messages.extend(context_messages)messages.append(Message(role=Role.USER, content=input))diff --git a/src/openjarvis/server/routes.py b/src/openjarvis/server/routes.pyindex 71608a7a..d2d6a4ec 100644--- a/src/openjarvis/server/routes.py+++ b/src/openjarvis/server/routes.py@@ -7,6 +7,7 @@ import loggingimport threadingimport uuidimport weakref+from dataclasses import replacefrom typing import Anyfrom fastapi import APIRouter, HTTPException, Request@@ -58,7 +59,7 @@ def _to_messages(chat_messages) -> list[Message]:def _ensure_identity_prompt(messages: list[Message], app_config) -> list[Message]:- """Prepend OpenJarvis's identity system prompt when the client omits one.+ """Return one leading system message, adding identity when needed.The desktop UI's chat backend posts only user/assistant turns to``/v1/chat/completions`` (see ``frontend/.../Chat/InputArea.tsx``), so@@ -69,9 +70,10 @@ def _ensure_identity_prompt(messages: list[Message], app_config) -> list[Messagedid not. This mirrors the agent fallback in ``agents/_stubs.py``.If any caller-supplied message already carries a system role, the caller- has supplied their own grounding and we leave the list untouched (no- double-prompting). Internally tagged memory context does not count as- caller grounding.+ has supplied their own grounding and we do not add the server identity.+ All system messages are still folded into one leading entry because some+ chat templates reject multiple or mid-history system roles. Internally+ tagged memory context does not count as caller grounding.Resolution of the identity text: the config comes from ``app.state`` whenwired, otherwise ``load_config()``; the prompt itself is assembled by@@ -86,37 +88,71 @@ def _ensure_identity_prompt(messages: list[Message], app_config) -> list[Messagedef _is_caller_system_prompt(m: Message) -> bool:return m.role == Role.SYSTEM and not m.metadata.get("memory_context")- if any(_is_caller_system_prompt(m) for m in messages):- return messages+ system_messages = [message for message in messages if message.role == Role.SYSTEM]+ caller_supplied_system = any(+ _is_caller_system_prompt(message) for message in system_messages+ )+ identity_already_applied = any(+ message.metadata.get("openjarvis_identity_prompt")+ for message in system_messages+ )prompt = ""- try:- cfg = app_config- if cfg is None:- from openjarvis.core.config import load_config+ if not caller_supplied_system and not identity_already_applied:+ try:+ cfg = app_config+ if cfg is None:+ from openjarvis.core.config import load_config++ cfg = load_config()- cfg = load_config()+ from openjarvis.prompt.builder import SystemPromptBuilder++ builder = SystemPromptBuilder(+ agent_template=cfg.agent.default_system_prompt or "",+ memory_files_config=getattr(cfg, "memory_files", None),+ system_prompt_config=getattr(cfg, "system_prompt", None),+ )+ prompt = builder.build()+ except Exception:+ logging.getLogger("openjarvis.server").debug(+ "Identity system prompt resolution failed; "+ "serving request without identity grounding",+ exc_info=True,+ )- from openjarvis.prompt.builder import SystemPromptBuilder+ system_parts = []+ if prompt:+ system_parts.append(prompt)+ system_parts.extend(message.text for message in system_messages if message.text)+ non_system_messages = [+ message for message in messages if message.role != Role.SYSTEM+ ]- builder = SystemPromptBuilder(- agent_template=cfg.agent.default_system_prompt or "",- memory_files_config=getattr(cfg, "memory_files", None),- system_prompt_config=getattr(cfg, "system_prompt", None),+ if not system_parts:+ return non_system_messages++ if system_messages:+ first_system = system_messages[0]+ metadata = dict(first_system.metadata)+ if prompt:+ # Preserve the first system message's metadata while tagging the+ # server-built identity so BaseAgent does not build it a second+ # time if this normalized conversation later reaches agent code.+ metadata["openjarvis_identity_prompt"] = True+ combined_system = replace(+ first_system,+ content="\n\n".join(system_parts),+ metadata=metadata,)- prompt = builder.build()- except Exception:- logging.getLogger("openjarvis.server").debug(- "Identity system prompt resolution failed; "- "serving request without identity grounding",- exc_info=True,+ else:+ combined_system = Message(+ role=Role.SYSTEM,+ content=prompt,+ metadata={"openjarvis_identity_prompt": True},)- return messages- if not prompt:- return messages-- return [Message(role=Role.SYSTEM, content=prompt), *messages]+ return [combined_system, *non_system_messages]@router.post("/v1/chat/completions")@@ -125,6 +161,11 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Requestengine = request.app.state.engineagent = getattr(request.app.state, "agent", None)model = request_body.model+ use_server_agent = (+ agent is not None+ and not request_body.tools+ and (not request_body.stream or bool(getattr(agent, "_tools", None)))+ )# Inject memory context into messages before dispatchingconfig = getattr(request.app.state, "config", None)@@ -149,7 +190,12 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Requestif query_text:messages = _to_messages(request_body.messages)- messages = _ensure_identity_prompt(messages, config)+ # Direct engine paths need the server identity folded into+ # memory here so adapters receive one leading system message.+ # Agent paths build that same identity in BaseAgent; injecting+ # it at both layers duplicates the prompt around memory.+ if not use_server_agent:+ messages = _ensure_identity_prompt(messages, config)ctx_cfg = ContextConfig(top_k=config.memory.context_top_k,min_score=config.memory.context_min_score,@@ -251,7 +297,7 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Requestbus=getattr(request.app.state, "bus", None),memory_service=getattr(request.app.state, "memory_service", None),)- if agent is not None and getattr(agent, "_tools", None):+ if use_server_agent:return await _handle_agent_stream(agent,model,@@ -292,7 +338,7 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Request# ``engine.generate()``) both make blocking upstream calls; run them in a# worker thread so a slow/wedged non-streaming request can't stall the# event loop and every other concurrent request with it.- if agent is not None and not request_body.tools:+ if use_server_agent:response = await asyncio.to_thread(_handle_agent,agent,diff --git a/src/openjarvis/tools/storage/context.py b/src/openjarvis/tools/storage/context.pyindex 7f539cea..56746e8e 100644--- a/src/openjarvis/tools/storage/context.py+++ b/src/openjarvis/tools/storage/context.py@@ -101,16 +101,12 @@ def _merge_context_message(if part)combined = replace(system_messages[0], content=content)- merged: List[Message] = []- inserted = False- for message in messages:- if message.role == Role.SYSTEM:- if not inserted:- merged.append(combined)- inserted = True- continue- merged.append(message)- return merged+ # Always place the merged system message FIRST. Ollama (and many chat+ # templates) reject a non-initial system message with+ # "system message must be at the beginning", so preserving the original+ # system position is not safe — a system message may arrive mid-list+ # (e.g. after caller-provided history).+ return [combined] + [m for m in messages if m.role != Role.SYSTEM]def inject_context(diff --git a/tests/agents/test_base_agent.py b/tests/agents/test_base_agent.pyindex 7c04289c..94f1c4cf 100644--- a/tests/agents/test_base_agent.py+++ b/tests/agents/test_base_agent.py@@ -237,7 +237,93 @@ class TestBuildMessages:messages = agent._build_messages("new", AgentContext(conversation=conv))- assert any(message.content == "You are helpful." for message in messages)+ assert any("You are helpful." in message.content for message in messages)++ def test_context_system_messages_are_folded_without_a_built_prompt(self):+ engine = MagicMock()+ agent = _ConcreteAgent(engine, "m")+ conv = Conversation()+ conv.add(Message(role=Role.USER, content="previous question"))+ conv.add(Message(role=Role.SYSTEM, content="First instruction."))+ conv.add(Message(role=Role.ASSISTANT, content="previous answer"))+ conv.add(Message(role=Role.SYSTEM, content="Second instruction."))++ messages = agent._build_messages("new", AgentContext(conversation=conv))++ assert [message.role for message in messages] == [+ Role.SYSTEM,+ Role.USER,+ Role.ASSISTANT,+ Role.USER,+ ]+ assert messages[0].content == "First instruction.\n\nSecond instruction."+ assert [message.content for message in messages[1:]] == [+ "previous question",+ "previous answer",+ "new",+ ]++ def test_empty_built_prompt_still_folds_context_system_messages(self):+ engine = MagicMock()+ prompt_builder = MagicMock()+ prompt_builder.build.return_value = ""+ agent = _ConcreteAgent(engine, "m", prompt_builder=prompt_builder)+ conv = Conversation()+ conv.add(Message(role=Role.SYSTEM, content="Context instruction."))++ messages = agent._build_messages("new", AgentContext(conversation=conv))++ assert [message.role for message in messages] == [Role.SYSTEM, Role.USER]+ assert messages[0].content == "Context instruction."++ def test_empty_context_system_message_is_removed(self):+ engine = MagicMock()+ agent = _ConcreteAgent(engine, "m")+ conv = Conversation()+ conv.add(Message(role=Role.SYSTEM, content=""))++ messages = agent._build_messages(+ "new",+ AgentContext(conversation=conv),+ system_prompt="Agent instructions.",+ )++ assert [message.role for message in messages] == [Role.SYSTEM, Role.USER]+ assert messages[0].content == "Agent instructions."++ def test_server_identity_already_merged_with_memory_is_not_duplicated(self):+ engine = MagicMock()+ prompt_builder = MagicMock()+ prompt_builder.build.return_value = "OpenJarvis identity."+ agent = _ConcreteAgent(engine, "m", prompt_builder=prompt_builder)+ conv = Conversation()+ conv.add(+ Message(+ role=Role.SYSTEM,+ content="OpenJarvis identity.\n\nRemembered preference.",+ metadata={"openjarvis_identity_prompt": True},+ )+ )++ messages = agent._build_messages("new", AgentContext(conversation=conv))++ assert [message.role for message in messages] == [Role.SYSTEM, Role.USER]+ assert messages[0].content == ("OpenJarvis identity.\n\nRemembered preference.")+ assert messages[0].content.count("OpenJarvis identity.") == 1++ def test_only_empty_context_system_messages_emit_no_system_message(self):+ engine = MagicMock()+ prompt_builder = MagicMock()+ prompt_builder.build.return_value = None+ agent = _ConcreteAgent(engine, "m", prompt_builder=prompt_builder)+ conv = Conversation()+ conv.add(Message(role=Role.SYSTEM, content=""))+ conv.add(Message(role=Role.USER, content="previous"))++ messages = agent._build_messages("new", AgentContext(conversation=conv))++ assert [message.role for message in messages] == [Role.USER, Role.USER]+ assert [message.content for message in messages] == ["previous", "new"]class TestGenerate:diff --git a/tests/integration/test_integration_extended.py b/tests/integration/test_integration_extended.pyindex bec9b97d..cf88df0f 100644--- a/tests/integration/test_integration_extended.py+++ b/tests/integration/test_integration_extended.py@@ -547,8 +547,11 @@ class TestAgentRoutingMatrix:result = agent.run("Hello", context=ctx)assert result.content == "Got context."- # Verify engine received system message+ # The agent prompt and caller context are folded into one leading+ # system message so strict chat templates (for example Qwen) accept it.call_args = engine.generate.call_argsmsgs = call_args[0][0]- # First message is ReAct system prompt, then context- assert any(m.content == "You are helpful." for m in msgs)+ system_messages = [m for m in msgs if m.role == Role.SYSTEM]+ assert system_messages == [msgs[0]]+ assert "You are a ReAct agent." in msgs[0].content+ assert "You are helpful." in msgs[0].contentdiff --git a/tests/memory/test_context.py b/tests/memory/test_context.pyindex 6d2d8172..ac078e66 100644--- a/tests/memory/test_context.py+++ b/tests/memory/test_context.py@@ -271,6 +271,42 @@ def test_inject_context_collapses_multiple_system_messages():assert "User likes jazz" in system_messages[0].content+def test_inject_context_moves_mid_history_system_messages_to_front_in_order():+ messages = [+ Message(role=Role.USER, content="Earlier question"),+ Message(+ role=Role.SYSTEM,+ content="Identity.",+ metadata={"origin": "caller"},+ ),+ Message(role=Role.ASSISTANT, content="Earlier answer"),+ Message(role=Role.SYSTEM, content="Persona."),+ ]++ augmented = inject_context(+ "remember",+ messages,+ None,+ facts=[Fact(text="User likes jazz")],+ )++ assert [message.role for message in augmented] == [+ Role.SYSTEM,+ Role.USER,+ Role.ASSISTANT,+ ]+ assert augmented[0].content == (+ "Identity.\n\nPersona.\n\n"+ "The following durable facts were remembered from prior conversations. "+ "Use them when relevant to the user's request:\n\n- User likes jazz"+ )+ assert augmented[0].metadata == {"origin": "caller"}+ assert [message.content for message in augmented[1:]] == [+ "Earlier question",+ "Earlier answer",+ ]++def test_inject_context_reserves_budget_for_retrieved_documents():backend = _FakeMemory([RetrievalResult(content="d1 d2 d3 d4 d5", score=1.0, source="doc")]diff --git a/tests/server/test_routes.py b/tests/server/test_routes.pyindex c66a3764..1b23310d 100644--- a/tests/server/test_routes.py+++ b/tests/server/test_routes.py@@ -928,6 +928,125 @@ class TestIdentityPromptInjection:assert len(system_msgs) == 1assert system_msgs[0].content == "Be terse."+ def test_direct_normalizes_mid_history_system_messages(self):+ captured: list = []+ engine = _make_capturing_engine(captured)+ client = TestClient(create_app(engine, "test-model", config=_identity_config()))++ resp = client.post(+ "/v1/chat/completions",+ json={+ "model": "test-model",+ "messages": [+ {"role": "user", "content": "before"},+ {"role": "system", "content": "First instruction."},+ {"role": "assistant", "content": "reply"},+ {"role": "system", "content": "Second instruction."},+ {"role": "user", "content": "after"},+ ],+ },+ )++ assert resp.status_code == 200+ messages = engine.generate.call_args.args[0]+ assert [message.role for message in messages] == [+ Role.SYSTEM,+ Role.USER,+ Role.ASSISTANT,+ Role.USER,+ ]+ assert messages[0].content == "First instruction.\n\nSecond instruction."+ assert [message.content for message in messages[1:]] == [+ "before",+ "reply",+ "after",+ ]++ def test_stream_normalizes_mid_history_system_messages(self):+ captured: list = []+ engine = _make_capturing_engine(captured)+ client = TestClient(create_app(engine, "test-model", config=_identity_config()))++ resp = client.post(+ "/v1/chat/completions",+ json={+ "model": "test-model",+ "messages": [+ {"role": "user", "content": "before"},+ {"role": "system", "content": "First instruction."},+ {"role": "assistant", "content": "reply"},+ {"role": "system", "content": "Second instruction."},+ {"role": "user", "content": "after"},+ ],+ "stream": True,+ },+ )++ assert resp.status_code == 200+ _ = resp.text+ messages = captured[-1]+ assert [message.role for message in messages] == [+ Role.SYSTEM,+ Role.USER,+ Role.ASSISTANT,+ Role.USER,+ ]+ assert messages[0].content == "First instruction.\n\nSecond instruction."+ assert [message.content for message in messages[1:]] == [+ "before",+ "reply",+ "after",+ ]++ def test_agent_path_folds_history_into_one_identity_system_message(self):+ from openjarvis.agents.simple import SimpleAgent+ from openjarvis.prompt.builder import SystemPromptBuilder++ captured: list = []+ engine = _make_capturing_engine(captured)+ cfg = _identity_config()+ agent = SimpleAgent(+ engine,+ "test-model",+ prompt_builder=SystemPromptBuilder(+ agent_template=cfg.agent.default_system_prompt,+ memory_files_config=cfg.memory_files,+ system_prompt_config=cfg.system_prompt,+ ),+ )+ client = TestClient(create_app(engine, "test-model", agent=agent, config=cfg))++ resp = client.post(+ "/v1/chat/completions",+ json={+ "model": "test-model",+ "messages": [+ {"role": "user", "content": "before"},+ {"role": "system", "content": "First instruction."},+ {"role": "assistant", "content": "reply"},+ {"role": "system", "content": "Second instruction."},+ {"role": "user", "content": "after"},+ ],+ },+ )++ assert resp.status_code == 200+ messages = engine.generate.call_args.args[0]+ assert [message.role for message in messages] == [+ Role.SYSTEM,+ Role.USER,+ Role.ASSISTANT,+ Role.USER,+ ]+ assert messages[0].content.count("You are OpenJarvis.") == 1+ assert "First instruction." in messages[0].content+ assert "Second instruction." in messages[0].content+ assert [message.content for message in messages[1:]] == [+ "before",+ "reply",+ "after",+ ]+def test_direct_merges_identity_and_auto_memory_into_one_system_message(self):from openjarvis.memory.store import Fact@@ -1063,6 +1182,53 @@ class TestIdentityPromptInjection:assert assistant.tool_calls[0].name == "lookup"assert assistant.tool_calls[0].arguments == '{"query":"jazz"}'+ def test_agent_does_not_rebuild_identity_already_merged_with_memory(self):+ from openjarvis.agents.simple import SimpleAgent+ from openjarvis.memory.store import Fact+ from openjarvis.prompt.builder import SystemPromptBuilder++ class _MemoryService:+ def list_facts(self):+ return [Fact(text="The user's favorite color is blue")]++ captured: list = []+ engine = _make_capturing_engine(captured)+ cfg = _identity_config()+ cfg.agent.context_from_memory = True+ agent = SimpleAgent(+ engine,+ "test-model",+ prompt_builder=SystemPromptBuilder(+ agent_template=cfg.agent.default_system_prompt,+ memory_files_config=cfg.memory_files,+ system_prompt_config=cfg.system_prompt,+ ),+ )+ client = TestClient(+ create_app(+ engine,+ "test-model",+ agent=agent,+ config=cfg,+ memory_service=_MemoryService(),+ )+ )++ resp = client.post(+ "/v1/chat/completions",+ json={+ "model": "test-model",+ "messages": [{"role": "user", "content": "What is my color?"}],+ },+ )++ assert resp.status_code == 200+ messages = engine.generate.call_args.args[0]+ system_messages = [m for m in messages if m.role == Role.SYSTEM]+ assert len(system_messages) == 1+ assert system_messages[0].content.count("You are OpenJarvis.") == 1+ assert "favorite color is blue" in system_messages[0].content+def test_direct_injects_soul_persona_when_present(self, tmp_path):"""Regression: /v1/chat/completions previously injected only the bare``default_system_prompt`` blurb via a hand-rolled lookup, bypassing@@ -1127,6 +1293,52 @@ class TestIdentityPromptInjection:assert msgs[0].role.value == "system"assert "OpenJarvis" in msgs[0].content+ def test_stream_tools_normalizes_mid_history_system_messages(self):+ captured: list = []+ engine = _make_capturing_engine(captured)+ client = TestClient(create_app(engine, "test-model", config=_identity_config()))++ resp = client.post(+ "/v1/chat/completions",+ json={+ "model": "test-model",+ "messages": [+ {"role": "user", "content": "before"},+ {"role": "system", "content": "First instruction."},+ {"role": "assistant", "content": "reply"},+ {"role": "system", "content": "Second instruction."},+ {"role": "user", "content": "after"},+ ],+ "tools": [+ {+ "type": "function",+ "function": {+ "name": "dummy",+ "description": "dummy",+ "parameters": {"type": "object", "properties": {}},+ },+ }+ ],+ "stream": True,+ },+ )++ assert resp.status_code == 200+ _ = resp.text+ messages = captured[-1]+ assert [message.role for message in messages] == [+ Role.SYSTEM,+ Role.USER,+ Role.ASSISTANT,+ Role.USER,+ ]+ assert messages[0].content == "First instruction.\n\nSecond instruction."+ assert [message.content for message in messages[1:]] == [+ "before",+ "reply",+ "after",+ ]+def test_memory_context_does_not_suppress_identity_injection(self):from openjarvis.core.types import Message, Rolefrom openjarvis.server.routes import _ensure_identity_prompt@@ -1136,8 +1348,38 @@ class TestIdentityPromptInjection:messages = [ctx_msg, Message(role=Role.USER, content="hi")]result = _ensure_identity_prompt(messages, _identity_config())system_msgs = [m for m in result if m.role == Role.SYSTEM]- assert len(system_msgs) == 2- assert any("OpenJarvis" in m.content for m in system_msgs)+ assert len(system_msgs) == 1+ assert "OpenJarvis" in system_msgs[0].content+ assert system_msgs[0].metadata["memory_context"] is True+ assert system_msgs[0].metadata["openjarvis_identity_prompt"] is True++ normalized_again = _ensure_identity_prompt(result, _identity_config())+ assert len([m for m in normalized_again if m.role == Role.SYSTEM]) == 1+ assert normalized_again[0].content.count("You are OpenJarvis.") == 1++ def test_normalization_preserves_first_metadata_and_non_system_order(self):+ from openjarvis.core.types import Message, Role+ from openjarvis.server.routes import _ensure_identity_prompt++ first_system = Message(+ role=Role.SYSTEM,+ content="First instruction.",+ metadata={"source": "first"},+ )+ first_user = Message(role=Role.USER, content="before")+ assistant = Message(role=Role.ASSISTANT, content="reply")+ second_system = Message(role=Role.SYSTEM, content="Second instruction.")+ final_user = Message(role=Role.USER, content="after")++ result = _ensure_identity_prompt(+ [first_user, first_system, assistant, second_system, final_user],+ _identity_config(),+ )++ assert result[0].role == Role.SYSTEM+ assert result[0].content == "First instruction.\n\nSecond instruction."+ assert result[0].metadata == {"source": "first"}+ assert result[1:] == [first_user, assistant, final_user]def test_caller_system_prompt_cannot_impersonate_memory_context(self):from openjarvis.core.types import Message, Role