Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(mcp): honor a leading UTF-8 BOM in chat streams (#4262)

Rudy Mizrahi Celekli496d25dc3ca5
diff --git a/mcp-server/test_chat_sse.py b/mcp-server/test_chat_sse.py
index e4d85b699..f57bc8c8f 100644
--- a/mcp-server/test_chat_sse.py
+++ b/mcp-server/test_chat_sse.py
@@ -246,5 +246,153 @@ class ChatSSETest(unittest.TestCase):
response.raw.release_conn.assert_called_once_with()
+ def consume_wire(self, wire, *, agent=False, chunks=None):
+ response = requests.Response()
+ response.status_code = 200
+ response._content_consumed = True
+ with mock.patch.object(
+ response, "iter_content", return_value=iter(chunks or [wire])
+ ), mock.patch.object(
+ self.client.session, "post", return_value=response
+ ), mock.patch.object(response, "close", wraps=response.close) as close:
+ self.wire_close = close
+ if agent:
+ return self.client.agent_chat("session-1", "question", "agent-1")
+ return self.client.chat("session-1", "question")
+
+ def test_initial_bom_answer_with_fragmented_transport_in_both_chat_paths(self):
+ wire = (
+ '\ufeffdata: {"response_type":"answer","content":"你好"}\r\n\r\n'
+ 'data: {"response_type":"references",'
+ '"knowledge_references":[{"id":"chunk-1"}]}\r\n\r\n'
+ 'data: {"response_type":"complete"}\r\n\r\n'
+ ).encode()
+ split = wire.index("你".encode()) + 1
+ chunks = [wire[:1], wire[1:2], wire[2:split], wire[split:]]
+ for agent in (False, True):
+ with self.subTest(agent=agent):
+ result = self.consume_wire(wire, agent=agent, chunks=chunks)
+ self.assertEqual(result["answer"], "你好")
+ self.assertEqual(result["references"], [{"id": "chunk-1"}])
+ self.assertEqual(result["session_id"], "session-1")
+ self.wire_close.assert_called_once_with()
+
+ def test_initial_bom_references_in_both_chat_paths(self):
+ wire = (
+ '\ufeffdata: {"response_type":"references",'
+ '"knowledge_references":[{"id":"first-reference"}]}\n\n'
+ 'data: {"response_type":"complete"}\n\n'
+ ).encode()
+ for agent in (False, True):
+ with self.subTest(agent=agent):
+ result = self.consume_wire(wire, agent=agent)
+ self.assertEqual(result["answer"], "")
+ self.assertEqual(result["references"], [{"id": "first-reference"}])
+
+ def test_initial_bom_error_propagates_and_closes_in_both_chat_paths(self):
+ wire = (
+ '\ufeffdata: {"response_type":"error","content":"upstream failed"}\n\n'
+ 'data: {"response_type":"complete"}\n\n'
+ ).encode()
+ for agent in (False, True):
+ with self.subTest(agent=agent):
+ with self.assertRaisesRegex(requests.RequestException, "upstream failed"):
+ self.consume_wire(wire, agent=agent)
+ self.wire_close.assert_called_once_with()
+
+ def test_initial_bom_complete_stops_before_later_events_in_both_chat_paths(self):
+ wire = (
+ '\ufeffdata: {"response_type":"complete"}\n\n'
+ 'data: {"response_type":"answer","content":"must not read"}\n\n'
+ ).encode()
+ for agent in (False, True):
+ with self.subTest(agent=agent):
+ result = self.consume_wire(wire, agent=agent)
+ self.assertEqual(result["answer"], "")
+ self.wire_close.assert_called_once_with()
+
+ def test_only_one_initial_bom_is_ignored(self):
+ wire = (
+ '\ufeff\ufeffdata: {"response_type":"answer","content":"unknown field"}\n\n'
+ 'data: {"response_type":"answer","content":"valid"}\n\n'
+ 'data: {"response_type":"complete"}\n\n'
+ ).encode()
+ for agent in (False, True):
+ with self.subTest(agent=agent):
+ self.assertEqual(self.consume_wire(wire, agent=agent)["answer"], "valid")
+
+ def test_later_bom_field_and_literal_content_keep_their_meaning(self):
+ wire = (
+ ': heartbeat\n'
+ '\ufeffdata: {"response_type":"answer","content":"unknown field"}\n\n'
+ 'data: {"response_type":"answer","content":"left\ufeffright"}\n\n'
+ 'data: {"response_type":"complete"}\n\n'
+ ).encode()
+ for agent in (False, True):
+ with self.subTest(agent=agent):
+ result = self.consume_wire(wire, agent=agent)
+ self.assertEqual(result["answer"], "left\ufeffright")
+
+ def test_plain_and_bom_metadata_streams_remain_healthy(self):
+ data = (
+ 'data: {"response_type":"answer","content":"plain"}\n\n'
+ 'data: {"response_type":"complete"}\n\n'
+ )
+ for prefix in ("", "\ufeffevent: message\n"):
+ for agent in (False, True):
+ with self.subTest(prefix=prefix, agent=agent):
+ result = self.consume_wire((prefix + data).encode(), agent=agent)
+ self.assertEqual(result["answer"], "plain")
+ self.assertEqual(result["session_id"], "session-1")
+ self.wire_close.assert_called_once_with()
+
+
+ def test_real_http_initial_bom_stream_in_both_chat_paths(self):
+ wire = (
+ '\ufeffdata: {"response_type":"answer","content":"你好"}\n\n'
+ 'data: {"response_type":"references",'
+ '"knowledge_references":[{"id":"wire-reference"}]}\n\n'
+ 'data: {"response_type":"complete"}\n\n'
+ ).encode()
+ paths = []
+
+ class SSEHandler(BaseHTTPRequestHandler):
+ def do_POST(self):
+ self.rfile.read(int(self.headers.get("Content-Length", "0")))
+ paths.append(self.path)
+ self.send_response(200)
+ self.send_header("Content-Type", "text/event-stream; charset=utf-8")
+ self.send_header("Content-Length", str(len(wire)))
+ self.end_headers()
+ self.wfile.write(wire)
+
+ def log_message(self, format, *args):
+ pass
+
+ server = ThreadingHTTPServer(("127.0.0.1", 0), SSEHandler)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ self.addCleanup(thread.join, 5)
+ self.addCleanup(server.server_close)
+ self.addCleanup(server.shutdown)
+ port = server.server_address[1]
+ client = srv.WeKnoraClient(f"http://127.0.0.1:{port}/api/v1", "test-key")
+ client.session.trust_env = False
+ self.addCleanup(client.session.close)
+ for agent in (False, True):
+ with self.subTest(agent=agent):
+ if agent:
+ result = client.agent_chat("session-1", "question", "agent-1")
+ else:
+ result = client.chat("session-1", "question")
+ self.assertEqual(result["answer"], "你好")
+ self.assertEqual(result["references"], [{"id": "wire-reference"}])
+ self.assertEqual(result["session_id"], "session-1")
+ self.assertEqual(
+ paths,
+ ["/api/v1/knowledge-chat/session-1", "/api/v1/agent-chat/session-1"],
+ )
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/mcp-server/weknora_mcp_server.py b/mcp-server/weknora_mcp_server.py
index cc4d46d6a..3bf17063f 100644
--- a/mcp-server/weknora_mcp_server.py
+++ b/mcp-server/weknora_mcp_server.py
@@ -531,9 +531,14 @@ class WeKnoraClient:
# even when breaking early on a 'complete' event.
with response:
response.raise_for_status()
+ first_line = True
for raw_line in response.iter_lines():
if isinstance(raw_line, bytes):
raw_line = raw_line.decode("utf-8")
+ if first_line:
+ # SSE ignores exactly one BOM at the start of the stream.
+ raw_line = raw_line.removeprefix("\ufeff")
+ first_line = False
if raw_line:
field, _, value = raw_line.partition(":")
if field == "data":