Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(session): split JSONL on newline only, not Unicode line boundaries (#3984) (#3988)

now-ing996128abcc33
diff --git a/openviking/session/session.py b/openviking/session/session.py
index 917ff5f80..6622eba1c 100644
--- a/openviking/session/session.py
+++ b/openviking/session/session.py
@@ -3984,7 +3984,13 @@ class Session:
ctx=self.ctx,
)
messages: List[Message] = []
- for line_number, line in enumerate(content.splitlines(), start=1):
+ # Split on "\n" only, not str.splitlines(): the latter also treats
+ # U+2028 / U+2029 / NEL (\x85) / \r / \v / \f as line boundaries, which
+ # would cut a JSONL record in half when those characters appear inside a
+ # JSON string value (e.g. assistant tool_output) and break json.loads()
+ # with "Unterminated string". Normalize CRLF first so a trailing "\r"
+ # does not leak into the record. See issue #3984.
+ for line_number, line in enumerate(content.replace("\r\n", "\n").split("\n"), start=1):
if not line.strip():
continue
try:
diff --git a/tests/session/test_session_commit.py b/tests/session/test_session_commit.py
index 02cb5206b..969b8ce55 100644
--- a/tests/session/test_session_commit.py
+++ b/tests/session/test_session_commit.py
@@ -37,8 +37,14 @@ async def _marker_exists(session, archive_uri: str, name: str) -> bool:
class TestCommit:
"""Test commit"""
- async def test_commit_success(self, session_with_messages: Session):
- """Test successful commit returns accepted with task_id"""
+ async def test_commit_preserves_unicode_separators_and_accepts_later_messages(
+ self, session_with_messages: Session
+ ):
+ """Unicode separators inside a message must not split its JSONL record."""
+ unicode_text = "before\u2028middle\u2029after\u0085tail"
+ session_with_messages.add_message("assistant", [TextPart(unicode_text)])
+ session_with_messages.add_message("user", [TextPart("continue")])
+
result = await session_with_messages.commit_async()
assert isinstance(result, dict)
@@ -47,6 +53,19 @@ class TestCommit:
assert result.get("task_id") is not None
assert "memory_diff_uri" not in result
assert "memories_extracted" not in result
+ archive_content = await session_with_messages._viking_fs.read_file(
+ f"{result['archive_uri']}/messages.jsonl",
+ ctx=session_with_messages.ctx,
+ )
+ archived_messages = [
+ json.loads(line) for line in archive_content.split("\n") if line.strip()
+ ]
+ assert unicode_text in {
+ part["text"]
+ for message in archived_messages
+ for part in message["parts"]
+ if part["type"] == "text"
+ }
async def test_commit_extracts_memories(
self,