Skip to content
Python
13.4 KB
Raw Blame
# -*- coding: utf-8 -*-
"""
raw 在写入时就裁剪好;历史拼接不再二次裁剪。

规则(hy_memory/utils/raw_trim.py):
  - user / system 完整保留
  - assistant 与对方真人(非已知 role 的 label)strip 后超过 500 字符截成前 500 + "..."
  - tool 头尾保留、中段替换为 "[...N bytes omitted by truncate...]"
    (上限 coding.tool_result_max_bytes)
  - 全部幂等

覆盖:
  1. 规则本身 + 幂等(含 coding preproc 的两个截断函数)
  2. lite:L1_RAW 存的是裁剪后的文本(tool 头尾截断 + 标记),带 raw_trimmed 标记;
     审计 history 的 ADD 记录同样裁剪
  3. coding_trace 存的 tool result 已头尾截断
  4. pro:extractor 输入仍是完整 messages;历史 = 已存 raw 直接拼接;
     没有标记的旧 raw 仍按同一规则兼容裁剪
"""

import asyncio
import json
import sqlite3
import types
import uuid
from datetime import datetime, timedelta
from typing import Any, Dict, List, Optional

import pytest

from hy_memory.coding.preproc import (
    truncate_assistant_text,
    truncate_tool_result_text,
)
from hy_memory.models.memory import MemoryLayer, MemoryNode, MemoryStatus
from hy_memory.pipelines.base import ChatMessage, WriteRequest
from hy_memory.utils.raw_trim import (
    DEFAULT_RAW_ASSISTANT_MAX_CHARS,
    RAW_TRIMMED_KEY,
    trim_raw_body,
    trim_raw_messages,
)

_MARKER = "bytes omitted by truncate...]"
_LONG_USER = "我把整段日志贴给你:" + "U" * 5000
_LONG_ASSISTANT = "A" * 1200
_LONG_TOOL = "H" * 3000 + "MIDDLE" + "T" * 3000


def _messages() -> List[ChatMessage]:
    return [
        ChatMessage(role="system", content="S" * 900),
        ChatMessage(role="user", content=_LONG_USER),
        ChatMessage(role="assistant", content=_LONG_ASSISTANT),
        ChatMessage(role="tool", content=_LONG_TOOL, tool_call_id="c1", tool_name="Bash"),
        ChatMessage(role="assistant", name="John", content="J" * 800),
    ]


# ---------------------------------------------------------------- 1. 规则


class TestTrimRule:
    def test_user_kept_in_full(self):
        assert trim_raw_body("user", _LONG_USER) == _LONG_USER

    def test_system_kept_in_full(self):
        assert trim_raw_body("system", "S" * 900) == "S" * 900

    def test_long_assistant_cut_at_limit(self):
        out = trim_raw_body("assistant", _LONG_ASSISTANT)
        assert out == "A" * DEFAULT_RAW_ASSISTANT_MAX_CHARS + "..."

    def test_short_assistant_untouched(self):
        assert trim_raw_body("assistant", "  short  ") == "  short  "

    def test_other_speaker_cut_like_assistant(self):
        assert trim_raw_body("John", "J" * 800) == "J" * 500 + "..."

    def test_tool_head_tail_with_marker(self):
        out = trim_raw_body("tool", _LONG_TOOL, tool_max_bytes=2048)
        assert out.startswith("H" * 100) and out.endswith("T" * 100)
        assert "MIDDLE" not in out and _MARKER in out
        assert len(out.encode("utf-8")) <= 2048

    def test_messages_trimmed_without_mutating_input(self):
        msgs = _messages()
        out = trim_raw_messages(msgs, tool_max_bytes=512)
        assert [m.content for m in msgs][1] == _LONG_USER
        assert msgs[2].content == _LONG_ASSISTANT, "入参不能被修改"
        assert out[1].content == _LONG_USER
        assert out[2].content == "A" * 500 + "..."
        assert _MARKER in out[3].content and out[3].tool_call_id == "c1"
        assert out[4].content == "J" * 500 + "..." and out[4].name == "John"

    @pytest.mark.parametrize("max_bytes", [48, 64, 200, 512, 2048])
    def test_idempotent(self, max_bytes):
        once = trim_raw_messages(_messages(), tool_max_bytes=max_bytes)
        twice = trim_raw_messages(once, tool_max_bytes=max_bytes)
        assert [m.content for m in twice] == [m.content for m in once]
        for label in ("assistant", "John", "tool", "user", "system"):
            for body in (_LONG_ASSISTANT, _LONG_TOOL, "中文" * 2000, " x " * 400):
                t1 = trim_raw_body(label, body, tool_max_bytes=max_bytes)
                assert trim_raw_body(label, t1, tool_max_bytes=max_bytes) == t1

    def test_preproc_truncations_idempotent(self):
        t = truncate_tool_result_text(_LONG_TOOL, max_bytes=64)
        assert truncate_tool_result_text(t, max_bytes=64) == t
        a = truncate_assistant_text("x" * 10000, head_chars=30, tail_chars=30)
        assert truncate_assistant_text(a, head_chars=30, tail_chars=30) == a


# ---------------------------------------------------------------- 2. lite 端到端


def _lite_config(tmp_path):
    from hy_memory import MemoryConfig

    return MemoryConfig.from_dict({
        "vector_store": {
            "provider": "chroma",
            "collection_name": f"test_raw_trim_{uuid.uuid4().hex[:8]}",
            "persist_directory": str(tmp_path / "chroma"),
            "embedding_dims": 8,
        },
        "embedder": {"embedding_dims": 8},
        "cache": {"backend": "sqlite", "sqlite_path": str(tmp_path / "cache.db")},
        "history": {"enable": True, "db_path": str(tmp_path / "history.db")},
        "coding": {"enable": True, "tool_result_max_bytes": 512},
    })


def _install_fixed_embed(client):
    vector = [1.0] + [0.0] * 7

    async def one(_self, _text, use_cache=True):
        return list(vector)

    async def one_meta(_self, _text, use_cache=True):
        return list(vector), {"cache_hit": False, "provider": "test"}

    async def batch(_self, texts, use_cache=True):
        return [list(vector) for _ in texts]

    s = client._embed_service
    s.embed = types.MethodType(one, s)
    s.embed_queued = types.MethodType(one, s)
    s.embed_with_meta = types.MethodType(one_meta, s)
    s.embed_queued_with_meta = types.MethodType(one_meta, s)
    s.embed_batch = types.MethodType(batch, s)


def test_lite_stores_trimmed_raw(tmp_path, monkeypatch):
    from hy_memory import HyMemoryClient

    monkeypatch.setenv("MEMORY_DATA_DIR", str(tmp_path / "data"))
    monkeypatch.setenv("MEMORY_LOG_DIR", str(tmp_path / "logs"))
    monkeypatch.setenv("MEMORY_PIPELINE_TRACE_ENABLED", "false")
    client = HyMemoryClient(_lite_config(tmp_path), mode="lite")
    try:
        _install_fixed_embed(client)
        resp = client.add(
            [
                {"role": "user", "content": _LONG_USER},
                {"role": "assistant", "content": _LONG_ASSISTANT,
                 "tool_calls": [{"id": "c1", "type": "function",
                                 "function": {"name": "Bash", "arguments": "{}"}}]},
                {"role": "tool", "tool_call_id": "c1", "content": _LONG_TOOL},
            ],
            user_id="u1",
        )
        assert resp["success"] is True, resp

        async def _raw_nodes():
            return await client._vector_store.list_by_user(
                user_id="u1", layers=[MemoryLayer.L1_RAW],
                with_vectors=False, allow_raw_layer=True,
            )

        nodes = client._loop_thread.run(_raw_nodes())
        assert len(nodes) == 1
        content = nodes[0].content
        assert f"[user]: {_LONG_USER}" in content, "user 必须完整保留"
        assert f"[assistant]: {'A' * 500}...\n" in content
        tool_line = content.split("[tool]: ", 1)[1]
        assert tool_line.startswith("H") and tool_line.endswith("T")
        assert _MARKER in tool_line and "MIDDLE" not in tool_line
        assert len(tool_line.encode("utf-8")) <= 512
        assert (nodes[0].custom or {}).get(RAW_TRIMMED_KEY) is True

        conn = sqlite3.connect(str(tmp_path / "history.db"))
        try:
            (audit,) = conn.execute(
                "SELECT new_memory FROM memory_history WHERE event='ADD'"
            ).fetchone()
        finally:
            conn.close()
        assert _LONG_USER in audit and "MIDDLE" not in audit
        assert "A" * 501 not in audit
    finally:
        client.close()


# ---------------------------------------------------------------- 3. coding_trace


def test_coding_trace_stores_truncated_tool_result(tmp_path):
    from hy_memory.coding.writer import CodingWriter

    captured: Dict[str, Any] = {}

    class _Store:
        config = types.SimpleNamespace(
            coding=types.SimpleNamespace(tool_result_max_bytes=512),
            extractor=types.SimpleNamespace(history_turns=5),
        )

        async def get_recent_trace_messages(self, **kw):
            return []

        async def append_trace(self, messages, **kw):
            captured["messages"] = messages

    class _Extractor:
        async def extract(self, messages, **kw):
            return [], []

    w = CodingWriter.__new__(CodingWriter)
    w.variant = "working"
    w.store = _Store()
    w.extractor = _Extractor()
    asyncio.run(w.write(_messages(), user_id="u1"))

    stored = {m.role: m.content for m in captured["messages"] if m.role != "assistant"}
    assert stored["user"] == _LONG_USER
    assert _MARKER in stored["tool"] and "MIDDLE" not in stored["tool"]
    assert len(stored["tool"].encode("utf-8")) <= 512


# ---------------------------------------------------------------- 4. pro writer


class _Store:
    supports_fulltext = False

    def __init__(self):
        self.nodes: Dict[str, MemoryNode] = {}

    async def initialize(self):
        pass

    async def upsert(self, node: MemoryNode) -> str:
        node.node_id = node.node_id or uuid.uuid4().hex
        if node.gmt_created is None:
            node.gmt_created = datetime.now()
        self.nodes[node.node_id] = node
        return node.node_id

    async def get_by_id(self, node_id):
        return self.nodes.get(node_id)

    async def list_by_user(self, *a, layers=None, **kw):
        return [
            n for n in self.nodes.values()
            if not layers or n.layer in layers
        ]

    async def search(self, *a, **kw):
        return []


class _Embed:
    async def embed_queued_with_meta(self, text):
        return [0.0] * 8, {"provider": "test"}

    async def embed_queued(self, text):
        return [0.0] * 8

    async def embed_batch(self, texts):
        return [[0.0] * 8 for _ in texts]


class _CapturingAgent:
    def __init__(self):
        self.calls: List[Dict[str, Any]] = []

    async def process_add(self, **kwargs):
        from tests.test_shadow_l1_raw_on_empty_extract import _FakeAgentResult

        self.calls.append(kwargs)
        return _FakeAgentResult(success=True, extracted_info={"memory": [], "intentions": []})


def _pro_writer(store):
    from hy_memory.config import MemoryConfig
    from hy_memory.pipelines.writer import MemoryWriter

    w = MemoryWriter(MemoryConfig(), embed_service=_Embed(), vector_store=store, cache=None)
    w._mem_agent = _CapturingAgent()
    w._vector_store = store

    async def _empty_tags(*a, **kw):
        return []

    w._collect_existing_tags = _empty_tags
    return w


def _write(w, msgs: List[ChatMessage]):
    req = WriteRequest(
        content="", messages=msgs, user_id="u1", agent_id="default_agent",
        session_id="s1", content_type="messages",
        extra={"agent_mode": "full", "mode": "pro"},
        request_id=uuid.uuid4().hex,
    )
    return asyncio.run(w.write(req))


def test_pro_extraction_input_unchanged_and_history_concatenates():
    store = _Store()
    w = _pro_writer(store)
    turn1 = [
        ChatMessage(role="user", content=_LONG_USER),
        ChatMessage(role="assistant", content=_LONG_ASSISTANT),
    ]
    turn2 = [
        ChatMessage(role="user", content="第二轮"),
        ChatMessage(role="assistant", content="B" * 900),
    ]
    r1 = _write(w, turn1)
    assert r1.memory_id
    r2 = _write(w, turn2)
    assert r2.memory_id

    calls = w._mem_agent.calls
    assert len(calls) == 2
    # extractor 收到的仍是完整 messages(没有被落库裁剪影响)
    for call, msgs in zip(calls, (turn1, turn2)):
        assert json.loads(call["content"]) == [
            {"role": m.role, "content": m.content} for m in msgs
        ]

    raw1 = store.nodes[r1.memory_id]
    assert raw1.content == f"[user]: {_LONG_USER}\n[assistant]: {'A' * 500}..."
    assert raw1.custom.get(RAW_TRIMMED_KEY) is True
    # 历史 = 已存 raw 直接拼接(第一次写入时没有历史)
    assert calls[0]["history_context"] == ""
    assert calls[1]["history_context"] == raw1.content


def test_history_does_not_retrim_new_raw_but_trims_legacy_raw():
    """新 raw(带标记)原样拼接;旧 raw(无标记)按同一规则兼容裁剪。"""
    from hy_memory.pipelines.writer import MemoryWriter

    store = _Store()
    base = datetime.now()
    # 新 raw:内容里刻意放一条超过 500 的 assistant——带标记就不应再被裁
    new_content = "[user]: 新问题\n[assistant]: " + "N" * 700
    legacy_content = "[user]: 旧问题\n[assistant]: " + "O" * 700 + "\n[tool]: " + _LONG_TOOL
    store.nodes["legacy"] = MemoryNode(
        node_id="legacy", user_id="u1", agent_id="default_agent",
        layer=MemoryLayer.L1_RAW, content=legacy_content,
        status=MemoryStatus.ACTIVE, gmt_created=base - timedelta(seconds=2),
    )
    store.nodes["new"] = MemoryNode(
        node_id="new", user_id="u1", agent_id="default_agent",
        layer=MemoryLayer.L1_RAW, content=new_content,
        status=MemoryStatus.ACTIVE, gmt_created=base - timedelta(seconds=1),
        custom={RAW_TRIMMED_KEY: True},
    )
    w = MemoryWriter.__new__(MemoryWriter)
    history = asyncio.run(w._get_recent_history(store, "u1", "default_agent", max_turns=20))

    assert history.endswith(new_content), "新 raw 必须原样拼接"
    assert "[assistant]: " + "O" * 500 + "...\n" in history
    assert "MIDDLE" not in history and _MARKER in history