Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

14.9 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
"""Tests for memory-context semantic enqueue deduplication (#769)."""

from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
from uuid import uuid4

import pytest

from openviking.storage.abstract_overview import (
    AbstractOverviewWriteResult,
    parse_abstract_overview,
)
from openviking.storage.queuefs.named_queue import NamedQueue
from openviking.storage.queuefs.semantic_msg import SemanticMsg
from openviking.storage.queuefs.semantic_processor import SemanticProcessor
from openviking.storage.queuefs.semantic_queue import SemanticQueue, is_semantic_msg_stale


def _processor(**kwargs):
    resolver = SimpleNamespace(get_vlm=AsyncMock(return_value=SimpleNamespace()))
    return SemanticProcessor(vlm_resolver=resolver, **kwargs)


@pytest.mark.asyncio
async def test_memory_semantic_enqueue_deduped_within_window():
    mock_agfs = MagicMock()
    with patch.object(NamedQueue, "enqueue", new_callable=AsyncMock) as named_enqueue:
        named_enqueue.return_value = "queued-id"
        q = SemanticQueue(mock_agfs, "/queue", "semantic")
        msg = SemanticMsg(
            uri="viking://user/default/memories/entities",
            context_type="memory",
            account_id="acc",
            user_id="u1",
            peer_id="p1",
        )
        r1 = await q.enqueue(msg)
        r2 = await q.enqueue(
            SemanticMsg(
                uri="viking://user/default/memories/entities",
                context_type="memory",
                account_id="acc",
                user_id="u1",
                peer_id="p1",
            )
        )
        assert r1 == "queued-id"
        assert r2 == "deduplicated"
        assert named_enqueue.call_count == 1


@pytest.mark.asyncio
async def test_memory_semantic_worker_retry_bypasses_dedupe_window():
    mock_agfs = MagicMock()
    with patch.object(NamedQueue, "enqueue", new_callable=AsyncMock) as named_enqueue:
        named_enqueue.return_value = "queued-id"
        q = SemanticQueue(mock_agfs, "/queue", "semantic")
        msg = SemanticMsg(
            uri="viking://user/default/memories/entities",
            context_type="memory",
            account_id="acc",
            user_id="u1",
            peer_id="p1",
        )

        await q.enqueue(msg)
        retry = await q.enqueue(msg, deduplicate_memory=False)

        assert retry == "queued-id"
        assert named_enqueue.call_count == 2


@pytest.mark.asyncio
async def test_worker_without_lock_does_not_dedupe_a_memory_retry(monkeypatch):
    """The circuit-breaker requeue runs before a lease is acquired.

    That is the branch `SemanticMessageWork` takes when it owns no lock, so the
    worker itself - not just the queue API - has to bypass the memory window.
    """
    from openviking.storage.queuefs.semantic_work import SemanticMessageWork

    processor = _processor()
    with patch.object(NamedQueue, "enqueue", new_callable=AsyncMock) as named_enqueue:
        named_enqueue.return_value = "queued-id"
        queue = SemanticQueue(MagicMock(), "/queue", "semantic")
        msg = SemanticMsg(
            uri="viking://user/default/memories/entities",
            context_type="memory",
            account_id="acc",
            user_id="u1",
            peer_id="p1",
        )
        assert await queue.enqueue(msg) == "queued-id"

        monkeypatch.setattr(
            "openviking.storage.queuefs.get_queue_manager",
            lambda: SimpleNamespace(SEMANTIC="semantic", get_queue=lambda _: queue),
        )

        work = SemanticMessageWork(processor, msg, None)
        await work.reenqueue()

        # Without the flag the retry would be coalesced against the enqueue it
        # is retrying, and the delivery would be ACKed with no replacement.
        assert named_enqueue.call_count == 2


@pytest.mark.asyncio
async def test_memory_semantic_enqueue_different_uri_not_deduped():
    mock_agfs = MagicMock()
    with patch.object(NamedQueue, "enqueue", new_callable=AsyncMock) as named_enqueue:
        named_enqueue.return_value = "queued-id"
        q = SemanticQueue(mock_agfs, "/queue", "semantic")
        await q.enqueue(
            SemanticMsg(
                uri="viking://user/default/memories/entities",
                context_type="memory",
            )
        )
        await q.enqueue(
            SemanticMsg(
                uri="viking://user/default/memories/patterns",
                context_type="memory",
            )
        )
        assert named_enqueue.call_count == 2


@pytest.mark.asyncio
async def test_non_memory_context_not_deduped():
    mock_agfs = MagicMock()
    with patch.object(NamedQueue, "enqueue", new_callable=AsyncMock) as named_enqueue:
        named_enqueue.return_value = "queued-id"
        q = SemanticQueue(mock_agfs, "/queue", "semantic")
        uri = "viking://resources/docs"
        await q.enqueue(SemanticMsg(uri=uri, context_type="resource"))
        await q.enqueue(SemanticMsg(uri=uri, context_type="resource"))
        assert named_enqueue.call_count == 2


@pytest.mark.asyncio
async def test_coalesced_semantic_messages_mark_old_version_stale():
    mock_agfs = MagicMock()
    with patch.object(NamedQueue, "enqueue", new_callable=AsyncMock) as named_enqueue:
        named_enqueue.return_value = "queued-id"
        q = SemanticQueue(mock_agfs, "/queue", "semantic")
        coalesce_key = f"resource|acc|u|p|viking://resources/docs/{uuid4().hex}"
        first = SemanticMsg(
            uri="viking://resources/docs",
            context_type="resource",
            coalesce_key=coalesce_key,
        )
        second = SemanticMsg(
            uri="viking://resources/docs",
            context_type="resource",
            coalesce_key=first.coalesce_key,
        )

        await q.enqueue(first)
        await q.enqueue(second)

        assert first.coalesce_version == 1
        assert second.coalesce_version == 2
        assert is_semantic_msg_stale(first)
        assert not is_semantic_msg_stale(second)


class _FakePathLock:
    """Mock for _async_agfs pathlock operations."""

    def __init__(self):
        self.acquired_batches = []
        self.release_calls = []

    async def pathlock_acquire_exact_batch(self, paths):
        self.acquired_batches.append(paths)
        return {"id": "lock-1"}

    async def pathlock_release(self, lease):
        self.release_calls.append(lease["id"])


class _FakeVikingFS:
    def __init__(self, pathlock=None):
        self._async_agfs = pathlock or _FakePathLock()
        self.writes = []

    def _uri_to_path(self, uri, ctx=None):
        del ctx
        return f"/fake/{uri.replace('://', '/').strip('/')}"

    async def write_file(self, uri, content, ctx=None, lease_ref=None):
        del ctx, lease_ref
        self.writes.append((uri, content))


class _FakeMemoryDirFS:
    async def ls(self, uri, node_limit=None, ctx=None):
        del uri, node_limit, ctx
        return [
            {"name": "first.md", "isDir": False},
            {"name": "second.md", "isDir": False},
        ]


def _patch_semantic_config(monkeypatch, *, overview_sample_limit=32):
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_processor.get_openviking_config",
        lambda: SimpleNamespace(semantic=SimpleNamespace(overview_sample_limit=overview_sample_limit)),
    )


@pytest.mark.asyncio
async def test_stale_memory_semantic_write_is_success(monkeypatch):
    pathlock = _FakePathLock()
    viking_fs = _FakeVikingFS(pathlock)
    processor = _processor()
    coalesce_key = f"memory|acc|u|p|viking://user/default/memories/preferences/{uuid4().hex}"

    with patch.object(NamedQueue, "enqueue", new_callable=AsyncMock):
        q = SemanticQueue(MagicMock(), "/queue", "semantic")
        first = SemanticMsg(
            uri="viking://user/default/memories/preferences",
            context_type="memory",
            coalesce_key=coalesce_key,
        )
        latest = SemanticMsg(
            uri="viking://user/default/memories/preferences",
            context_type="memory",
            coalesce_key=coalesce_key,
        )
        await q.enqueue(first)
        await q.enqueue(latest)

    wrote_first = await processor._write_memory_directory_semantics(
        msg=first,
        viking_fs=viking_fs,
        dir_uri=first.uri,
        overview="old overview",
        abstract="old abstract",
        ctx=None,
    )
    wrote_latest = await processor._write_memory_directory_semantics(
        msg=latest,
        viking_fs=viking_fs,
        dir_uri=latest.uri,
        overview="latest overview",
        abstract="latest abstract",
        ctx=None,
    )

    assert not wrote_first
    assert wrote_latest
    assert pathlock.acquired_batches == [
        [
            "/fake/viking/user/default/memories/preferences/.overview.md",
            "/fake/viking/user/default/memories/preferences/.abstract.md",
        ]
    ]
    assert [uri for uri, _ in viking_fs.writes] == [
        "viking://user/default/memories/preferences/.overview.md",
        "viking://user/default/memories/preferences/.abstract.md",
    ]
    assert [parse_abstract_overview(raw).body.strip() for _, raw in viking_fs.writes] == [
        "latest overview",
        "latest abstract",
    ]


@pytest.mark.asyncio
async def test_memory_directory_summarizes_all_uncached_files(monkeypatch):
    processor = _processor(max_concurrent_llm=4)
    summaries = []

    async def generate_file_summary(file_path, llm_sem=None, ctx=None):
        del llm_sem, ctx
        name = file_path.rsplit("/", 1)[-1]
        return {"name": name, "summary": f"summary:{name}"}

    async def generate_overview(
        dir_uri, file_summaries, children_abstracts, llm_sem=None, **kwargs
    ):
        del dir_uri, children_abstracts, llm_sem, kwargs
        summaries.extend(file_summaries)
        return "overview"

    async def write_semantics(**kwargs):
        del kwargs
        return AbstractOverviewWriteResult(
            wrote=True, overview_body_changed=True, abstract_body_changed=True
        )

    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_processor.get_viking_fs",
        lambda: _FakeMemoryDirFS(),
    )
    _patch_semantic_config(monkeypatch)
    monkeypatch.setattr(processor, "_generate_single_file_summary", generate_file_summary)
    monkeypatch.setattr(processor, "_generate_overview", generate_overview)
    monkeypatch.setattr(
        processor,
        "_normalize_overview_generation",
        lambda overview: (overview, "abstract"),
    )
    monkeypatch.setattr(processor, "_write_memory_directory_semantics", write_semantics)

    await processor._process_memory_directory(
        SemanticMsg(
            uri="viking://user/default/memories/preferences",
            context_type="memory",
            skip_vectorization=True,
        )
    )

    assert [item["name"] for item in summaries] == ["first.md", "second.md"]


@pytest.mark.asyncio
async def test_memory_directory_vectorizes_changed_files_with_generated_summary(monkeypatch):
    processor = _processor(max_concurrent_llm=4)
    dir_uri = "viking://user/default/memories/preferences"
    changed_uri = f"{dir_uri}/first.md"
    captured_file_vectorize = []
    captured_directory_vectorize = []

    async def generate_file_summary(file_path, llm_sem=None, ctx=None):
        del llm_sem, ctx
        name = file_path.rsplit("/", 1)[-1]
        return {"name": name, "summary": f"summary:{name}", "content": "raw content"}

    async def generate_overview(
        dir_uri, file_summaries, children_abstracts, llm_sem=None, **kwargs
    ):
        del dir_uri, children_abstracts, llm_sem, kwargs
        assert len(captured_file_vectorize) == 1
        assert all("content" not in summary for summary in file_summaries)
        return "overview"

    async def write_semantics(**kwargs):
        del kwargs
        return AbstractOverviewWriteResult(
            wrote=True, overview_body_changed=True, abstract_body_changed=True
        )

    async def vectorize_single_file(**kwargs):
        captured_file_vectorize.append(kwargs)

    async def vectorize_directory(**kwargs):
        captured_directory_vectorize.append(kwargs)

    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_processor.get_viking_fs",
        lambda: _FakeMemoryDirFS(),
    )
    _patch_semantic_config(monkeypatch)
    monkeypatch.setattr(processor, "_generate_single_file_summary", generate_file_summary)
    monkeypatch.setattr(processor, "_generate_overview", generate_overview)
    monkeypatch.setattr(
        processor,
        "_normalize_overview_generation",
        lambda overview: (overview, "abstract"),
    )
    monkeypatch.setattr(processor, "_write_memory_directory_semantics", write_semantics)
    monkeypatch.setattr(processor, "_vectorize_single_file", vectorize_single_file)
    monkeypatch.setattr(processor, "_vectorize_directory", vectorize_directory)

    await processor._process_memory_directory(
        SemanticMsg(
            uri=dir_uri,
            context_type="memory",
            changes={"modified": [changed_uri]},
        )
    )

    assert len(captured_file_vectorize) == 1
    assert captured_file_vectorize[0]["parent_uri"] == dir_uri
    assert captured_file_vectorize[0]["context_type"] == "memory"
    assert captured_file_vectorize[0]["file_path"] == changed_uri
    assert captured_file_vectorize[0]["summary_dict"] == {
        "name": "first.md",
        "summary": "summary:first.md",
        "content": "raw content",
    }
    assert captured_file_vectorize[0]["preserve_existing_created_at"] is True
    assert len(captured_directory_vectorize) == 1
    assert captured_directory_vectorize[0]["uri"] == dir_uri


@pytest.mark.asyncio
async def test_memory_directory_skips_vectorization_when_visible_semantics_are_unchanged(
    monkeypatch,
):
    processor = _processor(max_concurrent_llm=4)
    vectorize_directory = AsyncMock()
    vectorize_file = AsyncMock()

    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_processor.get_viking_fs",
        lambda: _FakeMemoryDirFS(),
    )
    _patch_semantic_config(monkeypatch)
    monkeypatch.setattr(
        processor,
        "_generate_single_file_summary",
        AsyncMock(return_value={"name": "file.md", "summary": "summary"}),
    )
    monkeypatch.setattr(processor, "_generate_overview", AsyncMock(return_value="overview"))
    monkeypatch.setattr(
        processor, "_normalize_overview_generation", lambda overview: (overview, "abstract")
    )
    monkeypatch.setattr(
        processor,
        "_write_memory_directory_semantics",
        AsyncMock(return_value=AbstractOverviewWriteResult(wrote=True)),
    )
    monkeypatch.setattr(processor, "_vectorize_single_file", vectorize_file)
    monkeypatch.setattr(processor, "_vectorize_directory", vectorize_directory)

    await processor._process_memory_directory(
        SemanticMsg(
            uri="viking://user/default/memories/preferences",
            context_type="memory",
        )
    )

    assert vectorize_file.await_count == 2
    vectorize_directory.assert_not_awaited()