Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(session): add memory cold-storage archival via hotness scoring (#620)
Matt Van Horn5225ef4aac8b
diff --git a/openviking/session/__init__.py b/openviking/session/__init__.pyindex 6add1ab9b..d9a8d9d3a 100644--- a/openviking/session/__init__.py+++ b/openviking/session/__init__.py@@ -3,6 +3,11 @@"""Session management module."""from openviking.session.compressor import ExtractionStats, SessionCompressor+from openviking.session.memory_archiver import (+ ArchivalCandidate,+ ArchivalResult,+ MemoryArchiver,+)from openviking.session.memory_deduplicator import (DedupDecision,DedupResult,@@ -26,6 +31,10 @@ __all__ = [# Compressor"SessionCompressor","ExtractionStats",+ # Memory Archiver+ "MemoryArchiver",+ "ArchivalCandidate",+ "ArchivalResult",# Memory Extractor"MemoryExtractor","MemoryCategory",diff --git a/openviking/session/memory_archiver.py b/openviking/session/memory_archiver.pynew file mode 100644index 000000000..680ae4028--- /dev/null+++ b/openviking/session/memory_archiver.py@@ -0,0 +1,328 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0+"""Cold-storage archival for stale memories based on hotness scoring.++Moves memories with low hotness scores to an archive directory,+reducing token consumption from stale abstracts and overviews during+retrieval. Archived memories can be restored to their original location.+"""++from dataclasses import dataclass, field+from datetime import datetime, timezone+from typing import Any, List, Optional++from openviking.retrieve.memory_lifecycle import hotness_score+from openviking.server.identity import RequestContext+from openviking.storage.expr import And, Eq+from openviking.utils.time_utils import parse_iso_datetime+from openviking_cli.utils.logger import get_logger++logger = get_logger(__name__)++# Directory name for archived memories within each scope.+ARCHIVE_DIR = "_archive"+++@dataclass+class ArchivalCandidate:+ """A memory that qualifies for archival."""++ uri: str+ active_count: int+ updated_at: Optional[datetime]+ score: float+ context_type: str = ""+ parent_uri: str = ""+++@dataclass+class ArchivalResult:+ """Summary of an archival operation."""++ scanned: int = 0+ archived: int = 0+ skipped: int = 0+ errors: int = 0+ candidates: List[ArchivalCandidate] = field(default_factory=list)+++class MemoryArchiver:+ """Archives cold memories based on hotness scoring.++ Uses ``hotness_score()`` from ``memory_lifecycle`` to identify memories+ whose access frequency and recency have fallen below a threshold.+ Moves them to ``{scope}/_archive/`` using ``viking_fs.mv()`` so+ they remain recoverable but are excluded from default retrieval.+ """++ DEFAULT_THRESHOLD: float = 0.1+ DEFAULT_MIN_AGE_DAYS: int = 7+ DEFAULT_BATCH_SIZE: int = 100++ def __init__(+ self,+ viking_fs: Any,+ storage: Any,+ threshold: float = DEFAULT_THRESHOLD,+ min_age_days: int = DEFAULT_MIN_AGE_DAYS,+ ):+ """Initialize the archiver.++ Args:+ viking_fs: VikingFS instance for filesystem operations.+ storage: VikingDBManagerProxy for vector index queries.+ threshold: Hotness score below which memories are archived.+ min_age_days: Skip memories updated within this many days.+ """+ self.viking_fs = viking_fs+ self.storage = storage+ self.threshold = threshold+ self.min_age_days = min_age_days++ async def scan(+ self,+ scope_uri: str,+ ctx: Optional[RequestContext] = None,+ now: Optional[datetime] = None,+ ) -> List[ArchivalCandidate]:+ """Scan a scope for cold memories.++ Queries the vector index for all L2 memories under *scope_uri*,+ computes their hotness score, and returns those below the threshold+ that are older than ``min_age_days``.++ Args:+ scope_uri: Root URI to scan (e.g. ``viking://memories/``).+ ctx: Request context for tenant isolation.+ now: Override current time (for deterministic tests).++ Returns:+ List of candidates eligible for archival, sorted by score+ ascending (coldest first).+ """+ if now is None:+ now = datetime.now(timezone.utc)++ candidates: List[ArchivalCandidate] = []++ # Only scan L2 content -- never archive L0 abstracts or L1 overviews.+ filter_expr = And(conds=[Eq("level", 2)])++ cursor: Optional[str] = None+ total_scanned = 0++ while True:+ records, next_cursor = await self.storage.scroll(+ filter=filter_expr,+ limit=self.DEFAULT_BATCH_SIZE,+ cursor=cursor,+ output_fields=[+ "uri",+ "active_count",+ "updated_at",+ "context_type",+ "parent_uri",+ ],+ )++ if not records:+ break++ for record in records:+ uri = record.get("uri", "")++ # Skip entries already in an archive directory.+ if f"/{ARCHIVE_DIR}/" in uri:+ continue++ # Skip entries outside the requested scope.+ if not uri.startswith(scope_uri):+ continue++ total_scanned += 1++ active_count = int(record.get("active_count", 0) or 0)+ updated_at_raw = record.get("updated_at")+ updated_at = _parse_datetime(updated_at_raw)++ # Respect minimum age.+ if updated_at is not None:+ age_days = (now - updated_at).total_seconds() / 86400.0+ if age_days < self.min_age_days:+ continue++ score = hotness_score(+ active_count=active_count,+ updated_at=updated_at,+ now=now,+ )++ if score < self.threshold:+ candidates.append(+ ArchivalCandidate(+ uri=uri,+ active_count=active_count,+ updated_at=updated_at,+ score=score,+ context_type=record.get("context_type", ""),+ parent_uri=record.get("parent_uri", ""),+ )+ )++ cursor = next_cursor+ if cursor is None:+ break++ # Coldest first.+ candidates.sort(key=lambda c: c.score)++ logger.info(+ f"[MemoryArchiver] Scanned {total_scanned} memories under {scope_uri}, "+ f"found {len(candidates)} archival candidates (threshold={self.threshold})"+ )+ return candidates++ async def archive(+ self,+ candidates: List[ArchivalCandidate],+ ctx: Optional[RequestContext] = None,+ dry_run: bool = False,+ ) -> ArchivalResult:+ """Archive the given candidates.++ Moves each candidate to ``{parent}/_archive/{filename}`` using+ ``viking_fs.mv()``, which atomically updates the vector index.++ Args:+ candidates: Output of ``scan()``.+ ctx: Request context for tenant isolation.+ dry_run: If True, log what would happen without moving files.++ Returns:+ Summary of the operation.+ """+ result = ArchivalResult(scanned=len(candidates), candidates=candidates)++ for candidate in candidates:+ archive_uri = _build_archive_uri(candidate.uri)++ if dry_run:+ logger.info(+ f"[MemoryArchiver] DRY-RUN would archive {candidate.uri} "+ f"(score={candidate.score:.4f}) -> {archive_uri}"+ )+ result.skipped += 1+ continue++ try:+ await self.viking_fs.mv(candidate.uri, archive_uri, ctx=ctx)+ result.archived += 1+ logger.info(+ f"[MemoryArchiver] Archived {candidate.uri} "+ f"(score={candidate.score:.4f}) -> {archive_uri}"+ )+ except Exception:+ logger.exception(f"[MemoryArchiver] Failed to archive {candidate.uri}")+ result.errors += 1++ logger.info(+ f"[MemoryArchiver] Archive complete: "+ f"{result.archived} archived, {result.skipped} skipped, "+ f"{result.errors} errors"+ )+ return result++ async def restore(+ self,+ archived_uri: str,+ ctx: Optional[RequestContext] = None,+ ) -> bool:+ """Restore an archived memory to its original location.++ The original location is derived by removing the ``_archive/``+ path segment from the URI.++ Args:+ archived_uri: URI of the archived memory.+ ctx: Request context for tenant isolation.++ Returns:+ True if the memory was restored successfully.+ """+ original_uri = _build_restore_uri(archived_uri)+ if original_uri is None:+ logger.warning(+ f"[MemoryArchiver] Cannot restore {archived_uri}: not in an archive directory"+ )+ return False++ try:+ await self.viking_fs.mv(archived_uri, original_uri, ctx=ctx)+ logger.info(f"[MemoryArchiver] Restored {archived_uri} -> {original_uri}")+ return True+ except Exception:+ logger.exception(f"[MemoryArchiver] Failed to restore {archived_uri}")+ return False++ async def scan_and_archive(+ self,+ scope_uri: str,+ ctx: Optional[RequestContext] = None,+ dry_run: bool = False,+ now: Optional[datetime] = None,+ ) -> ArchivalResult:+ """Convenience method: scan then archive in one call."""+ candidates = await self.scan(scope_uri, ctx=ctx, now=now)+ return await self.archive(candidates, ctx=ctx, dry_run=dry_run)+++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------+++def _build_archive_uri(uri: str) -> str:+ """Insert ``_archive/`` before the filename in a URI.++ ``viking://memories/facts/greeting.md``+ -> ``viking://memories/facts/_archive/greeting.md``+ """+ last_slash = uri.rfind("/")+ if last_slash == -1:+ return f"{ARCHIVE_DIR}/{uri}"+ parent = uri[:last_slash]+ filename = uri[last_slash + 1 :]+ return f"{parent}/{ARCHIVE_DIR}/{filename}"+++def _build_restore_uri(archived_uri: str) -> Optional[str]:+ """Remove the ``_archive/`` segment to recover the original URI.++ ``viking://memories/facts/_archive/greeting.md``+ -> ``viking://memories/facts/greeting.md``++ Returns None if the URI does not contain ``_archive/``.+ """+ marker = f"/{ARCHIVE_DIR}/"+ idx = archived_uri.find(marker)+ if idx == -1:+ return None+ parent = archived_uri[:idx]+ filename = archived_uri[idx + len(marker) :]+ return f"{parent}/{filename}"+++def _parse_datetime(value: Any) -> Optional[datetime]:+ """Best-effort parse of a datetime value from the vector store."""+ if value is None:+ return None+ if isinstance(value, datetime):+ if value.tzinfo is None:+ return value.replace(tzinfo=timezone.utc)+ return value+ if isinstance(value, str):+ try:+ return parse_iso_datetime(value)+ except Exception:+ return None+ return Nonediff --git a/tests/unit/session/test_memory_archiver.py b/tests/unit/session/test_memory_archiver.pynew file mode 100644index 000000000..e8d81a0b4--- /dev/null+++ b/tests/unit/session/test_memory_archiver.py@@ -0,0 +1,423 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0+"""Unit tests for memory cold-storage archival."""++from datetime import datetime, timedelta, timezone+from unittest.mock import AsyncMock++import pytest++from openviking.session.memory_archiver import (+ ArchivalCandidate,+ MemoryArchiver,+ _build_archive_uri,+ _build_restore_uri,+ _parse_datetime,+)++# ---------------------------------------------------------------------------+# Helper URI functions+# ---------------------------------------------------------------------------+++class TestBuildArchiveUri:+ def test_simple_file(self):+ assert (+ _build_archive_uri("viking://memories/facts/greeting.md")+ == "viking://memories/facts/_archive/greeting.md"+ )++ def test_nested_path(self):+ assert (+ _build_archive_uri("viking://memories/user/prefs/theme.md")+ == "viking://memories/user/prefs/_archive/theme.md"+ )++ def test_root_level_file(self):+ assert (+ _build_archive_uri("viking://memories/note.md") == "viking://memories/_archive/note.md"+ )++ def test_no_slash(self):+ assert _build_archive_uri("note.md") == "_archive/note.md"+++class TestBuildRestoreUri:+ def test_simple_restore(self):+ assert (+ _build_restore_uri("viking://memories/facts/_archive/greeting.md")+ == "viking://memories/facts/greeting.md"+ )++ def test_nested_restore(self):+ assert (+ _build_restore_uri("viking://memories/user/_archive/pref.md")+ == "viking://memories/user/pref.md"+ )++ def test_not_archived_returns_none(self):+ assert _build_restore_uri("viking://memories/facts/greeting.md") is None++ def test_roundtrip(self):+ original = "viking://memories/deep/path/to/file.md"+ archived = _build_archive_uri(original)+ restored = _build_restore_uri(archived)+ assert restored == original+++# ---------------------------------------------------------------------------+# Datetime parsing+# ---------------------------------------------------------------------------+++class TestParseDatetime:+ def test_none(self):+ assert _parse_datetime(None) is None++ def test_datetime_object(self):+ dt = datetime(2026, 1, 1, tzinfo=timezone.utc)+ assert _parse_datetime(dt) == dt++ def test_naive_datetime_gets_utc(self):+ dt = datetime(2026, 1, 1)+ result = _parse_datetime(dt)+ assert result is not None+ assert result.tzinfo == timezone.utc++ def test_iso_string(self):+ result = _parse_datetime("2026-01-01T00:00:00+00:00")+ assert result is not None+ assert result.year == 2026++ def test_invalid_string(self):+ assert _parse_datetime("not-a-date") is None++ def test_integer_returns_none(self):+ assert _parse_datetime(12345) is None+++# ---------------------------------------------------------------------------+# MemoryArchiver.scan+# ---------------------------------------------------------------------------+++def _make_storage(records):+ """Create a mock storage that returns records from scroll()."""+ storage = AsyncMock()+ storage.scroll = AsyncMock(return_value=(records, None))+ return storage+++def _make_viking_fs():+ """Create a mock VikingFS."""+ vfs = AsyncMock()+ vfs.mv = AsyncMock(return_value={"status": "ok"})+ return vfs+++NOW = datetime(2026, 3, 14, 12, 0, 0, tzinfo=timezone.utc)+OLD_DATE = NOW - timedelta(days=30)+RECENT_DATE = NOW - timedelta(days=2)+++class TestScan:+ @pytest.mark.asyncio+ async def test_scan_finds_cold_memories(self):+ records = [+ {+ "uri": "viking://memories/fact1.md",+ "active_count": 0,+ "updated_at": OLD_DATE,+ "context_type": "memory",+ "parent_uri": "viking://memories/",+ },+ ]+ archiver = MemoryArchiver(+ viking_fs=_make_viking_fs(),+ storage=_make_storage(records),+ threshold=0.5,+ min_age_days=7,+ )+ candidates = await archiver.scan("viking://memories/", now=NOW)+ assert len(candidates) == 1+ assert candidates[0].uri == "viking://memories/fact1.md"+ assert candidates[0].score < 0.5++ @pytest.mark.asyncio+ async def test_scan_skips_recent_memories(self):+ records = [+ {+ "uri": "viking://memories/recent.md",+ "active_count": 0,+ "updated_at": RECENT_DATE,+ "context_type": "memory",+ "parent_uri": "viking://memories/",+ },+ ]+ archiver = MemoryArchiver(+ viking_fs=_make_viking_fs(),+ storage=_make_storage(records),+ threshold=0.5,+ min_age_days=7,+ )+ candidates = await archiver.scan("viking://memories/", now=NOW)+ assert len(candidates) == 0++ @pytest.mark.asyncio+ async def test_scan_skips_already_archived(self):+ records = [+ {+ "uri": "viking://memories/_archive/old.md",+ "active_count": 0,+ "updated_at": OLD_DATE,+ "context_type": "memory",+ "parent_uri": "viking://memories/_archive/",+ },+ ]+ archiver = MemoryArchiver(+ viking_fs=_make_viking_fs(),+ storage=_make_storage(records),+ threshold=0.5,+ min_age_days=7,+ )+ candidates = await archiver.scan("viking://memories/", now=NOW)+ assert len(candidates) == 0++ @pytest.mark.asyncio+ async def test_scan_skips_out_of_scope(self):+ records = [+ {+ "uri": "viking://resources/doc.md",+ "active_count": 0,+ "updated_at": OLD_DATE,+ "context_type": "resource",+ "parent_uri": "viking://resources/",+ },+ ]+ archiver = MemoryArchiver(+ viking_fs=_make_viking_fs(),+ storage=_make_storage(records),+ threshold=0.5,+ min_age_days=7,+ )+ candidates = await archiver.scan("viking://memories/", now=NOW)+ assert len(candidates) == 0++ @pytest.mark.asyncio+ async def test_scan_keeps_hot_memories(self):+ records = [+ {+ "uri": "viking://memories/hot.md",+ "active_count": 100,+ "updated_at": NOW - timedelta(days=1),+ "context_type": "memory",+ "parent_uri": "viking://memories/",+ },+ ]+ archiver = MemoryArchiver(+ viking_fs=_make_viking_fs(),+ storage=_make_storage(records),+ threshold=0.5,+ min_age_days=0,+ )+ candidates = await archiver.scan("viking://memories/", now=NOW)+ # High active_count + recent = hot, should not be a candidate+ assert len(candidates) == 0++ @pytest.mark.asyncio+ async def test_scan_sorts_coldest_first(self):+ records = [+ {+ "uri": "viking://memories/warm.md",+ "active_count": 5,+ "updated_at": OLD_DATE,+ "context_type": "memory",+ "parent_uri": "viking://memories/",+ },+ {+ "uri": "viking://memories/cold.md",+ "active_count": 0,+ "updated_at": OLD_DATE - timedelta(days=60),+ "context_type": "memory",+ "parent_uri": "viking://memories/",+ },+ ]+ archiver = MemoryArchiver(+ viking_fs=_make_viking_fs(),+ storage=_make_storage(records),+ threshold=0.5,+ min_age_days=7,+ )+ candidates = await archiver.scan("viking://memories/", now=NOW)+ assert len(candidates) == 2+ assert candidates[0].uri == "viking://memories/cold.md"+ assert candidates[0].score <= candidates[1].score++ @pytest.mark.asyncio+ async def test_scan_empty_store(self):+ archiver = MemoryArchiver(+ viking_fs=_make_viking_fs(),+ storage=_make_storage([]),+ threshold=0.5,+ min_age_days=7,+ )+ candidates = await archiver.scan("viking://memories/", now=NOW)+ assert candidates == []+++# ---------------------------------------------------------------------------+# MemoryArchiver.archive+# ---------------------------------------------------------------------------+++class TestArchive:+ @pytest.mark.asyncio+ async def test_archive_moves_files(self):+ vfs = _make_viking_fs()+ archiver = MemoryArchiver(viking_fs=vfs, storage=_make_storage([]))+ candidates = [+ ArchivalCandidate(+ uri="viking://memories/fact1.md",+ active_count=0,+ updated_at=OLD_DATE,+ score=0.01,+ ),+ ]+ result = await archiver.archive(candidates)+ assert result.archived == 1+ assert result.errors == 0+ vfs.mv.assert_called_once_with(+ "viking://memories/fact1.md",+ "viking://memories/_archive/fact1.md",+ ctx=None,+ )++ @pytest.mark.asyncio+ async def test_archive_dry_run(self):+ vfs = _make_viking_fs()+ archiver = MemoryArchiver(viking_fs=vfs, storage=_make_storage([]))+ candidates = [+ ArchivalCandidate(+ uri="viking://memories/fact1.md",+ active_count=0,+ updated_at=OLD_DATE,+ score=0.01,+ ),+ ]+ result = await archiver.archive(candidates, dry_run=True)+ assert result.archived == 0+ assert result.skipped == 1+ vfs.mv.assert_not_called()++ @pytest.mark.asyncio+ async def test_archive_handles_mv_error(self):+ vfs = _make_viking_fs()+ vfs.mv = AsyncMock(side_effect=RuntimeError("AGFS error"))+ archiver = MemoryArchiver(viking_fs=vfs, storage=_make_storage([]))+ candidates = [+ ArchivalCandidate(+ uri="viking://memories/fact1.md",+ active_count=0,+ updated_at=OLD_DATE,+ score=0.01,+ ),+ ]+ result = await archiver.archive(candidates)+ assert result.archived == 0+ assert result.errors == 1++ @pytest.mark.asyncio+ async def test_archive_empty_candidates(self):+ archiver = MemoryArchiver(+ viking_fs=_make_viking_fs(),+ storage=_make_storage([]),+ )+ result = await archiver.archive([])+ assert result.archived == 0+ assert result.scanned == 0+++# ---------------------------------------------------------------------------+# MemoryArchiver.restore+# ---------------------------------------------------------------------------+++class TestRestore:+ @pytest.mark.asyncio+ async def test_restore_moves_back(self):+ vfs = _make_viking_fs()+ archiver = MemoryArchiver(viking_fs=vfs, storage=_make_storage([]))+ ok = await archiver.restore("viking://memories/_archive/fact1.md")+ assert ok is True+ vfs.mv.assert_called_once_with(+ "viking://memories/_archive/fact1.md",+ "viking://memories/fact1.md",+ ctx=None,+ )++ @pytest.mark.asyncio+ async def test_restore_non_archived_uri(self):+ vfs = _make_viking_fs()+ archiver = MemoryArchiver(viking_fs=vfs, storage=_make_storage([]))+ ok = await archiver.restore("viking://memories/fact1.md")+ assert ok is False+ vfs.mv.assert_not_called()++ @pytest.mark.asyncio+ async def test_restore_handles_error(self):+ vfs = _make_viking_fs()+ vfs.mv = AsyncMock(side_effect=RuntimeError("AGFS error"))+ archiver = MemoryArchiver(viking_fs=vfs, storage=_make_storage([]))+ ok = await archiver.restore("viking://memories/_archive/fact1.md")+ assert ok is False+++# ---------------------------------------------------------------------------+# scan_and_archive convenience+# ---------------------------------------------------------------------------+++class TestScanAndArchive:+ @pytest.mark.asyncio+ async def test_scan_and_archive(self):+ records = [+ {+ "uri": "viking://memories/cold.md",+ "active_count": 0,+ "updated_at": OLD_DATE,+ "context_type": "memory",+ "parent_uri": "viking://memories/",+ },+ ]+ vfs = _make_viking_fs()+ archiver = MemoryArchiver(+ viking_fs=vfs,+ storage=_make_storage(records),+ threshold=0.5,+ min_age_days=7,+ )+ result = await archiver.scan_and_archive("viking://memories/", now=NOW)+ assert result.archived == 1++ @pytest.mark.asyncio+ async def test_scan_and_archive_dry_run(self):+ records = [+ {+ "uri": "viking://memories/cold.md",+ "active_count": 0,+ "updated_at": OLD_DATE,+ "context_type": "memory",+ "parent_uri": "viking://memories/",+ },+ ]+ vfs = _make_viking_fs()+ archiver = MemoryArchiver(+ viking_fs=vfs,+ storage=_make_storage(records),+ threshold=0.5,+ min_age_days=7,+ )+ result = await archiver.scan_and_archive("viking://memories/", dry_run=True, now=NOW)+ assert result.archived == 0+ assert result.skipped == 1+ vfs.mv.assert_not_called()