Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

Add memory observability system (Phase 1)

chopratejase16691dd38f9
diff --git a/headroom/cache/compression_store.py b/headroom/cache/compression_store.py
index de2a6f022..64aedfeea 100644
--- a/headroom/cache/compression_store.py
+++ b/headroom/cache/compression_store.py
@@ -46,6 +46,7 @@ from typing import TYPE_CHECKING, Any
from ..relevance.bm25 import BM25Scorer
if TYPE_CHECKING:
+ from ..memory.tracker import ComponentStats
from .backends import CompressionStoreBackend
logger = logging.getLogger(__name__)
@@ -505,6 +506,39 @@ class CompressionStore:
"backend": backend_stats,
}
+ def get_memory_stats(self) -> ComponentStats:
+ """Get memory statistics for the MemoryTracker.
+
+ Returns:
+ ComponentStats with current memory usage.
+ """
+ from ..memory.tracker import ComponentStats
+
+ with self._lock:
+ # Get backend stats which include bytes_used
+ backend_stats = self._backend.get_stats()
+ bytes_used = backend_stats.get("bytes_used", 0)
+
+ # Add retrieval events memory
+ import sys
+
+ bytes_used += sys.getsizeof(self._retrieval_events)
+ for event in self._retrieval_events:
+ bytes_used += sys.getsizeof(event)
+
+ # Add eviction heap memory
+ bytes_used += sys.getsizeof(self._eviction_heap)
+
+ return ComponentStats(
+ name="compression_store",
+ entry_count=self._backend.count(),
+ size_bytes=bytes_used,
+ budget_bytes=None, # No budget set yet
+ hits=sum(1 for _, e in self._backend.items() if e.retrieval_count > 0),
+ misses=0, # CompressionStore doesn't track misses directly
+ evictions=0, # Would need to track this separately
+ )
+
def get_retrieval_events(
self,
limit: int = 100,
diff --git a/headroom/ccr/batch_store.py b/headroom/ccr/batch_store.py
index f2ff3c73e..895fdd4ec 100644
--- a/headroom/ccr/batch_store.py
+++ b/headroom/ccr/batch_store.py
@@ -15,7 +15,10 @@ import asyncio
import logging
import time
from dataclasses import dataclass, field
-from typing import Any
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+ from ..memory.tracker import ComponentStats
logger = logging.getLogger(__name__)
@@ -234,6 +237,51 @@ class BatchContextStore:
counts[ctx.provider] = counts.get(ctx.provider, 0) + 1
return counts
+ def get_memory_stats(self) -> ComponentStats:
+ """Get memory statistics for the MemoryTracker.
+
+ Returns:
+ ComponentStats with current memory usage.
+ """
+ import sys
+
+ from ..memory.tracker import ComponentStats
+
+ # Calculate size
+ size_bytes = sys.getsizeof(self._contexts)
+
+ for batch_id, ctx in self._contexts.items():
+ size_bytes += len(batch_id)
+ size_bytes += sys.getsizeof(ctx)
+
+ # Add request contexts
+ for req_id, req in ctx.requests.items():
+ size_bytes += len(req_id)
+ size_bytes += sys.getsizeof(req)
+ # Messages can be large
+ size_bytes += sys.getsizeof(req.messages)
+ for msg in req.messages:
+ size_bytes += sys.getsizeof(msg)
+ for _k, v in msg.items():
+ if isinstance(v, str):
+ size_bytes += len(v)
+ elif isinstance(v, list):
+ size_bytes += sys.getsizeof(v)
+
+ # Tools
+ if req.tools:
+ size_bytes += sys.getsizeof(req.tools)
+
+ return ComponentStats(
+ name="batch_context_store",
+ entry_count=len(self._contexts),
+ size_bytes=size_bytes,
+ budget_bytes=None,
+ hits=0,
+ misses=0,
+ evictions=0,
+ )
+
# Global store instance
_batch_context_store: BatchContextStore | None = None
diff --git a/headroom/memory/adapters/graph.py b/headroom/memory/adapters/graph.py
index 6f9bc6ea3..cd7476e47 100644
--- a/headroom/memory/adapters/graph.py
+++ b/headroom/memory/adapters/graph.py
@@ -13,7 +13,7 @@ from typing import TYPE_CHECKING
from .graph_models import Entity, Relationship, RelationshipDirection, Subgraph
if TYPE_CHECKING:
- pass
+ from ..tracker import ComponentStats
class InMemoryGraphStore:
@@ -572,3 +572,63 @@ class InMemoryGraphStore:
"source_index_size": len(self._relationships_by_source),
"target_index_size": len(self._relationships_by_target),
}
+
+ def get_memory_stats(self) -> ComponentStats:
+ """Get memory statistics for the MemoryTracker.
+
+ Returns:
+ ComponentStats with current memory usage.
+ """
+ import sys
+
+ from ..tracker import ComponentStats
+
+ with self._lock:
+ # Calculate size of all data structures
+ size_bytes = 0
+
+ # Entities
+ size_bytes += sys.getsizeof(self._entities)
+ for entity_id, entity in self._entities.items():
+ size_bytes += len(entity_id)
+ size_bytes += sys.getsizeof(entity)
+ size_bytes += len(entity.id) + len(entity.user_id) + len(entity.name)
+ size_bytes += len(entity.entity_type)
+ if entity.properties:
+ size_bytes += sys.getsizeof(entity.properties)
+
+ # Relationships
+ size_bytes += sys.getsizeof(self._relationships)
+ for rel_id, rel in self._relationships.items():
+ size_bytes += len(rel_id)
+ size_bytes += sys.getsizeof(rel)
+ size_bytes += len(rel.id) + len(rel.source_id) + len(rel.target_id)
+ size_bytes += len(rel.relation_type)
+ if rel.properties:
+ size_bytes += sys.getsizeof(rel.properties)
+
+ # Indexes
+ size_bytes += sys.getsizeof(self._entities_by_user)
+ for user_id, entity_ids in self._entities_by_user.items():
+ size_bytes += len(user_id)
+ size_bytes += sys.getsizeof(entity_ids)
+
+ size_bytes += sys.getsizeof(self._entities_by_name)
+ for user_id, name_map in self._entities_by_name.items():
+ size_bytes += len(user_id)
+ size_bytes += sys.getsizeof(name_map)
+
+ size_bytes += sys.getsizeof(self._relationships_by_source)
+ size_bytes += sys.getsizeof(self._relationships_by_target)
+
+ entry_count = len(self._entities) + len(self._relationships)
+
+ return ComponentStats(
+ name="graph_store",
+ entry_count=entry_count,
+ size_bytes=size_bytes,
+ budget_bytes=None,
+ hits=0,
+ misses=0,
+ evictions=0,
+ )
diff --git a/headroom/memory/adapters/hnsw.py b/headroom/memory/adapters/hnsw.py
index c82be8904..e7787edbe 100644
--- a/headroom/memory/adapters/hnsw.py
+++ b/headroom/memory/adapters/hnsw.py
@@ -91,7 +91,7 @@ def _check_hnswlib_available() -> bool:
if TYPE_CHECKING:
- pass
+ from ..tracker import ComponentStats
@dataclass
@@ -854,6 +854,64 @@ class HNSWVectorIndex:
),
}
+ def get_memory_stats(self) -> ComponentStats:
+ """Get memory statistics for the MemoryTracker.
+
+ Returns:
+ ComponentStats with current memory usage.
+ """
+ import sys
+
+ from ..tracker import ComponentStats
+
+ with self._lock:
+ size_bytes = 0
+
+ # ID mappings
+ size_bytes += sys.getsizeof(self._memory_to_hnsw)
+ for mem_id, hnsw_id in self._memory_to_hnsw.items():
+ size_bytes += len(mem_id) + sys.getsizeof(hnsw_id)
+
+ size_bytes += sys.getsizeof(self._hnsw_to_memory)
+ for hnsw_id, mem_id in self._hnsw_to_memory.items():
+ size_bytes += sys.getsizeof(hnsw_id) + len(mem_id)
+
+ # Metadata storage
+ size_bytes += sys.getsizeof(self._metadata)
+ for mem_id, meta in self._metadata.items():
+ size_bytes += len(mem_id)
+ size_bytes += sys.getsizeof(meta)
+ # Estimate metadata fields
+ if meta.content:
+ size_bytes += len(meta.content)
+ if meta.entity_refs:
+ size_bytes += sys.getsizeof(meta.entity_refs)
+ if meta.metadata:
+ size_bytes += sys.getsizeof(meta.metadata)
+
+ # Embeddings storage (numpy arrays)
+ size_bytes += sys.getsizeof(self._embeddings)
+ for mem_id, embedding in self._embeddings.items():
+ size_bytes += len(mem_id)
+ # numpy array size: dtype size * number of elements
+ size_bytes += embedding.nbytes
+
+ # HNSW index size estimate
+ # The actual index is in hnswlib C++ memory, so we estimate:
+ # Each element uses approximately: dimension * 4 bytes (float32) + M * 8 bytes (neighbors)
+ index_size_estimate = len(self._memory_to_hnsw) * (self._dimension * 4 + self._m * 8)
+ size_bytes += index_size_estimate
+
+ return ComponentStats(
+ name="vector_index",
+ entry_count=len(self._memory_to_hnsw),
+ size_bytes=size_bytes,
+ budget_bytes=None,
+ hits=0,
+ misses=0,
+ evictions=0,
+ )
+
def set_ef_search(self, ef_search: int) -> None:
"""Update the ef_search parameter for query time.
diff --git a/headroom/memory/tracker.py b/headroom/memory/tracker.py
new file mode 100644
index 000000000..a0ac01ca2
--- /dev/null
+++ b/headroom/memory/tracker.py
@@ -0,0 +1,388 @@
+"""Memory tracking infrastructure for headroom.
+
+This module provides centralized memory tracking across all components,
+enabling observability into memory usage patterns and budget enforcement.
+"""
+
+from __future__ import annotations
+
+import sys
+import threading
+import time
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+# Try to import psutil for process memory tracking
+try:
+ import psutil
+
+ PSUTIL_AVAILABLE = True
+except ImportError:
+ PSUTIL_AVAILABLE = False
+
+
+@dataclass
+class ComponentStats:
+ """Statistics for a single memory component."""
+
+ name: str
+ entry_count: int
+ size_bytes: int
+ budget_bytes: int | None = None
+ hits: int = 0
+ misses: int = 0
+ evictions: int = 0
+ last_updated: float = field(default_factory=time.time)
+
+ @property
+ def size_mb(self) -> float:
+ """Size in megabytes."""
+ return self.size_bytes / (1024 * 1024)
+
+ @property
+ def budget_mb(self) -> float | None:
+ """Budget in megabytes."""
+ return self.budget_bytes / (1024 * 1024) if self.budget_bytes else None
+
+ @property
+ def budget_used_percent(self) -> float | None:
+ """Percentage of budget used."""
+ if self.budget_bytes and self.budget_bytes > 0:
+ return (self.size_bytes / self.budget_bytes) * 100
+ return None
+
+ @property
+ def hit_rate(self) -> float | None:
+ """Cache hit rate as percentage."""
+ total = self.hits + self.misses
+ if total > 0:
+ return (self.hits / total) * 100
+ return None
+
+ def to_dict(self) -> dict[str, Any]:
+ """Convert to dictionary for JSON serialization."""
+ return {
+ "name": self.name,
+ "entry_count": self.entry_count,
+ "size_bytes": self.size_bytes,
+ "size_mb": round(self.size_mb, 2),
+ "budget_bytes": self.budget_bytes,
+ "budget_mb": round(self.budget_mb, 2) if self.budget_mb else None,
+ "budget_used_percent": round(self.budget_used_percent, 2)
+ if self.budget_used_percent
+ else None,
+ "hits": self.hits,
+ "misses": self.misses,
+ "evictions": self.evictions,
+ "hit_rate": round(self.hit_rate, 2) if self.hit_rate else None,
+ "last_updated": self.last_updated,
+ }
+
+
+@dataclass
+class ProcessStats:
+ """Process-level memory statistics."""
+
+ rss_bytes: int
+ vms_bytes: int
+ percent: float
+ available_bytes: int
+ total_bytes: int
+
+ @property
+ def rss_mb(self) -> float:
+ """Resident set size in MB."""
+ return self.rss_bytes / (1024 * 1024)
+
+ @property
+ def vms_mb(self) -> float:
+ """Virtual memory size in MB."""
+ return self.vms_bytes / (1024 * 1024)
+
+ @property
+ def available_mb(self) -> float:
+ """Available system memory in MB."""
+ return self.available_bytes / (1024 * 1024)
+
+ @property
+ def total_mb(self) -> float:
+ """Total system memory in MB."""
+ return self.total_bytes / (1024 * 1024)
+
+ def to_dict(self) -> dict[str, Any]:
+ """Convert to dictionary for JSON serialization."""
+ return {
+ "rss_bytes": self.rss_bytes,
+ "rss_mb": round(self.rss_mb, 2),
+ "vms_bytes": self.vms_bytes,
+ "vms_mb": round(self.vms_mb, 2),
+ "percent": round(self.percent, 2),
+ "available_mb": round(self.available_mb, 2),
+ "total_mb": round(self.total_mb, 2),
+ }
+
+
+@dataclass
+class MemoryReport:
+ """Complete memory report including process and component stats."""
+
+ process: ProcessStats
+ components: dict[str, ComponentStats]
+ total_tracked_bytes: int
+ target_budget_bytes: int | None
+ timestamp: float = field(default_factory=time.time)
+
+ @property
+ def total_tracked_mb(self) -> float:
+ """Total tracked memory in MB."""
+ return self.total_tracked_bytes / (1024 * 1024)
+
+ @property
+ def target_budget_mb(self) -> float | None:
+ """Target budget in MB."""
+ return self.target_budget_bytes / (1024 * 1024) if self.target_budget_bytes else None
+
+ @property
+ def is_over_budget(self) -> bool:
+ """Check if tracked memory exceeds target budget."""
+ if self.target_budget_bytes:
+ return self.total_tracked_bytes > self.target_budget_bytes
+ return False
+
+ def to_dict(self) -> dict[str, Any]:
+ """Convert to dictionary for JSON serialization."""
+ return {
+ "process": self.process.to_dict(),
+ "components": {name: stats.to_dict() for name, stats in self.components.items()},
+ "total_tracked_bytes": self.total_tracked_bytes,
+ "total_tracked_mb": round(self.total_tracked_mb, 2),
+ "target_budget_bytes": self.target_budget_bytes,
+ "target_budget_mb": round(self.target_budget_mb, 2) if self.target_budget_mb else None,
+ "is_over_budget": self.is_over_budget,
+ "timestamp": self.timestamp,
+ }
+
+
+class MemoryTracker:
+ """Singleton that tracks memory usage across all components.
+
+ Usage:
+ # Register a component
+ tracker = MemoryTracker.get()
+ tracker.register("my_store", my_store.get_memory_stats)
+
+ # Get all stats
+ report = tracker.get_report()
+
+ # Get specific component
+ stats = tracker.get_component_stats("my_store")
+ """
+
+ _instance: MemoryTracker | None = None
+ _lock: threading.Lock = threading.Lock()
+
+ def __init__(self, target_budget_mb: float | None = None):
+ """Initialize the tracker.
+
+ Args:
+ target_budget_mb: Target memory budget in MB for all tracked components.
+ """
+ self._components: dict[str, Callable[[], ComponentStats]] = {}
+ self._target_budget_bytes: int | None = (
+ int(target_budget_mb * 1024 * 1024) if target_budget_mb else None
+ )
+ self._component_lock = threading.Lock()
+
+ @classmethod
+ def get(cls, target_budget_mb: float | None = None) -> MemoryTracker:
+ """Get or create the singleton instance.
+
+ Args:
+ target_budget_mb: Target memory budget (only used on first call).
+
+ Returns:
+ The singleton MemoryTracker instance.
+ """
+ if cls._instance is None:
+ with cls._lock:
+ if cls._instance is None:
+ cls._instance = cls(target_budget_mb=target_budget_mb)
+ return cls._instance
+
+ @classmethod
+ def reset(cls) -> None:
+ """Reset the singleton instance. Useful for testing."""
+ with cls._lock:
+ cls._instance = None
+
+ def set_target_budget(self, budget_mb: float) -> None:
+ """Set the target memory budget.
+
+ Args:
+ budget_mb: Target budget in megabytes.
+ """
+ self._target_budget_bytes = int(budget_mb * 1024 * 1024)
+
+ def register(self, name: str, stats_fn: Callable[[], ComponentStats]) -> None:
+ """Register a component's stats function.
+
+ Args:
+ name: Unique name for the component.
+ stats_fn: Function that returns ComponentStats for this component.
+ """
+ with self._component_lock:
+ self._components[name] = stats_fn
+
+ def unregister(self, name: str) -> bool:
+ """Unregister a component.
+
+ Args:
+ name: Name of the component to unregister.
+
+ Returns:
+ True if component was unregistered, False if not found.
+ """
+ with self._component_lock:
+ if name in self._components:
+ del self._components[name]
+ return True
+ return False
+
+ def get_component_stats(self, name: str) -> ComponentStats | None:
+ """Get stats for a specific component.
+
+ Args:
+ name: Name of the component.
+
+ Returns:
+ ComponentStats or None if component not found.
+ """
+ with self._component_lock:
+ if name in self._components:
+ try:
+ return self._components[name]()
+ except Exception:
+ return None
+ return None
+
+ def get_all_component_stats(self) -> dict[str, ComponentStats]:
+ """Get stats for all registered components.
+
+ Returns:
+ Dictionary mapping component names to their stats.
+ """
+ stats: dict[str, ComponentStats] = {}
+ with self._component_lock:
+ for name, fn in self._components.items():
+ try:
+ stats[name] = fn()
+ except Exception:
+ # Skip components that fail to report stats
+ pass
+ return stats
+
+ def get_process_stats(self) -> ProcessStats:
+ """Get process-level memory statistics.
+
+ Returns:
+ ProcessStats with current memory usage.
+ """
+ if PSUTIL_AVAILABLE:
+ process = psutil.Process()
+ mem_info = process.memory_info()
+ sys_mem = psutil.virtual_memory()
+ return ProcessStats(
+ rss_bytes=mem_info.rss,
+ vms_bytes=mem_info.vms,
+ percent=process.memory_percent(),
+ available_bytes=sys_mem.available,
+ total_bytes=sys_mem.total,
+ )
+ else:
+ # Fallback when psutil not available
+ return ProcessStats(
+ rss_bytes=0,
+ vms_bytes=0,
+ percent=0.0,
+ available_bytes=0,
+ total_bytes=0,
+ )
+
+ def get_total_tracked_bytes(self) -> int:
+ """Get total memory used by all tracked components.
+
+ Returns:
+ Total bytes used by tracked components.
+ """
+ stats = self.get_all_component_stats()
+ return sum(s.size_bytes for s in stats.values())
+
+ def get_report(self) -> MemoryReport:
+ """Get a complete memory report.
+
+ Returns:
+ MemoryReport with process and component statistics.
+ """
+ process_stats = self.get_process_stats()
+ component_stats = self.get_all_component_stats()
+ total_tracked = sum(s.size_bytes for s in component_stats.values())
+
+ return MemoryReport(
+ process=process_stats,
+ components=component_stats,
+ total_tracked_bytes=total_tracked,
+ target_budget_bytes=self._target_budget_bytes,
+ )
+
+ @property
+ def registered_components(self) -> list[str]:
+ """Get list of registered component names."""
+ with self._component_lock:
+ return list(self._components.keys())
+
+ @property
+ def target_budget_mb(self) -> float | None:
+ """Get target budget in MB."""
+ return self._target_budget_bytes / (1024 * 1024) if self._target_budget_bytes else None
+
+
+def estimate_object_size(obj: Any, seen: set | None = None) -> int:
+ """Estimate the memory size of a Python object recursively.
+
+ This provides a rough estimate by traversing the object graph.
+ For more accurate measurements, use tracemalloc or memory_profiler.
+
+ Args:
+ obj: Object to measure.
+ seen: Set of already-seen object ids (for cycle detection).
+
+ Returns:
+ Estimated size in bytes.
+ """
+ if seen is None:
+ seen = set()
+
+ obj_id = id(obj)
+ if obj_id in seen:
+ return 0
+ seen.add(obj_id)
+
+ size = sys.getsizeof(obj)
+
+ if isinstance(obj, dict):
+ size += sum(
+ estimate_object_size(k, seen) + estimate_object_size(v, seen) for k, v in obj.items()
+ )
+ elif isinstance(obj, (list, tuple, set, frozenset)):
+ size += sum(estimate_object_size(item, seen) for item in obj)
+ elif hasattr(obj, "__dict__"):
+ size += estimate_object_size(obj.__dict__, seen)
+ elif hasattr(obj, "__slots__"):
+ size += sum(
+ estimate_object_size(getattr(obj, slot, None), seen)
+ for slot in obj.__slots__
+ if hasattr(obj, slot)
+ )
+
+ return size
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 3b48bcc4d..786d27fb5 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -36,7 +36,10 @@ from collections import OrderedDict, defaultdict, deque
from dataclasses import asdict, dataclass
from datetime import datetime, timedelta
from pathlib import Path
-from typing import Any, Literal
+from typing import TYPE_CHECKING, Any, Literal
+
+if TYPE_CHECKING:
+ from ..memory.tracker import ComponentStats, MemoryTracker
import httpx
@@ -402,6 +405,37 @@ class SemanticCache:
async with self._lock:
self._cache.clear()
+ def get_memory_stats(self) -> ComponentStats:
+ """Get memory statistics for the MemoryTracker.
+
+ Returns:
+ ComponentStats with current memory usage.
+ """
+ from ..memory.tracker import ComponentStats
+
+ # Calculate size - this is sync but we access _cache directly
+ # Note: This is a rough estimate, not perfectly accurate under async load
+ size_bytes = sys.getsizeof(self._cache)
+ total_hits = 0
+
+ for entry in self._cache.values():
+ size_bytes += sys.getsizeof(entry)
+ size_bytes += len(entry.response_body)
+ size_bytes += sys.getsizeof(entry.response_headers)
+ for k, v in entry.response_headers.items():
+ size_bytes += len(k) + len(v)
+ total_hits += entry.hit_count
+
+ return ComponentStats(
+ name="semantic_cache",
+ entry_count=len(self._cache),
+ size_bytes=size_bytes,
+ budget_bytes=None,
+ hits=total_hits,
+ misses=0, # Would need to track this separately
+ evictions=0, # Would need to track this separately
+ )
+
# =============================================================================
# Rate Limiting
@@ -865,6 +899,44 @@ class RequestLogger:
"log_file": str(self.log_file) if self.log_file else None,
}
+ def get_memory_stats(self) -> ComponentStats:
+ """Get memory statistics for the MemoryTracker.
+
+ Returns:
+ ComponentStats with current memory usage.
+ """
+ from ..memory.tracker import ComponentStats
+
+ # Calculate size
+ size_bytes = sys.getsizeof(self._logs)
+
+ for log_entry in self._logs:
+ size_bytes += sys.getsizeof(log_entry)
+ # Add string fields
+ if log_entry.request_id:
+ size_bytes += len(log_entry.request_id)
+ if log_entry.provider:
+ size_bytes += len(log_entry.provider)
+ if log_entry.model:
+ size_bytes += len(log_entry.model)
+ if log_entry.error:
+ size_bytes += len(log_entry.error)
+ # Messages and response can be large
+ if log_entry.request_messages:
+ size_bytes += sys.getsizeof(log_entry.request_messages)
+ if log_entry.response_content:
+ size_bytes += len(log_entry.response_content)
+
+ return ComponentStats(
+ name="request_logger",
+ entry_count=len(self._logs),
+ size_bytes=size_bytes,
+ budget_bytes=None,
+ hits=0,
+ misses=0,
+ evictions=0,
+ )
+
# =============================================================================
# Main Proxy
@@ -5333,6 +5405,44 @@ async def _log_toin_stats_periodically(interval_seconds: int = 300) -> None:
logger.debug("Failed to log TOIN stats: %s", e)
+def _register_memory_components(proxy: HeadroomProxy, tracker: MemoryTracker) -> None:
+ """Register all memory-tracked components with the tracker.
+
+ This function is idempotent - it checks if components are already registered.
+
+ Args:
+ proxy: The HeadroomProxy instance.
+ tracker: The MemoryTracker instance.
+ """
+ # Register compression store (global singleton)
+ if "compression_store" not in tracker.registered_components:
+ store = get_compression_store()
+ tracker.register("compression_store", store.get_memory_stats)
+
+ # Register semantic cache (instance on proxy)
+ if proxy.cache and "semantic_cache" not in tracker.registered_components:
+ tracker.register("semantic_cache", proxy.cache.get_memory_stats)
+
+ # Register request logger (instance on proxy)
+ if proxy.logger and "request_logger" not in tracker.registered_components:
+ tracker.register("request_logger", proxy.logger.get_memory_stats)
+
+ # Register batch context store (global singleton)
+ if "batch_context_store" not in tracker.registered_components:
+ try:
+ from ..ccr.batch_store import get_batch_context_store
+
+ batch_store = get_batch_context_store()
+ if hasattr(batch_store, "get_memory_stats"):
+ tracker.register("batch_context_store", batch_store.get_memory_stats)
+ except ImportError:
+ pass
+
+ # Note: graph_store and vector_index are created per-user within the
+ # LocalMemoryBackend, not as global singletons. They would need to be
+ # registered when the memory system is initialized with specific backends.
+
+
def create_app(config: ProxyConfig | None = None) -> FastAPI:
"""Create FastAPI application."""
if not FASTAPI_AVAILABLE:
@@ -5508,6 +5618,30 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
media_type="text/plain; version=0.0.4",
)
+ # Debug endpoints
+ @app.get("/debug/memory")
+ async def debug_memory():
+ """Get detailed memory usage statistics.
+
+ Returns memory usage for all tracked components including:
+ - Process-level memory (RSS, VMS, percent)
+ - Per-component memory usage and budgets
+ - Cache hit/miss statistics
+ - Total tracked vs target budget
+
+ This endpoint is useful for debugging memory issues and
+ monitoring memory budgets.
+ """
+ from ..memory.tracker import MemoryTracker
+
+ tracker = MemoryTracker.get()
+
+ # Register components if not already registered
+ _register_memory_components(proxy, tracker)
+
+ report = tracker.get_report()
+ return report.to_dict()
+
@app.post("/cache/clear")
async def clear_cache():
"""Clear the response cache."""
diff --git a/tests/test_memory_tracker.py b/tests/test_memory_tracker.py
new file mode 100644
index 000000000..7eea8cccc
--- /dev/null
+++ b/tests/test_memory_tracker.py
@@ -0,0 +1,399 @@
+"""Tests for memory tracking functionality.
+
+These tests verify that the MemoryTracker correctly tracks memory usage
+across all components without mocks or simulations.
+"""
+
+from __future__ import annotations
+
+import sys
+
+import pytest
+
+from headroom.memory.tracker import (
+ ComponentStats,
+ MemoryReport,
+ MemoryTracker,
+ ProcessStats,
+ estimate_object_size,
+)
+
+
+class TestComponentStats:
+ """Tests for ComponentStats dataclass."""
+
+ def test_basic_properties(self):
+ """Test basic property calculations."""
+ stats = ComponentStats(
+ name="test_store",
+ entry_count=100,
+ size_bytes=1024 * 1024, # 1 MB
+ budget_bytes=2 * 1024 * 1024, # 2 MB
+ hits=80,
+ misses=20,
+ evictions=5,
+ )
+
+ assert stats.name == "test_store"
+ assert stats.entry_count == 100
+ assert stats.size_mb == 1.0
+ assert stats.budget_mb == 2.0
+ assert stats.budget_used_percent == 50.0
+ assert stats.hit_rate == 80.0
+
+ def test_no_budget(self):
+ """Test when no budget is set."""
+ stats = ComponentStats(
+ name="test_store",
+ entry_count=100,
+ size_bytes=1024 * 1024,
+ budget_bytes=None,
+ )
+
+ assert stats.budget_mb is None
+ assert stats.budget_used_percent is None
+
+ def test_no_hits_misses(self):
+ """Test when no hits or misses recorded."""
+ stats = ComponentStats(
+ name="test_store",
+ entry_count=100,
+ size_bytes=1024,
+ hits=0,
+ misses=0,
+ )
+
+ assert stats.hit_rate is None
+
+ def test_to_dict(self):
+ """Test serialization to dictionary."""
+ stats = ComponentStats(
+ name="test_store",
+ entry_count=100,
+ size_bytes=1024 * 1024,
+ budget_bytes=2 * 1024 * 1024,
+ hits=80,
+ misses=20,
+ evictions=5,
+ )
+
+ d = stats.to_dict()
+
+ assert d["name"] == "test_store"
+ assert d["entry_count"] == 100
+ assert d["size_bytes"] == 1024 * 1024
+ assert d["size_mb"] == 1.0
+ assert d["budget_mb"] == 2.0
+ assert d["budget_used_percent"] == 50.0
+ assert d["hit_rate"] == 80.0
+
+
+class TestProcessStats:
+ """Tests for ProcessStats dataclass."""
+
+ def test_basic_properties(self):
+ """Test basic property calculations."""
+ stats = ProcessStats(
+ rss_bytes=500 * 1024 * 1024, # 500 MB
+ vms_bytes=1024 * 1024 * 1024, # 1 GB
+ percent=5.0,
+ available_bytes=8 * 1024 * 1024 * 1024, # 8 GB
+ total_bytes=16 * 1024 * 1024 * 1024, # 16 GB
+ )
+
+ assert stats.rss_mb == 500.0
+ assert stats.vms_mb == 1024.0
+ assert stats.percent == 5.0
+ assert stats.available_mb == 8192.0
+ assert stats.total_mb == 16384.0
+
+ def test_to_dict(self):
+ """Test serialization to dictionary."""
+ stats = ProcessStats(
+ rss_bytes=500 * 1024 * 1024,
+ vms_bytes=1024 * 1024 * 1024,
+ percent=5.0,
+ available_bytes=8 * 1024 * 1024 * 1024,
+ total_bytes=16 * 1024 * 1024 * 1024,
+ )
+
+ d = stats.to_dict()
+
+ assert d["rss_mb"] == 500.0
+ assert d["vms_mb"] == 1024.0
+ assert d["percent"] == 5.0
+
+
+class TestMemoryTracker:
+ """Tests for MemoryTracker singleton."""
+
+ @pytest.fixture(autouse=True)
+ def reset_tracker(self):
+ """Reset the tracker singleton before each test."""
+ MemoryTracker.reset()
+ yield
+ MemoryTracker.reset()
+
+ def test_singleton_pattern(self):
+ """Test that MemoryTracker is a singleton."""
+ tracker1 = MemoryTracker.get()
+ tracker2 = MemoryTracker.get()
+
+ assert tracker1 is tracker2
+
+ def test_reset(self):
+ """Test singleton reset."""
+ tracker1 = MemoryTracker.get()
+ MemoryTracker.reset()
+ tracker2 = MemoryTracker.get()
+
+ assert tracker1 is not tracker2
+
+ def test_register_component(self):
+ """Test registering a component."""
+ tracker = MemoryTracker.get()
+
+ def get_stats() -> ComponentStats:
+ return ComponentStats(
+ name="test_component",
+ entry_count=10,
+ size_bytes=1024,
+ )
+
+ tracker.register("test_component", get_stats)
+
+ assert "test_component" in tracker.registered_components
+
+ def test_unregister_component(self):
+ """Test unregistering a component."""
+ tracker = MemoryTracker.get()
+
+ def get_stats() -> ComponentStats:
+ return ComponentStats(
+ name="test_component",
+ entry_count=10,
+ size_bytes=1024,
+ )
+
+ tracker.register("test_component", get_stats)
+ assert "test_component" in tracker.registered_components
+
+ result = tracker.unregister("test_component")
+ assert result is True
+ assert "test_component" not in tracker.registered_components
+
+ # Unregistering non-existent component returns False
+ result = tracker.unregister("non_existent")
+ assert result is False
+
+ def test_get_component_stats(self):
+ """Test getting stats for a specific component."""
+ tracker = MemoryTracker.get()
+
+ def get_stats() -> ComponentStats:
+ return ComponentStats(
+ name="test_component",
+ entry_count=42,
+ size_bytes=2048,
+ )
+
+ tracker.register("test_component", get_stats)
+
+ stats = tracker.get_component_stats("test_component")
+
+ assert stats is not None
+ assert stats.name == "test_component"
+ assert stats.entry_count == 42
+ assert stats.size_bytes == 2048
+
+ def test_get_component_stats_not_found(self):
+ """Test getting stats for non-existent component."""
+ tracker = MemoryTracker.get()
+
+ stats = tracker.get_component_stats("non_existent")
+
+ assert stats is None
+
+ def test_get_all_component_stats(self):
+ """Test getting stats for all components."""
+ tracker = MemoryTracker.get()
+
+ def get_stats_a() -> ComponentStats:
+ return ComponentStats(name="component_a", entry_count=10, size_bytes=1024)
+
+ def get_stats_b() -> ComponentStats:
+ return ComponentStats(name="component_b", entry_count=20, size_bytes=2048)
+
+ tracker.register("component_a", get_stats_a)
+ tracker.register("component_b", get_stats_b)
+
+ all_stats = tracker.get_all_component_stats()
+
+ assert len(all_stats) == 2
+ assert "component_a" in all_stats
+ assert "component_b" in all_stats
+ assert all_stats["component_a"].entry_count == 10
+ assert all_stats["component_b"].entry_count == 20
+
+ def test_get_process_stats(self):
+ """Test getting process-level stats."""
+ tracker = MemoryTracker.get()
+
+ stats = tracker.get_process_stats()
+
+ # Should return ProcessStats (may be zero if psutil not available)
+ assert isinstance(stats, ProcessStats)
+ assert stats.rss_bytes >= 0
+ assert stats.vms_bytes >= 0
+
+ def test_get_total_tracked_bytes(self):
+ """Test getting total tracked bytes."""
+ tracker = MemoryTracker.get()
+
+ def get_stats_a() -> ComponentStats:
+ return ComponentStats(name="a", entry_count=10, size_bytes=1000)
+
+ def get_stats_b() -> ComponentStats:
+ return ComponentStats(name="b", entry_count=20, size_bytes=2000)
+
+ tracker.register("a", get_stats_a)
+ tracker.register("b", get_stats_b)
+
+ total = tracker.get_total_tracked_bytes()
+
+ assert total == 3000
+
+ def test_get_report(self):
+ """Test getting full memory report."""
+ tracker = MemoryTracker.get(target_budget_mb=100.0)
+
+ def get_stats() -> ComponentStats:
+ return ComponentStats(name="test", entry_count=10, size_bytes=50 * 1024 * 1024)
+
+ tracker.register("test", get_stats)
+
+ report = tracker.get_report()
+
+ assert isinstance(report, MemoryReport)
+ assert isinstance(report.process, ProcessStats)
+ assert "test" in report.components
+ assert report.total_tracked_bytes == 50 * 1024 * 1024
+ assert report.target_budget_bytes == 100 * 1024 * 1024
+ assert report.is_over_budget is False
+
+ def test_is_over_budget(self):
+ """Test budget checking."""
+ tracker = MemoryTracker.get(target_budget_mb=10.0) # 10 MB budget
+
+ def get_stats() -> ComponentStats:
+ return ComponentStats(
+ name="large", entry_count=10, size_bytes=20 * 1024 * 1024
+ ) # 20 MB
+
+ tracker.register("large", get_stats)
+
+ report = tracker.get_report()
+
+ assert report.is_over_budget is True
+
+ def test_set_target_budget(self):
+ """Test setting target budget after creation."""
+ tracker = MemoryTracker.get()
+
+ assert tracker.target_budget_mb is None
+
+ tracker.set_target_budget(500.0)
+
+ assert tracker.target_budget_mb == 500.0
+
+
+class TestEstimateObjectSize:
+ """Tests for the estimate_object_size utility function."""
+
+ def test_simple_types(self):
+ """Test size estimation for simple types."""
+ # Integer
+ int_size = estimate_object_size(42)
+ assert int_size == sys.getsizeof(42)
+
+ # String
+ s = "hello world"
+ str_size = estimate_object_size(s)
+ assert str_size == sys.getsizeof(s)
+
+ def test_dict(self):
+ """Test size estimation for dictionaries."""
+ d = {"a": 1, "b": 2, "c": 3}
+ size = estimate_object_size(d)
+
+ # Size should be at least the base dict size
+ assert size >= sys.getsizeof(d)
+ # Size should include keys and values
+ assert size > sys.getsizeof({})
+
+ def test_list(self):
+ """Test size estimation for lists."""
+ lst = [1, 2, 3, "hello", "world"]
+ size = estimate_object_size(lst)
+
+ assert size >= sys.getsizeof(lst)
+ assert size > sys.getsizeof([])
+
+ def test_nested_structure(self):
+ """Test size estimation for nested structures."""
+ nested = {
+ "items": [{"id": 1, "name": "first"}, {"id": 2, "name": "second"}],
+ "metadata": {"count": 2, "tags": ["a", "b", "c"]},
+ }
+ size = estimate_object_size(nested)
+
+ # Should be larger than just the outer dict
+ assert size > sys.getsizeof(nested)
+
+ def test_circular_reference(self):
+ """Test that circular references don't cause infinite loop."""
+ d: dict = {"a": 1}
+ d["self"] = d # Circular reference
+
+ # Should not hang or crash
+ size = estimate_object_size(d)
+ assert size > 0
+
+
+class TestMemoryReportSerialization:
+ """Tests for MemoryReport serialization."""
+
+ def test_to_dict(self):
+ """Test that MemoryReport serializes correctly."""
+ process = ProcessStats(
+ rss_bytes=100 * 1024 * 1024,
+ vms_bytes=200 * 1024 * 1024,
+ percent=1.0,
+ available_bytes=8 * 1024 * 1024 * 1024,
+ total_bytes=16 * 1024 * 1024 * 1024,
+ )
+
+ components = {
+ "store_a": ComponentStats(name="store_a", entry_count=100, size_bytes=10 * 1024 * 1024),
+ "store_b": ComponentStats(name="store_b", entry_count=200, size_bytes=20 * 1024 * 1024),
+ }
+
+ report = MemoryReport(
+ process=process,
+ components=components,
+ total_tracked_bytes=30 * 1024 * 1024,
+ target_budget_bytes=50 * 1024 * 1024,
+ )
+
+ d = report.to_dict()
+
+ assert "process" in d
+ assert "components" in d
+ assert "total_tracked_mb" in d
+ assert "target_budget_mb" in d
+ assert "is_over_budget" in d
+
+ assert d["process"]["rss_mb"] == 100.0
+ assert d["total_tracked_mb"] == 30.0
+ assert d["target_budget_mb"] == 50.0
+ assert d["is_over_budget"] is False
diff --git a/tests/test_memory_tracker_integration.py b/tests/test_memory_tracker_integration.py
new file mode 100644
index 000000000..fbaa4cf88
--- /dev/null
+++ b/tests/test_memory_tracker_integration.py
@@ -0,0 +1,451 @@
+"""Integration tests for memory tracking with real stores.
+
+These tests verify that memory tracking works correctly with actual
+store implementations - no mocks, no simulations.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from headroom.memory.tracker import MemoryTracker
+
+
+class TestCompressionStoreMemoryTracking:
+ """Tests for CompressionStore memory tracking integration."""
+
+ @pytest.fixture(autouse=True)
+ def reset_tracker(self):
+ """Reset the tracker singleton before each test."""
+ MemoryTracker.reset()
+ yield
+ MemoryTracker.reset()
+
+ def test_compression_store_reports_memory_stats(self):
+ """Test that CompressionStore correctly reports memory stats."""
+ from headroom.cache.compression_store import CompressionStore
+
+ store = CompressionStore(max_entries=100)
+
+ # Add some data - store(original, compressed, ...)
+ store.store("original content 1" * 100, "compressed1")
+ store.store("original content 2" * 100, "compressed2")
+
+ stats = store.get_memory_stats()
+
+ assert stats.name == "compression_store"
+ assert stats.entry_count == 2
+ assert stats.size_bytes > 0
+ # budget_bytes is None since CompressionStore uses entry count limit not byte limit
+
+ def test_compression_store_tracks_hits(self):
+ """Test that CompressionStore tracks cache hits."""
+ from headroom.cache.compression_store import CompressionStore
+
+ store = CompressionStore(max_entries=100)
+
+ # Store and retrieve (hit)
+ hash_key = store.store("original content", "compressed")
+ store.retrieve(hash_key) # Hit - increments retrieval_count
+ store.retrieve(hash_key) # Another retrieval
+ store.retrieve("nonexistent_hash") # Miss (not tracked)
+
+ stats = store.get_memory_stats()
+
+ # Hits counts entries with retrieval_count > 0, not total retrievals
+ assert stats.hits == 1 # 1 entry has been retrieved
+ # CompressionStore doesn't track misses
+ assert stats.misses == 0
+
+ def test_compression_store_registers_with_tracker(self):
+ """Test that CompressionStore can register with MemoryTracker."""
+ from headroom.cache.compression_store import CompressionStore
+
+ tracker = MemoryTracker.get()
+ store = CompressionStore(max_entries=100)
+
+ # Register the store
+ tracker.register("compression_store", store.get_memory_stats)
+
+ # Verify it's registered
+ assert "compression_store" in tracker.registered_components
+
+ # Get stats through tracker
+ stats = tracker.get_component_stats("compression_store")
+ assert stats is not None
+ assert stats.name == "compression_store"
+
+
+class TestBatchContextStoreMemoryTracking:
+ """Tests for BatchContextStore memory tracking integration."""
+
+ @pytest.fixture(autouse=True)
+ def reset_tracker(self):
+ """Reset the tracker singleton before each test."""
+ MemoryTracker.reset()
+ yield
+ MemoryTracker.reset()
+
+ def test_batch_context_store_reports_memory_stats(self):
+ """Test that BatchContextStore correctly reports memory stats."""
+ from headroom.ccr.batch_store import (
+ BatchContext,
+ BatchContextStore,
+ BatchRequestContext,
+ )
+
+ store = BatchContextStore(ttl=3600, max_contexts=100)
+
+ # Add some batch contexts
+ ctx1 = BatchContext(batch_id="batch_1", provider="anthropic")
+ ctx1.add_request(
+ BatchRequestContext(
+ custom_id="req_1",
+ messages=[{"role": "user", "content": "Hello world"}],
+ model="claude-3-opus",
+ )
+ )
+
+ ctx2 = BatchContext(batch_id="batch_2", provider="openai")
+ ctx2.add_request(
+ BatchRequestContext(
+ custom_id="req_2",
+ messages=[{"role": "user", "content": "Test message"}],
+ model="gpt-4",
+ )
+ )
+
+ # Store them (sync for testing - accessing internal dict)
+ store._contexts["batch_1"] = ctx1
+ store._contexts["batch_2"] = ctx2
+
+ stats = store.get_memory_stats()
+
+ assert stats.name == "batch_context_store"
+ assert stats.entry_count == 2
+ assert stats.size_bytes > 0
+
+ def test_batch_context_store_registers_with_tracker(self):
+ """Test that BatchContextStore can register with MemoryTracker."""
+ from headroom.ccr.batch_store import BatchContextStore
+
+ tracker = MemoryTracker.get()
+ store = BatchContextStore()
+
+ # Register the store
+ tracker.register("batch_context_store", store.get_memory_stats)
+
+ # Verify it's registered
+ assert "batch_context_store" in tracker.registered_components
+
+
+class TestGraphStoreMemoryTracking:
+ """Tests for InMemoryGraphStore memory tracking integration."""
+
+ @pytest.fixture(autouse=True)
+ def reset_tracker(self):
+ """Reset the tracker singleton before each test."""
+ MemoryTracker.reset()
+ yield
+ MemoryTracker.reset()
+
+ @pytest.mark.asyncio
+ async def test_graph_store_reports_memory_stats(self):
+ """Test that InMemoryGraphStore correctly reports memory stats."""
+ from headroom.memory.adapters.graph import InMemoryGraphStore
+ from headroom.memory.adapters.graph_models import Entity, Relationship
+
+ store = InMemoryGraphStore()
+
+ # Add some entities using the correct API
+ entity1 = Entity(id="node1", user_id="test", name="Test Entity 1", entity_type="entity")
+ entity2 = Entity(id="node2", user_id="test", name="Test Entity 2", entity_type="entity")
+ entity3 = Entity(id="node3", user_id="test", name="Test Concept", entity_type="concept")
+
+ await store.add_entity(entity1)
+ await store.add_entity(entity2)
+ await store.add_entity(entity3)
+
+ # Add a relationship
+ rel = Relationship(
+ source_id="node1",
+ target_id="node2",
+ relation_type="related_to",
+ user_id="test",
+ )
+ await store.add_relationship(rel)
+
+ stats = store.get_memory_stats()
+
+ assert stats.name == "graph_store"
+ assert stats.entry_count == 4 # 3 entities + 1 relationship
+ assert stats.size_bytes > 0
+
+ @pytest.mark.asyncio
+ async def test_graph_store_size_grows_with_data(self):
+ """Test that reported size grows as data is added."""
+ from headroom.memory.adapters.graph import InMemoryGraphStore
+ from headroom.memory.adapters.graph_models import Entity
+
+ store = InMemoryGraphStore()
+
+ # Get initial size
+ initial_stats = store.get_memory_stats()
+ initial_size = initial_stats.size_bytes
+
+ # Add data
+ for i in range(100):
+ entity = Entity(
+ id=f"node_{i}",
+ user_id="test",
+ name=f"Entity {i}",
+ entity_type="entity",
+ properties={"data": "x" * 100},
+ )
+ await store.add_entity(entity)
+
+ # Get new size
+ final_stats = store.get_memory_stats()
+
+ assert final_stats.size_bytes > initial_size
+ assert final_stats.entry_count == 100
+
+
+class TestHNSWVectorIndexMemoryTracking:
+ """Tests for HNSWVectorIndex memory tracking integration."""
+
+ @pytest.fixture(autouse=True)
+ def reset_tracker(self):
+ """Reset the tracker singleton before each test."""
+ MemoryTracker.reset()
+ yield
+ MemoryTracker.reset()
+
+ @pytest.mark.asyncio
+ async def test_hnsw_index_reports_memory_stats(self):
+ """Test that HNSWVectorIndex correctly reports memory stats."""
+ from headroom.memory.adapters.hnsw import HNSWVectorIndex
+ from headroom.memory.models import Memory
+
+ index = HNSWVectorIndex(dimension=128)
+
+ # Add some vectors using Memory objects
+ import numpy as np
+
+ for i in range(10):
+ embedding = np.random.rand(128).astype(np.float32).tolist()
+ memory = Memory(
+ id=f"mem_{i}",
+ content=f"Test memory {i}",
+ user_id="test_user",
+ embedding=embedding,
+ )
+ await index.index(memory)
+
+ stats = index.get_memory_stats()
+
+ assert stats.name == "vector_index"
+ assert stats.entry_count == 10
+ assert stats.size_bytes > 0
+
+ @pytest.mark.asyncio
+ async def test_hnsw_index_size_grows_with_vectors(self):
+ """Test that reported size grows as vectors are added."""
+ from headroom.memory.adapters.hnsw import HNSWVectorIndex
+ from headroom.memory.models import Memory
+
+ index = HNSWVectorIndex(dimension=256)
+
+ # Get initial size
+ initial_stats = index.get_memory_stats()
+ initial_size = initial_stats.size_bytes
+
+ # Add vectors
+ import numpy as np
+
+ for i in range(100):
+ embedding = np.random.rand(256).astype(np.float32).tolist()
+ memory = Memory(
+ id=f"mem_{i}",
+ content=f"Test memory {i}",
+ user_id="test_user",
+ embedding=embedding,
+ )
+ await index.index(memory)
+
+ # Get new size
+ final_stats = index.get_memory_stats()
+
+ assert final_stats.size_bytes > initial_size
+ assert final_stats.entry_count == 100
+
+
+class TestTrackerIntegrationWithMultipleStores:
+ """Tests for MemoryTracker with multiple real stores."""
+
+ @pytest.fixture(autouse=True)
+ def reset_tracker(self):
+ """Reset the tracker singleton before each test."""
+ MemoryTracker.reset()
+ yield
+ MemoryTracker.reset()
+
+ @pytest.mark.asyncio
+ async def test_tracker_aggregates_multiple_stores(self):
+ """Test that tracker correctly aggregates stats from multiple stores."""
+ from headroom.cache.compression_store import CompressionStore
+ from headroom.ccr.batch_store import BatchContextStore
+ from headroom.memory.adapters.graph import InMemoryGraphStore
+ from headroom.memory.adapters.graph_models import Entity
+
+ tracker = MemoryTracker.get()
+
+ # Create stores
+ compression_store = CompressionStore(max_entries=100)
+ batch_store = BatchContextStore()
+ graph_store = InMemoryGraphStore()
+
+ # Add some data
+ compression_store.store("original content" * 50, "compressed")
+ entity = Entity(id="node1", user_id="test", name="Test", entity_type="entity")
+ await graph_store.add_entity(entity)
+
+ # Register all stores
+ tracker.register("compression_store", compression_store.get_memory_stats)
+ tracker.register("batch_context_store", batch_store.get_memory_stats)
+ tracker.register("graph_store", graph_store.get_memory_stats)
+
+ # Get total
+ total = tracker.get_total_tracked_bytes()
+
+ # Should be sum of all stores
+ cs_stats = compression_store.get_memory_stats()
+ bs_stats = batch_store.get_memory_stats()
+ gs_stats = graph_store.get_memory_stats()
+
+ expected_total = cs_stats.size_bytes + bs_stats.size_bytes + gs_stats.size_bytes
+ assert total == expected_total
+
+ @pytest.mark.asyncio
+ async def test_full_memory_report(self):
+ """Test generating a full memory report with real stores."""
+ from headroom.cache.compression_store import CompressionStore
+ from headroom.memory.adapters.graph import InMemoryGraphStore
+ from headroom.memory.adapters.graph_models import Entity
+
+ tracker = MemoryTracker.get(target_budget_mb=100.0)
+
+ # Create and register stores
+ compression_store = CompressionStore(max_entries=1000)
+ graph_store = InMemoryGraphStore()
+
+ # Add data
+ for i in range(10):
+ compression_store.store(f"original content {i}" * 100, f"compressed_{i}")
+ entity = Entity(
+ id=f"node_{i}",
+ user_id="test",
+ name=f"Entity {i}",
+ entity_type="entity",
+ )
+ await graph_store.add_entity(entity)
+
+ tracker.register("compression_store", compression_store.get_memory_stats)
+ tracker.register("graph_store", graph_store.get_memory_stats)
+
+ # Get full report
+ report = tracker.get_report()
+
+ # Verify report structure
+ assert report.process is not None
+ assert report.process.rss_bytes >= 0
+ assert len(report.components) == 2
+ assert "compression_store" in report.components
+ assert "graph_store" in report.components
+ assert report.total_tracked_bytes > 0
+ assert report.target_budget_bytes == 100 * 1024 * 1024
+
+ # Verify serialization
+ d = report.to_dict()
+ assert "process" in d
+ assert "components" in d
+ assert "total_tracked_mb" in d
+
+
+class TestMemoryBudgetEnforcement:
+ """Tests for memory budget checking."""
+
+ @pytest.fixture(autouse=True)
+ def reset_tracker(self):
+ """Reset the tracker singleton before each test."""
+ MemoryTracker.reset()
+ yield
+ MemoryTracker.reset()
+
+ def test_under_budget(self):
+ """Test that under-budget is correctly detected."""
+ from headroom.cache.compression_store import CompressionStore
+
+ tracker = MemoryTracker.get(target_budget_mb=100.0) # 100 MB budget
+
+ store = CompressionStore(max_entries=10) # Small store
+ store.store("original", "compressed")
+
+ tracker.register("compression_store", store.get_memory_stats)
+
+ report = tracker.get_report()
+
+ # Small store should be under budget
+ assert report.is_over_budget is False
+
+ def test_over_budget_detection(self):
+ """Test that over-budget is correctly detected."""
+ tracker = MemoryTracker.get(target_budget_mb=0.001) # Very small budget (1 KB)
+
+ # Create a component that reports large size
+ from headroom.memory.tracker import ComponentStats
+
+ def large_component_stats() -> ComponentStats:
+ return ComponentStats(
+ name="large_component",
+ entry_count=1000,
+ size_bytes=10 * 1024 * 1024, # 10 MB
+ )
+
+ tracker.register("large_component", large_component_stats)
+
+ report = tracker.get_report()
+
+ # Should be over budget
+ assert report.is_over_budget is True
+
+
+class TestProcessStatsCollection:
+ """Tests for process-level memory stats."""
+
+ @pytest.fixture(autouse=True)
+ def reset_tracker(self):
+ """Reset the tracker singleton before each test."""
+ MemoryTracker.reset()
+ yield
+ MemoryTracker.reset()
+
+ def test_process_stats_collected(self):
+ """Test that process stats are collected from the real process."""
+ tracker = MemoryTracker.get()
+
+ stats = tracker.get_process_stats()
+
+ # Should have real values from the current process
+ assert stats.rss_bytes > 0 # Process must use some memory
+ assert stats.vms_bytes > 0
+ assert stats.percent >= 0 # Could be 0 on some systems
+
+ def test_process_stats_in_report(self):
+ """Test that process stats are included in report."""
+ tracker = MemoryTracker.get()
+
+ report = tracker.get_report()
+
+ assert report.process.rss_bytes > 0
+ assert report.process.rss_mb > 0
diff --git a/tests/test_memory_usage_integration.py b/tests/test_memory_usage_integration.py
new file mode 100644
index 000000000..d6903cae8
--- /dev/null
+++ b/tests/test_memory_usage_integration.py
@@ -0,0 +1,667 @@
+"""Comprehensive integration tests for memory tracking with real components.
+
+These tests exercise the full system including:
+- Memory system (GraphStore, HNSWVectorIndex)
+- CCR (Compress-Cache-Retrieve)
+- Compression store
+- Real API calls through the proxy
+
+Tests track memory usage throughout to verify our tracking is accurate.
+
+Requirements:
+ - ANTHROPIC_API_KEY in .env
+ - Run with: uv run pytest tests/test_memory_usage_integration.py -v -s
+"""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+# Load .env file
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+def get_process_memory_mb() -> float:
+ """Get current process memory in MB."""
+ import psutil
+
+ return psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024
+
+
+def get_tracked_memory() -> dict:
+ """Get memory stats from the tracker."""
+ from headroom.memory.tracker import MemoryTracker
+
+ tracker = MemoryTracker.get()
+ report = tracker.get_report()
+ return report.to_dict()
+
+
+class TestMemorySystemIntegration:
+ """Tests for the memory system (GraphStore + HNSWVectorIndex)."""
+
+ @pytest.fixture(autouse=True)
+ def reset_tracker(self):
+ """Reset the tracker singleton before each test."""
+ from headroom.memory.tracker import MemoryTracker
+
+ MemoryTracker.reset()
+ yield
+ MemoryTracker.reset()
+
+ @pytest.mark.asyncio
+ async def test_graph_store_memory_growth(self):
+ """Test that graph store memory is tracked as entities are added."""
+ from headroom.memory.adapters.graph import InMemoryGraphStore
+ from headroom.memory.adapters.graph_models import Entity, Relationship
+ from headroom.memory.tracker import MemoryTracker
+
+ tracker = MemoryTracker.get()
+ store = InMemoryGraphStore()
+ tracker.register("graph_store", store.get_memory_stats)
+
+ print("\n=== Graph Store Memory Growth Test ===")
+
+ # Track memory at each stage
+ memory_snapshots = []
+
+ # Initial state
+ stats = store.get_memory_stats()
+ memory_snapshots.append(("initial", stats.entry_count, stats.size_bytes))
+ print(f"Initial: {stats.entry_count} entries, {stats.size_bytes} bytes")
+
+ # Add 100 entities
+ for i in range(100):
+ entity = Entity(
+ id=f"entity_{i}",
+ user_id="test_user",
+ name=f"Test Entity {i}",
+ entity_type="concept",
+ description=f"This is a detailed description for entity {i} " * 10,
+ properties={"index": i, "data": "x" * 200},
+ )
+ await store.add_entity(entity)
+
+ stats = store.get_memory_stats()
+ memory_snapshots.append(("100 entities", stats.entry_count, stats.size_bytes))
+ print(f"After 100 entities: {stats.entry_count} entries, {stats.size_bytes} bytes")
+
+ # Add 200 relationships
+ for i in range(200):
+ rel = Relationship(
+ id=f"rel_{i}",
+ user_id="test_user",
+ source_id=f"entity_{i % 100}",
+ target_id=f"entity_{(i + 1) % 100}",
+ relation_type="related_to",
+ properties={"weight": 0.5, "metadata": "y" * 100},
+ )
+ await store.add_relationship(rel)
+
+ stats = store.get_memory_stats()
+ memory_snapshots.append(("+ 200 relationships", stats.entry_count, stats.size_bytes))
+ print(f"After 200 relationships: {stats.entry_count} entries, {stats.size_bytes} bytes")
+
+ # Verify memory grew
+ assert memory_snapshots[1][2] > memory_snapshots[0][2], (
+ "Memory should grow after adding entities"
+ )
+ assert memory_snapshots[2][2] > memory_snapshots[1][2], (
+ "Memory should grow after adding relationships"
+ )
+
+ # Verify tracker reports correctly
+ report = tracker.get_report()
+ assert "graph_store" in report.components
+ assert (
+ report.components["graph_store"].entry_count == 300
+ ) # 100 entities + 200 relationships
+
+ print(f"\nTotal tracked memory: {report.total_tracked_mb:.4f} MB")
+ print(f"Process RSS: {report.process.rss_mb:.1f} MB")
+
+ @pytest.mark.asyncio
+ async def test_hnsw_vector_index_memory_growth(self):
+ """Test that HNSW vector index memory is tracked as vectors are added."""
+ from headroom.memory.adapters.hnsw import HNSWVectorIndex
+ from headroom.memory.models import Memory
+ from headroom.memory.tracker import MemoryTracker
+
+ tracker = MemoryTracker.get()
+
+ # Use 384 dimensions (common for MiniLM embeddings)
+ index = HNSWVectorIndex(dimension=384)
+ tracker.register("vector_index", index.get_memory_stats)
+
+ print("\n=== HNSW Vector Index Memory Growth Test ===")
+
+ import numpy as np
+
+ # Track memory at each stage
+ memory_snapshots = []
+
+ # Initial state
+ stats = index.get_memory_stats()
+ memory_snapshots.append(("initial", stats.entry_count, stats.size_bytes))
+ print(f"Initial: {stats.entry_count} entries, {stats.size_bytes} bytes")
+
+ # Add 100 vectors
+ for i in range(100):
+ embedding = np.random.rand(384).astype(np.float32).tolist()
+ memory = Memory(
+ id=f"mem_{i}",
+ content=f"This is memory content {i} with some additional text " * 5,
+ user_id="test_user",
+ embedding=embedding,
+ importance=0.5 + (i % 10) / 20,
+ )
+ await index.index(memory)
+
+ stats = index.get_memory_stats()
+ memory_snapshots.append(("100 vectors", stats.entry_count, stats.size_bytes))
+ print(f"After 100 vectors: {stats.entry_count} entries, {stats.size_bytes} bytes")
+
+ # Add 400 more vectors
+ for i in range(100, 500):
+ embedding = np.random.rand(384).astype(np.float32).tolist()
+ memory = Memory(
+ id=f"mem_{i}",
+ content=f"This is memory content {i} with some additional text " * 5,
+ user_id="test_user",
+ embedding=embedding,
+ )
+ await index.index(memory)
+
+ stats = index.get_memory_stats()
+ memory_snapshots.append(("500 vectors", stats.entry_count, stats.size_bytes))
+ print(f"After 500 vectors: {stats.entry_count} entries, {stats.size_bytes} bytes")
+
+ # Verify memory grew
+ assert memory_snapshots[1][2] > memory_snapshots[0][2], (
+ "Memory should grow after adding vectors"
+ )
+ assert memory_snapshots[2][2] > memory_snapshots[1][2], (
+ "Memory should grow with more vectors"
+ )
+
+ # Verify tracker reports correctly
+ report = tracker.get_report()
+ assert "vector_index" in report.components
+ assert report.components["vector_index"].entry_count == 500
+
+ print(f"\nTotal tracked memory: {report.total_tracked_mb:.4f} MB")
+ print(f"Process RSS: {report.process.rss_mb:.1f} MB")
+
+
+class TestCCRIntegration:
+ """Tests for CCR (Compress-Cache-Retrieve) memory tracking."""
+
+ @pytest.fixture(autouse=True)
+ def reset_stores(self):
+ """Reset stores before each test."""
+ from headroom.ccr.batch_store import reset_batch_context_store
+ from headroom.memory.tracker import MemoryTracker
+
+ MemoryTracker.reset()
+ reset_batch_context_store()
+ yield
+ MemoryTracker.reset()
+ reset_batch_context_store()
+
+ def test_compression_store_memory_growth(self):
+ """Test that compression store memory is tracked correctly."""
+ from headroom.cache.compression_store import CompressionStore
+ from headroom.memory.tracker import MemoryTracker
+
+ tracker = MemoryTracker.get()
+ store = CompressionStore(max_entries=1000, default_ttl=3600)
+ tracker.register("compression_store", store.get_memory_stats)
+
+ print("\n=== Compression Store Memory Growth Test ===")
+
+ memory_snapshots = []
+
+ # Initial state
+ stats = store.get_memory_stats()
+ memory_snapshots.append(("initial", stats.entry_count, stats.size_bytes))
+ print(f"Initial: {stats.entry_count} entries, {stats.size_bytes} bytes")
+
+ # Add compressed content (simulating tool outputs)
+ for i in range(50):
+ original = f"Original tool output {i}: " + "data " * 500
+ compressed = f"Compressed {i}: " + "data " * 50
+ store.store(
+ original=original,
+ compressed=compressed,
+ original_tokens=len(original.split()),
+ compressed_tokens=len(compressed.split()),
+ tool_name=f"tool_{i % 5}",
+ )
+
+ stats = store.get_memory_stats()
+ memory_snapshots.append(("50 entries", stats.entry_count, stats.size_bytes))
+ print(f"After 50 entries: {stats.entry_count} entries, {stats.size_bytes} bytes")
+
+ # Add more with larger content
+ for i in range(50, 150):
+ original = f"Large tool output {i}: " + "data " * 2000
+ compressed = f"Compressed {i}: " + "data " * 200
+ store.store(
+ original=original,
+ compressed=compressed,
+ original_tokens=len(original.split()),
+ compressed_tokens=len(compressed.split()),
+ tool_name=f"tool_{i % 5}",
+ )
+
+ stats = store.get_memory_stats()
+ memory_snapshots.append(("150 entries", stats.entry_count, stats.size_bytes))
+ print(f"After 150 entries: {stats.entry_count} entries, {stats.size_bytes} bytes")
+
+ # Verify memory grew
+ assert memory_snapshots[1][2] > memory_snapshots[0][2]
+ assert memory_snapshots[2][2] > memory_snapshots[1][2]
+
+ # Test retrieval (should register hits)
+ # Get a key from the first entry
+ first_key = store.store("test original", "test compressed")
+ store.retrieve(first_key)
+ store.retrieve(first_key)
+ store.retrieve("nonexistent")
+
+ stats = store.get_memory_stats()
+ print(f"\nAfter retrievals - Hits: {stats.hits}, Misses: {stats.misses}")
+
+ report = tracker.get_report()
+ print(f"Total tracked memory: {report.total_tracked_mb:.4f} MB")
+
+ def test_batch_context_store_memory_growth(self):
+ """Test that batch context store memory is tracked correctly."""
+ from headroom.ccr.batch_store import (
+ BatchContext,
+ BatchContextStore,
+ BatchRequestContext,
+ )
+ from headroom.memory.tracker import MemoryTracker
+
+ tracker = MemoryTracker.get()
+ store = BatchContextStore(ttl=3600, max_contexts=1000)
+ tracker.register("batch_context_store", store.get_memory_stats)
+
+ print("\n=== Batch Context Store Memory Growth Test ===")
+
+ memory_snapshots = []
+
+ # Initial state
+ stats = store.get_memory_stats()
+ memory_snapshots.append(("initial", stats.entry_count, stats.size_bytes))
+ print(f"Initial: {stats.entry_count} entries, {stats.size_bytes} bytes")
+
+ # Add batch contexts (simulating batch API submissions)
+ for batch_num in range(20):
+ ctx = BatchContext(
+ batch_id=f"batch_{batch_num}",
+ provider="anthropic",
+ )
+ # Each batch has multiple requests
+ for req_num in range(10):
+ ctx.add_request(
+ BatchRequestContext(
+ custom_id=f"req_{batch_num}_{req_num}",
+ messages=[
+ {"role": "system", "content": "You are a helpful assistant."},
+ {"role": "user", "content": f"Request {req_num}: " + "context " * 100},
+ ],
+ model="claude-sonnet-4-20250514",
+ tools=[
+ {
+ "name": "search",
+ "description": "Search the web",
+ "input_schema": {"type": "object", "properties": {}},
+ }
+ ],
+ )
+ )
+ # Store directly (bypassing async for testing)
+ store._contexts[ctx.batch_id] = ctx
+
+ stats = store.get_memory_stats()
+ memory_snapshots.append(("20 batches", stats.entry_count, stats.size_bytes))
+ print(
+ f"After 20 batches (200 requests): {stats.entry_count} entries, {stats.size_bytes} bytes"
+ )
+
+ # Verify memory grew
+ assert memory_snapshots[1][2] > memory_snapshots[0][2]
+
+ report = tracker.get_report()
+ print(f"Total tracked memory: {report.total_tracked_mb:.4f} MB")
+
+
+ not os.environ.get("ANTHROPIC_API_KEY"),
+ reason="ANTHROPIC_API_KEY not set in environment",
+)
+class TestProxyMemoryIntegration:
+ """Tests that exercise the proxy with real API calls and track memory."""
+
+ @pytest.fixture
+ def api_key(self):
+ """Get API key from environment."""
+ return os.environ.get("ANTHROPIC_API_KEY")
+
+ @pytest.fixture(autouse=True)
+ def reset_tracker(self):
+ """Reset the tracker singleton before each test."""
+ from headroom.memory.tracker import MemoryTracker
+
+ MemoryTracker.reset()
+ yield
+ MemoryTracker.reset()
+
+ def test_real_api_calls_memory_tracking(self, api_key):
+ """Test memory tracking with real API calls."""
+ import httpx
+
+ from headroom.memory.tracker import MemoryTracker
+
+ tracker = MemoryTracker.get()
+
+ print("\n=== Real API Calls Memory Tracking Test ===")
+
+ # Note: This test requires a running proxy
+ # We'll test the components directly instead
+
+ # Create and register stores
+ from headroom.cache.compression_store import CompressionStore
+ from headroom.ccr.batch_store import BatchContextStore
+
+ compression_store = CompressionStore(max_entries=100)
+ batch_store = BatchContextStore()
+
+ tracker.register("compression_store", compression_store.get_memory_stats)
+ tracker.register("batch_context_store", batch_store.get_memory_stats)
+
+ initial_report = tracker.get_report()
+ print(f"Initial tracked: {initial_report.total_tracked_mb:.4f} MB")
+ print(f"Initial RSS: {initial_report.process.rss_mb:.1f} MB")
+
+ # Make real API call using httpx directly
+ headers = {
+ "x-api-key": api_key,
+ "anthropic-version": "2023-06-01",
+ "content-type": "application/json",
+ }
+
+ messages_list = [
+ [{"role": "user", "content": f"Say 'test {i}' and nothing else."}] for i in range(3)
+ ]
+
+ with httpx.Client(timeout=60.0) as client:
+ for i, messages in enumerate(messages_list):
+ response = client.post(
+ "https://api.anthropic.com/v1/messages",
+ headers=headers,
+ json={
+ "model": "claude-sonnet-4-20250514",
+ "max_tokens": 50,
+ "messages": messages,
+ },
+ )
+ assert response.status_code == 200, f"API call failed: {response.text}"
+
+ # Simulate storing compressed response (as CCR would)
+ response_text = response.text
+ compression_store.store(
+ original=response_text,
+ compressed=response_text[:100], # Simulated compression
+ tool_name="api_response",
+ )
+
+ report = tracker.get_report()
+ print(
+ f"After request {i + 1}: tracked={report.total_tracked_mb:.4f} MB, RSS={report.process.rss_mb:.1f} MB"
+ )
+
+ final_report = tracker.get_report()
+ print(f"\nFinal tracked: {final_report.total_tracked_mb:.4f} MB")
+ print(f"Final RSS: {final_report.process.rss_mb:.1f} MB")
+
+ # Verify stores have entries
+ assert final_report.components["compression_store"].entry_count == 3
+
+
+class TestCombinedMemoryTracking:
+ """Tests that combine multiple components and track total memory."""
+
+ @pytest.fixture(autouse=True)
+ def reset_all(self):
+ """Reset all stores."""
+ from headroom.ccr.batch_store import reset_batch_context_store
+ from headroom.memory.tracker import MemoryTracker
+
+ MemoryTracker.reset()
+ reset_batch_context_store()
+ yield
+ MemoryTracker.reset()
+ reset_batch_context_store()
+
+ @pytest.mark.asyncio
+ async def test_all_components_memory_tracking(self):
+ """Test memory tracking with all components active."""
+ import numpy as np
+
+ from headroom.cache.compression_store import CompressionStore
+ from headroom.ccr.batch_store import BatchContext, BatchContextStore, BatchRequestContext
+ from headroom.memory.adapters.graph import InMemoryGraphStore
+ from headroom.memory.adapters.graph_models import Entity, Relationship
+ from headroom.memory.adapters.hnsw import HNSWVectorIndex
+ from headroom.memory.models import Memory
+ from headroom.memory.tracker import MemoryTracker
+
+ tracker = MemoryTracker.get(target_budget_mb=50.0) # Set a 50MB budget
+
+ print("\n=== Combined Memory Tracking Test ===")
+
+ # Create all components
+ compression_store = CompressionStore(max_entries=500)
+ batch_store = BatchContextStore(max_contexts=100)
+ graph_store = InMemoryGraphStore()
+ vector_index = HNSWVectorIndex(dimension=384)
+
+ # Register all with tracker
+ tracker.register("compression_store", compression_store.get_memory_stats)
+ tracker.register("batch_context_store", batch_store.get_memory_stats)
+ tracker.register("graph_store", graph_store.get_memory_stats)
+ tracker.register("vector_index", vector_index.get_memory_stats)
+
+ # Initial state
+ report = tracker.get_report()
+ print("\nInitial state:")
+ print(f" Total tracked: {report.total_tracked_mb:.4f} MB")
+ print(f" Budget: {report.target_budget_mb:.1f} MB")
+ print(f" Over budget: {report.is_over_budget}")
+
+ # Add data to all components
+ print("\nAdding data to components...")
+
+ # 1. Compression store - 100 entries (unique content for each)
+ for i in range(100):
+ compression_store.store(
+ original=f"unique content {i}: " + "x" * 1000,
+ compressed=f"compressed {i}: " + "x" * 100,
+ tool_name=f"tool_{i}",
+ )
+
+ # 2. Batch store - 10 batches with 5 requests each
+ for b in range(10):
+ ctx = BatchContext(batch_id=f"batch_{b}", provider="anthropic")
+ for r in range(5):
+ ctx.add_request(
+ BatchRequestContext(
+ custom_id=f"req_{b}_{r}",
+ messages=[{"role": "user", "content": "test " * 50}],
+ model="claude-sonnet-4-20250514",
+ )
+ )
+ batch_store._contexts[ctx.batch_id] = ctx
+
+ # 3. Graph store - 50 entities, 100 relationships
+ for i in range(50):
+ entity = Entity(
+ id=f"entity_{i}",
+ user_id="test",
+ name=f"Entity {i}",
+ entity_type="concept",
+ properties={"data": "y" * 200},
+ )
+ await graph_store.add_entity(entity)
+
+ for i in range(100):
+ rel = Relationship(
+ id=f"rel_{i}",
+ user_id="test",
+ source_id=f"entity_{i % 50}",
+ target_id=f"entity_{(i + 1) % 50}",
+ relation_type="related",
+ )
+ await graph_store.add_relationship(rel)
+
+ # 4. Vector index - 200 vectors
+ for i in range(200):
+ embedding = np.random.rand(384).astype(np.float32).tolist()
+ memory = Memory(
+ id=f"mem_{i}",
+ content=f"Memory {i}",
+ user_id="test",
+ embedding=embedding,
+ )
+ await vector_index.index(memory)
+
+ # Final state
+ report = tracker.get_report()
+ print("\nAfter adding data:")
+ print(" Components:")
+ for name, comp in report.components.items():
+ print(f" {name}: {comp.entry_count} entries, {comp.size_bytes / 1024:.2f} KB")
+ print(f" Total tracked: {report.total_tracked_mb:.4f} MB")
+ print(f" Process RSS: {report.process.rss_mb:.1f} MB")
+ print(f" Over budget: {report.is_over_budget}")
+
+ # Verify all components are tracked
+ assert len(report.components) == 4
+ assert report.components["compression_store"].entry_count == 100
+ assert report.components["batch_context_store"].entry_count == 10
+ assert report.components["graph_store"].entry_count == 150 # 50 + 100
+ assert report.components["vector_index"].entry_count == 200
+
+ # Verify total is sum of components
+ total_from_components = sum(c.size_bytes for c in report.components.values())
+ assert report.total_tracked_bytes == total_from_components
+
+ @pytest.mark.asyncio
+ async def test_memory_budget_enforcement(self):
+ """Test that budget enforcement works correctly."""
+ import numpy as np
+
+ from headroom.memory.adapters.hnsw import HNSWVectorIndex
+ from headroom.memory.models import Memory
+ from headroom.memory.tracker import MemoryTracker
+
+ # Set a very small budget (1 MB)
+ tracker = MemoryTracker.get(target_budget_mb=1.0)
+
+ vector_index = HNSWVectorIndex(dimension=384)
+ tracker.register("vector_index", vector_index.get_memory_stats)
+
+ print("\n=== Budget Enforcement Test ===")
+
+ # Add vectors until we exceed budget
+ for i in range(1000):
+ embedding = np.random.rand(384).astype(np.float32).tolist()
+ memory = Memory(
+ id=f"mem_{i}",
+ content=f"Memory {i} with extra content " * 10,
+ user_id="test",
+ embedding=embedding,
+ )
+ await vector_index.index(memory)
+
+ if i % 100 == 0:
+ report = tracker.get_report()
+ print(
+ f"After {i} vectors: {report.total_tracked_mb:.4f} MB, over_budget={report.is_over_budget}"
+ )
+ if report.is_over_budget:
+ print(f" Budget exceeded at {i} vectors!")
+ break
+
+ report = tracker.get_report()
+ print(
+ f"\nFinal: {report.total_tracked_mb:.4f} MB (budget: {report.target_budget_mb:.1f} MB)"
+ )
+
+ # With 1MB budget and 384-dim vectors, we should exceed budget
+ # Each vector is ~1.5KB (384 floats * 4 bytes + metadata)
+ # 1000 vectors = ~1.5MB, so we should exceed 1MB budget
+
+
+class TestMemoryReportEndpoint:
+ """Test the /debug/memory endpoint format."""
+
+ @pytest.fixture(autouse=True)
+ def reset_tracker(self):
+ """Reset the tracker singleton before each test."""
+ from headroom.memory.tracker import MemoryTracker
+
+ MemoryTracker.reset()
+ yield
+ MemoryTracker.reset()
+
+ def test_memory_report_serialization(self):
+ """Test that memory report serializes correctly for API response."""
+ from headroom.cache.compression_store import CompressionStore
+ from headroom.memory.tracker import MemoryTracker
+
+ tracker = MemoryTracker.get(target_budget_mb=100.0)
+
+ store = CompressionStore(max_entries=10)
+ store.store("original", "compressed")
+ tracker.register("compression_store", store.get_memory_stats)
+
+ report = tracker.get_report()
+ data = report.to_dict()
+
+ # Verify structure matches what API returns
+ assert "process" in data
+ assert "rss_mb" in data["process"]
+ assert "vms_mb" in data["process"]
+ assert "percent" in data["process"]
+
+ assert "components" in data
+ assert "compression_store" in data["components"]
+ comp = data["components"]["compression_store"]
+ assert "name" in comp
+ assert "entry_count" in comp
+ assert "size_bytes" in comp
+ assert "size_mb" in comp
+ assert "hits" in comp
+ assert "misses" in comp
+
+ assert "total_tracked_mb" in data
+ assert "target_budget_mb" in data
+ assert "is_over_budget" in data
+ assert "timestamp" in data
+
+ print("\n=== Memory Report Format ===")
+ import json
+
+ print(json.dumps(data, indent=2))
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-v", "-s"])