Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix: register memory monitor at startup (#6195)
Kian Jonesa5e435c56fc8
diff --git a/letta/monitoring/memory_tracker.py b/letta/monitoring/memory_tracker.pyindex a0670ecaf..10fe19532 100644--- a/letta/monitoring/memory_tracker.py+++ b/letta/monitoring/memory_tracker.py@@ -87,11 +87,12 @@ class MemoryTracker:# Check if there's a running event looploop = asyncio.get_running_loop()# Create the monitor task- asyncio.create_task(self.start_background_monitor())+ task = asyncio.create_task(self.start_background_monitor())self._monitor_started = True- except RuntimeError:+ logger.debug("Monitor start task created from _ensure_monitor_started")+ except RuntimeError as e:# No event loop running yet, will try again later- pass+ logger.debug(f"No event loop available yet for monitor start: {e}")def get_memory_info(self) -> Dict[str, Any]:"""Get current memory information."""@@ -303,12 +304,20 @@ class MemoryTracker:async def start_background_monitor(self):"""Start the background memory monitoring task."""if self._monitoring:+ logger.info("Background monitor already running, skipping start")returnself._monitoring = Trueself._monitor_started = True- self._monitor_task = asyncio.create_task(self._monitor_loop())- logger.info(f"Background memory monitor started (interval: {self.monitor_interval}s)")++ try:+ self._monitor_task = asyncio.create_task(self._monitor_loop())+ logger.info(f"Background memory monitor task created successfully (interval: {self.monitor_interval}s)")+ except Exception as e:+ logger.error(f"Failed to create monitor task: {e}", exc_info=True)+ self._monitoring = False+ self._monitor_started = False+ raiseasync def stop_background_monitor(self):"""Stop the background memory monitoring task."""@@ -326,9 +335,13 @@ class MemoryTracker:"""Background monitoring loop that runs continuously using asyncio."""consecutive_high_memory = 0last_gc_time = time.time()+ iteration_count = 0++ logger.info(f"Memory monitor loop started (PID: {os.getpid()})")while self._monitoring:try:+ iteration_count += 1mem_info = self.get_memory_info()current_mb = mem_info.get("rss_mb", 0)system_percent = mem_info.get("system_percent", 0)@@ -339,6 +352,14 @@ class MemoryTracker:if len(self.memory_history) > self.max_history_size:self.memory_history.pop(0)+ # Log periodic memory status+ # Using INFO since production logging is at INFO level+ percent = (current_mb / self.CRITICAL_THRESHOLD_MB) * 100+ logger.info(+ f"Memory Status: RSS: {current_mb:.2f} MB ({percent:.1f}% of {self.CRITICAL_THRESHOLD_MB} MB limit), "+ f"System: {system_percent:.1f}%"+ )+# Check memory levelsself._check_memory_thresholds(mem_info)@@ -364,11 +385,14 @@ class MemoryTracker:await asyncio.sleep(self.monitor_interval)except asyncio.CancelledError:+ logger.info(f"Memory monitor loop cancelled after {iteration_count} iterations")breakexcept Exception as e:- logger.error(f"Error in memory monitor loop: {e}")+ logger.error(f"Error in memory monitor loop (iteration {iteration_count}): {e}", exc_info=True)await asyncio.sleep(self.monitor_interval)+ logger.info(f"Memory monitor loop exited after {iteration_count} iterations")+def _check_memory_thresholds(self, mem_info: Dict[str, Any]):"""Check memory against thresholds and log appropriately."""current_mb = mem_info.get("rss_mb", 0)diff --git a/letta/server/rest_api/app.py b/letta/server/rest_api/app.pyindex e97f564b0..d944a5eec 100644--- a/letta/server/rest_api/app.py+++ b/letta/server/rest_api/app.py@@ -144,8 +144,10 @@ async def lifespan(app_: FastAPI):# Initialize memory trackingif MEMORY_TRACKING_ENABLED:logger.info(f"[Worker {worker_id}] Initializing memory tracking")- # Get the global tracker instance (will start background monitor automatically)+ # Get the global tracker instancetracker = get_memory_tracker(enable_background_monitor=True, monitor_interval=5)+ # Explicitly start the background monitor (won't wait for first tracked operation)+ await tracker.start_background_monitor()logger.info(f"[Worker {worker_id}] Memory tracking enabled - monitoring every 5s with proactive alerts")if telemetry_settings.profiler: