Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
chore: add traces to polling (#2713)
cthomas3a26e8173b5e
diff --git a/letta/jobs/llm_batch_job_polling.py b/letta/jobs/llm_batch_job_polling.pyindex 6e0ba172c..cfaae2ab2 100644--- a/letta/jobs/llm_batch_job_polling.py+++ b/letta/jobs/llm_batch_job_polling.py@@ -6,6 +6,7 @@ from letta.agents.letta_agent_batch import LettaAgentBatchfrom letta.jobs.helpers import map_anthropic_batch_job_status_to_job_status, map_anthropic_individual_batch_item_status_to_job_statusfrom letta.jobs.types import BatchPollingResult, ItemUpdateInfofrom letta.log import get_logger+from letta.otel.tracing import trace_methodfrom letta.schemas.enums import JobStatus, ProviderTypefrom letta.schemas.letta_response import LettaBatchResponsefrom letta.schemas.llm_batch_job import LLMBatchJob@@ -37,6 +38,7 @@ class BatchPollingMetrics:logger.info(f"[Poll BatchJob] Updated {self.updated_items_count} items for newly completed batch(es).")+@trace_methodasync def fetch_batch_status(server: SyncServer, batch_job: LLMBatchJob) -> BatchPollingResult:"""Fetch the current status of a single batch job from the provider.@@ -60,6 +62,7 @@ async def fetch_batch_status(server: SyncServer, batch_job: LLMBatchJob) -> Batcreturn BatchPollingResult(batch_job.id, JobStatus.running, None)+@trace_methodasync def fetch_batch_items(server: SyncServer, batch_id: str, batch_resp_id: str) -> List[ItemUpdateInfo]:"""Fetch individual item results for a completed batch.@@ -86,6 +89,7 @@ async def fetch_batch_items(server: SyncServer, batch_id: str, batch_resp_id: streturn updates+@trace_methodasync def poll_batch_updates(server: SyncServer, batch_jobs: List[LLMBatchJob], metrics: BatchPollingMetrics) -> List[BatchPollingResult]:"""Poll for updates to multiple batch jobs concurrently.@@ -113,6 +117,7 @@ async def poll_batch_updates(server: SyncServer, batch_jobs: List[LLMBatchJob],return results+@trace_methodasync def process_completed_batches(server: SyncServer, batch_results: List[BatchPollingResult], metrics: BatchPollingMetrics) -> List[ItemUpdateInfo]:@@ -161,6 +166,7 @@ async def process_completed_batches(return item_updates+@trace_methodasync def poll_running_llm_batches(server: "SyncServer") -> List[LettaBatchResponse]:"""Cron job to poll all running LLM batch jobs and update their polling responses in bulk.diff --git a/letta/jobs/scheduler.py b/letta/jobs/scheduler.pyindex 056167fa4..df220e4b1 100644--- a/letta/jobs/scheduler.py+++ b/letta/jobs/scheduler.py@@ -45,6 +45,7 @@ async def _try_acquire_lock_and_start_scheduler(server: SyncServer) -> bool:text("SELECT pg_try_advisory_lock(CAST(:lock_key AS bigint))"), {"lock_key": ADVISORY_LOCK_KEY})acquired_lock = result.scalar()+ await lock_session.commit()if not acquired_lock:if lock_session:@@ -90,7 +91,7 @@ async def _try_acquire_lock_and_start_scheduler(server: SyncServer) -> bool:logger.warning("Attempting to release lock due to error during startup.")try:_advisory_lock_session = lock_session- await _release_advisory_lock()+ await _release_advisory_lock(lock_session)except Exception as unlock_err:logger.error(f"Failed to release lock during error handling: {unlock_err}", exc_info=True)finally:@@ -140,11 +141,11 @@ async def _background_lock_retry_loop(server: SyncServer):await asyncio.sleep(settings.poll_lock_retry_interval_seconds)-async def _release_advisory_lock():+async def _release_advisory_lock(lock_session=None):"""Releases the advisory lock using the stored session."""global _advisory_lock_session- lock_session = _advisory_lock_session+ lock_session = _advisory_lock_session or lock_session_advisory_lock_session = Noneif lock_session is not None:@@ -152,6 +153,7 @@ async def _release_advisory_lock():try:await lock_session.execute(text("SELECT pg_advisory_unlock(CAST(:lock_key AS bigint))"), {"lock_key": ADVISORY_LOCK_KEY})logger.info(f"Executed pg_advisory_unlock for lock {ADVISORY_LOCK_KEY}")+ await lock_session.commit()except Exception as e:logger.error(f"Error executing pg_advisory_unlock: {e}", exc_info=True)finally: