Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
chore: bump letta version 0.8.11 (#2711)
cthomasca6424470fc2
diff --git a/letta/constants.py b/letta/constants.pyindex bd8d2c49d..5b7725bbe 100644--- a/letta/constants.py+++ b/letta/constants.py@@ -371,3 +371,4 @@ PINECONE_TEXT_FIELD_NAME = "chunk_text"PINECONE_METRIC = "cosine"PINECONE_CLOUD = "aws"PINECONE_REGION = "us-east-1"+PINECONE_MAX_BATCH_SIZE = 96diff --git a/letta/helpers/pinecone_utils.py b/letta/helpers/pinecone_utils.pyindex d7cd1d4ae..d51b071d4 100644--- a/letta/helpers/pinecone_utils.py+++ b/letta/helpers/pinecone_utils.py@@ -2,7 +2,14 @@ from typing import Any, Dict, Listfrom pinecone import PineconeAsyncio-from letta.constants import PINECONE_CLOUD, PINECONE_EMBEDDING_MODEL, PINECONE_METRIC, PINECONE_REGION, PINECONE_TEXT_FIELD_NAME+from letta.constants import (+ PINECONE_CLOUD,+ PINECONE_EMBEDDING_MODEL,+ PINECONE_MAX_BATCH_SIZE,+ PINECONE_METRIC,+ PINECONE_REGION,+ PINECONE_TEXT_FIELD_NAME,+)from letta.log import get_loggerfrom letta.schemas.user import Userfrom letta.settings import settings@@ -90,7 +97,10 @@ async def upsert_records_to_pinecone_index(records: List[dict], actor: User):async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:description = await pc.describe_index(name=settings.pinecone_source_index)async with pc.IndexAsyncio(host=description.index.host) as dense_index:- await dense_index.upsert_records(actor.organization_id, records)+ # Process records in batches to avoid exceeding Pinecone limits+ for i in range(0, len(records), PINECONE_MAX_BATCH_SIZE):+ batch = records[i : i + PINECONE_MAX_BATCH_SIZE]+ await dense_index.upsert_records(actor.organization_id, batch)async def search_pinecone_index(query: str, limit: int, filter: Dict[str, Any], actor: User) -> Dict[str, Any]:diff --git a/letta/jobs/scheduler.py b/letta/jobs/scheduler.pyindex b453aea95..056167fa4 100644--- a/letta/jobs/scheduler.py+++ b/letta/jobs/scheduler.py@@ -4,10 +4,11 @@ from typing import Optionalfrom apscheduler.schedulers.asyncio import AsyncIOSchedulerfrom apscheduler.triggers.interval import IntervalTrigger+from sqlalchemy import textfrom letta.jobs.llm_batch_job_polling import poll_running_llm_batchesfrom letta.log import get_logger-from letta.server.db import db_context+from letta.server.db import db_registryfrom letta.server.server import SyncServerfrom letta.settings import settings@@ -16,68 +17,54 @@ scheduler = AsyncIOScheduler()logger = get_logger(__name__)ADVISORY_LOCK_KEY = 0x12345678ABCDEF00-_advisory_lock_conn = None # Holds the raw DB connection if leader-_advisory_lock_cur = None # Holds the cursor for the lock connection if leader+_advisory_lock_session = None # Holds the async session if leader_lock_retry_task: Optional[asyncio.Task] = None # Background task handle for non-leaders_is_scheduler_leader = False # Flag indicating if this instance runs the schedulerasync def _try_acquire_lock_and_start_scheduler(server: SyncServer) -> bool:"""Attempts to acquire lock, starts scheduler if successful."""- global _advisory_lock_conn, _advisory_lock_cur, _is_scheduler_leader, scheduler+ global _advisory_lock_session, _is_scheduler_leader, schedulerif _is_scheduler_leader:return True # Already leading- raw_conn = None- cur = None+ lock_session = Noneacquired_lock = Falsetry:- # Use a temporary connection context for the attempt initially- with db_context() as session:+ async with db_registry.async_session() as session:engine = session.get_bind()engine_name = engine.namelogger.info(f"Database engine type: {engine_name}")-if engine_name != "postgresql":logger.warning(f"Advisory locks not supported for {engine_name} database. Starting scheduler without leader election.")- acquired_lock = True # For SQLite, assume we can start the scheduler+ acquired_lock = Trueelse:- # Get raw connection - MUST be kept open if lock is acquired- raw_conn = engine.raw_connection()- cur = raw_conn.cursor()-- cur.execute("SELECT pg_try_advisory_lock(CAST(%s AS bigint))", (ADVISORY_LOCK_KEY,))- acquired_lock = cur.fetchone()[0]+ lock_session = db_registry.get_async_session_factory()()+ result = await lock_session.execute(+ text("SELECT pg_try_advisory_lock(CAST(:lock_key AS bigint))"), {"lock_key": ADVISORY_LOCK_KEY}+ )+ acquired_lock = result.scalar()if not acquired_lock:- if cur:- cur.close()- if raw_conn:- raw_conn.close()+ if lock_session:+ await lock_session.close()logger.info("Scheduler lock held by another instance.")return False- # --- Lock Acquired ---if engine_name == "postgresql":logger.info("Acquired PostgreSQL advisory lock.")- _advisory_lock_conn = raw_conn # Keep connection for lock duration- _advisory_lock_cur = cur # Keep cursor for lock duration- raw_conn = None # Prevent closing in finally block- cur = None # Prevent closing in finally block+ _advisory_lock_session = lock_session+ lock_session = Noneelse:logger.info("Starting scheduler for non-PostgreSQL database.")- # For SQLite, we don't need to keep the connection open- if cur:- cur.close()- if raw_conn:- raw_conn.close()- raw_conn = None- cur = None+ if lock_session:+ await lock_session.close()+ lock_session = Nonetrigger = IntervalTrigger(seconds=settings.poll_running_llm_batches_interval_seconds,- jitter=10, # Jitter for the job execution+ jitter=10,)scheduler.add_job(poll_running_llm_batches,@@ -91,7 +78,7 @@ async def _try_acquire_lock_and_start_scheduler(server: SyncServer) -> bool:if not scheduler.running:scheduler.start()- elif scheduler.state == 2: # PAUSED+ elif scheduler.state == 2:scheduler.resume()_is_scheduler_leader = True@@ -99,38 +86,27 @@ async def _try_acquire_lock_and_start_scheduler(server: SyncServer) -> bool:except Exception as e:logger.error(f"Error during lock acquisition/scheduler start: {e}", exc_info=True)- if acquired_lock: # If lock was acquired before error, try to release+ if acquired_lock:logger.warning("Attempting to release lock due to error during startup.")try:- # Use the cursor/connection we were about to store- _advisory_lock_cur = cur- _advisory_lock_conn = raw_conn- await _release_advisory_lock() # Attempt cleanup+ _advisory_lock_session = lock_session+ await _release_advisory_lock()except Exception as unlock_err:logger.error(f"Failed to release lock during error handling: {unlock_err}", exc_info=True)finally:- # Ensure globals are cleared after failed attempt- _advisory_lock_cur = None- _advisory_lock_conn = None+ _advisory_lock_session = None_is_scheduler_leader = False- # Ensure scheduler is stopped if we failed partiallyif scheduler.running:try:scheduler.shutdown(wait=False)except:- pass # Best effort+ passreturn Falsefinally:- # Clean up temporary resources if lock wasn't acquired or error occurred before storing- if cur:- try:- cur.close()- except:- pass- if raw_conn:+ if lock_session:try:- raw_conn.close()+ await lock_session.close()except:pass@@ -141,63 +117,50 @@ async def _background_lock_retry_loop(server: SyncServer):logger.info("Starting background task to periodically check for scheduler lock.")while True:- if _is_scheduler_leader: # Should be cancelled first, but safety check+ if _is_scheduler_leader:breaktry:wait_time = settings.poll_lock_retry_interval_secondsawait asyncio.sleep(wait_time)- # Re-check state before attempting lockif _is_scheduler_leader or _lock_retry_task is None:- break # Stop if became leader or task was cancelled+ breakacquired = await _try_acquire_lock_and_start_scheduler(server)if acquired:logger.info("Background task acquired lock and started scheduler.")- _lock_retry_task = None # Clear self handle- break # Exit loop, we are now the leader+ _lock_retry_task = None+ breakexcept asyncio.CancelledError:logger.info("Background lock retry task cancelled.")breakexcept Exception as e:logger.error(f"Error in background lock retry loop: {e}", exc_info=True)- # Avoid tight loop on persistent errorsawait asyncio.sleep(settings.poll_lock_retry_interval_seconds)async def _release_advisory_lock():- """Releases the advisory lock using the stored connection."""- global _advisory_lock_conn, _advisory_lock_cur+ """Releases the advisory lock using the stored session."""+ global _advisory_lock_session- lock_cur = _advisory_lock_cur- lock_conn = _advisory_lock_conn- _advisory_lock_cur = None # Clear global immediately- _advisory_lock_conn = None # Clear global immediately+ lock_session = _advisory_lock_session+ _advisory_lock_session = None- if lock_cur is not None and lock_conn is not None:+ if lock_session is not None:logger.info(f"Attempting to release PostgreSQL advisory lock {ADVISORY_LOCK_KEY}")try:- # Try to execute unlock - connection/cursor validity is checked by attempting the operation- lock_cur.execute("SELECT pg_advisory_unlock(CAST(%s AS bigint))", (ADVISORY_LOCK_KEY,))- lock_cur.fetchone() # Consume result- lock_conn.commit()+ await lock_session.execute(text("SELECT pg_advisory_unlock(CAST(:lock_key AS bigint))"), {"lock_key": ADVISORY_LOCK_KEY})logger.info(f"Executed pg_advisory_unlock for lock {ADVISORY_LOCK_KEY}")except Exception as e:logger.error(f"Error executing pg_advisory_unlock: {e}", exc_info=True)finally:- # Ensure resources are closed regardless of unlock success- try:- if lock_cur:- lock_cur.close()- except Exception as e:- logger.error(f"Error closing advisory lock cursor: {e}", exc_info=True)try:- if lock_conn:- lock_conn.close()- logger.info("Closed database connection that held advisory lock.")+ if lock_session:+ await lock_session.close()+ logger.info("Closed database session that held advisory lock.")except Exception as e:- logger.error(f"Error closing advisory lock connection: {e}", exc_info=True)+ logger.error(f"Error closing advisory lock session: {e}", exc_info=True)else:logger.info("No PostgreSQL advisory lock to release (likely using SQLite or non-PostgreSQL database).")@@ -220,7 +183,6 @@ async def start_scheduler_with_leader_election(server: SyncServer):acquired_immediately = await _try_acquire_lock_and_start_scheduler(server)if not acquired_immediately and _lock_retry_task is None:- # Failed initial attempt, start background retry taskloop = asyncio.get_running_loop()_lock_retry_task = loop.create_task(_background_lock_retry_loop(server))@@ -232,48 +194,40 @@ async def shutdown_scheduler_and_release_lock():"""global _is_scheduler_leader, _lock_retry_task, scheduler- # 1. Cancel retry task if running (for non-leaders)if _lock_retry_task is not None:logger.info("Shutting down: Cancelling background lock retry task.")current_task = _lock_retry_task- _lock_retry_task = None # Clear handle first+ _lock_retry_task = Nonecurrent_task.cancel()try:- await current_task # Wait for cancellation+ await current_taskexcept asyncio.CancelledError:logger.info("Background lock retry task successfully cancelled.")except Exception as e:logger.warning(f"Exception waiting for cancelled retry task: {e}", exc_info=True)- # 2. Shutdown scheduler and release lock if we were the leaderif _is_scheduler_leader:logger.info("Shutting down: Leader instance stopping scheduler and releasing lock.")if scheduler.running:try:- # Force synchronous shutdown to prevent callback schedulingscheduler.shutdown(wait=True)- # wait for any internal cleanup to completeawait asyncio.sleep(0.1)logger.info("APScheduler shut down.")except Exception as e:- # Handle SchedulerNotRunningError and other shutdown exceptionslogger.warning(f"Exception during APScheduler shutdown: {e}")if "not running" not in str(e).lower():logger.error(f"Unexpected error shutting down APScheduler: {e}", exc_info=True)await _release_advisory_lock()- _is_scheduler_leader = False # Update state after cleanup+ _is_scheduler_leader = Falseelse:logger.info("Shutting down: Non-leader instance.")- # Final cleanup check for scheduler state (belt and suspenders)- # This should rarely be needed if shutdown logic above worked correctlytry:if scheduler.running:logger.warning("Scheduler still running after shutdown logic completed? Forcing shutdown.")scheduler.shutdown(wait=False)except Exception as e:- # Catch SchedulerNotRunningError and other shutdown exceptionslogger.debug(f"Expected exception during final scheduler cleanup: {e}")diff --git a/letta/schemas/source_metadata.py b/letta/schemas/source_metadata.pynew file mode 100644index 000000000..d395e188b--- /dev/null+++ b/letta/schemas/source_metadata.py@@ -0,0 +1,32 @@+from typing import List, Optional++from pydantic import Field++from letta.schemas.letta_base import LettaBase+++class FileStats(LettaBase):+ """File statistics for metadata endpoint"""++ file_id: str = Field(..., description="Unique identifier of the file")+ file_name: str = Field(..., description="Name of the file")+ file_size: Optional[int] = Field(None, description="Size of the file in bytes")+++class SourceStats(LettaBase):+ """Aggregated metadata for a source"""++ source_id: str = Field(..., description="Unique identifier of the source")+ source_name: str = Field(..., description="Name of the source")+ file_count: int = Field(0, description="Number of files in the source")+ total_size: int = Field(0, description="Total size of all files in bytes")+ files: List[FileStats] = Field(default_factory=list, description="List of file statistics")+++class OrganizationSourcesStats(LettaBase):+ """Complete metadata response for organization sources"""++ total_sources: int = Field(0, description="Total number of sources")+ total_files: int = Field(0, description="Total number of files across all sources")+ total_size: int = Field(0, description="Total size of all files in bytes")+ sources: List[SourceStats] = Field(default_factory=list, description="List of source metadata")diff --git a/letta/server/db.py b/letta/server/db.pyindex cf423c59e..ff76a0007 100644--- a/letta/server/db.py+++ b/letta/server/db.py@@ -226,6 +226,32 @@ class DatabaseRegistry:@contextmanagerdef session(self, name: str = "default") -> Generator[Any, None, None]:"""Context manager for database sessions."""+ caller_info = "unknown caller"+ try:+ import inspect++ frame = inspect.currentframe()+ stack = inspect.getouterframes(frame)++ for i, frame_info in enumerate(stack):+ module = inspect.getmodule(frame_info.frame)+ module_name = module.__name__ if module else "unknown"++ if module_name != "contextlib" and "db.py" not in frame_info.filename:+ caller_module = module_name+ caller_function = frame_info.function+ caller_lineno = frame_info.lineno+ caller_file = frame_info.filename.split("/")[-1]++ caller_info = f"{caller_module}.{caller_function}:{caller_lineno} ({caller_file})"+ break+ except:+ pass+ finally:+ del frame++ self.session_caller_trace(caller_info)+session_factory = self.get_session_factory(name)if not session_factory:raise ValueError(f"No session factory found for '{name}'")@@ -250,6 +276,11 @@ class DatabaseRegistry:finally:await session.close()+ @trace_method+ def session_caller_trace(self, caller_info: str):+ """Trace sync db caller information for debugging purposes."""+ pass # wrapper used for otel tracing only+# Create a singleton instancedb_registry = DatabaseRegistry()diff --git a/letta/server/rest_api/routers/v1/sources.py b/letta/server/rest_api/routers/v1/sources.pyindex bb6694ebb..48049d730 100644--- a/letta/server/rest_api/routers/v1/sources.py+++ b/letta/server/rest_api/routers/v1/sources.py@@ -23,10 +23,10 @@ from letta.schemas.enums import FileProcessingStatusfrom letta.schemas.file import FileMetadatafrom letta.schemas.passage import Passagefrom letta.schemas.source import Source, SourceCreate, SourceUpdate+from letta.schemas.source_metadata import OrganizationSourcesStatsfrom letta.schemas.user import Userfrom letta.server.rest_api.utils import get_letta_serverfrom letta.server.server import SyncServer-from letta.services.file_processor.chunker.llama_index_chunker import LlamaIndexChunkerfrom letta.services.file_processor.embedder.openai_embedder import OpenAIEmbedderfrom letta.services.file_processor.embedder.pinecone_embedder import PineconeEmbedderfrom letta.services.file_processor.file_processor import FileProcessor@@ -95,6 +95,24 @@ async def get_source_id_by_name(return source.id[email protected]("/metadata", response_model=OrganizationSourcesStats, operation_id="get_sources_metadata")+async def get_sources_metadata(+ server: "SyncServer" = Depends(get_letta_server),+ actor_id: Optional[str] = Header(None, alias="user_id"),+):+ """+ Get aggregated metadata for all sources in an organization.++ Returns structured metadata including:+ - Total number of sources+ - Total number of files across all sources+ - Total size of all files+ - Per-source breakdown with file details (file_name, file_size per file)+ """+ actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)+ return await server.file_manager.get_organization_sources_metadata(actor=actor)++@router.get("/", response_model=List[Source], operation_id="list_sources")async def list_sources(server: "SyncServer" = Depends(get_letta_server),@@ -344,7 +362,9 @@ async def get_file_metadata(if should_use_pinecone() and not file_metadata.is_processing_terminal():ids = await list_pinecone_index_for_files(file_id=file_id, actor=actor, limit=file_metadata.total_chunks)- logger.info(f"Embedded chunks {len(ids)}/{file_metadata.total_chunks} for {file_id} in organization {actor.organization_id}")+ logger.info(+ f"Embedded chunks {len(ids)}/{file_metadata.total_chunks} for {file_id} ({file_metadata.file_name}) in organization {actor.organization_id}"+ )if len(ids) != file_metadata.chunks_embedded or len(ids) == file_metadata.total_chunks:if len(ids) != file_metadata.total_chunks:@@ -424,15 +444,12 @@ async def load_file_to_source_cloud(file_metadata: FileMetadata,):file_processor = MistralFileParser()- text_chunker = LlamaIndexChunker(chunk_size=embedding_config.embedding_chunk_size)using_pinecone = should_use_pinecone()if using_pinecone:embedder = PineconeEmbedder()else:embedder = OpenAIEmbedder(embedding_config=embedding_config)- file_processor = FileProcessor(- file_parser=file_processor, text_chunker=text_chunker, embedder=embedder, actor=actor, using_pinecone=using_pinecone- )+ file_processor = FileProcessor(file_parser=file_processor, embedder=embedder, actor=actor, using_pinecone=using_pinecone)await file_processor.process(server=server, agent_states=agent_states, source_id=source_id, content=content, file_metadata=file_metadata)diff --git a/letta/services/file_manager.py b/letta/services/file_manager.pyindex ed1ee7bd7..f4a84fb3c 100644--- a/letta/services/file_manager.py+++ b/letta/services/file_manager.py@@ -16,6 +16,7 @@ from letta.otel.tracing import trace_methodfrom letta.schemas.enums import FileProcessingStatusfrom letta.schemas.file import FileMetadata as PydanticFileMetadatafrom letta.schemas.source import Source as PydanticSource+from letta.schemas.source_metadata import FileStats, OrganizationSourcesStats, SourceStatsfrom letta.schemas.user import User as PydanticUserfrom letta.server.db import db_registryfrom letta.utils import enforce_types@@ -272,3 +273,72 @@ class FileManager:else:# Add numeric suffixreturn f"{source.name}/{base}_({count}){ext}"++ @enforce_types+ @trace_method+ async def get_organization_sources_metadata(self, actor: PydanticUser) -> OrganizationSourcesStats:+ """+ Get aggregated metadata for all sources in an organization with optimized queries.++ Returns structured metadata including:+ - Total number of sources+ - Total number of files across all sources+ - Total size of all files+ - Per-source breakdown with file details+ """+ async with db_registry.async_session() as session:+ # Import here to avoid circular imports+ from letta.orm.source import Source as SourceModel++ # Single optimized query to get all sources with their file aggregations+ query = (+ select(+ SourceModel.id,+ SourceModel.name,+ func.count(FileMetadataModel.id).label("file_count"),+ func.coalesce(func.sum(FileMetadataModel.file_size), 0).label("total_size"),+ )+ .outerjoin(FileMetadataModel, (FileMetadataModel.source_id == SourceModel.id) & (FileMetadataModel.is_deleted == False))+ .where(SourceModel.organization_id == actor.organization_id)+ .where(SourceModel.is_deleted == False)+ .group_by(SourceModel.id, SourceModel.name)+ .order_by(SourceModel.name)+ )++ result = await session.execute(query)+ source_aggregations = result.fetchall()++ # Build response+ metadata = OrganizationSourcesStats()++ for row in source_aggregations:+ source_id, source_name, file_count, total_size = row++ # Get individual file details for this source+ files_query = (+ select(FileMetadataModel.id, FileMetadataModel.file_name, FileMetadataModel.file_size)+ .where(+ FileMetadataModel.source_id == source_id,+ FileMetadataModel.organization_id == actor.organization_id,+ FileMetadataModel.is_deleted == False,+ )+ .order_by(FileMetadataModel.file_name)+ )++ files_result = await session.execute(files_query)+ files_rows = files_result.fetchall()++ # Build file stats+ files = [FileStats(file_id=file_row[0], file_name=file_row[1], file_size=file_row[2]) for file_row in files_rows]++ # Build source metadata+ source_metadata = SourceStats(+ source_id=source_id, source_name=source_name, file_count=file_count, total_size=total_size, files=files+ )++ metadata.sources.append(source_metadata)+ metadata.total_files += file_count+ metadata.total_size += total_size++ metadata.total_sources = len(metadata.sources)+ return metadatadiff --git a/letta/services/file_processor/chunker/line_chunker.py b/letta/services/file_processor/chunker/line_chunker.pyindex 3b49a43f9..c06f024b3 100644--- a/letta/services/file_processor/chunker/line_chunker.py+++ b/letta/services/file_processor/chunker/line_chunker.py@@ -99,7 +99,12 @@ class LineChunker:return [line for line in lines if line.strip()]def chunk_text(- self, file_metadata: FileMetadata, start: Optional[int] = None, end: Optional[int] = None, add_metadata: bool = True+ self,+ file_metadata: FileMetadata,+ start: Optional[int] = None,+ end: Optional[int] = None,+ add_metadata: bool = True,+ validate_range: bool = False,) -> List[str]:"""Content-aware text chunking based on file type"""strategy = self._determine_chunking_strategy(file_metadata)@@ -116,11 +121,31 @@ class LineChunker:content_lines = self._chunk_by_lines(text, preserve_indentation=False)total_chunks = len(content_lines)+ chunk_type = (+ "sentences" if strategy == ChunkingStrategy.DOCUMENTATION else "chunks" if strategy == ChunkingStrategy.PROSE else "lines"+ )++ # Validate range if requested+ if validate_range and (start is not None or end is not None):+ if start is not None and start >= total_chunks:+ # Convert to 1-indexed for user-friendly error message+ start_display = start + 1+ raise ValueError(+ f"File {file_metadata.file_name} has only {total_chunks} lines, but requested offset {start_display} is out of range"+ )++ if start is not None and end is not None and end > total_chunks:+ # Convert to 1-indexed for user-friendly error message+ start_display = start + 1+ end_display = end+ raise ValueError(+ f"File {file_metadata.file_name} has only {total_chunks} lines, but requested range {start_display} to {end_display} extends beyond file bounds"+ )# Handle start/end slicing- if start is not None and end is not None:+ if start is not None or end is not None:content_lines = content_lines[start:end]- line_offset = start+ line_offset = start if start is not None else 0else:line_offset = 0@@ -129,14 +154,15 @@ class LineChunker:# Add metadata about total chunksif add_metadata:- chunk_type = (- "sentences" if strategy == ChunkingStrategy.DOCUMENTATION else "chunks" if strategy == ChunkingStrategy.PROSE else "lines"- )if start is not None and end is not None:# Display 1-indexed ranges for usersstart_display = start + 1end_display = endcontent_lines.insert(0, f"[Viewing {chunk_type} {start_display} to {end_display} (out of {total_chunks} {chunk_type})]")+ elif start is not None:+ # Only start specified - viewing from start to end+ start_display = start + 1+ content_lines.insert(0, f"[Viewing {chunk_type} {start_display} to end (out of {total_chunks} {chunk_type})]")else:content_lines.insert(0, f"[Viewing file start (out of {total_chunks} {chunk_type})]")diff --git a/letta/services/file_processor/chunker/llama_index_chunker.py b/letta/services/file_processor/chunker/llama_index_chunker.pyindex e62b9ceb6..ab6ea4a61 100644--- a/letta/services/file_processor/chunker/llama_index_chunker.py+++ b/letta/services/file_processor/chunker/llama_index_chunker.py@@ -1,119 +1,169 @@-from typing import List, Tuple+from typing import List, Optional, Unionfrom mistralai import OCRPageObjectfrom letta.log import get_loggerfrom letta.otel.tracing import trace_method+from letta.services.file_processor.file_types import ChunkingStrategy, file_type_registrylogger = get_logger(__name__)class LlamaIndexChunker:- """LlamaIndex-based text chunking"""+ """LlamaIndex-based text chunking with automatic splitter selection"""- def __init__(self, chunk_size: int = 512, chunk_overlap: int = 50):+ # Conservative default chunk sizes for fallback scenarios+ DEFAULT_CONSERVATIVE_CHUNK_SIZE = 384+ DEFAULT_CONSERVATIVE_CHUNK_OVERLAP = 25++ def __init__(self, chunk_size: int = 512, chunk_overlap: int = 50, file_type: Optional[str] = None):self.chunk_size = chunk_sizeself.chunk_overlap = chunk_overlap+ self.file_type = file_type- from llama_index.core.node_parser import SentenceSplitter+ # Create appropriate parser based on file type+ self.parser = self._create_parser_for_file_type(file_type, chunk_size, chunk_overlap)- self.parser = SentenceSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)+ # Log which parser was selected+ parser_name = type(self.parser).__name__+ logger.info(f"LlamaIndexChunker initialized with {parser_name} for file type: {file_type}")++ def _create_parser_for_file_type(self, file_type: Optional[str], chunk_size: int, chunk_overlap: int):+ """Create appropriate parser based on file type"""+ if not file_type:+ # Default fallback+ from llama_index.core.node_parser import SentenceSplitter++ return SentenceSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)- # TODO: Make this more general beyond Mistral- @trace_method- def chunk_text(self, page: OCRPageObject) -> List[str]:- """Chunk text using LlamaIndex splitter"""try:- return self.parser.split_text(page.markdown)+ # Get chunking strategy from file type registry+ chunking_strategy = file_type_registry.get_chunking_strategy_by_mime_type(file_type)+ logger.debug(f"Chunking strategy for {file_type}: {chunking_strategy}")- except Exception as e:- logger.error(f"Chunking failed: {str(e)}")- raise+ if chunking_strategy == ChunkingStrategy.CODE:+ from llama_index.core.node_parser import CodeSplitter+ return CodeSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)-class MarkdownChunker:- """Markdown-specific chunker that preserves line numbers for citation purposes"""+ elif chunking_strategy == ChunkingStrategy.DOCUMENTATION:+ if file_type in ["text/markdown", "text/x-markdown"]:+ from llama_index.core.node_parser import MarkdownNodeParser- def __init__(self, chunk_size: int = 2048):- self.chunk_size = chunk_size- # No overlap for line-based citations to avoid ambiguity+ return MarkdownNodeParser()+ elif file_type in ["text/html"]:+ from llama_index.core.node_parser import HTMLNodeParser++ return HTMLNodeParser()+ else:+ # Fall back to sentence splitter for other documentation+ from llama_index.core.node_parser import SentenceSplitter++ return SentenceSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)++ elif chunking_strategy == ChunkingStrategy.STRUCTURED_DATA:+ if file_type in ["application/json", "application/jsonl"]:+ from llama_index.core.node_parser import JSONNodeParser- from llama_index.core.node_parser import MarkdownNodeParser+ return JSONNodeParser()+ else:+ # Fall back to sentence splitter for other structured data+ from llama_index.core.node_parser import SentenceSplitter- self.parser = MarkdownNodeParser()+ return SentenceSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)- def chunk_markdown_with_line_numbers(self, markdown_content: str) -> List[Tuple[str, int, int]]:- """- Chunk markdown content while preserving line number mappings.+ else:+ # Default to sentence splitter for PROSE and LINE_BASED+ from llama_index.core.node_parser import SentenceSplitter- Returns:- List of tuples: (chunk_text, start_line, end_line)- """+ return SentenceSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)++ except Exception as e:+ logger.warning(f"Failed to create specialized parser for {file_type}: {str(e)}. Using default SentenceSplitter.")+ from llama_index.core.node_parser import SentenceSplitter++ return SentenceSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)++ @trace_method+ def chunk_text(self, content: Union[OCRPageObject, str]) -> List[str]:+ """Chunk text using LlamaIndex splitter"""try:- # Split content into lines for line number tracking- lines = markdown_content.split("\n")+ # Handle different input types+ if isinstance(content, OCRPageObject):+ # Extract markdown from OCR page object+ text_content = content.markdown+ else:+ # Assume it's a string+ text_content = content++ # Use the selected parser+ if hasattr(self.parser, "split_text"):+ # Most parsers have split_text method+ return self.parser.split_text(text_content)+ elif hasattr(self.parser, "get_nodes_from_documents"):+ # Some parsers need Document objects+ from llama_index.core import Document+ from llama_index.core.node_parser import SentenceSplitter++ document = Document(text=text_content)+ nodes = self.parser.get_nodes_from_documents([document])++ # Further split nodes that exceed chunk_size using SentenceSplitter+ final_chunks = []+ sentence_splitter = SentenceSplitter(chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap)++ for node in nodes:+ if len(node.text) > self.chunk_size:+ # Split oversized nodes with sentence splitter+ sub_chunks = sentence_splitter.split_text(node.text)+ final_chunks.extend(sub_chunks)+ else:+ final_chunks.append(node.text)++ return final_chunks+ else:+ # Fallback - try to call the parser directly+ return self.parser(text_content)- # Create nodes using MarkdownNodeParser- from llama_index.core import Document+ except Exception as e:+ logger.error(f"Chunking failed with {type(self.parser).__name__}: {str(e)}")+ # Try fallback with SentenceSplitter+ try:+ logger.info("Attempting fallback to SentenceSplitter")+ from llama_index.core.node_parser import SentenceSplitter- document = Document(text=markdown_content)- nodes = self.parser.get_nodes_from_documents([document])+ fallback_parser = SentenceSplitter(chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap)- chunks_with_line_numbers = []+ # Extract text content if needed+ if isinstance(content, OCRPageObject):+ text_content = content.markdown+ else:+ text_content = content- for node in nodes:- chunk_text = node.text+ return fallback_parser.split_text(text_content)+ except Exception as fallback_error:+ logger.error(f"Fallback chunking also failed: {str(fallback_error)}")+ raise e # Raise the original error- # Find the line numbers for this chunk- start_line, end_line = self._find_line_numbers(chunk_text, lines)+ @trace_method+ def default_chunk_text(self, content: Union[OCRPageObject, str], chunk_size: int = None, chunk_overlap: int = None) -> List[str]:+ """Chunk text using default SentenceSplitter regardless of file type with conservative defaults"""+ try:+ from llama_index.core.node_parser import SentenceSplitter- chunks_with_line_numbers.append((chunk_text, start_line, end_line))+ # Use provided defaults or fallback to conservative values+ chunk_size = chunk_size if chunk_size is not None else self.DEFAULT_CONSERVATIVE_CHUNK_SIZE+ chunk_overlap = chunk_overlap if chunk_overlap is not None else self.DEFAULT_CONSERVATIVE_CHUNK_OVERLAP+ default_parser = SentenceSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)- return chunks_with_line_numbers+ # Handle different input types+ if isinstance(content, OCRPageObject):+ text_content = content.markdown+ else:+ text_content = content++ return default_parser.split_text(text_content)except Exception as e:- logger.error(f"Markdown chunking failed: {str(e)}")- # Fallback to simple line-based chunking- return self._fallback_line_chunking(markdown_content)-- def _find_line_numbers(self, chunk_text: str, lines: List[str]) -> Tuple[int, int]:- """Find the start and end line numbers for a given chunk of text."""- chunk_lines = chunk_text.split("\n")-- # Find the first line of the chunk in the original document- start_line = 1- for i, line in enumerate(lines):- if chunk_lines[0].strip() in line.strip() and len(chunk_lines[0].strip()) > 10: # Avoid matching short lines- start_line = i + 1- break-- # Calculate end line- end_line = start_line + len(chunk_lines) - 1-- return start_line, min(end_line, len(lines))-- def _fallback_line_chunking(self, markdown_content: str) -> List[Tuple[str, int, int]]:- """Fallback chunking method that simply splits by lines with no overlap."""- lines = markdown_content.split("\n")- chunks = []-- i = 0- while i < len(lines):- chunk_lines = []- start_line = i + 1- char_count = 0-- # Build chunk until we hit size limit- while i < len(lines) and char_count < self.chunk_size:- line = lines[i]- chunk_lines.append(line)- char_count += len(line) + 1 # +1 for newline- i += 1-- end_line = i- chunk_text = "\n".join(chunk_lines)- chunks.append((chunk_text, start_line, end_line))-- # No overlap - continue from where we left off-- return chunks+ logger.error(f"Default chunking failed: {str(e)}")+ raisediff --git a/letta/services/file_processor/embedder/openai_embedder.py b/letta/services/file_processor/embedder/openai_embedder.pyindex ce43a72d4..5a888549c 100644--- a/letta/services/file_processor/embedder/openai_embedder.py+++ b/letta/services/file_processor/embedder/openai_embedder.py@@ -91,7 +91,7 @@ class OpenAIEmbedder(BaseEmbedder):try:return await self._embed_batch(batch, indices)except Exception as e:- logger.error(f"Failed to embed batch of size {len(batch)}: {str(e)}")+ logger.error("Failed to embed batch of size %s: %s", len(batch), e)log_event("embedder.batch_failed", {"batch_size": len(batch), "error": str(e), "error_type": type(e).__name__})raisediff --git a/letta/services/file_processor/file_processor.py b/letta/services/file_processor/file_processor.pyindex cbf70b1c4..07b324c97 100644--- a/letta/services/file_processor/file_processor.py+++ b/letta/services/file_processor/file_processor.py@@ -26,14 +26,12 @@ class FileProcessor:def __init__(self,file_parser: MistralFileParser,- text_chunker: LlamaIndexChunker,embedder: BaseEmbedder,actor: User,using_pinecone: bool,max_file_size: int = 50 * 1024 * 1024, # 50MB default):self.file_parser = file_parser- self.text_chunker = text_chunkerself.line_chunker = LineChunker()self.embedder = embedderself.max_file_size = max_file_size@@ -44,6 +42,61 @@ class FileProcessor:self.actor = actorself.using_pinecone = using_pinecone+ async def _chunk_and_embed_with_fallback(self, file_metadata: FileMetadata, ocr_response, source_id: str) -> List:+ """Chunk text and generate embeddings with fallback to default chunker if needed"""+ filename = file_metadata.file_name++ # Create file-type-specific chunker+ text_chunker = LlamaIndexChunker(file_type=file_metadata.file_type)++ # First attempt with file-specific chunker+ try:+ all_chunks = []+ for page in ocr_response.pages:+ chunks = text_chunker.chunk_text(page)+ if not chunks:+ log_event("file_processor.chunking_failed", {"filename": filename, "page_index": ocr_response.pages.index(page)})+ raise ValueError("No chunks created from text")+ all_chunks.extend(chunks)++ all_passages = await self.embedder.generate_embedded_passages(+ file_id=file_metadata.id, source_id=source_id, chunks=all_chunks, actor=self.actor+ )+ return all_passages++ except Exception as e:+ logger.warning(f"Failed to chunk/embed with file-specific chunker for {filename}: {str(e)}. Retrying with default chunker.")+ log_event("file_processor.embedding_failed_retrying", {"filename": filename, "error": str(e), "error_type": type(e).__name__})++ # Retry with default chunker+ try:+ logger.info(f"Retrying chunking with default SentenceSplitter for {filename}")+ all_chunks = []++ for page in ocr_response.pages:+ chunks = text_chunker.default_chunk_text(page)+ if not chunks:+ log_event(+ "file_processor.default_chunking_failed", {"filename": filename, "page_index": ocr_response.pages.index(page)}+ )+ raise ValueError("No chunks created from text with default chunker")+ all_chunks.extend(chunks)++ all_passages = await self.embedder.generate_embedded_passages(+ file_id=file_metadata.id, source_id=source_id, chunks=all_chunks, actor=self.actor+ )+ logger.info(f"Successfully generated passages with default chunker for {filename}")+ log_event("file_processor.default_chunking_success", {"filename": filename, "total_chunks": len(all_chunks)})+ return all_passages++ except Exception as fallback_error:+ logger.error("Default chunking also failed for %s: %s", filename, fallback_error)+ log_event(+ "file_processor.default_chunking_also_failed",+ {"filename": filename, "fallback_error": str(fallback_error), "fallback_error_type": type(fallback_error).__name__},+ )+ raise fallback_error+# TODO: Factor this function out of SyncServer@trace_methodasync def process(@@ -111,19 +164,10 @@ class FileProcessor:logger.info("Chunking extracted text")log_event("file_processor.chunking_started", {"filename": filename, "pages_to_process": len(ocr_response.pages)})- all_chunks = []-- for page in ocr_response.pages:- chunks = self.text_chunker.chunk_text(page)- if not chunks:- log_event("file_processor.chunking_failed", {"filename": filename, "page_index": ocr_response.pages.index(page)})- raise ValueError("No chunks created from text")-- all_chunks.extend(self.text_chunker.chunk_text(page))-- all_passages = await self.embedder.generate_embedded_passages(- file_id=file_metadata.id, source_id=source_id, chunks=all_chunks, actor=self.actor+ # Chunk and embed with fallback logic+ all_passages = await self._chunk_and_embed_with_fallback(+ file_metadata=file_metadata, ocr_response=ocr_response, source_id=source_id)if not self.using_pinecone:@@ -156,7 +200,7 @@ class FileProcessor:return all_passagesexcept Exception as e:- logger.error(f"File processing failed for {filename}: {str(e)}")+ logger.error("File processing failed for %s: %s", filename, e)log_event("file_processor.processing_failed",{diff --git a/letta/services/tool_executor/files_tool_executor.py b/letta/services/tool_executor/files_tool_executor.pyindex 1de53724e..4815243a4 100644--- a/letta/services/tool_executor/files_tool_executor.py+++ b/letta/services/tool_executor/files_tool_executor.py@@ -175,7 +175,7 @@ class LettaFileToolExecutor(ToolExecutor):file = await self.file_manager.get_file_by_id(file_id=file_id, actor=self.actor, include_content=True)# Process file content- content_lines = LineChunker().chunk_text(file_metadata=file, start=start, end=end)+ content_lines = LineChunker().chunk_text(file_metadata=file, start=start, end=end, validate_range=True)visible_content = "\n".join(content_lines)# Handle LRU eviction and file openingdiff --git a/letta/settings.py b/letta/settings.pyindex fb2751c30..7a86605ef 100644--- a/letta/settings.py+++ b/letta/settings.py@@ -247,7 +247,7 @@ class Settings(BaseSettings):# cron job parametersenable_batch_job_polling: bool = Falsepoll_running_llm_batches_interval_seconds: int = 5 * 60- poll_lock_retry_interval_seconds: int = 5 * 60+ poll_lock_retry_interval_seconds: int = 8 * 60batch_job_polling_lookback_weeks: int = 2batch_job_polling_batch_size: Optional[int] = Nonediff --git a/tests/test_managers.py b/tests/test_managers.pyindex 9edc8af31..8b88000ae 100644--- a/tests/test_managers.py+++ b/tests/test_managers.py@@ -5299,6 +5299,95 @@ async def test_upsert_file_content_basic(server: SyncServer, default_user, defauassert orm_file.updated_at > orm_file.created_at+async def test_get_organization_sources_metadata(server, default_user):+ """Test getting organization sources metadata with aggregated file information."""+ # Create test sources+ source1 = await server.source_manager.create_source(+ source=PydanticSource(+ name="test_source_1",+ embedding_config=DEFAULT_EMBEDDING_CONFIG,+ ),+ actor=default_user,+ )++ source2 = await server.source_manager.create_source(+ source=PydanticSource(+ name="test_source_2",+ embedding_config=DEFAULT_EMBEDDING_CONFIG,+ ),+ actor=default_user,+ )++ # Create test files for source1+ file1_meta = PydanticFileMetadata(+ source_id=source1.id,+ file_name="file1.txt",+ file_type="text/plain",+ file_size=1024,+ )+ file1 = await server.file_manager.create_file(file_metadata=file1_meta, actor=default_user)++ file2_meta = PydanticFileMetadata(+ source_id=source1.id,+ file_name="file2.txt",+ file_type="text/plain",+ file_size=2048,+ )+ file2 = await server.file_manager.create_file(file_metadata=file2_meta, actor=default_user)++ # Create test file for source2+ file3_meta = PydanticFileMetadata(+ source_id=source2.id,+ file_name="file3.txt",+ file_type="text/plain",+ file_size=512,+ )+ file3 = await server.file_manager.create_file(file_metadata=file3_meta, actor=default_user)++ # Get organization metadata+ metadata = await server.file_manager.get_organization_sources_metadata(actor=default_user)++ # Verify top-level aggregations+ assert metadata.total_sources >= 2 # May have other sources from other tests+ assert metadata.total_files >= 3+ assert metadata.total_size >= 3584++ # Find our test sources in the results+ source1_meta = next((s for s in metadata.sources if s.source_id == source1.id), None)+ source2_meta = next((s for s in metadata.sources if s.source_id == source2.id), None)++ assert source1_meta is not None+ assert source1_meta.source_name == "test_source_1"+ assert source1_meta.file_count == 2+ assert source1_meta.total_size == 3072 # 1024 + 2048+ assert len(source1_meta.files) == 2++ # Verify file details in source1+ file1_stats = next((f for f in source1_meta.files if f.file_id == file1.id), None)+ file2_stats = next((f for f in source1_meta.files if f.file_id == file2.id), None)++ assert file1_stats is not None+ assert file1_stats.file_name == "file1.txt"+ assert file1_stats.file_size == 1024++ assert file2_stats is not None+ assert file2_stats.file_name == "file2.txt"+ assert file2_stats.file_size == 2048++ assert source2_meta is not None+ assert source2_meta.source_name == "test_source_2"+ assert source2_meta.file_count == 1+ assert source2_meta.total_size == 512+ assert len(source2_meta.files) == 1++ # Verify file details in source2+ file3_stats = source2_meta.files[0]+ assert file3_stats.file_id == file3.id+ assert file3_stats.file_name == "file3.txt"+ assert file3_stats.file_size == 512++# ======================================================================================================================# SandboxConfigManager Tests - Sandbox Configs# ======================================================================================================================diff --git a/tests/test_utils.py b/tests/test_utils.pyindex 1af23e62c..5b0e724cb 100644--- a/tests/test_utils.py+++ b/tests/test_utils.py@@ -2,6 +2,8 @@ import pytestfrom letta.constants import MAX_FILENAME_LENGTHfrom letta.functions.ast_parsers import coerce_dict_args_by_annotations, get_function_annotations_from_source+from letta.schemas.file import FileMetadata+from letta.services.file_processor.chunker.line_chunker import LineChunkerfrom letta.services.helpers.agent_manager_helper import safe_formatfrom letta.utils import sanitize_filename@@ -407,3 +409,103 @@ def test_formatter():"""assert UNUSED_AND_EMPRY_VAR_SOL == safe_format(UNUSED_AND_EMPRY_VAR, VARS_DICT)+++# ---------------------- LineChunker TESTS ---------------------- #+++def test_line_chunker_valid_range():+ """Test that LineChunker works correctly with valid ranges"""+ file = FileMetadata(file_name="test.py", source_id="test_source", content="line1\nline2\nline3\nline4")+ chunker = LineChunker()++ # Test valid range with validation+ result = chunker.chunk_text(file, start=1, end=3, validate_range=True)+ # Should return lines 2 and 3 (0-indexed 1:3)+ assert "[Viewing lines 2 to 3 (out of 4 lines)]" in result[0]+ assert "2: line2" in result[1]+ assert "3: line3" in result[2]+++def test_line_chunker_valid_range_no_validation():+ """Test that LineChunker works the same without validation for valid ranges"""+ file = FileMetadata(file_name="test.py", source_id="test_source", content="line1\nline2\nline3\nline4")+ chunker = LineChunker()++ # Test same range without validation+ result = chunker.chunk_text(file, start=1, end=3, validate_range=False)+ assert "[Viewing lines 2 to 3 (out of 4 lines)]" in result[0]+ assert "2: line2" in result[1]+ assert "3: line3" in result[2]+++def test_line_chunker_out_of_range_start():+ """Test that LineChunker throws error when start is out of range"""+ file = FileMetadata(file_name="test.py", source_id="test_source", content="line1\nline2\nline3")+ chunker = LineChunker()++ # Test with start beyond file length (3 lines, requesting start=5 which is 0-indexed 4)+ with pytest.raises(ValueError, match="File test.py has only 3 lines, but requested offset 6 is out of range"):+ chunker.chunk_text(file, start=5, end=6, validate_range=True)+++def test_line_chunker_out_of_range_end():+ """Test that LineChunker throws error when end extends beyond file bounds"""+ file = FileMetadata(file_name="test.py", source_id="test_source", content="line1\nline2\nline3")+ chunker = LineChunker()++ # Test with end beyond file length (3 lines, requesting 1 to 10)+ with pytest.raises(ValueError, match="File test.py has only 3 lines, but requested range 1 to 10 extends beyond file bounds"):+ chunker.chunk_text(file, start=0, end=10, validate_range=True)+++def test_line_chunker_edge_case_empty_file():+ """Test that LineChunker handles empty files correctly"""+ file = FileMetadata(file_name="empty.py", source_id="test_source", content="")+ chunker = LineChunker()++ # Test requesting lines from empty file+ with pytest.raises(ValueError, match="File empty.py has only 0 lines, but requested offset 1 is out of range"):+ chunker.chunk_text(file, start=0, end=1, validate_range=True)+++def test_line_chunker_edge_case_single_line():+ """Test that LineChunker handles single line files correctly"""+ file = FileMetadata(file_name="single.py", source_id="test_source", content="only line")+ chunker = LineChunker()++ # Test valid single line access+ result = chunker.chunk_text(file, start=0, end=1, validate_range=True)+ assert "1: only line" in result[1]++ # Test out of range for single line file+ with pytest.raises(ValueError, match="File single.py has only 1 lines, but requested offset 2 is out of range"):+ chunker.chunk_text(file, start=1, end=2, validate_range=True)+++def test_line_chunker_validation_disabled_allows_out_of_range():+ """Test that when validation is disabled, out of range silently returns partial results"""+ file = FileMetadata(file_name="test.py", source_id="test_source", content="line1\nline2\nline3")+ chunker = LineChunker()++ # Test with validation disabled - should not raise error+ result = chunker.chunk_text(file, start=5, end=10, validate_range=False)+ # Should return empty content (except metadata header) since slice is out of bounds+ assert len(result) == 1 # Only metadata header+ assert "[Viewing lines 6 to 10 (out of 3 lines)]" in result[0]+++def test_line_chunker_only_start_parameter():+ """Test validation with only start parameter specified"""+ file = FileMetadata(file_name="test.py", source_id="test_source", content="line1\nline2\nline3")+ chunker = LineChunker()++ # Test valid start only+ result = chunker.chunk_text(file, start=1, validate_range=True)+ assert "[Viewing lines 2 to end (out of 3 lines)]" in result[0]+ assert "2: line2" in result[1]+ assert "3: line3" in result[2]++ # Test invalid start only+ with pytest.raises(ValueError, match="File test.py has only 3 lines, but requested offset 4 is out of range"):+ chunker.chunk_text(file, start=3, validate_range=True)