The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
core: docstrings `indexing` (#23785)
Leonid Ganeline716a31665497
diff --git a/libs/core/langchain_core/indexing/__init__.py b/libs/core/langchain_core/indexing/__init__.pyindex bc581297d7..3643b13041 100644--- a/libs/core/langchain_core/indexing/__init__.py+++ b/libs/core/langchain_core/indexing/__init__.py@@ -4,6 +4,7 @@ This package contains helper logic to help deal with indexing data intoa vectorstore while avoiding duplicated content and over-writing contentif it's unchanged."""+from langchain_core.indexing.api import IndexingResult, aindex, indexfrom langchain_core.indexing.base import InMemoryRecordManager, RecordManagerdiff --git a/libs/core/langchain_core/indexing/api.py b/libs/core/langchain_core/indexing/api.pyindex 86741ae3a2..0462e0838b 100644--- a/libs/core/langchain_core/indexing/api.py+++ b/libs/core/langchain_core/indexing/api.py@@ -1,4 +1,5 @@"""Module contains logic for indexing documents into vector stores."""+from __future__ import annotationsimport hashlib@@ -232,8 +233,8 @@ def index(record_manager: Timestamped set to keep track of which documents wereupdated.vector_store: Vector store to index the documents into.- batch_size: Batch size to use when indexing.- cleanup: How to handle clean up of documents.+ batch_size: Batch size to use when indexing. Default is 100.+ cleanup: How to handle clean up of documents. Default is None.- Incremental: Cleans up all documents that haven't been updated ANDthat are associated with source ids that were seenduring indexing.@@ -246,14 +247,23 @@ def index(This means that users may see duplicated content during indexing.- None: Do not delete any documents.source_id_key: Optional key that helps identify the original source- of the document.+ of the document. Default is None.cleanup_batch_size: Batch size to use when cleaning up documents.+ Default is 1_000.force_update: Force update documents even if they are present in therecord manager. Useful if you are re-indexing with updated embeddings.+ Default is False.Returns:Indexing result which contains information about how many documentswere added, updated, deleted, or skipped.++ Raises:+ ValueError: If cleanup mode is not one of 'incremental', 'full' or None+ ValueError: If cleanup mode is incremental and source_id_key is None.+ ValueError: If vectorstore does not have+ "delete" and "add_documents" required methods.+ ValueError: If source_id_key is not None, but is not a string or callable."""if cleanup not in {"incremental", "full", None}:raise ValueError(@@ -415,7 +425,7 @@ async def aindex(cleanup_batch_size: int = 1_000,force_update: bool = False,) -> IndexingResult:- """Index data from the loader into the vector store.+ """Async index data from the loader into the vector store.Indexing functionality uses a manager to keep track of which documentsare in the vector store.@@ -437,8 +447,8 @@ async def aindex(record_manager: Timestamped set to keep track of which documents wereupdated.vector_store: Vector store to index the documents into.- batch_size: Batch size to use when indexing.- cleanup: How to handle clean up of documents.+ batch_size: Batch size to use when indexing. Default is 100.+ cleanup: How to handle clean up of documents. Default is None.- Incremental: Cleans up all documents that haven't been updated ANDthat are associated with source ids that were seenduring indexing.@@ -450,14 +460,23 @@ async def aindex(This means that users may see duplicated content during indexing.- None: Do not delete any documents.source_id_key: Optional key that helps identify the original source- of the document.+ of the document. Default is None.cleanup_batch_size: Batch size to use when cleaning up documents.+ Default is 1_000.force_update: Force update documents even if they are present in therecord manager. Useful if you are re-indexing with updated embeddings.+ Default is False.Returns:Indexing result which contains information about how many documentswere added, updated, deleted, or skipped.++ Raises:+ ValueError: If cleanup mode is not one of 'incremental', 'full' or None+ ValueError: If cleanup mode is incremental and source_id_key is None.+ ValueError: If vectorstore does not have+ "adelete" and "aadd_documents" required methods.+ ValueError: If source_id_key is not None, but is not a string or callable."""if cleanup not in {"incremental", "full", None}:diff --git a/libs/core/langchain_core/indexing/base.py b/libs/core/langchain_core/indexing/base.pyindex aae9a32824..15912358ba 100644--- a/libs/core/langchain_core/indexing/base.py+++ b/libs/core/langchain_core/indexing/base.py@@ -37,7 +37,7 @@ class RecordManager(ABC):2. The record manager is currently implemented separately from thevectorstore, which means that the overall system becomes distributedand may create issues with consistency. For example, writing to- record manager succeeds but corresponding writing to vectorstore fails.+ record manager succeeds, but corresponding writing to vectorstore fails."""def __init__(@@ -227,6 +227,11 @@ class InMemoryRecordManager(RecordManager):"""An in-memory record manager for testing purposes."""def __init__(self, namespace: str) -> None:+ """Initialize the in-memory record manager.++ Args:+ namespace (str): The namespace for the record manager.+ """super().__init__(namespace)# Each key points to a dictionary# of {'group_id': group_id, 'updated_at': timestamp}@@ -237,14 +242,16 @@ class InMemoryRecordManager(RecordManager):"""In-memory schema creation is simply ensuring the structure is initialized."""async def acreate_schema(self) -> None:- """In-memory schema creation is simply ensuring the structure is initialized."""+ """Async in-memory schema creation is simply ensuring+ the structure is initialized.+ """def get_time(self) -> float:"""Get the current server time as a high resolution timestamp!"""return time.time()async def aget_time(self) -> float:- """Get the current server time as a high resolution timestamp!"""+ """Async get the current server time as a high resolution timestamp!"""return self.get_time()def update(@@ -254,6 +261,27 @@ class InMemoryRecordManager(RecordManager):group_ids: Optional[Sequence[Optional[str]]] = None,time_at_least: Optional[float] = None,) -> None:+ """Upsert records into the database.++ Args:+ keys: A list of record keys to upsert.+ group_ids: A list of group IDs corresponding to the keys.+ Defaults to None.+ time_at_least: Optional timestamp. Implementation can use this+ to optionally verify that the timestamp IS at least this time+ in the system that stores. Defaults to None.+ E.g., use to validate that the time in the postgres database+ is equal to or larger than the given timestamp, if not+ raise an error.+ This is meant to help prevent time-drift issues since+ time may not be monotonically increasing!++ Raises:+ ValueError: If the length of keys doesn't match the length of group+ ids.+ ValueError: If time_at_least is in the future.+ """+if group_ids and len(keys) != len(group_ids):raise ValueError("Length of keys must match length of group_ids")for index, key in enumerate(keys):@@ -269,12 +297,48 @@ class InMemoryRecordManager(RecordManager):group_ids: Optional[Sequence[Optional[str]]] = None,time_at_least: Optional[float] = None,) -> None:+ """Async upsert records into the database.++ Args:+ keys: A list of record keys to upsert.+ group_ids: A list of group IDs corresponding to the keys.+ Defaults to None.+ time_at_least: Optional timestamp. Implementation can use this+ to optionally verify that the timestamp IS at least this time+ in the system that stores. Defaults to None.+ E.g., use to validate that the time in the postgres database+ is equal to or larger than the given timestamp, if not+ raise an error.+ This is meant to help prevent time-drift issues since+ time may not be monotonically increasing!++ Raises:+ ValueError: If the length of keys doesn't match the length of group+ ids.+ ValueError: If time_at_least is in the future.+ """self.update(keys, group_ids=group_ids, time_at_least=time_at_least)def exists(self, keys: Sequence[str]) -> List[bool]:+ """Check if the provided keys exist in the database.++ Args:+ keys: A list of keys to check.++ Returns:+ A list of boolean values indicating the existence of each key.+ """return [key in self.records for key in keys]async def aexists(self, keys: Sequence[str]) -> List[bool]:+ """Async check if the provided keys exist in the database.++ Args:+ keys: A list of keys to check.++ Returns:+ A list of boolean values indicating the existence of each key.+ """return self.exists(keys)def list_keys(@@ -285,6 +349,21 @@ class InMemoryRecordManager(RecordManager):group_ids: Optional[Sequence[str]] = None,limit: Optional[int] = None,) -> List[str]:+ """List records in the database based on the provided filters.++ Args:+ before: Filter to list records updated before this time.+ Defaults to None.+ after: Filter to list records updated after this time.+ Defaults to None.+ group_ids: Filter to list records with specific group IDs.+ Defaults to None.+ limit: optional limit on the number of records to return.+ Defaults to None.++ Returns:+ A list of keys for the matching records.+ """result = []for key, data in self.records.items():if before and data["updated_at"] >= before:@@ -306,14 +385,39 @@ class InMemoryRecordManager(RecordManager):group_ids: Optional[Sequence[str]] = None,limit: Optional[int] = None,) -> List[str]:+ """Async list records in the database based on the provided filters.++ Args:+ before: Filter to list records updated before this time.+ Defaults to None.+ after: Filter to list records updated after this time.+ Defaults to None.+ group_ids: Filter to list records with specific group IDs.+ Defaults to None.+ limit: optional limit on the number of records to return.+ Defaults to None.++ Returns:+ A list of keys for the matching records.+ """return self.list_keys(before=before, after=after, group_ids=group_ids, limit=limit)def delete_keys(self, keys: Sequence[str]) -> None:+ """Delete specified records from the database.++ Args:+ keys: A list of keys to delete.+ """for key in keys:if key in self.records:del self.records[key]async def adelete_keys(self, keys: Sequence[str]) -> None:+ """Async delete specified records from the database.++ Args:+ keys: A list of keys to delete.+ """self.delete_keys(keys)diff --git a/libs/core/langchain_core/language_models/__init__.py b/libs/core/langchain_core/language_models/__init__.pyindex daa481ba2d..4781f98cc9 100644--- a/libs/core/langchain_core/language_models/__init__.py+++ b/libs/core/langchain_core/language_models/__init__.py@@ -1,15 +1,15 @@"""**Language Model** is a type of model that can generate text or complete-text prompts.+text prompts.-LangChain has two main classes to work with language models: **Chat Models**+LangChain has two main classes to work with language models: **Chat Models**and "old-fashioned" **LLMs**.## Chat Models-Language models that use a sequence of messages as inputs and return chat messages+Language models that use a sequence of messages as inputs and return chat messagesas outputs (as opposed to using plain text). These are traditionally newer models (-older models are generally LLMs, see below). Chat models support the assignment of-distinct roles to conversation messages, helping to distinguish messages from the AI,+older models are generally LLMs, see below). Chat models support the assignment of+distinct roles to conversation messages, helping to distinguish messages from the AI,users, and instructions such as system messages.The key abstraction for chat models is `BaseChatModel`. Implementations@@ -23,15 +23,15 @@ https://python.langchain.com/v0.2/docs/how_to/custom_chat_model/## LLMs-Language models that takes a string as input and returns a string.+Language models that takes a string as input and returns a string.These are traditionally older models (newer models generally are Chat Models, see below).-Although the underlying models are string in, string out, the LangChain wrappers-also allow these models to take messages as input. This gives them the same interface-as Chat Models. When messages are passed in as input, they will be formatted into a+Although the underlying models are string in, string out, the LangChain wrappers+also allow these models to take messages as input. This gives them the same interface+as Chat Models. When messages are passed in as input, they will be formatted into astring under the hood before being passed to the underlying model.-To implement a custom LLM, inherit from `BaseLLM` or `LLM`.+To implement a custom LLM, inherit from `BaseLLM` or `LLM`.Please see the following guide for more information on how to implement a custom LLM:https://python.langchain.com/v0.2/docs/how_to/custom_llm/diff --git a/libs/core/langchain_core/language_models/base.py b/libs/core/langchain_core/language_models/base.pyindex 52bd626177..262a3b4e54 100644--- a/libs/core/langchain_core/language_models/base.py+++ b/libs/core/langchain_core/language_models/base.py@@ -39,6 +39,11 @@ if TYPE_CHECKING:@lru_cache(maxsize=None) # Cache the tokenizerdef get_tokenizer() -> Any:+ """Get a GPT-2 tokenizer instance.++ This function is cached to avoid re-loading the tokenizer+ every time it is called.+ """try:from transformers import GPT2TokenizerFast # type: ignore[import]except ImportError:@@ -77,7 +82,7 @@ class BaseLanguageModel():"""Abstract base class for interfacing with language models.- All language model wrappers inherit from BaseLanguageModel.+ All language model wrappers inherited from BaseLanguageModel."""cache: Union[BaseCache, bool, None] = None@@ -108,6 +113,12 @@ class BaseLanguageModel("""If verbose is None, set it.This allows users to pass in None as verbose to access the global setting.++ Args:+ verbose: The verbosity setting to use.++ Returns:+ The verbosity setting to use."""if verbose is None:return _get_verbosity()@@ -324,7 +335,7 @@ class BaseLanguageModel(def get_num_tokens(self, text: str) -> int:"""Get the number of tokens present in the text.- Useful for checking if an input will fit in a model's context window.+ Useful for checking if an input fits in a model's context window.Args:text: The string input to tokenize.@@ -337,7 +348,7 @@ class BaseLanguageModel(def get_num_tokens_from_messages(self, messages: List[BaseMessage]) -> int:"""Get the number of tokens in the messages.- Useful for checking if an input will fit in a model's context window.+ Useful for checking if an input fits in a model's context window.Args:messages: The message inputs to tokenize.diff --git a/libs/core/langchain_core/language_models/chat_models.py b/libs/core/langchain_core/language_models/chat_models.pyindex 8dd6dcc771..13c68dded9 100644--- a/libs/core/langchain_core/language_models/chat_models.py+++ b/libs/core/langchain_core/language_models/chat_models.py@@ -68,16 +68,31 @@ if TYPE_CHECKING:class LangSmithParams(TypedDict, total=False):+ """LangSmith parameters for tracing."""+ls_provider: str+ """Provider of the model."""ls_model_name: str+ """Name of the model."""ls_model_type: Literal["chat"]+ """Type of the model. Should be 'chat'."""ls_temperature: Optional[float]+ """Temperature for generation."""ls_max_tokens: Optional[int]+ """Max tokens for generation."""ls_stop: Optional[List[str]]+ """Stop words for generation."""def generate_from_stream(stream: Iterator[ChatGenerationChunk]) -> ChatResult:- """Generate from a stream."""+ """Generate from a stream.++ Args:+ stream: Iterator of ChatGenerationChunk.++ Returns:+ ChatResult: Chat result.+ """generation: Optional[ChatGenerationChunk] = Nonefor chunk in stream:@@ -99,7 +114,14 @@ def generate_from_stream(stream: Iterator[ChatGenerationChunk]) -> ChatResult:async def agenerate_from_stream(stream: AsyncIterator[ChatGenerationChunk],) -> ChatResult:- """Async generate from a stream."""+ """Async generate from a stream.++ Args:+ stream: Iterator of ChatGenerationChunk.++ Returns:+ ChatResult: Chat result.+ """generation: Optional[ChatGenerationChunk] = Noneasync for chunk in stream:@@ -200,7 +222,17 @@ class BaseChatModel(BaseLanguageModel[BaseMessage], ABC):@root_validator(pre=True)def raise_deprecation(cls, values: Dict) -> Dict:- """Raise deprecation warning if callback_manager is used."""+ """Raise deprecation warning if callback_manager is used.++ Args:+ values (Dict): Values to validate.++ Returns:+ Dict: Validated values.++ Raises:+ DeprecationWarning: If callback_manager is used.+ """if values.get("callback_manager") is not None:warnings.warn("callback_manager is deprecated. Please use callbacks instead.",diff --git a/libs/core/langchain_core/language_models/fake_chat_models.py b/libs/core/langchain_core/language_models/fake_chat_models.pyindex 1b265ec4c1..92b0e58993 100644--- a/libs/core/langchain_core/language_models/fake_chat_models.py+++ b/libs/core/langchain_core/language_models/fake_chat_models.py@@ -1,4 +1,5 @@"""Fake ChatModel for testing purposes."""+import asyncioimport reimport timediff --git a/libs/core/langchain_core/language_models/llms.py b/libs/core/langchain_core/language_models/llms.pyindex e52ea156e3..1b1e014a8e 100644--- a/libs/core/langchain_core/language_models/llms.py+++ b/libs/core/langchain_core/language_models/llms.py@@ -78,7 +78,20 @@ def create_base_retry_decorator(Union[AsyncCallbackManagerForLLMRun, CallbackManagerForLLMRun]] = None,) -> Callable[[Any], Any]:- """Create a retry decorator for a given LLM and provided list of error types."""+ """Create a retry decorator for a given LLM and provided+ a list of error types.++ Args:+ error_types: List of error types to retry on.+ max_retries: Number of retries. Default is 1.+ run_manager: Callback manager for the run. Default is None.++ Returns:+ A retry decorator.++ Raises:+ ValueError: If the cache is not set and cache is True.+ """_logging = before_sleep_log(logger, logging.WARNING)@@ -141,7 +154,20 @@ def get_prompts(prompts: List[str],cache: Optional[Union[BaseCache, bool, None]] = None,) -> Tuple[Dict[int, List], str, List[int], List[str]]:- """Get prompts that are already cached."""+ """Get prompts that are already cached.++ Args:+ params: Dictionary of parameters.+ prompts: List of prompts.+ cache: Cache object. Default is None.++ Returns:+ A tuple of existing prompts, llm_string, missing prompt indexes,+ and missing prompts.++ Raises:+ ValueError: If the cache is not set and cache is True.+ """llm_string = str(sorted([(k, v) for k, v in params.items()]))missing_prompts = []missing_prompt_idxs = []@@ -164,7 +190,20 @@ async def aget_prompts(prompts: List[str],cache: Optional[Union[BaseCache, bool, None]] = None,) -> Tuple[Dict[int, List], str, List[int], List[str]]:- """Get prompts that are already cached. Async version."""+ """Get prompts that are already cached. Async version.++ Args:+ params: Dictionary of parameters.+ prompts: List of prompts.+ cache: Cache object. Default is None.++ Returns:+ A tuple of existing prompts, llm_string, missing prompt indexes,+ and missing prompts.++ Raises:+ ValueError: If the cache is not set and cache is True.+ """llm_string = str(sorted([(k, v) for k, v in params.items()]))missing_prompts = []missing_prompt_idxs = []@@ -189,7 +228,22 @@ def update_cache(new_results: LLMResult,prompts: List[str],) -> Optional[dict]:- """Update the cache and get the LLM output."""+ """Update the cache and get the LLM output.++ Args:+ cache: Cache object.+ existing_prompts: Dictionary of existing prompts.+ llm_string: LLM string.+ missing_prompt_idxs: List of missing prompt indexes.+ new_results: LLMResult object.+ prompts: List of prompts.++ Returns:+ LLM output.++ Raises:+ ValueError: If the cache is not set and cache is True.+ """llm_cache = _resolve_cache(cache)for i, result in enumerate(new_results.generations):existing_prompts[missing_prompt_idxs[i]] = result@@ -208,7 +262,23 @@ async def aupdate_cache(new_results: LLMResult,prompts: List[str],) -> Optional[dict]:- """Update the cache and get the LLM output. Async version"""+ """Update the cache and get the LLM output. Async version.++ Args:+ cache: Cache object.+ existing_prompts: Dictionary of existing prompts.+ llm_string: LLM string.+ missing_prompt_idxs: List of missing prompt indexes.+ new_results: LLMResult object.+ prompts: List of prompts.++ Returns:+ LLM output.++ Raises:+ ValueError: If the cache is not set and cache is True.+ """+llm_cache = _resolve_cache(cache)for i, result in enumerate(new_results.generations):existing_prompts[missing_prompt_idxs[i]] = result@@ -706,6 +776,15 @@ class BaseLLM(BaseLanguageModel[str], ABC):first occurrence of any of these substrings.callbacks: Callbacks to pass through. Used for executing additionalfunctionality, such as logging or streaming, throughout generation.+ tags: List of tags to associate with each prompt. If provided, the length+ of the list must match the length of the prompts list.+ metadata: List of metadata dictionaries to associate with each prompt. If+ provided, the length of the list must match the length of the prompts+ list.+ run_name: List of run names to associate with each prompt. If provided, the+ length of the list must match the length of the prompts list.+ run_id: List of run IDs to associate with each prompt. If provided, the+ length of the list must match the length of the prompts list.**kwargs: Arbitrary additional keyword arguments. These are usually passedto the model provider API call.@@ -924,6 +1003,15 @@ class BaseLLM(BaseLanguageModel[str], ABC):first occurrence of any of these substrings.callbacks: Callbacks to pass through. Used for executing additionalfunctionality, such as logging or streaming, throughout generation.+ tags: List of tags to associate with each prompt. If provided, the length+ of the list must match the length of the prompts list.+ metadata: List of metadata dictionaries to associate with each prompt. If+ provided, the length of the list must match the length of the prompts+ list.+ run_name: List of run names to associate with each prompt. If provided, the+ length of the list must match the length of the prompts list.+ run_id: List of run IDs to associate with each prompt. If provided, the+ length of the list must match the length of the prompts list.**kwargs: Arbitrary additional keyword arguments. These are usually passedto the model provider API call.@@ -1075,7 +1163,25 @@ class BaseLLM(BaseLanguageModel[str], ABC):metadata: Optional[Dict[str, Any]] = None,**kwargs: Any,) -> str:- """Check Cache and run the LLM on the given prompt and input."""+ """Check Cache and run the LLM on the given prompt and input.++ Args:+ prompt: The prompt to generate from.+ stop: Stop words to use when generating. Model output is cut off at the+ first occurrence of any of these substrings.+ callbacks: Callbacks to pass through. Used for executing additional+ functionality, such as logging or streaming, throughout generation.+ tags: List of tags to associate with the prompt.+ metadata: Metadata to associate with the prompt.+ **kwargs: Arbitrary additional keyword arguments. These are usually passed+ to the model provider API call.++ Returns:+ The generated text.++ Raises:+ ValueError: If the prompt is not a string.+ """if not isinstance(prompt, str):raise ValueError("Argument `prompt` is expected to be a string. Instead found "@@ -1190,6 +1296,9 @@ class BaseLLM(BaseLanguageModel[str], ABC):Args:file_path: Path to file to save the LLM to.+ Raises:+ ValueError: If the file path is not a string or Path object.+Example:.. code-block:: python@@ -1333,7 +1442,7 @@ class LLM(BaseLLM):run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,**kwargs: Any,) -> LLMResult:- """Run the LLM on the given prompt and input."""+ """Async run the LLM on the given prompt and input."""generations = []new_arg_supported = inspect.signature(self._acall).parameters.get("run_manager")for prompt in prompts: