Skip to content

The agent engineering platform.

qdrant: new Qdrant implementation (#24164)

Anush7014d07cab70
diff --git a/libs/partners/qdrant/langchain_qdrant/__init__.py b/libs/partners/qdrant/langchain_qdrant/__init__.py
index 74ac0ff84c..f826630848 100644
--- a/libs/partners/qdrant/langchain_qdrant/__init__.py
+++ b/libs/partners/qdrant/langchain_qdrant/__init__.py
@@ -1,5 +1,13 @@
from langchain_qdrant.fastembed_sparse import FastEmbedSparse
+from langchain_qdrant.qdrant import QdrantVectorStore, RetrievalMode
from langchain_qdrant.sparse_embeddings import SparseEmbeddings, SparseVector
from langchain_qdrant.vectorstores import Qdrant
-__all__ = ["Qdrant", "SparseEmbeddings", "SparseVector", "FastEmbedSparse"]
+__all__ = [
+ "Qdrant",
+ "QdrantVectorStore",
+ "SparseEmbeddings",
+ "SparseVector",
+ "FastEmbedSparse",
+ "RetrievalMode",
+]
diff --git a/libs/partners/qdrant/langchain_qdrant/qdrant.py b/libs/partners/qdrant/langchain_qdrant/qdrant.py
new file mode 100644
index 0000000000..d7b5970be5
--- /dev/null
+++ b/libs/partners/qdrant/langchain_qdrant/qdrant.py
@@ -0,0 +1,1050 @@
+from __future__ import annotations
+
+import uuid
+from enum import Enum
+from itertools import islice
+from operator import itemgetter
+from typing import (
+ Any,
+ Callable,
+ Dict,
+ Generator,
+ Iterable,
+ List,
+ Optional,
+ Sequence,
+ Tuple,
+ Type,
+ Union,
+)
+
+import numpy as np
+from langchain_core.documents import Document
+from langchain_core.embeddings import Embeddings
+from langchain_core.vectorstores import VectorStore
+from qdrant_client import QdrantClient, models
+
+from langchain_qdrant._utils import maximal_marginal_relevance
+from langchain_qdrant.sparse_embeddings import SparseEmbeddings
+
+
+class QdrantVectorStoreError(Exception):
+ """`QdrantVectorStore` related exceptions."""
+
+
+class RetrievalMode(str, Enum):
+ DENSE = "dense"
+ SPARSE = "sparse"
+ HYBRID = "hybrid"
+
+
+class QdrantVectorStore(VectorStore):
+ """`QdrantVectorStore` - Vector store implementation using https://qdrant.tech/
+
+ Example:
+ .. code-block:: python
+ from langchain_qdrant import QdrantVectorStore
+
+ store = QdrantVectorStore.from_existing_collection("my-collection", embedding, url="http://localhost:6333")
+ """
+
+ CONTENT_KEY: str = "page_content"
+ METADATA_KEY: str = "metadata"
+ VECTOR_NAME: str = "" # The default/unnamed vector - https://qdrant.tech/documentation/concepts/collections/#create-a-collection
+ SPARSE_VECTOR_NAME: str = "langchain-sparse"
+
+ def __init__(
+ self,
+ client: QdrantClient,
+ collection_name: str,
+ embedding: Optional[Embeddings] = None,
+ retrieval_mode: RetrievalMode = RetrievalMode.DENSE,
+ vector_name: str = VECTOR_NAME,
+ content_payload_key: str = CONTENT_KEY,
+ metadata_payload_key: str = METADATA_KEY,
+ distance: models.Distance = models.Distance.COSINE,
+ sparse_embedding: Optional[SparseEmbeddings] = None,
+ sparse_vector_name: str = SPARSE_VECTOR_NAME,
+ validate_embeddings: bool = True,
+ validate_collection_config: bool = True,
+ ):
+ """Initialize a new instance of `QdrantVectorStore`.
+
+ Example:
+ .. code-block:: python
+ qdrant = Qdrant(
+ client=client,
+ collection_name="my-collection",
+ embedding=OpenAIEmbeddings(),
+ retrieval_mode=RetrievalMode.HYBRID,
+ sparse_embedding=FastEmbedSparse(),
+ )
+ """
+ if validate_embeddings:
+ self._validate_embeddings(retrieval_mode, embedding, sparse_embedding)
+
+ if validate_collection_config:
+ self._validate_collection_config(
+ client,
+ collection_name,
+ retrieval_mode,
+ vector_name,
+ sparse_vector_name,
+ distance,
+ embedding,
+ )
+
+ self._client = client
+ self.collection_name = collection_name
+ self._embeddings = embedding
+ self.retrieval_mode = retrieval_mode
+ self.vector_name = vector_name
+ self.content_payload_key = content_payload_key
+ self.metadata_payload_key = metadata_payload_key
+ self.distance = distance
+ self._sparse_embeddings = sparse_embedding
+ self.sparse_vector_name = sparse_vector_name
+
+ @property
+ def client(self) -> QdrantClient:
+ """Get the Qdrant client instance that is being used.
+
+ Returns:
+ QdrantClient: An instance of `QdrantClient`.
+ """
+ return self._client
+
+ @property
+ def embeddings(self) -> Embeddings:
+ """Get the dense embeddings instance that is being used.
+
+ Raises:
+ ValueError: If embeddings are `None`.
+
+ Returns:
+ Embeddings: An instance of `Embeddings`.
+ """
+ if self._embeddings is None:
+ raise ValueError(
+ "Embeddings are `None`. Please set using the `embedding` parameter."
+ )
+ return self._embeddings
+
+ @property
+ def sparse_embeddings(self) -> SparseEmbeddings:
+ """Get the sparse embeddings instance that is being used.
+
+ Raises:
+ ValueError: If sparse embeddings are `None`.
+
+ Returns:
+ SparseEmbeddings: An instance of `SparseEmbeddings`.
+ """
+ if self._sparse_embeddings is None:
+ raise ValueError(
+ "Sparse embeddings are `None`. "
+ "Please set using the `sparse_embedding` parameter."
+ )
+ return self._sparse_embeddings
+
+ @classmethod
+ def from_texts(
+ cls: Type[QdrantVectorStore],
+ texts: List[str],
+ embedding: Optional[Embeddings] = None,
+ metadatas: Optional[List[dict]] = None,
+ ids: Optional[Sequence[str | int]] = None,
+ collection_name: Optional[str] = None,
+ location: Optional[str] = None,
+ url: Optional[str] = None,
+ port: Optional[int] = 6333,
+ grpc_port: int = 6334,
+ prefer_grpc: bool = False,
+ https: Optional[bool] = None,
+ api_key: Optional[str] = None,
+ prefix: Optional[str] = None,
+ timeout: Optional[int] = None,
+ host: Optional[str] = None,
+ path: Optional[str] = None,
+ distance: models.Distance = models.Distance.COSINE,
+ content_payload_key: str = CONTENT_KEY,
+ metadata_payload_key: str = METADATA_KEY,
+ vector_name: str = VECTOR_NAME,
+ retrieval_mode: RetrievalMode = RetrievalMode.DENSE,
+ sparse_embedding: Optional[SparseEmbeddings] = None,
+ sparse_vector_name: str = SPARSE_VECTOR_NAME,
+ collection_create_options: Dict[str, Any] = {},
+ vector_params: Dict[str, Any] = {},
+ sparse_vector_params: Dict[str, Any] = {},
+ batch_size: int = 64,
+ force_recreate: bool = False,
+ validate_embeddings: bool = True,
+ validate_collection_config: bool = True,
+ **kwargs: Any,
+ ) -> QdrantVectorStore:
+ """Construct an instance of `QdrantVectorStore` from a list of texts.
+
+ This is a user-friendly interface that:
+ 1. Creates embeddings, one for each text
+ 2. Creates a Qdrant collection if it doesn't exist.
+ 3. Adds the text embeddings to the Qdrant database
+
+ This is intended to be a quick way to get started.
+
+ Example:
+ .. code-block:: python
+
+ from langchain_qdrant import Qdrant
+ from langchain_openai import OpenAIEmbeddings
+ embeddings = OpenAIEmbeddings()
+ qdrant = Qdrant.from_texts(texts, embeddings, url="http://localhost:6333")
+ """
+ client_options = {
+ "location": location,
+ "url": url,
+ "port": port,
+ "grpc_port": grpc_port,
+ "prefer_grpc": prefer_grpc,
+ "https": https,
+ "api_key": api_key,
+ "prefix": prefix,
+ "timeout": timeout,
+ "host": host,
+ "path": path,
+ **kwargs,
+ }
+
+ qdrant = cls.construct_instance(
+ embedding,
+ retrieval_mode,
+ sparse_embedding,
+ client_options,
+ collection_name,
+ distance,
+ content_payload_key,
+ metadata_payload_key,
+ vector_name,
+ sparse_vector_name,
+ force_recreate,
+ collection_create_options,
+ vector_params,
+ sparse_vector_params,
+ validate_embeddings,
+ validate_collection_config,
+ )
+ qdrant.add_texts(texts, metadatas, ids, batch_size)
+ return qdrant
+
+ @classmethod
+ def from_existing_collection(
+ cls: Type[QdrantVectorStore],
+ collection_name: str,
+ embedding: Optional[Embeddings] = None,
+ retrieval_mode: RetrievalMode = RetrievalMode.DENSE,
+ location: Optional[str] = None,
+ url: Optional[str] = None,
+ port: Optional[int] = 6333,
+ grpc_port: int = 6334,
+ prefer_grpc: bool = False,
+ https: Optional[bool] = None,
+ api_key: Optional[str] = None,
+ prefix: Optional[str] = None,
+ timeout: Optional[int] = None,
+ host: Optional[str] = None,
+ path: Optional[str] = None,
+ distance: models.Distance = models.Distance.COSINE,
+ content_payload_key: str = CONTENT_KEY,
+ metadata_payload_key: str = METADATA_KEY,
+ vector_name: str = VECTOR_NAME,
+ sparse_vector_name: str = SPARSE_VECTOR_NAME,
+ sparse_embedding: Optional[SparseEmbeddings] = None,
+ validate_embeddings: bool = True,
+ validate_collection_config: bool = True,
+ **kwargs: Any,
+ ) -> QdrantVectorStore:
+ """Construct an instance of `QdrantVectorStore` from an existing collection
+ without adding any data.
+
+ Returns:
+ QdrantVectorStore: A new instance of `QdrantVectorStore`.
+ """
+ client = QdrantClient(
+ location=location,
+ url=url,
+ port=port,
+ grpc_port=grpc_port,
+ prefer_grpc=prefer_grpc,
+ https=https,
+ api_key=api_key,
+ prefix=prefix,
+ timeout=timeout,
+ host=host,
+ path=path,
+ **kwargs,
+ )
+
+ return cls(
+ client=client,
+ collection_name=collection_name,
+ embedding=embedding,
+ retrieval_mode=retrieval_mode,
+ content_payload_key=content_payload_key,
+ metadata_payload_key=metadata_payload_key,
+ distance=distance,
+ vector_name=vector_name,
+ sparse_embedding=sparse_embedding,
+ sparse_vector_name=sparse_vector_name,
+ validate_embeddings=validate_embeddings,
+ validate_collection_config=validate_collection_config,
+ )
+
+ def add_texts( # type: ignore
+ self,
+ texts: Iterable[str],
+ metadatas: Optional[List[dict]] = None,
+ ids: Optional[Sequence[str | int]] = None,
+ batch_size: int = 64,
+ **kwargs: Any,
+ ) -> List[str | int]:
+ """Add texts with embeddings to the vectorstore.
+
+ Returns:
+ List of ids from adding the texts into the vectorstore.
+ """
+ added_ids = []
+ for batch_ids, points in self._generate_batches(
+ texts, metadatas, ids, batch_size
+ ):
+ self.client.upsert(
+ collection_name=self.collection_name, points=points, **kwargs
+ )
+ added_ids.extend(batch_ids)
+
+ return added_ids
+
+ def similarity_search(
+ self,
+ query: str,
+ k: int = 4,
+ filter: Optional[models.Filter] = None,
+ search_params: Optional[models.SearchParams] = None,
+ offset: int = 0,
+ score_threshold: Optional[float] = None,
+ consistency: Optional[models.ReadConsistency] = None,
+ hybrid_fusion: Optional[models.FusionQuery] = None,
+ **kwargs: Any,
+ ) -> List[Document]:
+ """Return docs most similar to query.
+
+ Returns:
+ List of Documents most similar to the query.
+ """
+ results = self.similarity_search_with_score(
+ query,
+ k,
+ filter=filter,
+ search_params=search_params,
+ offset=offset,
+ score_threshold=score_threshold,
+ consistency=consistency,
+ hybrid_fusion=hybrid_fusion,
+ **kwargs,
+ )
+ return list(map(itemgetter(0), results))
+
+ def similarity_search_with_score(
+ self,
+ query: str,
+ k: int = 4,
+ filter: Optional[models.Filter] = None,
+ search_params: Optional[models.SearchParams] = None,
+ offset: int = 0,
+ score_threshold: Optional[float] = None,
+ consistency: Optional[models.ReadConsistency] = None,
+ hybrid_fusion: Optional[models.FusionQuery] = None,
+ **kwargs: Any,
+ ) -> List[Tuple[Document, float]]:
+ """Return docs most similar to query.
+
+ Returns:
+ List of documents most similar to the query text and distance for each.
+ """
+ query_options = {
+ "collection_name": self.collection_name,
+ "query_filter": filter,
+ "search_params": search_params,
+ "limit": k,
+ "offset": offset,
+ "with_payload": True,
+ "with_vectors": False,
+ "score_threshold": score_threshold,
+ "consistency": consistency,
+ **kwargs,
+ }
+ if self.retrieval_mode == RetrievalMode.DENSE:
+ query_dense_embedding = self.embeddings.embed_query(query)
+ results = self.client.query_points(
+ query=query_dense_embedding,
+ using=self.vector_name,
+ **query_options,
+ ).points
+
+ elif self.retrieval_mode == RetrievalMode.SPARSE:
+ query_sparse_embedding = self.sparse_embeddings.embed_query(query)
+ results = self.client.query_points(
+ query=models.SparseVector(
+ indices=query_sparse_embedding.indices,
+ values=query_sparse_embedding.values,
+ ),
+ using=self.sparse_vector_name,
+ **query_options,
+ ).points
+
+ elif self.retrieval_mode == RetrievalMode.HYBRID:
+ query_dense_embedding = self.embeddings.embed_query(query)
+ query_sparse_embedding = self.sparse_embeddings.embed_query(query)
+ results = self.client.query_points(
+ prefetch=[
+ models.Prefetch(
+ using=self.vector_name,
+ query=query_dense_embedding,
+ filter=filter,
+ limit=k,
+ params=search_params,
+ ),
+ models.Prefetch(
+ using=self.sparse_vector_name,
+ query=models.SparseVector(
+ indices=query_sparse_embedding.indices,
+ values=query_sparse_embedding.values,
+ ),
+ filter=filter,
+ limit=k,
+ params=search_params,
+ ),
+ ],
+ query=hybrid_fusion or models.FusionQuery(fusion=models.Fusion.RRF),
+ **query_options,
+ ).points
+
+ else:
+ raise ValueError(f"Invalid retrieval mode. {self.retrieval_mode}.")
+ return [
+ (
+ self._document_from_point(
+ result,
+ self.collection_name,
+ self.content_payload_key,
+ self.metadata_payload_key,
+ ),
+ result.score,
+ )
+ for result in results
+ ]
+
+ def similarity_search_by_vector(
+ self,
+ embedding: List[float],
+ k: int = 4,
+ filter: Optional[models.Filter] = None,
+ search_params: Optional[models.SearchParams] = None,
+ offset: int = 0,
+ score_threshold: Optional[float] = None,
+ consistency: Optional[models.ReadConsistency] = None,
+ **kwargs: Any,
+ ) -> List[Document]:
+ """Return docs most similar to embedding vector.
+
+ Returns:
+ List of Documents most similar to the query.
+ """
+ qdrant_filter = filter
+
+ self._validate_collection_for_dense(
+ client=self.client,
+ collection_name=self.collection_name,
+ vector_name=self.vector_name,
+ distance=self.distance,
+ dense_embeddings=embedding,
+ )
+ results = self.client.query_points(
+ collection_name=self.collection_name,
+ query=embedding,
+ using=self.vector_name,
+ query_filter=qdrant_filter,
+ search_params=search_params,
+ limit=k,
+ offset=offset,
+ with_payload=True,
+ with_vectors=False,
+ score_threshold=score_threshold,
+ consistency=consistency,
+ **kwargs,
+ ).points
+
+ return [
+ self._document_from_point(
+ result,
+ self.collection_name,
+ self.content_payload_key,
+ self.metadata_payload_key,
+ )
+ for result in results
+ ]
+
+ def max_marginal_relevance_search(
+ self,
+ query: str,
+ k: int = 4,
+ fetch_k: int = 20,
+ lambda_mult: float = 0.5,
+ filter: Optional[models.Filter] = None,
+ search_params: Optional[models.SearchParams] = None,
+ score_threshold: Optional[float] = None,
+ consistency: Optional[models.ReadConsistency] = None,
+ **kwargs: Any,
+ ) -> List[Document]:
+ """Return docs selected using the maximal marginal relevance with dense vectors.
+
+ Maximal marginal relevance optimizes for similarity to query AND diversity
+ among selected documents.
+
+
+ Returns:
+ List of Documents selected by maximal marginal relevance.
+ """
+ self._validate_collection_for_dense(
+ self.client,
+ self.collection_name,
+ self.vector_name,
+ self.distance,
+ self.embeddings,
+ )
+
+ query_embedding = self.embeddings.embed_query(query)
+ return self.max_marginal_relevance_search_by_vector(
+ query_embedding,
+ k=k,
+ fetch_k=fetch_k,
+ lambda_mult=lambda_mult,
+ filter=filter,
+ search_params=search_params,
+ score_threshold=score_threshold,
+ consistency=consistency,
+ **kwargs,
+ )
+
+ def max_marginal_relevance_search_by_vector(
+ self,
+ embedding: List[float],
+ k: int = 4,
+ fetch_k: int = 20,
+ lambda_mult: float = 0.5,
+ filter: Optional[models.Filter] = None,
+ search_params: Optional[models.SearchParams] = None,
+ score_threshold: Optional[float] = None,
+ consistency: Optional[models.ReadConsistency] = None,
+ **kwargs: Any,
+ ) -> List[Document]:
+ """Return docs selected using the maximal marginal relevance with dense vectors.
+
+ Maximal marginal relevance optimizes for similarity to query AND diversity
+ among selected documents.
+
+ Returns:
+ List of Documents selected by maximal marginal relevance.
+ """
+ results = self.max_marginal_relevance_search_with_score_by_vector(
+ embedding,
+ k=k,
+ fetch_k=fetch_k,
+ lambda_mult=lambda_mult,
+ filter=filter,
+ search_params=search_params,
+ score_threshold=score_threshold,
+ consistency=consistency,
+ **kwargs,
+ )
+ return list(map(itemgetter(0), results))
+
+ def max_marginal_relevance_search_with_score_by_vector(
+ self,
+ embedding: List[float],
+ k: int = 4,
+ fetch_k: int = 20,
+ lambda_mult: float = 0.5,
+ filter: Optional[models.Filter] = None,
+ search_params: Optional[models.SearchParams] = None,
+ score_threshold: Optional[float] = None,
+ consistency: Optional[models.ReadConsistency] = None,
+ **kwargs: Any,
+ ) -> List[Tuple[Document, float]]:
+ """Return docs selected using the maximal marginal relevance.
+ Maximal marginal relevance optimizes for similarity to query AND diversity
+ among selected documents.
+
+ Returns:
+ List of Documents selected by maximal marginal relevance and distance for
+ each.
+ """
+ results = self.client.query_points(
+ collection_name=self.collection_name,
+ query=embedding,
+ query_filter=filter,
+ search_params=search_params,
+ limit=fetch_k,
+ with_payload=True,
+ with_vectors=True,
+ score_threshold=score_threshold,
+ consistency=consistency,
+ using=self.vector_name,
+ **kwargs,
+ ).points
+
+ embeddings = [
+ result.vector
+ if isinstance(result.vector, list)
+ else result.vector.get(self.vector_name) # type: ignore
+ for result in results
+ ]
+ mmr_selected = maximal_marginal_relevance(
+ np.array(embedding), embeddings, k=k, lambda_mult=lambda_mult
+ )
+ return [
+ (
+ self._document_from_point(
+ results[i],
+ self.collection_name,
+ self.content_payload_key,
+ self.metadata_payload_key,
+ ),
+ results[i].score,
+ )
+ for i in mmr_selected
+ ]
+
+ def delete( # type: ignore
+ self,
+ ids: Optional[List[str | int]] = None,
+ **kwargs: Any,
+ ) -> Optional[bool]:
+ """Delete documents by their ids.
+
+ Args:
+ ids: List of ids to delete.
+ **kwargs: Other keyword arguments that subclasses might use.
+
+ Returns:
+ True if deletion is successful, False otherwise.
+ """
+ result = self.client.delete(
+ collection_name=self.collection_name,
+ points_selector=ids,
+ )
+ return result.status == models.UpdateStatus.COMPLETED
+
+ def get_by_ids(self, ids: Sequence[str | int], /) -> List[Document]:
+ results = self.client.retrieve(self.collection_name, ids, with_payload=True)
+
+ return [
+ self._document_from_point(
+ result,
+ self.collection_name,
+ self.content_payload_key,
+ self.metadata_payload_key,
+ )
+ for result in results
+ ]
+
+ @classmethod
+ def construct_instance(
+ cls: Type[QdrantVectorStore],
+ embedding: Optional[Embeddings] = None,
+ retrieval_mode: RetrievalMode = RetrievalMode.DENSE,
+ sparse_embedding: Optional[SparseEmbeddings] = None,
+ client_options: Dict[str, Any] = {},
+ collection_name: Optional[str] = None,
+ distance: models.Distance = models.Distance.COSINE,
+ content_payload_key: str = CONTENT_KEY,
+ metadata_payload_key: str = METADATA_KEY,
+ vector_name: str = VECTOR_NAME,
+ sparse_vector_name: str = SPARSE_VECTOR_NAME,
+ force_recreate: bool = False,
+ collection_create_options: Dict[str, Any] = {},
+ vector_params: Dict[str, Any] = {},
+ sparse_vector_params: Dict[str, Any] = {},
+ validate_embeddings: bool = True,
+ validate_collection_config: bool = True,
+ ) -> QdrantVectorStore:
+ if validate_embeddings:
+ cls._validate_embeddings(retrieval_mode, embedding, sparse_embedding)
+ collection_name = collection_name or uuid.uuid4().hex
+ client = QdrantClient(**client_options)
+
+ collection_exists = client.collection_exists(collection_name)
+
+ if collection_exists and force_recreate:
+ client.delete_collection(collection_name)
+ collection_exists = False
+ if collection_exists:
+ if validate_collection_config:
+ cls._validate_collection_config(
+ client,
+ collection_name,
+ retrieval_mode,
+ vector_name,
+ sparse_vector_name,
+ distance,
+ embedding,
+ )
+ else:
+ vectors_config, sparse_vectors_config = {}, {}
+ if retrieval_mode == RetrievalMode.DENSE:
+ partial_embeddings = embedding.embed_documents(["dummy_text"]) # type: ignore
+
+ vector_params["size"] = len(partial_embeddings[0])
+ vector_params["distance"] = distance
+
+ vectors_config = {
+ vector_name: models.VectorParams(
+ **vector_params,
+ )
+ }
+
+ elif retrieval_mode == RetrievalMode.SPARSE:
+ sparse_vectors_config = {
+ sparse_vector_name: models.SparseVectorParams(
+ **sparse_vector_params
+ )
+ }
+
+ elif retrieval_mode == RetrievalMode.HYBRID:
+ partial_embeddings = embedding.embed_documents(["dummy_text"]) # type: ignore
+
+ vector_params["size"] = len(partial_embeddings[0])
+ vector_params["distance"] = distance
+
+ vectors_config = {
+ vector_name: models.VectorParams(
+ **vector_params,
+ )
+ }
+
+ sparse_vectors_config = {
+ sparse_vector_name: models.SparseVectorParams(
+ **sparse_vector_params
+ )
+ }
+
+ collection_create_options["collection_name"] = collection_name
+ collection_create_options["vectors_config"] = vectors_config
+ collection_create_options["sparse_vectors_config"] = sparse_vectors_config
+
+ client.create_collection(**collection_create_options)
+
+ qdrant = cls(
+ client=client,
+ collection_name=collection_name,
+ embedding=embedding,
+ retrieval_mode=retrieval_mode,
+ content_payload_key=content_payload_key,
+ metadata_payload_key=metadata_payload_key,
+ distance=distance,
+ vector_name=vector_name,
+ sparse_embedding=sparse_embedding,
+ sparse_vector_name=sparse_vector_name,
+ validate_embeddings=False,
+ validate_collection_config=False,
+ )
+ return qdrant
+
+ @staticmethod
+ def _cosine_relevance_score_fn(distance: float) -> float:
+ """Normalize the distance to a score on a scale [0, 1]."""
+ return (distance + 1.0) / 2.0
+
+ def _select_relevance_score_fn(self) -> Callable[[float], float]:
+ """
+ The 'correct' relevance function
+ may differ depending on a few things, including:
+ - the distance / similarity metric used by the VectorStore
+ - the scale of your embeddings (OpenAI's are unit normed. Many others are not!)
+ - embedding dimensionality
+ - etc.
+ """
+
+ if self.distance == models.Distance.COSINE:
+ return self._cosine_relevance_score_fn
+ elif self.distance == models.Distance.DOT:
+ return self._max_inner_product_relevance_score_fn
+ elif self.distance == models.Distance.EUCLID:
+ return self._euclidean_relevance_score_fn
+ else:
+ raise ValueError(
+ "Unknown distance strategy, must be COSINE, DOT, or EUCLID."
+ )
+
+ @classmethod
+ def _document_from_point(
+ cls,
+ scored_point: Any,
+ collection_name: str,
+ content_payload_key: str,
+ metadata_payload_key: str,
+ ) -> Document:
+ metadata = scored_point.payload.get(metadata_payload_key) or {}
+ metadata["_id"] = scored_point.id
+ metadata["_collection_name"] = collection_name
+ return Document(
+ page_content=scored_point.payload.get(content_payload_key, ""),
+ metadata=metadata,
+ )
+
+ def _generate_batches(
+ self,
+ texts: Iterable[str],
+ metadatas: Optional[List[dict]] = None,
+ ids: Optional[Sequence[str | int]] = None,
+ batch_size: int = 64,
+ ) -> Generator[tuple[list[str | int], list[models.PointStruct]], Any, None]:
+ texts_iterator = iter(texts)
+ metadatas_iterator = iter(metadatas or [])
+ ids_iterator = iter(ids or [uuid.uuid4().hex for _ in iter(texts)])
+
+ while batch_texts := list(islice(texts_iterator, batch_size)):
+ batch_metadatas = list(islice(metadatas_iterator, batch_size)) or None
+ batch_ids = list(islice(ids_iterator, batch_size))
+ points = [
+ models.PointStruct(
+ id=point_id,
+ vector=vector,
+ payload=payload,
+ )
+ for point_id, vector, payload in zip(
+ batch_ids,
+ self._build_vectors(batch_texts),
+ self._build_payloads(
+ batch_texts,
+ batch_metadatas,
+ self.content_payload_key,
+ self.metadata_payload_key,
+ ),
+ )
+ ]
+
+ yield batch_ids, points
+
+ def _build_payloads(
+ self,
+ texts: Iterable[str],
+ metadatas: Optional[List[dict]],
+ content_payload_key: str,
+ metadata_payload_key: str,
+ ) -> List[dict]:
+ payloads = []
+ for i, text in enumerate(texts):
+ if text is None:
+ raise ValueError(
+ "At least one of the texts is None. Please remove it before "
+ "calling .from_texts or .add_texts."
+ )
+ metadata = metadatas[i] if metadatas is not None else None
+ payloads.append(
+ {
+ content_payload_key: text,
+ metadata_payload_key: metadata,
+ }
+ )
+
+ return payloads
+
+ def _build_vectors(
+ self,
+ texts: Iterable[str],
+ ) -> List[models.VectorStruct]:
+ if self.retrieval_mode == RetrievalMode.DENSE:
+ batch_embeddings = self.embeddings.embed_documents(list(texts))
+ return [
+ {
+ self.vector_name: vector,
+ }
+ for vector in batch_embeddings
+ ]
+
+ elif self.retrieval_mode == RetrievalMode.SPARSE:
+ batch_sparse_embeddings = self.sparse_embeddings.embed_documents(
+ list(texts)
+ )
+ return [
+ {
+ self.sparse_vector_name: models.SparseVector(
+ values=vector.values, indices=vector.indices
+ )
+ }
+ for vector in batch_sparse_embeddings
+ ]
+
+ elif self.retrieval_mode == RetrievalMode.HYBRID:
+ dense_embeddings = self.embeddings.embed_documents(list(texts))
+ sparse_embeddings = self.sparse_embeddings.embed_documents(list(texts))
+
+ assert len(dense_embeddings) == len(
+ sparse_embeddings
+ ), "Mismatched length between dense and sparse embeddings."
+
+ return [
+ {
+ self.vector_name: dense_vector,
+ self.sparse_vector_name: models.SparseVector(
+ values=sparse_vector.values, indices=sparse_vector.indices
+ ),
+ }
+ for dense_vector, sparse_vector in zip(
+ dense_embeddings, sparse_embeddings
+ )
+ ]
+
+ else:
+ raise ValueError(
+ f"Unknown retrieval mode. {self.retrieval_mode} to build vectors."
+ )
+
+ @classmethod
+ def _validate_collection_config(
+ cls: Type[QdrantVectorStore],
+ client: QdrantClient,
+ collection_name: str,
+ retrieval_mode: RetrievalMode,
+ vector_name: str,
+ sparse_vector_name: str,
+ distance: models.Distance,
+ embedding: Optional[Embeddings],
+ ) -> None:
+ if retrieval_mode == RetrievalMode.DENSE:
+ cls._validate_collection_for_dense(
+ client, collection_name, vector_name, distance, embedding
+ )
+
+ elif retrieval_mode == RetrievalMode.SPARSE:
+ cls._validate_collection_for_sparse(
+ client, collection_name, sparse_vector_name
+ )
+
+ elif retrieval_mode == RetrievalMode.HYBRID:
+ cls._validate_collection_for_dense(
+ client, collection_name, vector_name, distance, embedding
+ )
+ cls._validate_collection_for_sparse(
+ client, collection_name, sparse_vector_name
+ )
+
+ @classmethod
+ def _validate_collection_for_dense(
+ cls: Type[QdrantVectorStore],
+ client: QdrantClient,
+ collection_name: str,
+ vector_name: str,
+ distance: models.Distance,
+ dense_embeddings: Union[Embeddings, List[float], None],
+ ) -> None:
+ collection_info = client.get_collection(collection_name=collection_name)
+ vector_config = collection_info.config.params.vectors
+
+ if isinstance(vector_config, models.VectorParams) and vector_name != "":
+ # For single/unnamed vector,
+ # qdrant-client returns a single VectorParams object
+
+ raise QdrantVectorStoreError(
+ f"Existing Qdrant collection {collection_name} is built "
+ "with unnamed dense vector. "
+ f"If you want to reuse it, set `vector_name` to ''(empty string)."
+ f"If you want to recreate the collection, "
+ "set `force_recreate` to `True`."
+ )
+
+ else:
+ # vector_config is a Dict[str, VectorParams]
+ if isinstance(vector_config, dict) and vector_name not in vector_config:
+ raise QdrantVectorStoreError(
+ f"Existing Qdrant collection {collection_name} does not "
+ f"contain dense vector named {vector_name}. "
+ "Did you mean one of the "
+ f"existing vectors: {', '.join(vector_config.keys())}? " # type: ignore
+ f"If you want to recreate the collection, set `force_recreate` "
+ f"parameter to `True`."
+ )
+
+ # Get the VectorParams object for the specified vector_name
+ vector_config = vector_config[vector_name] # type: ignore
+
+ if isinstance(dense_embeddings, Embeddings):
+ vector_size = len(dense_embeddings.embed_documents(["dummy_text"])[0])
+ elif isinstance(dense_embeddings, list):
+ vector_size = len(dense_embeddings)
+ else:
+ raise ValueError("Invalid `embeddings` type.")
+
+ if vector_config.size != vector_size:
+ raise QdrantVectorStoreError(
+ f"Existing Qdrant collection is configured for dense vectors with "
+ f"{vector_config.size} dimensions. "
+ f"Selected embeddings are {vector_size}-dimensional. "
+ f"If you want to recreate the collection, set `force_recreate` "
+ f"parameter to `True`."
+ )
+
+ if vector_config.distance != distance:
+ raise QdrantVectorStoreError(
+ f"Existing Qdrant collection is configured for "
+ f"{vector_config.distance.name} similarity, but requested "
+ f"{distance.upper()}. Please set `distance` parameter to "
+ f"`{vector_config.distance.name}` if you want to reuse it. "
+ f"If you want to recreate the collection, set `force_recreate` "
+ f"parameter to `True`."
+ )
+
+ @classmethod
+ def _validate_collection_for_sparse(
+ cls: Type[QdrantVectorStore],
+ client: QdrantClient,
+ collection_name: str,
+ sparse_vector_name: str,
+ ) -> None:
+ collection_info = client.get_collection(collection_name=collection_name)
+ sparse_vector_config = collection_info.config.params.sparse_vectors
+
+ if (
+ sparse_vector_config is None
+ or sparse_vector_name not in sparse_vector_config
+ ):
+ raise QdrantVectorStoreError(
+ f"Existing Qdrant collection {collection_name} does not "
+ f"contain sparse vectors named {sparse_vector_config}. "
+ f"If you want to recreate the collection, set `force_recreate` "
+ f"parameter to `True`."
+ )
+
+ @classmethod
+ def _validate_embeddings(
+ cls: Type[QdrantVectorStore],
+ retrieval_mode: RetrievalMode,
+ embedding: Optional[Embeddings],
+ sparse_embedding: Optional[SparseEmbeddings],
+ ) -> None:
+ if retrieval_mode == RetrievalMode.DENSE and embedding is None:
+ raise ValueError(
+ "'embedding' cannot be None when retrieval mode is 'dense'"
+ )
+
+ elif retrieval_mode == RetrievalMode.SPARSE and sparse_embedding is None:
+ raise ValueError(
+ "'sparse_embedding' cannot be None when retrieval mode is 'sparse'"
+ )
+
+ elif retrieval_mode == RetrievalMode.HYBRID and any(
+ [embedding is None, sparse_embedding is None]
+ ):
+ raise ValueError(
+ "Both 'embedding' and 'sparse_embedding' cannot be None "
+ "when retrieval mode is 'hybrid'"
+ )
diff --git a/libs/partners/qdrant/langchain_qdrant/vectorstores.py b/libs/partners/qdrant/langchain_qdrant/vectorstores.py
index ee332c2cfe..e75d13c6c4 100644
--- a/libs/partners/qdrant/langchain_qdrant/vectorstores.py
+++ b/libs/partners/qdrant/langchain_qdrant/vectorstores.py
@@ -23,6 +23,7 @@ from typing import (
)
import numpy as np
+from langchain_core._api.deprecation import deprecated
from langchain_core.documents import Document
from langchain_core.embeddings import Embeddings
from langchain_core.runnables.config import run_in_executor
@@ -65,6 +66,7 @@ def sync_call_fallback(method: Callable) -> Callable:
return wrapper
+@deprecated(since="0.1.2", alternative="QdrantVectorStore", removal="0.5.0")
class Qdrant(VectorStore):
"""`Qdrant` vector store.
diff --git a/libs/partners/qdrant/poetry.lock b/libs/partners/qdrant/poetry.lock
index 11550f9c97..0cba3269ba 100644
--- a/libs/partners/qdrant/poetry.lock
+++ b/libs/partners/qdrant/poetry.lock
@@ -207,13 +207,13 @@ test = ["pytest (>=6)"]
[[package]]
name = "fastembed"
-version = "0.3.1"
+version = "0.3.3"
description = "Fast, light, accurate library built for retrieval embedding generation"
optional = true
python-versions = "<3.13,>=3.8.0"
files = [
- {file = "fastembed-0.3.1-py3-none-any.whl", hash = "sha256:5e9fecf6031a010eff75c02986f556d0635102210c0b1765e784cfca07df8975"},
- {file = "fastembed-0.3.1.tar.gz", hash = "sha256:01a52c5c9cdfe2504dfa56b3c1b032d1ad88d621de144fc9a571371a79df6457"},
+ {file = "fastembed-0.3.3-py3-none-any.whl", hash = "sha256:1282a44dac187bbb0765b7e05a0612f1a7eb28c0c65ccb10b2b963467807fef6"},
+ {file = "fastembed-0.3.3.tar.gz", hash = "sha256:2735777d8c4462a92ec289ad99534242deb2073cede7d4ad3b37d070994d767f"},
]
[package.dependencies]
@@ -622,7 +622,7 @@ files = [
[[package]]
name = "langchain-core"
-version = "0.2.11"
+version = "0.2.16"
description = "Building applications with LLMs through composability"
optional = false
python-versions = ">=3.8.1,<4.0"
@@ -1530,13 +1530,13 @@ files = [
[[package]]
name = "qdrant-client"
-version = "1.10.0"
+version = "1.10.1"
description = "Client library for the Qdrant vector search engine"
optional = false
python-versions = ">=3.8"
files = [
- {file = "qdrant_client-1.10.0-py3-none-any.whl", hash = "sha256:423c2586709ccf3db20850cd85c3d18954692a8faff98367dfa9dc82ab7f91d9"},
- {file = "qdrant_client-1.10.0.tar.gz", hash = "sha256:47c4f7abfab152fb7e5e4902ab0e2e9e33483c49ea5e80128ccd0295f342cf9b"},
+ {file = "qdrant_client-1.10.1-py3-none-any.whl", hash = "sha256:b9fb8fe50dd168d92b2998be7c6135d5a229b3a3258ad158cc69c8adf9ff1810"},
+ {file = "qdrant_client-1.10.1.tar.gz", hash = "sha256:2284c8c5bb1defb0d9dbacb07d16f344972f395f4f2ed062318476a7951fd84c"},
]
[package.dependencies]
@@ -1932,4 +1932,4 @@ fastembed = ["fastembed"]
[metadata]
lock-version = "2.0"
python-versions = ">=3.8.1,<4.0"
-content-hash = "192e50ec7768ff659cfe7d2bbc75497dbeba3ee9f1e245d12f2eb3cdf7fe07d9"
+content-hash = "a73d49a493892731f3fccc06217626d20b8114bc83ad107bc7336eabca479988"
diff --git a/libs/partners/qdrant/pyproject.toml b/libs/partners/qdrant/pyproject.toml
index 1db40f3e8c..0707f4b49b 100644
--- a/libs/partners/qdrant/pyproject.toml
+++ b/libs/partners/qdrant/pyproject.toml
@@ -23,9 +23,9 @@ disallow_untyped_defs = true
[tool.poetry.dependencies]
python = ">=3.8.1,<4.0"
langchain-core = ">=0.1.52,<0.3"
-qdrant-client = "^1.9.0"
+qdrant-client = "^1.10.1"
pydantic = "^2.7.4"
-fastembed = { version = "^0.3.1", python = ">=3.8.1,<3.13", optional = true}
+fastembed = { version = "^0.3.3", python = ">=3.8.1,<3.13", optional = true}
[tool.poetry.extras]
fastembed = [ "fastembed"]
diff --git a/libs/partners/qdrant/tests/integration_tests/common.py b/libs/partners/qdrant/tests/integration_tests/common.py
index 8336791f3d..f1dcc53a26 100644
--- a/libs/partners/qdrant/tests/integration_tests/common.py
+++ b/libs/partners/qdrant/tests/integration_tests/common.py
@@ -31,29 +31,7 @@ def assert_documents_equals(actual: List[Document], expected: List[Document]):
assert actual_doc.metadata == expected_doc.metadata
-class FakeEmbeddings(Embeddings):
- """Fake embeddings functionality for testing."""
-
- def embed_documents(self, texts: List[str]) -> List[List[float]]:
- """Return simple embeddings.
- Embeddings encode each text as its index."""
- return [[float(1.0)] * 9 + [float(i)] for i in range(len(texts))]
-
- async def aembed_documents(self, texts: List[str]) -> List[List[float]]:
- return self.embed_documents(texts)
-
- def embed_query(self, text: str) -> List[float]:
- """Return constant query embeddings.
- Embeddings are identical to embed_documents(texts)[0].
- Distance to each text will be that text's index,
- as it was passed to embed_documents."""
- return [float(1.0)] * 9 + [float(0.0)]
-
- async def aembed_query(self, text: str) -> List[float]:
- return self.embed_query(text)
-
-
-class ConsistentFakeEmbeddings(FakeEmbeddings):
+class ConsistentFakeEmbeddings(Embeddings):
"""Fake embeddings which remember all the texts seen so far to return consistent
vectors for the same texts."""
diff --git a/libs/partners/qdrant/tests/integration_tests/fastembed/test_fastembed_sparse.py b/libs/partners/qdrant/tests/integration_tests/fastembed/test_fastembed_sparse.py
index 313457b020..61e404272e 100644
--- a/libs/partners/qdrant/tests/integration_tests/fastembed/test_fastembed_sparse.py
+++ b/libs/partners/qdrant/tests/integration_tests/fastembed/test_fastembed_sparse.py
@@ -3,6 +3,8 @@ import pytest
from langchain_qdrant import FastEmbedSparse
+pytest.importorskip("fastembed", reason="'fastembed' package is not installed")
+
@pytest.mark.parametrize(
"model_name", ["Qdrant/bm25", "Qdrant/bm42-all-minilm-l6-v2-attentions"]
diff --git a/libs/partners/qdrant/tests/unit_tests/test_imports.py b/libs/partners/qdrant/tests/unit_tests/test_imports.py
index 41b18dd811..93a6393e41 100644
--- a/libs/partners/qdrant/tests/unit_tests/test_imports.py
+++ b/libs/partners/qdrant/tests/unit_tests/test_imports.py
@@ -1,6 +1,13 @@
from langchain_qdrant import __all__
-EXPECTED_ALL = ["Qdrant", "SparseEmbeddings", "SparseVector", "FastEmbedSparse"]
+EXPECTED_ALL = [
+ "Qdrant",
+ "QdrantVectorStore",
+ "SparseEmbeddings",
+ "SparseVector",
+ "FastEmbedSparse",
+ "RetrievalMode",
+]
def test_all_imports() -> None: