Python
# -*- coding: utf-8 -*-
"""
Qdrant BM25 稀疏向量编码器(fastembed Qdrant/bm25)单例封装。
Qdrant 原生 BM25 走 sparse vector:
- 写入:文档文本 → fastembed .embed() → sparse(indices, values),含 IDF/TF 权重
- 检索:query → .query_embed() → sparse,query_points(using="bm25")
BM25 是非对称的:文档侧用 embed()(带 IDF×TF 权重),查询侧用 query_embed()
(权重为 1)。两者不可混用。
SparseTextEmbedding('Qdrant/bm25') 首次会下载 ONNX 模型(数百 MB)并加载
~4.4s,必须进程内单例缓存。fastembed 缺失时优雅降级:返回 None,调用方据此
关闭 qdrant sparse 能力。
"""
from __future__ import annotations
import logging
import threading
from typing import List, Optional, Tuple
logger = logging.getLogger(__name__)
# (indices, values):可直接用于 qdrant SparseVector(indices=, values=)
SparseTuple = Tuple[List[int], List[float]]
_encoder = None # SparseTextEmbedding | None | False(=不可用)
_lock = threading.Lock()
_MODEL_NAME = "Qdrant/bm25"
def _get_encoder():
"""懒加载 fastembed BM25 单例。不可用时返回 False。"""
global _encoder
if _encoder is not None:
return _encoder
with _lock:
if _encoder is not None:
return _encoder
try:
from fastembed import SparseTextEmbedding
enc = SparseTextEmbedding(_MODEL_NAME)
logger.info(f"[bm25-fastembed] SparseTextEmbedding('{_MODEL_NAME}') loaded")
_encoder = enc
except Exception as e:
logger.warning(
f"[bm25-fastembed] fastembed unavailable ({e}); "
f"qdrant sparse BM25 disabled, keyword channel will be empty. "
f"pip install fastembed"
)
_encoder = False
return _encoder
def is_available() -> bool:
"""fastembed BM25 编码器是否可用。"""
return bool(_get_encoder())
def _to_tuple(sparse_embedding) -> Optional[SparseTuple]:
"""fastembed SparseEmbedding → (indices:List[int], values:List[float])。"""
try:
indices = [int(i) for i in sparse_embedding.indices]
values = [float(v) for v in sparse_embedding.values]
if not indices:
return None
return indices, values
except Exception as e:
logger.debug(f"[bm25-fastembed] sparse convert failed: {e}")
return None
def encode_doc(text: str) -> Optional[SparseTuple]:
"""文档侧编码(含 IDF×TF 权重);不可用或空文本返回 None。"""
enc = _get_encoder()
if not enc or not text:
return None
try:
out = list(enc.embed([text]))
return _to_tuple(out[0]) if out else None
except Exception as e:
logger.debug(f"[bm25-fastembed] encode_doc failed: {e}")
return None
def encode_docs(texts: List[str]) -> List[Optional[SparseTuple]]:
"""批量文档编码;与输入等长,空文本对应 None。"""
enc = _get_encoder()
if not enc:
return [None] * len(texts)
idxs = [i for i, t in enumerate(texts) if t]
result: List[Optional[SparseTuple]] = [None] * len(texts)
if not idxs:
return result
try:
encoded = list(enc.embed([texts[i] for i in idxs]))
for j, i in enumerate(idxs):
if j < len(encoded):
result[i] = _to_tuple(encoded[j])
except Exception as e:
logger.debug(f"[bm25-fastembed] encode_docs failed: {e}")
return result
def encode_query(text: str) -> Optional[SparseTuple]:
"""查询侧编码(query_embed,权重为 1);不可用或空文本返回 None。"""
enc = _get_encoder()
if not enc or not text:
return None
try:
out = list(enc.query_embed(text))
return _to_tuple(out[0]) if out else None
except Exception as e:
logger.debug(f"[bm25-fastembed] encode_query failed: {e}")
return None