Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

refactor(retrieval): simplify quick search flow (#2812)

Qin Haojie93dbd223abac
diff --git a/openviking/retrieve/hierarchical_retriever.py b/openviking/retrieve/hierarchical_retriever.py
index 62782bf36..a7aaadee4 100644
--- a/openviking/retrieve/hierarchical_retriever.py
+++ b/openviking/retrieve/hierarchical_retriever.py
@@ -22,6 +22,7 @@ from openviking.retrieve.memory_lifecycle import hotness_score
from openviking.retrieve.retrieval_stats import get_stats_collector
from openviking.server.identity import RequestContext
from openviking.storage import VikingDBManager, VikingDBManagerProxy
+from openviking.storage.expr import FilterExpr
from openviking.telemetry import get_current_telemetry
from openviking.utils.time_utils import parse_iso_datetime
from openviking_cli.retrieve.types import (
@@ -94,26 +95,27 @@ class HierarchicalRetriever:
query: TypedQuery,
ctx: RequestContext,
limit: int = 5,
- mode: str = RetrieverMode.THINKING,
+ mode: Optional[RetrieverMode] = None,
score_threshold: Optional[float] = None,
score_gte: bool = False,
- scope_dsl: Optional[Dict[str, Any]] = None,
+ scope_dsl: Optional[FilterExpr | Dict[str, Any]] = None,
level: Optional[List[int]] = None,
) -> QueryResult:
"""
Execute hierarchical retrieval.
Args:
- user: User ID (for permission filtering)
+ ctx: Request context used for tenant and permission filtering
score_threshold: Custom score threshold (overrides config)
score_gte: True uses >=, False uses >
- grep_patterns: Keyword match pattern list
scope_dsl: Additional scope constraints passed from public find/search filter
+ level: Optional result level filter (0=L0, 1=L1, 2=L2)
"""
t0 = time.monotonic()
telemetry = get_current_telemetry()
- # Use custom threshold or default threshold
- effective_threshold = score_threshold if score_threshold is not None else self.threshold
+ effective_threshold = self._resolve_threshold(score_threshold)
+ if mode is None:
+ mode = RetrieverMode.QUICK if not self._rerank_client else RetrieverMode.THINKING
# 创建 proxy 包装器,绑定当前 ctx
vector_proxy = VikingDBManagerProxy(self.vector_store, ctx)
@@ -146,90 +148,150 @@ class HierarchicalRetriever:
else:
root_uris = default_target_directories(ctx, context_type=query.context_type)
- # Step 2: Global vector search to supplement starting points
- with telemetry.measure("search.vector_retrieval"):
- global_results = await self._global_vector_search(
- vector_proxy=vector_proxy,
- query_vector=query_vector,
- sparse_query_vector=sparse_query_vector,
- context_type=query.context_type.value if query.context_type else None,
- target_dirs=target_dirs,
- scope_dsl=scope_dsl,
- limit=max(limit, self.GLOBAL_SEARCH_TOPK),
- )
+ context_type = query.context_type.value if query.context_type else None
- # Debug: Print all URIs in global_results
- if logger.isEnabledFor(logging.DEBUG):
- logger.debug(f"[retrieve] target_dirs: {target_dirs}")
- logger.debug(f"[retrieve] root_uris: {root_uris}")
- logger.debug(f"[retrieve] scope_dsl: {scope_dsl}")
- logger.debug(
- f"[retrieve] Step 2 completed, global_results contains {len(global_results)} items:"
- )
- for i, r in enumerate(global_results):
- uri = r.get("uri", "UNKNOWN_URI")
- score = r.get("_score", 0.0)
- result_level = r.get("level", "UNKNOWN_LEVEL")
- account_id = r.get("account_id", "UNKNOWN_ACCOUNT_ID")
- logger.debug(
- f" [{i}] URI: {uri}, score: {score:.4f}, level: {result_level}, account_id: {account_id}"
+ if mode == RetrieverMode.QUICK:
+ with telemetry.measure("search.vector_retrieval"):
+ quick_results = await vector_proxy.search_in_tenant(
+ query_vector=query_vector,
+ sparse_query_vector=sparse_query_vector,
+ context_type=context_type,
+ target_directories=target_dirs,
+ extra_filter=scope_dsl,
+ level=level,
+ limit=max(limit, self.GLOBAL_SEARCH_TOPK),
)
+ telemetry.count("vector.searches", 1)
+ telemetry.count("vector.scored", len(quick_results))
+ telemetry.count("vector.scanned", len(quick_results))
- # Step 3: Merge starting points
- starting_points = self._merge_starting_points(
- query.query,
- root_uris,
- global_results,
- mode=mode,
- )
+ collected_by_uri: Dict[str, Dict[str, Any]] = {}
+ for result in quick_results:
+ uri = result.get("uri", "")
+ if not uri:
+ continue
- # Add global hits to the result pool only when they match the requested level.
- if level is not None:
- initial_candidates = [r for r in global_results if r.get("level", 2) in level]
- else:
- initial_candidates = [r for r in global_results if r.get("level", 2) == 2]
+ score = self._finite_score(result.get("_score", 0.0))
+ if not self._passes_threshold(score, effective_threshold, score_gte):
+ continue
- initial_candidates = self._prepare_initial_candidates(
- query.query,
- initial_candidates,
- mode=mode,
- )
+ candidate = dict(result)
+ candidate["_score"] = score
+ candidate["_final_score"] = score
- # Step 4: Recursive search
- with telemetry.measure("search.vector_retrieval"):
- candidates = await self._recursive_search(
- vector_proxy=vector_proxy,
- query=query.query,
- query_vector=query_vector,
- sparse_query_vector=sparse_query_vector,
- starting_points=starting_points,
- limit=limit,
- mode=mode,
- threshold=effective_threshold,
- score_gte=score_gte,
- context_type=query.context_type.value if query.context_type else None,
- target_dirs=target_dirs,
- scope_dsl=scope_dsl,
- initial_candidates=initial_candidates,
- level=level,
+ previous = collected_by_uri.get(uri)
+ if previous is None or score > previous.get("_final_score", 0.0):
+ collected_by_uri[uri] = candidate
+
+ candidates = sorted(
+ collected_by_uri.values(),
+ key=lambda x: x.get("_final_score", 0.0),
+ reverse=True,
)
+ apply_hotness = False
+ rerank_used = False
+ else:
+ # Step 2: Global vector search to supplement starting points
+ with telemetry.measure("search.vector_retrieval"):
+ global_results = await vector_proxy.search_in_tenant(
+ query_vector=query_vector,
+ sparse_query_vector=sparse_query_vector,
+ context_type=context_type,
+ target_directories=target_dirs,
+ extra_filter=scope_dsl,
+ level=[0, 1],
+ limit=max(limit, self.GLOBAL_SEARCH_TOPK),
+ )
+ telemetry.count("vector.searches", 1)
+ telemetry.count("vector.scored", len(global_results))
+ telemetry.count("vector.scanned", len(global_results))
+
+ # Debug: Print all URIs in global_results
+ if logger.isEnabledFor(logging.DEBUG):
+ logger.debug(f"[retrieve] target_dirs: {target_dirs}")
+ logger.debug(f"[retrieve] root_uris: {root_uris}")
+ logger.debug(f"[retrieve] scope_dsl: {scope_dsl}")
+ logger.debug(
+ f"[retrieve] Step 2 completed, global_results contains {len(global_results)} items:"
+ )
+ for i, r in enumerate(global_results):
+ uri = r.get("uri", "UNKNOWN_URI")
+ score = r.get("_score", 0.0)
+ result_level = r.get("level", "UNKNOWN_LEVEL")
+ account_id = r.get("account_id", "UNKNOWN_ACCOUNT_ID")
+ logger.debug(
+ f" [{i}] URI: {uri}, score: {score:.4f}, level: {result_level}, account_id: {account_id}"
+ )
+
+ # Step 3: Pick recursive entry points from directory hits and explicit roots.
+ directory_scores = [self._finite_score(r.get("_score", 0.0)) for r in global_results]
+ if self._rerank_client and mode == RetrieverMode.THINKING:
+ directory_scores = self._rerank_scores(
+ query.query,
+ [str(r.get("abstract", "")) for r in global_results],
+ directory_scores,
+ )
+
+ starting_points = []
+ seen_starting_uris = set()
+ for result, score in zip(global_results, directory_scores, strict=True):
+ uri = result.get("uri", "")
+ if not uri or uri in seen_starting_uris:
+ continue
+ starting_points.append((uri, score))
+ seen_starting_uris.add(uri)
+
+ for uri in root_uris:
+ if uri not in seen_starting_uris:
+ starting_points.append((uri, 0.0))
+ seen_starting_uris.add(uri)
+
+ # Add directory hits to the result pool only when explicitly requested.
+ initial_candidates = []
+ if level is not None:
+ for result, score in zip(global_results, directory_scores, strict=True):
+ if result.get("level", 2) not in level:
+ continue
+ candidate = dict(result)
+ candidate["_score"] = score
+ initial_candidates.append(candidate)
+
+ # Step 4: Recursive search
+ with telemetry.measure("search.vector_retrieval"):
+ candidates = await self._recursive_search(
+ vector_proxy=vector_proxy,
+ query=query.query,
+ query_vector=query_vector,
+ sparse_query_vector=sparse_query_vector,
+ starting_points=starting_points,
+ limit=limit,
+ mode=mode,
+ threshold=effective_threshold,
+ score_gte=score_gte,
+ context_type=context_type,
+ target_dirs=target_dirs,
+ scope_dsl=scope_dsl,
+ initial_candidates=initial_candidates,
+ level=level,
+ )
+ apply_hotness = True
+ rerank_used = self._rerank_client is not None and mode == RetrieverMode.THINKING
# Step 6: Convert results
matched = await self._convert_to_matched_contexts(
candidates,
ctx=ctx,
+ apply_hotness=apply_hotness,
)
-
final = matched[:limit]
- # Record retrieval stats for the observer.
elapsed_ms = (time.monotonic() - t0) * 1000
get_stats_collector().record_query(
- context_type=query.context_type.value if query.context_type else "unknown",
+ context_type=context_type or "unknown",
result_count=len(final),
scores=[m.score for m in final],
latency_ms=elapsed_ms,
- rerank_used=self._rerank_client is not None and mode == RetrieverMode.THINKING,
+ rerank_used=rerank_used,
)
return QueryResult(
@@ -238,30 +300,23 @@ class HierarchicalRetriever:
searched_directories=root_uris,
)
- async def _global_vector_search(
- self,
- vector_proxy: VikingDBManagerProxy,
- query_vector: Optional[List[float]],
- sparse_query_vector: Optional[Dict[str, float]],
- context_type: Optional[str],
- target_dirs: List[str],
- scope_dsl: Optional[Dict[str, Any]],
- limit: int,
- ) -> List[Dict[str, Any]]:
- """Global vector search to locate initial directories."""
- results = await vector_proxy.search_global_roots_in_tenant(
- query_vector=query_vector,
- sparse_query_vector=sparse_query_vector,
- context_type=context_type,
- target_directories=target_dirs,
- extra_filter=scope_dsl,
- limit=limit,
- )
- telemetry = get_current_telemetry()
- telemetry.count("vector.searches", 1)
- telemetry.count("vector.scored", len(results))
- telemetry.count("vector.scanned", len(results))
- return results
+ def _resolve_threshold(self, threshold: Optional[float]) -> float:
+ resolved = threshold if threshold is not None else self.threshold
+ return resolved if resolved is not None else 0.0
+
+ @staticmethod
+ def _finite_score(value: Any, default: float = 0.0) -> float:
+ try:
+ score = float(value)
+ except (TypeError, ValueError):
+ return default
+ return score if math.isfinite(score) else default
+
+ @staticmethod
+ def _passes_threshold(score: float, threshold: float, score_gte: bool) -> bool:
+ if score_gte:
+ return score >= threshold
+ return score > threshold
def _rerank_scores(
self,
@@ -289,81 +344,9 @@ class HierarchicalRetriever:
normalized_scores: List[float] = []
for score, fallback in zip(scores, fallback_scores, strict=True):
- if isinstance(score, (int, float)):
- normalized_scores.append(float(score))
- else:
- normalized_scores.append(fallback)
+ normalized_scores.append(self._finite_score(score, fallback))
return normalized_scores
- def _merge_starting_points(
- self,
- query: str,
- root_uris: List[str],
- global_results: List[Dict[str, Any]],
- mode: str = "thinking",
- ) -> List[Tuple[str, float]]:
- """Merge starting points.
- Returns:
- List of (uri, parent_score) tuples
- """
- points = []
- seen = set()
-
- global_results = [r for r in global_results if r.get("level", 2) != 2]
-
- # Results from global search
- default_scores = [
- s if math.isfinite(s) else 0.0 for s in (r.get("_score", 0.0) for r in global_results)
- ]
- if self._rerank_client and mode == RetrieverMode.THINKING:
- docs = [str(r.get("abstract", "")) for r in global_results]
- query_scores = self._rerank_scores(query, docs, default_scores)
- for i, r in enumerate(global_results):
- # 只添加非 level 2 的项目到起始点
- if r.get("level", 2) != 2:
- points.append((r["uri"], query_scores[i]))
- seen.add(r["uri"])
- else:
- for r in global_results:
- # 只添加非 level 2 的项目到起始点
- if r.get("level", 2) != 2:
- points.append((r["uri"], r["_score"]))
- seen.add(r["uri"])
-
- # Root directories as starting points
- for uri in root_uris:
- if uri not in seen:
- points.append((uri, 0.0))
- seen.add(uri)
-
- return points
-
- def _prepare_initial_candidates(
- self,
- query: str,
- global_results: List[Dict[str, Any]],
- mode: str = RetrieverMode.THINKING,
- ) -> List[Dict[str, Any]]:
- """Preserve rerank scores for global hits added to the result pool."""
- initial_candidates = [dict(r) for r in global_results]
- if not initial_candidates:
- return []
-
- default_scores = [
- s if math.isfinite(s) else 0.0
- for s in (r.get("_score", 0.0) for r in initial_candidates)
- ]
- if self._rerank_client and mode == RetrieverMode.THINKING:
- docs = [str(r.get("abstract", "")) for r in initial_candidates]
- query_scores = self._rerank_scores(query, docs, default_scores)
- else:
- query_scores = default_scores
-
- for candidate, score in zip(initial_candidates, query_scores, strict=True):
- candidate["_score"] = score
-
- return initial_candidates
-
async def _recursive_search(
self,
vector_proxy: VikingDBManagerProxy,
@@ -377,7 +360,7 @@ class HierarchicalRetriever:
score_gte: bool = False,
context_type: Optional[str] = None,
target_dirs: Optional[List[str]] = None,
- scope_dsl: Optional[Dict[str, Any]] = None,
+ scope_dsl: Optional[FilterExpr | Dict[str, Any]] = None,
initial_candidates: Optional[List[Dict[str, Any]]] = None,
level: Optional[List[int]] = None,
) -> List[Dict[str, Any]]:
@@ -390,14 +373,7 @@ class HierarchicalRetriever:
grep_patterns: Keyword match patterns
scope_dsl: Additional scope constraints from public find/search filter
"""
- # Use passed threshold or default threshold
- effective_threshold = threshold if threshold is not None else self.threshold
-
- def passes_threshold(score: float) -> bool:
- """Check if score passes threshold."""
- if score_gte:
- return score >= effective_threshold
- return score > effective_threshold
+ effective_threshold = self._resolve_threshold(threshold)
sparse_query_vector = sparse_query_vector or None
@@ -416,8 +392,8 @@ class HierarchicalRetriever:
if not uri:
continue
if level is None or r.get("level", 2) in level:
- score = r.get("_score", 0.0)
- if not passes_threshold(score):
+ score = self._finite_score(r.get("_score", 0.0))
+ if not self._passes_threshold(score, effective_threshold, score_gte):
logger.debug(
f"[RecursiveSearch] Initial candidate URI {uri} score {score:.4f} did not pass threshold {effective_threshold}"
)
@@ -474,9 +450,7 @@ class HierarchicalRetriever:
if not results:
continue
- query_scores = [
- s if math.isfinite(s) else 0.0 for s in (r.get("_score", 0.0) for r in results)
- ]
+ query_scores = [self._finite_score(r.get("_score", 0.0)) for r in results]
if self._rerank_client and mode == RetrieverMode.THINKING:
documents = [str(r.get("abstract", "")) for r in results]
query_scores = self._rerank_scores(query, documents, query_scores)
@@ -487,7 +461,7 @@ class HierarchicalRetriever:
alpha * score + (1 - alpha) * current_score if current_score else score
)
- if not passes_threshold(final_score):
+ if not self._passes_threshold(final_score, effective_threshold, score_gte):
logger.debug(
f"[RecursiveSearch] URI {uri} score {final_score} did not pass threshold {effective_threshold}"
)
@@ -546,6 +520,7 @@ class HierarchicalRetriever:
self,
candidates: List[Dict[str, Any]],
ctx: RequestContext,
+ apply_hotness: bool = True,
) -> List[MatchedContext]:
"""Convert candidate results to MatchedContext list.
@@ -557,13 +532,11 @@ class HierarchicalRetriever:
for c in candidates:
relations = []
- semantic_score = c.get("_final_score", c.get("_score", 0.0))
# Fix: clamp inf/nan scores from vector search (#inf-score)
- if not math.isfinite(semantic_score):
- semantic_score = 0.0
+ semantic_score = self._finite_score(c.get("_final_score", c.get("_score", 0.0)))
alpha = self.hotness_alpha
- if alpha > 0:
+ if apply_hotness and alpha > 0:
updated_at_raw = c.get("updated_at")
if isinstance(updated_at_raw, str):
try:
diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.py
index 2b6921fe2..c3a5e083b 100644
--- a/openviking/storage/viking_vector_index_backend.py
+++ b/openviking/storage/viking_vector_index_backend.py
@@ -36,18 +36,6 @@ LOOKUP_OUTPUT_FIELDS = [
"active_count",
]
-MEMORY_DEDUP_OUTPUT_FIELDS = [
- "uri",
- "abstract",
- "context_type",
- "created_at",
- "updated_at",
- "active_count",
- "level",
- "account_id",
- "owner_user_id",
-]
-
FETCH_BY_URI_OUTPUT_FIELDS = [
"id",
"uri",
@@ -1106,6 +1094,7 @@ class VikingVectorIndexBackend:
context_type: Optional[str] = None,
target_directories: Optional[List[str]] = None,
extra_filter: Optional[FilterExpr | Dict[str, Any]] = None,
+ level: Optional[List[int]] = None,
limit: int = 10,
offset: int = 0,
) -> List[Dict[str, Any]]:
@@ -1114,6 +1103,7 @@ class VikingVectorIndexBackend:
context_type=context_type,
target_directories=target_directories,
extra_filter=extra_filter,
+ level=level,
)
return await self.search(
query_vector=query_vector,
@@ -1125,37 +1115,6 @@ class VikingVectorIndexBackend:
ctx=ctx,
)
- async def search_global_roots_in_tenant(
- self,
- ctx: RequestContext,
- query_vector: Optional[List[float]],
- sparse_query_vector: Optional[Dict[str, float]] = None,
- context_type: Optional[str] = None,
- target_directories: Optional[List[str]] = None,
- extra_filter: Optional[FilterExpr | Dict[str, Any]] = None,
- limit: int = 10,
- ) -> List[Dict[str, Any]]:
- if not query_vector:
- return []
-
- merged_filter = self._merge_filters(
- self._build_scope_filter(
- ctx=ctx,
- context_type=context_type,
- target_directories=target_directories,
- extra_filter=extra_filter,
- ),
- In("level", [0, 1, 2]), # TODO: smj fix this
- )
- return await self.search(
- query_vector=query_vector,
- sparse_query_vector=sparse_query_vector,
- filter=merged_filter,
- limit=limit,
- output_fields=RETRIEVAL_OUTPUT_FIELDS,
- ctx=ctx,
- )
-
async def search_children_in_tenant(
self,
ctx: RequestContext,
@@ -1201,31 +1160,6 @@ class VikingVectorIndexBackend:
ctx=ctx,
)
- async def search_similar_memories(
- self,
- owner_space: Optional[str],
- category_uri_prefix: str,
- query_vector: List[float],
- limit: int = 5,
- *,
- ctx: RequestContext,
- ) -> List[Dict[str, Any]]:
- conds: List[FilterExpr] = [
- Eq("context_type", "memory"),
- Eq("level", 2),
- Eq("account_id", ctx.account_id),
- ]
- if category_uri_prefix:
- conds.append(PathScope("uri", canonicalize_uri(category_uri_prefix, ctx), depth=-1))
-
- backend = self._get_backend_for_context(ctx)
- return await backend.search(
- query_vector=query_vector,
- filter=And(conds),
- limit=limit,
- output_fields=MEMORY_DEDUP_OUTPUT_FIELDS,
- )
-
async def get_context_by_uri(
self,
uri: str,
@@ -1386,6 +1320,7 @@ class VikingVectorIndexBackend:
context_type: Optional[str],
target_directories: Optional[List[str]],
extra_filter: Optional[FilterExpr | Dict[str, Any]],
+ level: Optional[List[int]] = None,
) -> Optional[FilterExpr]:
filters: List[FilterExpr] = []
if context_type:
@@ -1410,8 +1345,10 @@ class VikingVectorIndexBackend:
else:
filters.append(extra_filter)
- merged = self._merge_filters(*filters)
- return merged
+ if level:
+ filters.append(In("level", level))
+
+ return self._merge_filters(*filters)
@staticmethod
def _tenant_filter(
diff --git a/openviking/storage/vikingdb_manager.py b/openviking/storage/vikingdb_manager.py
index 5701403b8..39c1247e9 100644
--- a/openviking/storage/vikingdb_manager.py
+++ b/openviking/storage/vikingdb_manager.py
@@ -184,7 +184,6 @@ class VikingDBManagerProxy:
# 使用(无需传 ctx;仅在需要保留未显式传入字段时开启 partial_update)
await proxy.upsert(data, partial_update=True)
- results = await proxy.search_similar_memories(...)
```
"""
@@ -418,6 +417,7 @@ class VikingDBManagerProxy:
context_type: Optional[str] = None,
target_directories: Optional[List[str]] = None,
extra_filter: Optional[FilterExpr | Dict[str, Any]] = None,
+ level: Optional[List[int]] = None,
limit: int = 10,
offset: int = 0,
) -> List[Dict[str, Any]]:
@@ -428,29 +428,11 @@ class VikingDBManagerProxy:
context_type=context_type,
target_directories=target_directories,
extra_filter=extra_filter,
+ level=level,
limit=limit,
offset=offset,
)
- async def search_global_roots_in_tenant(
- self,
- query_vector: Optional[List[float]],
- sparse_query_vector: Optional[Dict[str, float]] = None,
- context_type: Optional[str] = None,
- target_directories: Optional[List[str]] = None,
- extra_filter: Optional[FilterExpr | Dict[str, Any]] = None,
- limit: int = 10,
- ) -> List[Dict[str, Any]]:
- return await self._manager.search_global_roots_in_tenant(
- self._ctx,
- query_vector=query_vector,
- sparse_query_vector=sparse_query_vector,
- context_type=context_type,
- target_directories=target_directories,
- extra_filter=extra_filter,
- limit=limit,
- )
-
async def search_children_in_tenant(
self,
parent_uri: str,
@@ -472,21 +454,6 @@ class VikingDBManagerProxy:
limit=limit,
)
- async def search_similar_memories(
- self,
- owner_space: Optional[str],
- category_uri_prefix: str,
- query_vector: List[float],
- limit: int = 5,
- ) -> List[Dict[str, Any]]:
- return await self._manager.search_similar_memories(
- owner_space=owner_space,
- category_uri_prefix=category_uri_prefix,
- query_vector=query_vector,
- limit=limit,
- ctx=self._ctx,
- )
-
async def get_context_by_uri(
self,
uri: str,
diff --git a/tests/retrieve/test_hierarchical_retriever_rerank.py b/tests/retrieve/test_hierarchical_retriever_rerank.py
index 1c5ab89de..09691dd84 100644
--- a/tests/retrieve/test_hierarchical_retriever_rerank.py
+++ b/tests/retrieve/test_hierarchical_retriever_rerank.py
@@ -12,6 +12,18 @@ from openviking_cli.session.user_id import UserIdentifier
from openviking_cli.utils.config import RerankConfig, RetrievalConfig
+def _result(uri, score, level=2, abstract=None, **extra):
+ result = {
+ "uri": uri,
+ "abstract": abstract if abstract is not None else uri.rsplit("/", 1)[-1],
+ "_score": score,
+ "level": level,
+ "context_type": "resource",
+ }
+ result.update(extra)
+ return result
+
+
class DummyEmbedResult:
def __init__(self) -> None:
self.dense_vector = [1.0]
@@ -32,13 +44,13 @@ class DummyEmbedder:
class DummyStorage:
def __init__(self) -> None:
self.collection_name = "context"
- self.global_search_calls = []
+ self.search_calls = []
self.child_search_calls = []
async def collection_exists_bound(self) -> bool:
return True
- async def search_global_roots_in_tenant(
+ async def search_in_tenant(
self,
ctx,
query_vector=None,
@@ -46,9 +58,11 @@ class DummyStorage:
context_type=None,
target_directories=None,
extra_filter=None,
+ level=None,
limit: int = 10,
+ offset: int = 0,
):
- self.global_search_calls.append(
+ self.search_calls.append(
{
"ctx": ctx,
"query_vector": query_vector,
@@ -56,24 +70,14 @@ class DummyStorage:
"context_type": context_type,
"target_directories": target_directories,
"extra_filter": extra_filter,
+ "level": level,
"limit": limit,
+ "offset": offset,
}
)
return [
- {
- "uri": "viking://resources/root-a",
- "abstract": "root A",
- "_score": 0.2,
- "level": 1,
- "context_type": "resource",
- },
- {
- "uri": "viking://resources/root-b",
- "abstract": "root B",
- "_score": 0.8,
- "level": 1,
- "context_type": "resource",
- },
+ _result("viking://resources/root-a", 0.2, level=1, abstract="root A"),
+ _result("viking://resources/root-b", 0.8, level=1, abstract="root B"),
]
async def search_children_in_tenant(
@@ -101,28 +105,18 @@ class DummyStorage:
)
if parent_uri == "viking://resources":
return [
- {
- "uri": "viking://resources/file-a",
- "abstract": "child A",
- "_score": 0.2,
- "level": 2,
- "context_type": "resource",
- "category": "doc",
- },
- {
- "uri": "viking://resources/file-b",
- "abstract": "child B",
- "_score": 0.8,
- "level": 2,
- "context_type": "resource",
- "category": "doc",
- },
+ _result("viking://resources/file-a", 0.2, abstract="child A", category="doc"),
+ _result("viking://resources/file-b", 0.8, abstract="child B", category="doc"),
]
return []
-class LevelTwoGlobalStorage(DummyStorage):
- async def search_global_roots_in_tenant(
+class QuickSearchStorage(DummyStorage):
+ def __init__(self, results):
+ super().__init__()
+ self.results = list(results)
+
+ async def search_in_tenant(
self,
ctx,
query_vector=None,
@@ -130,9 +124,11 @@ class LevelTwoGlobalStorage(DummyStorage):
context_type=None,
target_directories=None,
extra_filter=None,
+ level=None,
limit: int = 10,
+ offset: int = 0,
):
- self.global_search_calls.append(
+ self.search_calls.append(
{
"ctx": ctx,
"query_vector": query_vector,
@@ -140,26 +136,15 @@ class LevelTwoGlobalStorage(DummyStorage):
"context_type": context_type,
"target_directories": target_directories,
"extra_filter": extra_filter,
+ "level": level,
"limit": limit,
+ "offset": offset,
}
)
return [
- {
- "uri": "viking://resources/file-a",
- "abstract": "child A",
- "_score": 0.2,
- "level": 2,
- "context_type": "resource",
- "category": "doc",
- },
- {
- "uri": "viking://resources/file-b",
- "abstract": "child B",
- "_score": 0.8,
- "level": 2,
- "context_type": "resource",
- "category": "doc",
- },
+ dict(result)
+ for result in self.results
+ if level is None or result.get("level", 2) in level
]
async def search_children_in_tenant(
@@ -185,7 +170,7 @@ class LevelTwoGlobalStorage(DummyStorage):
"limit": limit,
}
)
- return []
+ return [_result(f"{parent_uri}/should-not-be-returned", 1.0, abstract="child")]
class DirectChildProxy:
@@ -200,20 +185,8 @@ class DirectChildProxy:
limit: int = 10,
):
return [
- {
- "uri": f"{parent_uri}/file-a",
- "abstract": "child A",
- "_score": 0.2,
- "level": 2,
- "context_type": "resource",
- },
- {
- "uri": f"{parent_uri}/file-b",
- "abstract": "child B",
- "_score": 0.8,
- "level": 2,
- "context_type": "resource",
- },
+ _result(f"{parent_uri}/file-a", 0.2, abstract="child A"),
+ _result(f"{parent_uri}/file-b", 0.8, abstract="child B"),
]
@@ -251,8 +224,9 @@ def test_retriever_initializes_rerank_client(monkeypatch):
lambda config: fake_client,
)
+ storage = DummyStorage()
retriever = HierarchicalRetriever(
- storage=DummyStorage(),
+ storage=storage,
embedder=DummyEmbedder(),
rerank_config=_config(),
)
@@ -260,49 +234,40 @@ def test_retriever_initializes_rerank_client(monkeypatch):
assert retriever._rerank_client is fake_client
-def test_merge_starting_points_prefers_rerank_scores_in_thinking_mode(monkeypatch):
- fake_client = FakeRerankClient([0.95, 0.05])
+async def test_retrieve_uses_rerank_scores_in_thinking_mode(monkeypatch):
+ fake_client = FakeRerankClient([0.95, 0.05, 0.11, 0.95])
monkeypatch.setattr(
"openviking.retrieve.hierarchical_retriever.RerankClient.from_config",
lambda config: fake_client,
)
+ storage = DummyStorage()
retriever = HierarchicalRetriever(
- storage=DummyStorage(),
+ storage=storage,
embedder=DummyEmbedder(),
rerank_config=_config(),
)
- starting_points = retriever._merge_starting_points(
- "hello",
- ["viking://resources"],
- [
- {
- "uri": "viking://resources/root-a",
- "abstract": "root A",
- "_score": 0.2,
- "level": 1,
- },
- {
- "uri": "viking://resources/root-b",
- "abstract": "root B",
- "_score": 0.8,
- "level": 1,
- },
- ],
- mode=RetrieverMode.THINKING,
- )
+ result = await retriever.retrieve(_query(), ctx=_ctx(), limit=2, mode=RetrieverMode.THINKING)
- assert starting_points[:2] == [
- ("viking://resources/root-a", 0.95),
- ("viking://resources/root-b", 0.05),
+ assert [ctx.uri for ctx in result.matched_contexts] == [
+ "viking://resources/file-b",
+ "viking://resources/file-a",
]
- assert fake_client.calls == [("hello", ["root A", "root B"])]
+ assert fake_client.calls[0] == ("hello", ["root A", "root B"])
+ assert fake_client.calls[1] == ("hello", ["child A", "child B"])
+ assert storage.search_calls[0]["level"] == [0, 1]
@pytest.mark.asyncio
-async def test_retrieve_uses_rerank_scores_in_thinking_mode(monkeypatch):
- fake_client = FakeRerankClient([0.95, 0.05, 0.11, 0.95])
+async def test_retrieve_falls_back_to_vector_scores_when_rerank_returns_none(monkeypatch):
+ class NoneRerankClient(FakeRerankClient):
+ def rerank_batch(self, query: str, documents: list[str]):
+ self.calls.append((query, list(documents)))
+ return None
+
+ fake_client = NoneRerankClient([])
monkeypatch.setattr(
"openviking.retrieve.hierarchical_retriever.RerankClient.from_config",
lambda config: fake_client,
@@ -320,82 +285,156 @@ async def test_retrieve_uses_rerank_scores_in_thinking_mode(monkeypatch):
"viking://resources/file-b",
"viking://resources/file-a",
]
- assert fake_client.calls[0] == ("hello", ["root A", "root B"])
- assert fake_client.calls[1] == ("hello", ["child A", "child B"])
+ assert fake_client.calls
@pytest.mark.asyncio
-async def test_retrieve_reranks_level_two_initial_candidates_in_thinking_mode(monkeypatch):
- fake_client = FakeRerankClient([0.11, 0.95])
+async def test_quick_mode_uses_single_vector_search_without_rerank_or_recursion(monkeypatch):
+ fake_client = FakeRerankClient([0.05, 0.95, 0.95])
monkeypatch.setattr(
"openviking.retrieve.hierarchical_retriever.RerankClient.from_config",
lambda config: fake_client,
)
+ storage = QuickSearchStorage(
+ [
+ _result("viking://resources/root", 0.95, level=0, abstract="root abstract"),
+ _result("viking://resources/file", 0.9, abstract="file abstract"),
+ _result("viking://resources/dir", 0.85, level=1, abstract="dir overview"),
+ ]
+ )
retriever = HierarchicalRetriever(
- storage=LevelTwoGlobalStorage(),
+ storage=storage,
embedder=DummyEmbedder(),
rerank_config=_config(),
)
- result = await retriever.retrieve(_query(), ctx=_ctx(), limit=2, mode=RetrieverMode.THINKING)
+ result = await retriever.retrieve(_query(), ctx=_ctx(), limit=3, mode=RetrieverMode.QUICK)
assert [ctx.uri for ctx in result.matched_contexts] == [
- "viking://resources/file-b",
- "viking://resources/file-a",
+ "viking://resources/root/.abstract.md",
+ "viking://resources/file",
+ "viking://resources/dir/.overview.md",
+ ]
+ assert [ctx.level for ctx in result.matched_contexts] == [0, 2, 1]
+ assert [ctx.score for ctx in result.matched_contexts] == [
+ pytest.approx(0.95),
+ pytest.approx(0.9),
+ pytest.approx(0.85),
]
- assert fake_client.calls == [("hello", ["child A", "child B"])]
+ assert len(storage.search_calls) == 1
+ assert storage.search_calls[0]["limit"] == retriever.GLOBAL_SEARCH_TOPK
+ assert storage.search_calls[0]["extra_filter"] is None
+ assert storage.search_calls[0]["level"] is None
+ assert storage.child_search_calls == []
+ assert fake_client.calls == []
@pytest.mark.asyncio
-async def test_retrieve_falls_back_to_vector_scores_when_rerank_returns_none(monkeypatch):
- class NoneRerankClient(FakeRerankClient):
- def rerank_batch(self, query: str, documents: list[str]):
- self.calls.append((query, list(documents)))
- return None
-
- fake_client = NoneRerankClient([])
- monkeypatch.setattr(
- "openviking.retrieve.hierarchical_retriever.RerankClient.from_config",
- lambda config: fake_client,
+async def test_quick_mode_pushes_explicit_level_filter_to_vector_search():
+ storage = QuickSearchStorage(
+ [
+ _result("viking://resources/root", 0.99, level=0, abstract="root abstract"),
+ _result("viking://resources/dir", 0.98, level=1, abstract="dir overview"),
+ _result("viking://resources/file-a", 0.5, abstract="file A"),
+ _result("viking://resources/file-b", 0.7, abstract="file B"),
+ ]
)
-
retriever = HierarchicalRetriever(
- storage=DummyStorage(),
+ storage=storage,
embedder=DummyEmbedder(),
- rerank_config=_config(),
+ rerank_config=None,
)
- result = await retriever.retrieve(_query(), ctx=_ctx(), limit=2, mode=RetrieverMode.THINKING)
+ result = await retriever.retrieve(
+ _query(),
+ ctx=_ctx(),
+ limit=3,
+ mode=RetrieverMode.QUICK,
+ scope_dsl={"op": "must", "field": "category", "conds": ["doc"]},
+ level=[2],
+ )
assert [ctx.uri for ctx in result.matched_contexts] == [
"viking://resources/file-b",
"viking://resources/file-a",
]
- assert fake_client.calls
+ assert len(storage.search_calls) == 1
+ assert storage.search_calls[0]["limit"] == retriever.GLOBAL_SEARCH_TOPK
+ assert storage.search_calls[0]["extra_filter"] == {
+ "op": "must",
+ "field": "category",
+ "conds": ["doc"],
+ }
+ assert storage.search_calls[0]["level"] == [2]
+ assert storage.child_search_calls == []
@pytest.mark.asyncio
-async def test_quick_mode_skips_rerank(monkeypatch):
- fake_client = FakeRerankClient([0.95, 0.05, 0.05, 0.95])
- monkeypatch.setattr(
- "openviking.retrieve.hierarchical_retriever.RerankClient.from_config",
- lambda config: fake_client,
+async def test_quick_mode_threshold_uses_raw_vector_score():
+ storage = QuickSearchStorage(
+ [
+ _result("viking://resources/high", 0.91, abstract="high"),
+ _result("viking://resources/exact", 0.9, abstract="exact"),
+ ]
)
-
retriever = HierarchicalRetriever(
- storage=DummyStorage(),
+ storage=storage,
embedder=DummyEmbedder(),
- rerank_config=_config(),
+ rerank_config=None,
)
- result = await retriever.retrieve(_query(), ctx=_ctx(), limit=2, mode=RetrieverMode.QUICK)
+ strict_result = await retriever.retrieve(
+ _query(),
+ ctx=_ctx(),
+ limit=2,
+ mode=RetrieverMode.QUICK,
+ score_threshold=0.9,
+ )
+ inclusive_result = await retriever.retrieve(
+ _query(),
+ ctx=_ctx(),
+ limit=2,
+ mode=RetrieverMode.QUICK,
+ score_threshold=0.9,
+ score_gte=True,
+ )
- assert [ctx.uri for ctx in result.matched_contexts] == [
- "viking://resources/file-b",
- "viking://resources/file-a",
+ assert [ctx.uri for ctx in strict_result.matched_contexts] == ["viking://resources/high"]
+ assert [ctx.uri for ctx in inclusive_result.matched_contexts] == [
+ "viking://resources/high",
+ "viking://resources/exact",
]
- assert fake_client.calls == []
+
+
+async def test_quick_mode_keeps_scores_pure_when_hotness_and_propagation_configured(monkeypatch):
+ monkeypatch.setattr(
+ "openviking.retrieve.hierarchical_retriever.hotness_score",
+ lambda *args, **kwargs: pytest.fail("hotness_score should not be called in QUICK mode"),
+ )
+ storage = QuickSearchStorage(
+ [
+ _result(
+ "viking://resources/file-a",
+ 0.8,
+ abstract="file A",
+ active_count=100,
+ updated_at="2026-01-01T00:00:00+00:00",
+ )
+ ]
+ )
+ retriever = HierarchicalRetriever(
+ storage=storage,
+ embedder=DummyEmbedder(),
+ rerank_config=None,
+ retrieval_config=RetrievalConfig(hotness_alpha=0.5, score_propagation_alpha=0.1),
+ )
+
+ result = await retriever.retrieve(_query(), ctx=_ctx(), limit=1, mode=RetrieverMode.QUICK)
+
+ assert result.matched_contexts[0].score == pytest.approx(0.8)
+ assert storage.child_search_calls == []
@pytest.mark.asyncio
@@ -434,15 +473,7 @@ async def test_default_retrieval_config_uses_semantic_score_without_hotness(monk
)
result = await retriever._convert_to_matched_contexts(
- [
- {
- "uri": "viking://resources/file-a",
- "abstract": "child A",
- "_score": 1.0,
- "level": 2,
- "context_type": "resource",
- }
- ],
+ [_result("viking://resources/file-a", 1.0, abstract="child A")],
ctx=_ctx(),
)
@@ -463,15 +494,7 @@ async def test_retrieval_hotness_alpha_blends_when_configured(monkeypatch):
)
result = await retriever._convert_to_matched_contexts(
- [
- {
- "uri": "viking://resources/file-a",
- "abstract": "child A",
- "_score": 1.0,
- "level": 2,
- "context_type": "resource",
- }
- ],
+ [_result("viking://resources/file-a", 1.0, abstract="child A")],
ctx=_ctx(),
)
@@ -487,15 +510,7 @@ async def test_convert_to_matched_contexts_returns_empty_relations():
)
result = await retriever._convert_to_matched_contexts(
- [
- {
- "uri": "viking://resources/file-a",
- "abstract": "child A",
- "_score": 1.0,
- "level": 2,
- "context_type": "resource",
- }
- ],
+ [_result("viking://resources/file-a", 1.0, abstract="child A")],
ctx=_ctx(),
)
diff --git a/tests/retrieve/test_hierarchical_retriever_target_dirs.py b/tests/retrieve/test_hierarchical_retriever_target_dirs.py
deleted file mode 100644
index 6bec55494..000000000
--- a/tests/retrieve/test_hierarchical_retriever_target_dirs.py
+++ /dev/null
@@ -1,95 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-
-"""Hierarchical retriever target_directories tests."""
-
-import pytest
-
-from openviking.retrieve.hierarchical_retriever import HierarchicalRetriever
-from openviking.server.identity import RequestContext, Role
-from openviking_cli.retrieve.types import ContextType, TypedQuery
-from openviking_cli.session.user_id import UserIdentifier
-
-
-class DummyStorage:
- """Minimal storage stub to capture search filters."""
-
- def __init__(self) -> None:
- self.collection_name = "context"
- self.global_search_calls = []
- self.child_search_calls = []
-
- async def collection_exists_bound(self) -> bool:
- return True
-
- async def search_global_roots_in_tenant(
- self,
- ctx,
- query_vector=None,
- sparse_query_vector=None,
- context_type=None,
- target_directories=None,
- extra_filter=None,
- limit: int = 10,
- ):
- self.global_search_calls.append(
- {
- "ctx": ctx,
- "query_vector": query_vector,
- "sparse_query_vector": sparse_query_vector,
- "context_type": context_type,
- "target_directories": target_directories,
- "extra_filter": extra_filter,
- "limit": limit,
- }
- )
- return []
-
- async def search_children_in_tenant(
- self,
- ctx,
- parent_uri: str,
- query_vector=None,
- sparse_query_vector=None,
- context_type=None,
- target_directories=None,
- extra_filter=None,
- limit: int = 10,
- ):
- self.child_search_calls.append(
- {
- "ctx": ctx,
- "parent_uri": parent_uri,
- "query_vector": query_vector,
- "sparse_query_vector": sparse_query_vector,
- "context_type": context_type,
- "target_directories": target_directories,
- "extra_filter": extra_filter,
- "limit": limit,
- }
- )
- return []
-
-
-async def test_retrieve_honors_target_directories_scope_filter():
- target_uri = "viking://resources/foo"
- storage = DummyStorage()
- retriever = HierarchicalRetriever(storage=storage, embedder=None, rerank_config=None)
- ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)
-
- query = TypedQuery(
- query="test",
- context_type=ContextType.RESOURCE,
- intent="",
- target_directories=[target_uri],
- )
-
- result = await retriever.retrieve(query, ctx=ctx, limit=3)
-
- assert result.searched_directories == [target_uri]
- assert storage.global_search_calls
- assert storage.global_search_calls[0]["target_directories"] == [target_uri]
- assert storage.child_search_calls
- assert storage.child_search_calls[0]["target_directories"] == [target_uri]
- assert storage.child_search_calls[0]["parent_uri"] == target_uri