Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(hermes-plugin): drop the query-token overlap boost from recall ranking (#5712)

Maksym Kozachuk6bd89a36be81
diff --git a/examples/hermes-plugin/__init__.py b/examples/hermes-plugin/__init__.py
index 2e338061c..33ab9da32 100644
--- a/examples/hermes-plugin/__init__.py
+++ b/examples/hermes-plugin/__init__.py
@@ -2453,14 +2453,12 @@ class OpenVikingMemoryProvider(MemoryProvider):
@classmethod
def _select_recall_candidates(cls, items: List[Dict[str, Any]], query: str, *, limit: int, score_threshold: float) -> List[Dict[str, Any]]:
"""Threshold + dedupe (uri, then abstract+category — events/cases stay URI-distinct),
- ranked by score + L2 leaf boost + query-token overlap."""
- tokens = ["".join(ch for ch in raw if ch.isalnum()) for raw in query.lower().replace("_", " ").split()]
- tokens = [token for token in tokens if len(token) >= 2][:8]
+ ranked by score + L2 leaf boost. No query-token overlap boost: server scores sit in a
+ band of about +/-0.01, so a literal-overlap bonus decided the order and buried
+ cross-language hits (queries and memories in different languages share no tokens)."""
def rank(item: Dict[str, Any]) -> float:
- text = f"{item.get('uri', '')} {cls._recall_abstract(item)}".lower()
- overlap_boost = min(0.2, sum(1 for token in tokens if token in text) * 0.05)
- return cls._clamp_score(item.get("score")) + (0.12 if item.get("level") == 2 else 0.0) + overlap_boost
+ return cls._clamp_score(item.get("score")) + (0.12 if item.get("level") == 2 else 0.0)
seen_uri, seen_key = set(), set()
filtered: List[Dict[str, Any]] = []
diff --git a/examples/hermes-plugin/tests/test_recall_rank.py b/examples/hermes-plugin/tests/test_recall_rank.py
new file mode 100644
index 000000000..a9a0057b9
--- /dev/null
+++ b/examples/hermes-plugin/tests/test_recall_rank.py
@@ -0,0 +1,96 @@
+"""Recall candidate ranking: server score + L2 leaf boost, no query-token overlap boost.
+
+The OpenViking server's similarity scores for one query typically sit in a narrow band
+(about +/-0.01 in practice). A literal query-token overlap bonus of 0.05/token was larger
+than that whole band, so it decided the order: memories that merely repeated the query's
+words (often raw chat-log events) outranked the semantically closer fact, and memories in a
+different language from the query (no shared tokens at all) were pushed out of the top N.
+"""
+
+import pytest
+
+
+def _item(uri, score, *, abstract="", level=2, category="preferences"):
+ return {
+ "uri": uri,
+ "score": score,
+ "abstract": abstract,
+ "level": level,
+ "category": category,
+ }
+
+
+def select(external_provider):
+ _, provider, _, _ = external_provider("recall-rank")
+ provider_cls = type(provider)
+
+ def run(items, query, limit=5, score_threshold=0.0):
+ selected = provider_cls._select_recall_candidates(
+ items, query, limit=limit, score_threshold=score_threshold
+ )
+ return [item["uri"] for item in selected]
+
+ return run
+
+
+def test_literal_overlap_does_not_outrank_a_higher_server_score(select):
+ query = "what hookah tobacco does the user like best"
+ items = [
+ # English summary of a fact the user stated in another language: no shared tokens.
+ _item(
+ "viking://user/memories/preferences/tobacco.md",
+ 0.512,
+ abstract="Prefers Darkside Bananapapa, mild blends",
+ ),
+ # Chat-log event that repeats the query's words but answers nothing.
+ _item(
+ "viking://user/memories/events/chat_1.md",
+ 0.505,
+ abstract="user asked what hookah tobacco does the user like best",
+ category="events",
+ ),
+ ]
+
+ assert select(items, query) == [
+ "viking://user/memories/preferences/tobacco.md",
+ "viking://user/memories/events/chat_1.md",
+ ]
+
+
+def test_ranking_matches_server_order_among_leaves_regardless_of_query_text(select):
+ items = [
+ _item("viking://user/memories/a.md", 0.53, abstract="alpha"),
+ _item("viking://user/memories/b.md", 0.52, abstract="beta beta beta"),
+ _item("viking://user/memories/c.md", 0.51, abstract="gamma"),
+ ]
+
+ assert (
+ select(items, "beta beta beta beta")
+ == select(items, "")
+ == [
+ "viking://user/memories/a.md",
+ "viking://user/memories/b.md",
+ "viking://user/memories/c.md",
+ ]
+ )
+
+
+def test_leaf_boost_threshold_dedupe_and_limit_are_kept(select):
+ items = [
+ _item("viking://user/memories/dir", 0.60, abstract="directory overview", level=1),
+ _item("viking://user/memories/leaf.md", 0.55, abstract="leaf fact"),
+ _item("viking://user/memories/leaf.md", 0.55, abstract="leaf fact"), # same uri
+ # Same abstract + category as leaf.md, different uri.
+ _item("viking://user/memories/copy.md", 0.54, abstract="Leaf fact"),
+ _item("viking://user/memories/low.md", 0.10, abstract="below threshold"),
+ _item("viking://user/memories/other.md", 0.50, abstract="other fact"),
+ ]
+
+ # L2 leaf (+0.12) beats the higher-scored L1 directory; duplicates and sub-threshold drop out.
+ assert select(items, "leaf", limit=5, score_threshold=0.2) == [
+ "viking://user/memories/leaf.md",
+ "viking://user/memories/other.md",
+ "viking://user/memories/dir",
+ ]
+ assert select(items, "leaf", limit=1, score_threshold=0.2) == ["viking://user/memories/leaf.md"]