Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(hermes-plugin): drop the query-token overlap boost from recall ranking (#5712)
Maksym Kozachuk6bd89a36be81
diff --git a/examples/hermes-plugin/__init__.py b/examples/hermes-plugin/__init__.pyindex 2e338061c..33ab9da32 100644--- a/examples/hermes-plugin/__init__.py+++ b/examples/hermes-plugin/__init__.py@@ -2453,14 +2453,12 @@ class OpenVikingMemoryProvider(MemoryProvider):@classmethoddef _select_recall_candidates(cls, items: List[Dict[str, Any]], query: str, *, limit: int, score_threshold: float) -> List[Dict[str, Any]]:"""Threshold + dedupe (uri, then abstract+category — events/cases stay URI-distinct),- ranked by score + L2 leaf boost + query-token overlap."""- tokens = ["".join(ch for ch in raw if ch.isalnum()) for raw in query.lower().replace("_", " ").split()]- tokens = [token for token in tokens if len(token) >= 2][:8]+ ranked by score + L2 leaf boost. No query-token overlap boost: server scores sit in a+ band of about +/-0.01, so a literal-overlap bonus decided the order and buried+ cross-language hits (queries and memories in different languages share no tokens)."""def rank(item: Dict[str, Any]) -> float:- text = f"{item.get('uri', '')} {cls._recall_abstract(item)}".lower()- overlap_boost = min(0.2, sum(1 for token in tokens if token in text) * 0.05)- return cls._clamp_score(item.get("score")) + (0.12 if item.get("level") == 2 else 0.0) + overlap_boost+ return cls._clamp_score(item.get("score")) + (0.12 if item.get("level") == 2 else 0.0)seen_uri, seen_key = set(), set()filtered: List[Dict[str, Any]] = []diff --git a/examples/hermes-plugin/tests/test_recall_rank.py b/examples/hermes-plugin/tests/test_recall_rank.pynew file mode 100644index 000000000..a9a0057b9--- /dev/null+++ b/examples/hermes-plugin/tests/test_recall_rank.py@@ -0,0 +1,96 @@+"""Recall candidate ranking: server score + L2 leaf boost, no query-token overlap boost.++The OpenViking server's similarity scores for one query typically sit in a narrow band+(about +/-0.01 in practice). A literal query-token overlap bonus of 0.05/token was larger+than that whole band, so it decided the order: memories that merely repeated the query's+words (often raw chat-log events) outranked the semantically closer fact, and memories in a+different language from the query (no shared tokens at all) were pushed out of the top N.+"""++import pytest+++def _item(uri, score, *, abstract="", level=2, category="preferences"):+ return {+ "uri": uri,+ "score": score,+ "abstract": abstract,+ "level": level,+ "category": category,+ }+++def select(external_provider):+ _, provider, _, _ = external_provider("recall-rank")+ provider_cls = type(provider)++ def run(items, query, limit=5, score_threshold=0.0):+ selected = provider_cls._select_recall_candidates(+ items, query, limit=limit, score_threshold=score_threshold+ )+ return [item["uri"] for item in selected]++ return run+++def test_literal_overlap_does_not_outrank_a_higher_server_score(select):+ query = "what hookah tobacco does the user like best"+ items = [+ # English summary of a fact the user stated in another language: no shared tokens.+ _item(+ "viking://user/memories/preferences/tobacco.md",+ 0.512,+ abstract="Prefers Darkside Bananapapa, mild blends",+ ),+ # Chat-log event that repeats the query's words but answers nothing.+ _item(+ "viking://user/memories/events/chat_1.md",+ 0.505,+ abstract="user asked what hookah tobacco does the user like best",+ category="events",+ ),+ ]++ assert select(items, query) == [+ "viking://user/memories/preferences/tobacco.md",+ "viking://user/memories/events/chat_1.md",+ ]+++def test_ranking_matches_server_order_among_leaves_regardless_of_query_text(select):+ items = [+ _item("viking://user/memories/a.md", 0.53, abstract="alpha"),+ _item("viking://user/memories/b.md", 0.52, abstract="beta beta beta"),+ _item("viking://user/memories/c.md", 0.51, abstract="gamma"),+ ]++ assert (+ select(items, "beta beta beta beta")+ == select(items, "")+ == [+ "viking://user/memories/a.md",+ "viking://user/memories/b.md",+ "viking://user/memories/c.md",+ ]+ )+++def test_leaf_boost_threshold_dedupe_and_limit_are_kept(select):+ items = [+ _item("viking://user/memories/dir", 0.60, abstract="directory overview", level=1),+ _item("viking://user/memories/leaf.md", 0.55, abstract="leaf fact"),+ _item("viking://user/memories/leaf.md", 0.55, abstract="leaf fact"), # same uri+ # Same abstract + category as leaf.md, different uri.+ _item("viking://user/memories/copy.md", 0.54, abstract="Leaf fact"),+ _item("viking://user/memories/low.md", 0.10, abstract="below threshold"),+ _item("viking://user/memories/other.md", 0.50, abstract="other fact"),+ ]++ # L2 leaf (+0.12) beats the higher-scored L1 directory; duplicates and sub-threshold drop out.+ assert select(items, "leaf", limit=5, score_threshold=0.2) == [+ "viking://user/memories/leaf.md",+ "viking://user/memories/other.md",+ "viking://user/memories/dir",+ ]+ assert select(items, "leaf", limit=1, score_threshold=0.2) == ["viking://user/memories/leaf.md"]