Skip to content
Python

fix(sdk): gate near-duplicate demotion with the MMR top-k threshold

alvinfeie540386db1be
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0240c6a..a3c4081 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,9 @@ Hy Memory and MemHub are versioned and released independently.
below 20 (`MMR_MAX_TOP_K`) and is skipped for larger limits, where its
quadratic token-overlap pass is slow and gains little. Previously it ran for
every limit. `MEMORY_HYBRID_V2_MMR_LAMBDA` still sets the weight.
+ Near-duplicate demotion follows the same gate: it compares each candidate
+ with every earlier non-duplicate, which is quadratic in a pool that grows
+ with `limit`, and large result pages should show similar items anyway.
- SDK: `search(scene="all")` searches chat, working and coding at the same time
and merges them with Reciprocal Rank Fusion (k=60) over each scene's own
ranking, so no scene is crowded out by the other scenes' score scales. Each
diff --git a/hy_memory/pipelines/_retrieval/diversity.py b/hy_memory/pipelines/_retrieval/diversity.py
index b81ede6..e759b95 100644
--- a/hy_memory/pipelines/_retrieval/diversity.py
+++ b/hy_memory/pipelines/_retrieval/diversity.py
@@ -171,6 +171,17 @@ def mmr_applies(top_k: int, lambda_: float) -> bool:
return 0.0 < float(lambda_) < 0.999 and int(top_k) < MMR_MAX_TOP_K
+def near_dup_applies(top_k: int) -> bool:
+ """本次检索是否走近重复降权:与 MMR 同一个 top-k 门槛。
+
+ near_dup_demote 让每个候选和此前所有"非重复"候选两两算 Jaccard,候选
+ 互不相似时是 O(pool^2);池子随 limit 放大(约 max(3*limit, 30) + 关键词
+ 命中),limit 达到 MMR_MAX_TOP_K 时代价同样陡增。大 k 的调用方(如记忆
+ 面板翻页)要的是全部相关项,相似项也应照常出现,降权没有收益。
+ """
+ return int(top_k) < MMR_MAX_TOP_K
+
+
def mmr_select(
hits: Sequence[Dict[str, Any]],
limit: int,
diff --git a/hy_memory/pipelines/reader_base.py b/hy_memory/pipelines/reader_base.py
index afeab3c..bc288d4 100644
--- a/hy_memory/pipelines/reader_base.py
+++ b/hy_memory/pipelines/reader_base.py
@@ -57,6 +57,7 @@ from ._retrieval.ner import (
memory_matches_time_hints,
)
from ._retrieval.diversity import (
+ near_dup_applies,
near_dup_demote,
mmr_applies,
mmr_select,
@@ -1438,6 +1439,8 @@ class BaseReadPipeline(ReadPipeline):
# MMR 只在调用方的 top-k(final_limit,即 search 的 limit)< MMR_MAX_TOP_K
# 时启用;池子(LOCAL_RERANK 下是 final_limit 的若干倍)不参与判定。
MMR_ON = mmr_applies(final_limit, MMR_LAMBDA)
+ # 近重复降权同样是候选两两比较,用同一个 top-k 门槛。
+ NEAR_DUP_ON = near_dup_applies(final_limit)
vdb_sem_limit = max(final_limit * 3, 30)
vdb_kw_limit = max(final_limit * 5, 80)
graph_limit = max(
@@ -2532,7 +2535,7 @@ class BaseReadPipeline(ReadPipeline):
df_thr=POOL_DF_THR,
penalty=POOL_DF_PENALTY,
)
- if NEAR_DUP_THR > 0 and NEAR_DUP_PENALTY > 0:
+ if NEAR_DUP_ON and NEAR_DUP_THR > 0 and NEAR_DUP_PENALTY > 0:
normal_results = near_dup_demote(
normal_results,
jaccard_thr=NEAR_DUP_THR,
@@ -3553,6 +3556,7 @@ class BaseReadPipeline(ReadPipeline):
"local_rerank_enabled": LOCAL_RERANK_ON,
"rerank": RERANK_ON,
"mmr_applied": MMR_ON,
+ "near_dup_applied": NEAR_DUP_ON,
"local_rerank_model": getattr(
self.config.recall,
"local_rerank_model",
diff --git a/tests/test_search_rerank_mmr_gates.py b/tests/test_search_rerank_mmr_gates.py
index aca395e..dbdad01 100644
--- a/tests/test_search_rerank_mmr_gates.py
+++ b/tests/test_search_rerank_mmr_gates.py
@@ -15,7 +15,11 @@ import pytest
from hy_memory.config import MemoryConfig
from hy_memory.models.memory import MemoryLayer, MemoryNode, MemoryStatus
from hy_memory.pipelines import reader_base
-from hy_memory.pipelines._retrieval.diversity import MMR_MAX_TOP_K, mmr_applies
+from hy_memory.pipelines._retrieval.diversity import (
+ MMR_MAX_TOP_K,
+ mmr_applies,
+ near_dup_applies,
+)
from hy_memory.pipelines.base import ReadRequest
DIMS = 32
@@ -126,6 +130,53 @@ def test_mmr_skipped_when_limit_reaches_threshold(config, mmr_spy):
assert mmr_spy == []
+def near_dup_spy(monkeypatch):
+ calls = []
+ real = reader_base.near_dup_demote
+
+ def _spy(hits, **kw):
+ calls.append(len(hits))
+ return real(hits, **kw)
+
+ monkeypatch.setattr(reader_base, "near_dup_demote", _spy)
+ return calls
+
+
+def test_near_dup_shares_the_mmr_threshold():
+ # Pairwise Jaccard against every kept candidate: quadratic in the pool,
+ # which grows with limit. Same gate as MMR.
+ assert near_dup_applies(MMR_MAX_TOP_K - 1)
+ assert not near_dup_applies(MMR_MAX_TOP_K)
+
+
+def test_near_dup_runs_when_limit_below_threshold(config, near_dup_spy):
+ async def _run():
+ reader, store = await _reader_with(config, 12)
+ try:
+ resp = await reader.read(_request(5))
+ assert resp.extra["config"]["near_dup_applied"] is True
+ finally:
+ await store.close()
+
+ asyncio.run(_run())
+ assert len(near_dup_spy) == 1
+
+
+def test_near_dup_skipped_when_limit_reaches_threshold(config, near_dup_spy):
+ async def _run():
+ reader, store = await _reader_with(config, MMR_MAX_TOP_K + 6)
+ try:
+ resp = await reader.read(_request(MMR_MAX_TOP_K))
+ assert resp.success
+ assert resp.extra["config"]["near_dup_applied"] is False
+ finally:
+ await store.close()
+
+ asyncio.run(_run())
+ assert near_dup_spy == []
+
+
class _FakeReranker:
model_name = "fake-cross-encoder"
device = "cpu"