Python
fix(sdk): gate near-duplicate demotion with the MMR top-k threshold
alvinfeie540386db1be
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 0240c6a..a3c4081 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -14,6 +14,9 @@ Hy Memory and MemHub are versioned and released independently.below 20 (`MMR_MAX_TOP_K`) and is skipped for larger limits, where itsquadratic token-overlap pass is slow and gains little. Previously it ran forevery limit. `MEMORY_HYBRID_V2_MMR_LAMBDA` still sets the weight.+ Near-duplicate demotion follows the same gate: it compares each candidate+ with every earlier non-duplicate, which is quadratic in a pool that grows+ with `limit`, and large result pages should show similar items anyway.- SDK: `search(scene="all")` searches chat, working and coding at the same timeand merges them with Reciprocal Rank Fusion (k=60) over each scene's ownranking, so no scene is crowded out by the other scenes' score scales. Eachdiff --git a/hy_memory/pipelines/_retrieval/diversity.py b/hy_memory/pipelines/_retrieval/diversity.pyindex b81ede6..e759b95 100644--- a/hy_memory/pipelines/_retrieval/diversity.py+++ b/hy_memory/pipelines/_retrieval/diversity.py@@ -171,6 +171,17 @@ def mmr_applies(top_k: int, lambda_: float) -> bool:return 0.0 < float(lambda_) < 0.999 and int(top_k) < MMR_MAX_TOP_K+def near_dup_applies(top_k: int) -> bool:+ """本次检索是否走近重复降权:与 MMR 同一个 top-k 门槛。++ near_dup_demote 让每个候选和此前所有"非重复"候选两两算 Jaccard,候选+ 互不相似时是 O(pool^2);池子随 limit 放大(约 max(3*limit, 30) + 关键词+ 命中),limit 达到 MMR_MAX_TOP_K 时代价同样陡增。大 k 的调用方(如记忆+ 面板翻页)要的是全部相关项,相似项也应照常出现,降权没有收益。+ """+ return int(top_k) < MMR_MAX_TOP_K++def mmr_select(hits: Sequence[Dict[str, Any]],limit: int,diff --git a/hy_memory/pipelines/reader_base.py b/hy_memory/pipelines/reader_base.pyindex afeab3c..bc288d4 100644--- a/hy_memory/pipelines/reader_base.py+++ b/hy_memory/pipelines/reader_base.py@@ -57,6 +57,7 @@ from ._retrieval.ner import (memory_matches_time_hints,)from ._retrieval.diversity import (+ near_dup_applies,near_dup_demote,mmr_applies,mmr_select,@@ -1438,6 +1439,8 @@ class BaseReadPipeline(ReadPipeline):# MMR 只在调用方的 top-k(final_limit,即 search 的 limit)< MMR_MAX_TOP_K# 时启用;池子(LOCAL_RERANK 下是 final_limit 的若干倍)不参与判定。MMR_ON = mmr_applies(final_limit, MMR_LAMBDA)+ # 近重复降权同样是候选两两比较,用同一个 top-k 门槛。+ NEAR_DUP_ON = near_dup_applies(final_limit)vdb_sem_limit = max(final_limit * 3, 30)vdb_kw_limit = max(final_limit * 5, 80)graph_limit = max(@@ -2532,7 +2535,7 @@ class BaseReadPipeline(ReadPipeline):df_thr=POOL_DF_THR,penalty=POOL_DF_PENALTY,)- if NEAR_DUP_THR > 0 and NEAR_DUP_PENALTY > 0:+ if NEAR_DUP_ON and NEAR_DUP_THR > 0 and NEAR_DUP_PENALTY > 0:normal_results = near_dup_demote(normal_results,jaccard_thr=NEAR_DUP_THR,@@ -3553,6 +3556,7 @@ class BaseReadPipeline(ReadPipeline):"local_rerank_enabled": LOCAL_RERANK_ON,"rerank": RERANK_ON,"mmr_applied": MMR_ON,+ "near_dup_applied": NEAR_DUP_ON,"local_rerank_model": getattr(self.config.recall,"local_rerank_model",diff --git a/tests/test_search_rerank_mmr_gates.py b/tests/test_search_rerank_mmr_gates.pyindex aca395e..dbdad01 100644--- a/tests/test_search_rerank_mmr_gates.py+++ b/tests/test_search_rerank_mmr_gates.py@@ -15,7 +15,11 @@ import pytestfrom hy_memory.config import MemoryConfigfrom hy_memory.models.memory import MemoryLayer, MemoryNode, MemoryStatusfrom hy_memory.pipelines import reader_base-from hy_memory.pipelines._retrieval.diversity import MMR_MAX_TOP_K, mmr_applies+from hy_memory.pipelines._retrieval.diversity import (+ MMR_MAX_TOP_K,+ mmr_applies,+ near_dup_applies,+)from hy_memory.pipelines.base import ReadRequestDIMS = 32@@ -126,6 +130,53 @@ def test_mmr_skipped_when_limit_reaches_threshold(config, mmr_spy):assert mmr_spy == []+def near_dup_spy(monkeypatch):+ calls = []+ real = reader_base.near_dup_demote++ def _spy(hits, **kw):+ calls.append(len(hits))+ return real(hits, **kw)++ monkeypatch.setattr(reader_base, "near_dup_demote", _spy)+ return calls+++def test_near_dup_shares_the_mmr_threshold():+ # Pairwise Jaccard against every kept candidate: quadratic in the pool,+ # which grows with limit. Same gate as MMR.+ assert near_dup_applies(MMR_MAX_TOP_K - 1)+ assert not near_dup_applies(MMR_MAX_TOP_K)+++def test_near_dup_runs_when_limit_below_threshold(config, near_dup_spy):+ async def _run():+ reader, store = await _reader_with(config, 12)+ try:+ resp = await reader.read(_request(5))+ assert resp.extra["config"]["near_dup_applied"] is True+ finally:+ await store.close()++ asyncio.run(_run())+ assert len(near_dup_spy) == 1+++def test_near_dup_skipped_when_limit_reaches_threshold(config, near_dup_spy):+ async def _run():+ reader, store = await _reader_with(config, MMR_MAX_TOP_K + 6)+ try:+ resp = await reader.read(_request(MMR_MAX_TOP_K))+ assert resp.success+ assert resp.extra["config"]["near_dup_applied"] is False+ finally:+ await store.close()++ asyncio.run(_run())+ assert near_dup_spy == []++class _FakeReranker:model_name = "fake-cross-encoder"device = "cpu"