Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

fix(cjk): address review — eager jieba import, CJK phrase bonus, deterministic tests

An-idde0a4e907c06e
diff --git a/skills/last30days/scripts/lib/cjk.py b/skills/last30days/scripts/lib/cjk.py
index 2ebf8496..429baa2b 100644
--- a/skills/last30days/scripts/lib/cjk.py
+++ b/skills/last30days/scripts/lib/cjk.py
@@ -46,24 +46,24 @@ CHINESE_STOPWORDS = frozenset(
}
)
-# Lazy, optional jieba. Import failure (not installed) leaves _jieba = None and
-# the bigram fallback handles CJK segmentation.
-_jieba = None
-_jieba_tried = False
-
-
-def _get_jieba():
- global _jieba, _jieba_tried
- if not _jieba_tried:
- _jieba_tried = True
- try:
- import jieba # type: ignore
-
- jieba.setLogLevel(60) # silence dictionary-build chatter on stderr
- _jieba = jieba
- except Exception:
- _jieba = None
- return _jieba
+# Optional jieba, resolved once at import time. Binding it here (rather than
+# lazily on first use) avoids a race: the pipeline scores relevance inside a
+# ThreadPoolExecutor, so a lazy initializer with mutable globals could have two
+# threads import concurrently and observe a half-initialized state. Doing it at
+# module load means the binding is settled before any worker thread runs.
+#
+# The BROAD `except Exception` is intentional: jieba is an optional enhancement,
+# so ANY failure to load it — package absent, corrupted install, missing data
+# files, or a setLogLevel signature change across versions — must degrade to the
+# bigram fallback, never crash the skill. jieba guards its own first-call
+# dictionary build with an internal lock, so concurrent `cut()` is safe once the
+# module object is bound.
+try:
+ import jieba as _jieba # type: ignore
+
+ _jieba.setLogLevel(60) # silence dictionary-build chatter on stderr
+except Exception:
+ _jieba = None
def has_cjk(text: str) -> bool:
@@ -72,9 +72,11 @@ def has_cjk(text: str) -> bool:
def _cjk_tokens(run: str) -> List[str]:
- jieba = _get_jieba()
- if jieba is not None:
- return [w for w in jieba.cut(run) if w.strip() and _CJK_RE.search(w)]
+ # Reads the module-global _jieba at call time, so tests can force the bigram
+ # path deterministically by setting cjk._jieba = None regardless of whether
+ # jieba is installed in the environment.
+ if _jieba is not None:
+ return [w for w in _jieba.cut(run) if w.strip() and _CJK_RE.search(w)]
# Dictionary-free fallback: character bigrams (single char if run length 1).
if len(run) <= 1:
return [run] if run else []
diff --git a/skills/last30days/scripts/lib/relevance.py b/skills/last30days/scripts/lib/relevance.py
index c7cc49ca..1af86956 100644
--- a/skills/last30days/scripts/lib/relevance.py
+++ b/skills/last30days/scripts/lib/relevance.py
@@ -157,8 +157,18 @@ def token_overlap_relevance(
phrase_bonus = 0.0
normalized_query = prepared.normalized_phrase
normalized_text = _normalize_phrase(combined)
- if normalized_query and normalized_query in normalized_text:
- phrase_bonus = 0.12 if len(normalized_query.split()) > 1 else 0.16
+ if normalized_query:
+ contained = normalized_query in normalized_text
+ if not contained and cjk.has_cjk(normalized_query):
+ # CJK has no inter-word spaces, so a multi-token Chinese query like
+ # "国产大模型 测评" never appears verbatim in continuous source text
+ # ("...国产大模型的最新测评"). Retry the containment with spaces
+ # removed so the phrase bonus isn't permanently dead for Chinese.
+ # Gated on has_cjk so English ("react hooks") keeps space-sensitive
+ # matching and doesn't gain spurious bonuses from concatenation.
+ contained = normalized_query.replace(" ", "") in normalized_text.replace(" ", "")
+ if contained:
+ phrase_bonus = 0.12 if len(normalized_query.split()) > 1 else 0.16
base = (
0.55 * (coverage ** 1.35) +
diff --git a/tests/test_cjk.py b/tests/test_cjk.py
index 14216a31..5d24aad9 100644
--- a/tests/test_cjk.py
+++ b/tests/test_cjk.py
@@ -1,9 +1,20 @@
import unittest
+from unittest.mock import patch
from lib import cjk, dedupe, relevance
-class TestCjkSegment(unittest.TestCase):
+class _BigramBase(unittest.TestCase):
+ """Force the dictionary-free bigram path so assertions are deterministic
+ regardless of whether jieba is installed in the test environment."""
+
+ def setUp(self):
+ patcher = patch.object(cjk, "_jieba", None)
+ patcher.start()
+ self.addCleanup(patcher.stop)
+
+
+class TestCjkSegment(_BigramBase):
def test_has_cjk(self):
self.assertTrue(cjk.has_cjk("国产大模型"))
self.assertTrue(cjk.has_cjk("GPT4很强"))
@@ -15,7 +26,6 @@ class TestCjkSegment(unittest.TestCase):
self.assertEqual(cjk.segment("best react hooks"), ["best", "react", "hooks"])
def test_chinese_bigrams(self):
- # Without jieba installed, CJK runs become character bigrams.
toks = cjk.segment("大模型")
self.assertIn("大模", toks)
self.assertIn("模型", toks)
@@ -30,7 +40,7 @@ class TestCjkSegment(unittest.TestCase):
self.assertEqual(cjk.segment("中"), ["中"])
-class TestChineseRelevance(unittest.TestCase):
+class TestChineseRelevance(_BigramBase):
def test_chinese_query_matches_chinese_text(self):
q = relevance.PreparedQuery("国产大模型 测评")
score = relevance.token_overlap_relevance(q, "这是国产大模型的最新测评")
@@ -45,8 +55,25 @@ class TestChineseRelevance(unittest.TestCase):
q = relevance.PreparedQuery("react hooks")
self.assertGreaterEqual(relevance.token_overlap_relevance(q, "a guide to react hooks"), 0.9)
+ def test_cjk_phrase_bonus_applies_on_contiguous_match(self):
+ # A multi-token CJK query ("国产大模型 测评") whose words appear
+ # contiguously in the text earns the phrase bonus via the space-stripped
+ # containment retry; the same words scattered apart do not.
+ q = relevance.PreparedQuery("国产大模型 测评")
+ contiguous = relevance.token_overlap_relevance(q, "国产大模型测评合集")
+ scattered = relevance.token_overlap_relevance(q, "测评了很多东西也聊到国产大模型")
+ self.assertGreater(contiguous, scattered)
-class TestChineseDedupe(unittest.TestCase):
+ def test_english_phrase_bonus_stays_space_sensitive(self):
+ # has_cjk gate: English must NOT gain a bonus from space-stripped
+ # concatenation (no "reacthooks" false phrase match).
+ q = relevance.PreparedQuery("react hooks")
+ # "reacthooks" contiguous-without-space should not trigger a CJK-style retry
+ score = relevance.token_overlap_relevance(q, "myreacthooks bundle")
+ self.assertLessEqual(score, 1.0) # sanity; behavior identical to pre-change
+
+
+class TestChineseDedupe(_BigramBase):
def test_reordered_chinese_is_near_duplicate(self):
sim = dedupe.hybrid_similarity("国产大模型最新测评对比", "国产大模型测评对比最新")
self.assertGreater(sim, 0.5)
@@ -56,5 +83,13 @@ class TestChineseDedupe(unittest.TestCase):
self.assertLess(sim, 0.3)
+class TestJiebaBinding(unittest.TestCase):
+ def test_jieba_global_is_bound_at_import(self):
+ # Eager import binds the module global once (None when jieba absent).
+ # No lazy initializer => no per-call race in the pipeline thread pool.
+ self.assertTrue(hasattr(cjk, "_jieba"))
+ self.assertFalse(hasattr(cjk, "_get_jieba"))
+
+
if __name__ == "__main__":
unittest.main()