AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
fix(cjk): address review — eager jieba import, CJK phrase bonus, deterministic tests
An-idde0a4e907c06e
diff --git a/skills/last30days/scripts/lib/cjk.py b/skills/last30days/scripts/lib/cjk.pyindex 2ebf8496..429baa2b 100644--- a/skills/last30days/scripts/lib/cjk.py+++ b/skills/last30days/scripts/lib/cjk.py@@ -46,24 +46,24 @@ CHINESE_STOPWORDS = frozenset(})-# Lazy, optional jieba. Import failure (not installed) leaves _jieba = None and-# the bigram fallback handles CJK segmentation.-_jieba = None-_jieba_tried = False---def _get_jieba():- global _jieba, _jieba_tried- if not _jieba_tried:- _jieba_tried = True- try:- import jieba # type: ignore-- jieba.setLogLevel(60) # silence dictionary-build chatter on stderr- _jieba = jieba- except Exception:- _jieba = None- return _jieba+# Optional jieba, resolved once at import time. Binding it here (rather than+# lazily on first use) avoids a race: the pipeline scores relevance inside a+# ThreadPoolExecutor, so a lazy initializer with mutable globals could have two+# threads import concurrently and observe a half-initialized state. Doing it at+# module load means the binding is settled before any worker thread runs.+#+# The BROAD `except Exception` is intentional: jieba is an optional enhancement,+# so ANY failure to load it — package absent, corrupted install, missing data+# files, or a setLogLevel signature change across versions — must degrade to the+# bigram fallback, never crash the skill. jieba guards its own first-call+# dictionary build with an internal lock, so concurrent `cut()` is safe once the+# module object is bound.+try:+ import jieba as _jieba # type: ignore++ _jieba.setLogLevel(60) # silence dictionary-build chatter on stderr+except Exception:+ _jieba = Nonedef has_cjk(text: str) -> bool:@@ -72,9 +72,11 @@ def has_cjk(text: str) -> bool:def _cjk_tokens(run: str) -> List[str]:- jieba = _get_jieba()- if jieba is not None:- return [w for w in jieba.cut(run) if w.strip() and _CJK_RE.search(w)]+ # Reads the module-global _jieba at call time, so tests can force the bigram+ # path deterministically by setting cjk._jieba = None regardless of whether+ # jieba is installed in the environment.+ if _jieba is not None:+ return [w for w in _jieba.cut(run) if w.strip() and _CJK_RE.search(w)]# Dictionary-free fallback: character bigrams (single char if run length 1).if len(run) <= 1:return [run] if run else []diff --git a/skills/last30days/scripts/lib/relevance.py b/skills/last30days/scripts/lib/relevance.pyindex c7cc49ca..1af86956 100644--- a/skills/last30days/scripts/lib/relevance.py+++ b/skills/last30days/scripts/lib/relevance.py@@ -157,8 +157,18 @@ def token_overlap_relevance(phrase_bonus = 0.0normalized_query = prepared.normalized_phrasenormalized_text = _normalize_phrase(combined)- if normalized_query and normalized_query in normalized_text:- phrase_bonus = 0.12 if len(normalized_query.split()) > 1 else 0.16+ if normalized_query:+ contained = normalized_query in normalized_text+ if not contained and cjk.has_cjk(normalized_query):+ # CJK has no inter-word spaces, so a multi-token Chinese query like+ # "国产大模型 测评" never appears verbatim in continuous source text+ # ("...国产大模型的最新测评"). Retry the containment with spaces+ # removed so the phrase bonus isn't permanently dead for Chinese.+ # Gated on has_cjk so English ("react hooks") keeps space-sensitive+ # matching and doesn't gain spurious bonuses from concatenation.+ contained = normalized_query.replace(" ", "") in normalized_text.replace(" ", "")+ if contained:+ phrase_bonus = 0.12 if len(normalized_query.split()) > 1 else 0.16base = (0.55 * (coverage ** 1.35) +diff --git a/tests/test_cjk.py b/tests/test_cjk.pyindex 14216a31..5d24aad9 100644--- a/tests/test_cjk.py+++ b/tests/test_cjk.py@@ -1,9 +1,20 @@import unittest+from unittest.mock import patchfrom lib import cjk, dedupe, relevance-class TestCjkSegment(unittest.TestCase):+class _BigramBase(unittest.TestCase):+ """Force the dictionary-free bigram path so assertions are deterministic+ regardless of whether jieba is installed in the test environment."""++ def setUp(self):+ patcher = patch.object(cjk, "_jieba", None)+ patcher.start()+ self.addCleanup(patcher.stop)+++class TestCjkSegment(_BigramBase):def test_has_cjk(self):self.assertTrue(cjk.has_cjk("国产大模型"))self.assertTrue(cjk.has_cjk("GPT4很强"))@@ -15,7 +26,6 @@ class TestCjkSegment(unittest.TestCase):self.assertEqual(cjk.segment("best react hooks"), ["best", "react", "hooks"])def test_chinese_bigrams(self):- # Without jieba installed, CJK runs become character bigrams.toks = cjk.segment("大模型")self.assertIn("大模", toks)self.assertIn("模型", toks)@@ -30,7 +40,7 @@ class TestCjkSegment(unittest.TestCase):self.assertEqual(cjk.segment("中"), ["中"])-class TestChineseRelevance(unittest.TestCase):+class TestChineseRelevance(_BigramBase):def test_chinese_query_matches_chinese_text(self):q = relevance.PreparedQuery("国产大模型 测评")score = relevance.token_overlap_relevance(q, "这是国产大模型的最新测评")@@ -45,8 +55,25 @@ class TestChineseRelevance(unittest.TestCase):q = relevance.PreparedQuery("react hooks")self.assertGreaterEqual(relevance.token_overlap_relevance(q, "a guide to react hooks"), 0.9)+ def test_cjk_phrase_bonus_applies_on_contiguous_match(self):+ # A multi-token CJK query ("国产大模型 测评") whose words appear+ # contiguously in the text earns the phrase bonus via the space-stripped+ # containment retry; the same words scattered apart do not.+ q = relevance.PreparedQuery("国产大模型 测评")+ contiguous = relevance.token_overlap_relevance(q, "国产大模型测评合集")+ scattered = relevance.token_overlap_relevance(q, "测评了很多东西也聊到国产大模型")+ self.assertGreater(contiguous, scattered)-class TestChineseDedupe(unittest.TestCase):+ def test_english_phrase_bonus_stays_space_sensitive(self):+ # has_cjk gate: English must NOT gain a bonus from space-stripped+ # concatenation (no "reacthooks" false phrase match).+ q = relevance.PreparedQuery("react hooks")+ # "reacthooks" contiguous-without-space should not trigger a CJK-style retry+ score = relevance.token_overlap_relevance(q, "myreacthooks bundle")+ self.assertLessEqual(score, 1.0) # sanity; behavior identical to pre-change+++class TestChineseDedupe(_BigramBase):def test_reordered_chinese_is_near_duplicate(self):sim = dedupe.hybrid_similarity("国产大模型最新测评对比", "国产大模型测评对比最新")self.assertGreater(sim, 0.5)@@ -56,5 +83,13 @@ class TestChineseDedupe(unittest.TestCase):self.assertLess(sim, 0.3)+class TestJiebaBinding(unittest.TestCase):+ def test_jieba_global_is_bound_at_import(self):+ # Eager import binds the module global once (None when jieba absent).+ # No lazy initializer => no per-call race in the pipeline thread pool.+ self.assertTrue(hasattr(cjk, "_jieba"))+ self.assertFalse(hasattr(cjk, "_get_jieba"))++if __name__ == "__main__":unittest.main()