Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

3.8 KB
Raw Blame
import unittest
from unittest.mock import patch

from lib import cjk, dedupe, relevance


class _BigramBase(unittest.TestCase):
    """Force the dictionary-free bigram path so assertions are deterministic
    regardless of whether jieba is installed in the test environment."""

    def setUp(self):
        patcher = patch.object(cjk, "_jieba", None)
        patcher.start()
        self.addCleanup(patcher.stop)


class TestCjkSegment(_BigramBase):
    def test_has_cjk(self):
        self.assertTrue(cjk.has_cjk("国产大模型"))
        self.assertTrue(cjk.has_cjk("GPT4很强"))
        self.assertFalse(cjk.has_cjk("hello world"))
        self.assertFalse(cjk.has_cjk(""))

    def test_ascii_path_unchanged(self):
        # Non-CJK text keeps whitespace/word tokenization.
        self.assertEqual(cjk.segment("best react hooks"), ["best", "react", "hooks"])

    def test_chinese_bigrams(self):
        toks = cjk.segment("大模型")
        self.assertIn("大模", toks)
        self.assertIn("模型", toks)

    def test_mixed_language(self):
        toks = cjk.segment("GPT4很强 react")
        self.assertIn("gpt4", toks)
        self.assertIn("react", toks)
        self.assertIn("很强", toks)

    def test_single_cjk_char(self):
        self.assertEqual(cjk.segment("中"), ["中"])


class TestChineseRelevance(_BigramBase):
    def test_chinese_query_matches_chinese_text(self):
        q = relevance.PreparedQuery("国产大模型 测评")
        score = relevance.token_overlap_relevance(q, "这是国产大模型的最新测评")
        self.assertGreater(score, 0.5)

    def test_chinese_query_rejects_unrelated_text(self):
        q = relevance.PreparedQuery("国产大模型 测评")
        score = relevance.token_overlap_relevance(q, "今天天气很好适合出门散步")
        self.assertEqual(score, 0.0)

    def test_english_relevance_not_regressed(self):
        q = relevance.PreparedQuery("react hooks")
        self.assertGreaterEqual(relevance.token_overlap_relevance(q, "a guide to react hooks"), 0.9)

    def test_cjk_phrase_bonus_applies_on_contiguous_match(self):
        # A multi-token CJK query ("国产大模型 测评") whose words appear
        # contiguously in the text earns the phrase bonus via the space-stripped
        # containment retry; the same words scattered apart do not.
        q = relevance.PreparedQuery("国产大模型 测评")
        contiguous = relevance.token_overlap_relevance(q, "国产大模型测评合集")
        scattered = relevance.token_overlap_relevance(q, "测评了很多东西也聊到国产大模型")
        self.assertGreater(contiguous, scattered)

    def test_english_phrase_bonus_stays_space_sensitive(self):
        q = relevance.PreparedQuery("react hooks")
        joined = relevance.token_overlap_relevance(q, "react myreacthooks bundle")
        absent = relevance.token_overlap_relevance(q, "react unrelated bundle")
        self.assertEqual(joined, 0.58)
        self.assertEqual(absent, 0.58)


class TestChineseDedupe(_BigramBase):
    def test_reordered_chinese_is_near_duplicate(self):
        sim = dedupe.hybrid_similarity("国产大模型最新测评对比", "国产大模型测评对比最新")
        self.assertGreater(sim, 0.5)

    def test_distinct_chinese_is_not_duplicate(self):
        sim = dedupe.hybrid_similarity("国产大模型测评", "今天天气很好出门散步")
        self.assertLess(sim, 0.3)


class TestJiebaBinding(unittest.TestCase):
    def test_jieba_global_is_bound_at_import(self):
        # Eager import binds the module global once (None when jieba absent).
        # No lazy initializer => no per-call race in the pipeline thread pool.
        self.assertTrue(hasattr(cjk, "_jieba"))
        self.assertFalse(hasattr(cjk, "_get_jieba"))


if __name__ == "__main__":
    unittest.main()