AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
Add shared query.py and relevance.py modules
Jeffrey Sperlingd66758659703
diff --git a/scripts/lib/query.py b/scripts/lib/query.pynew file mode 100644index 00000000..1d488751--- /dev/null+++ b/scripts/lib/query.py@@ -0,0 +1,98 @@+"""Shared query utilities for /last30days search modules.++Consolidates duplicated _extract_core_subject() logic from bird_x, reddit,+youtube_yt, tiktok, instagram, bluesky, and scrapecreators_x into one+parameterized function. Each platform calls with its own overrides.+"""++from typing import FrozenSet, List, Optional, Set++# Common multi-word prefixes stripped from all queries (identical across modules)+PREFIXES = [+ 'what are the best', 'what is the best', 'what are the latest',+ 'what are people saying about', 'what do people think about',+ 'how do i use', 'how to use', 'how to',+ 'what are', 'what is', 'tips for', 'best practices for',+]++# Multi-word suffixes (used by bird_x)+SUFFIXES = [+ 'best practices', 'use cases', 'prompt techniques',+ 'prompting techniques', 'prompting tips',+]++# Base noise words shared across most modules+NOISE_WORDS = frozenset({+ # Articles/prepositions/conjunctions+ 'a', 'an', 'the', 'is', 'are', 'was', 'were', 'and', 'or',+ 'of', 'in', 'on', 'for', 'with', 'about', 'to',+ # Question words+ 'how', 'what', 'which', 'who', 'why', 'when', 'where',+ 'does', 'should', 'could', 'would',+ # Research/meta descriptors+ 'best', 'top', 'good', 'great', 'awesome', 'killer',+ 'latest', 'new', 'news', 'update', 'updates',+ 'trendiest', 'trending', 'hottest', 'hot', 'popular', 'viral',+ 'practices', 'features', 'guide', 'tutorial',+ 'recommendations', 'advice', 'review', 'reviews',+ 'usecases', 'examples', 'comparison', 'versus', 'vs',+ 'plugin', 'plugins', 'skill', 'skills', 'tool', 'tools',+ # Prompting meta words+ 'prompt', 'prompts', 'prompting', 'techniques', 'tips',+ 'tricks', 'methods', 'strategies', 'approaches',+ # Action words+ 'using', 'uses', 'use',+ # Misc filler+ 'people', 'saying', 'think', 'said', 'lately',+})+++def extract_core_subject(+ topic: str,+ *,+ noise: Optional[FrozenSet[str]] = None,+ max_words: Optional[int] = None,+ strip_suffixes: bool = False,+) -> str:+ """Extract core subject from a verbose search query.++ Strips common question/meta prefixes and noise words to produce a+ compact search-friendly query. Platforms customize via parameters.++ Args:+ topic: Raw user query+ noise: Override noise word set (default: NOISE_WORDS)+ max_words: Cap result to N words (default: no cap)+ strip_suffixes: Also strip trailing multi-word suffixes (bird_x uses this)++ Returns:+ Cleaned query string+ """+ text = topic.lower().strip()+ if not text:+ return text++ # Phase 1: Strip multi-word prefixes (longest first, stop after first match)+ for p in PREFIXES:+ if text.startswith(p + ' '):+ text = text[len(p):].strip()+ break++ # Phase 2: Strip multi-word suffixes (opt-in)+ if strip_suffixes:+ for s in SUFFIXES:+ if text.endswith(' ' + s):+ text = text[:-len(s)].strip()+ break++ # Phase 3: Filter individual noise words+ noise_set = noise if noise is not None else NOISE_WORDS+ words = text.split()+ filtered = [w for w in words if w not in noise_set]++ # Apply word cap if requested+ if max_words is not None and filtered:+ filtered = filtered[:max_words]++ result = ' '.join(filtered) if filtered else text+ return result.rstrip('?!.') if not max_words else (result or topic.lower().strip())diff --git a/scripts/lib/relevance.py b/scripts/lib/relevance.pynew file mode 100644index 00000000..f936b706--- /dev/null+++ b/scripts/lib/relevance.py@@ -0,0 +1,95 @@+"""Shared relevance scoring for /last30days search modules.++Consolidates duplicated _tokenize, _compute_relevance, STOPWORDS, and SYNONYMS+from youtube_yt, tiktok, instagram, and scrapecreators_x into one module.+"""++import re+from typing import List, Optional, Set++# Stopwords for relevance computation (common English words that dilute token overlap)+STOPWORDS = frozenset({+ 'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on',+ 'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my',+ 'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can',+ 'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about',+ 'all', 'just', 'get', 'has', 'have', 'was', 'will',+})++# Synonym groups for relevance scoring (bidirectional expansion)+# Superset of all platform-specific synonym dicts+SYNONYMS = {+ 'hip': {'rap', 'hiphop'},+ 'hop': {'rap', 'hiphop'},+ 'rap': {'hip', 'hop', 'hiphop'},+ 'hiphop': {'rap', 'hip', 'hop'},+ 'js': {'javascript'},+ 'javascript': {'js'},+ 'ts': {'typescript'},+ 'typescript': {'ts'},+ 'ai': {'artificial', 'intelligence'},+ 'ml': {'machine', 'learning'},+ 'react': {'reactjs'},+ 'reactjs': {'react'},+ 'svelte': {'sveltejs'},+ 'sveltejs': {'svelte'},+ 'vue': {'vuejs'},+ 'vuejs': {'vue'},+}+++def tokenize(text: str) -> Set[str]:+ """Lowercase, strip punctuation, remove stopwords, drop single-char tokens.++ Expands tokens with synonyms for better cross-domain matching.+ """+ words = re.sub(r'[^\w\s]', ' ', text.lower()).split()+ tokens = {w for w in words if w not in STOPWORDS and len(w) > 1}+ expanded = set(tokens)+ for t in tokens:+ if t in SYNONYMS:+ expanded.update(SYNONYMS[t])+ return expanded+++def token_overlap_relevance(+ query: str,+ text: str,+ hashtags: Optional[List[str]] = None,+) -> float:+ """Compute relevance as ratio of query tokens found in text.++ Uses ratio overlap (intersection / query_length) so short queries+ score higher when fully represented in the text. Floors at 0.1.++ Args:+ query: Search query+ text: Content text to match against+ hashtags: Optional list of hashtags (TikTok/Instagram). Concatenated+ hashtags are split to match query tokens (e.g. "claudecode" matches "claude").++ Returns:+ Float between 0.1 and 1.0 (0.5 for empty queries)+ """+ q_tokens = tokenize(query)++ # Combine text and hashtags for matching+ combined = text+ if hashtags:+ combined = f"{text} {' '.join(hashtags)}"+ t_tokens = tokenize(combined)++ # Split concatenated hashtags (e.g., "claudecode" -> matches "claude", "code")+ if hashtags:+ for tag in hashtags:+ tag_lower = tag.lower()+ for qt in q_tokens:+ if qt in tag_lower and qt != tag_lower:+ t_tokens.add(qt)++ if not q_tokens:+ return 0.5 # Neutral fallback for empty/stopword-only queries++ overlap = len(q_tokens & t_tokens)+ ratio = overlap / len(q_tokens)+ return max(0.1, min(1.0, ratio))diff --git a/tests/test_query.py b/tests/test_query.pynew file mode 100644index 00000000..a081f4ff--- /dev/null+++ b/tests/test_query.py@@ -0,0 +1,130 @@+"""Tests for query.py — shared query utilities."""++import sys+import unittest+from pathlib import Path++sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))++from lib.query import NOISE_WORDS, extract_core_subject+++class TestExtractCoreSubject(unittest.TestCase):+ """Tests for extract_core_subject() with default noise set."""++ def test_strips_what_are_prefix(self):+ self.assertEqual(extract_core_subject("what are the best AI tools"), "ai")++ def test_strips_how_to_prefix(self):+ self.assertEqual(extract_core_subject("how to use cursor IDE"), "cursor ide")++ def test_strips_what_do_people_think(self):+ result = extract_core_subject("what do people think about React Server Components")+ self.assertEqual(result, "react server components")++ def test_preserves_product_name(self):+ self.assertEqual(extract_core_subject("cursor IDE"), "cursor ide")++ def test_strips_trailing_punctuation(self):+ result = extract_core_subject("what is Claude?")+ self.assertFalse(result.endswith("?"))++ def test_empty_string(self):+ self.assertEqual(extract_core_subject(""), "")++ def test_all_noise_returns_original(self):+ # When all words are noise, fall back to original text+ result = extract_core_subject("best latest new")+ self.assertTrue(len(result) > 0)++ def test_only_first_prefix_stripped(self):+ # "how to" should match, stripping once, not recursively+ result = extract_core_subject("how to use how to debug")+ self.assertIn("debug", result)+++class TestMaxWords(unittest.TestCase):+ """Tests for max_words parameter."""++ def test_max_words_caps_output(self):+ result = extract_core_subject(+ "multi agent reinforcement learning framework",+ max_words=5,+ )+ self.assertLessEqual(len(result.split()), 5)++ def test_max_words_none_no_cap(self):+ result = extract_core_subject("cursor IDE react native components")+ # Without max_words, no cap applied+ self.assertGreaterEqual(len(result.split()), 3)++ def test_max_words_fallback_on_empty(self):+ # All words filtered + max_words should fall back to original+ result = extract_core_subject("best top latest", max_words=3)+ self.assertTrue(len(result) > 0)+++class TestStripSuffixes(unittest.TestCase):+ """Tests for strip_suffixes parameter."""++ def test_strips_best_practices(self):+ result = extract_core_subject(+ "claude code best practices",+ strip_suffixes=True,+ )+ self.assertNotIn("practices", result)++ def test_strips_use_cases(self):+ result = extract_core_subject(+ "react hooks use cases",+ strip_suffixes=True,+ )+ self.assertNotIn("cases", result)++ def test_no_strip_without_flag(self):+ result = extract_core_subject("claude code best practices")+ # "best" and "practices" are noise words so they get filtered anyway+ # but the suffix phase doesn't run+ self.assertIn("claude", result)+++class TestCustomNoise(unittest.TestCase):+ """Tests for noise override parameter."""++ def test_custom_noise_keeps_tips(self):+ # YouTube keeps tips/tricks/tutorial — pass a noise set without them+ youtube_noise = frozenset({+ 'best', 'top', 'good', 'great', 'awesome', 'killer',+ 'latest', 'new', 'news', 'update', 'updates',+ 'trending', 'hottest', 'popular', 'viral',+ 'practices', 'features',+ 'recommendations', 'advice',+ 'prompt', 'prompts', 'prompting',+ 'methods', 'strategies', 'approaches',+ })+ result = extract_core_subject("best react tips", noise=youtube_noise)+ self.assertIn("tips", result)++ def test_default_noise_removes_tips(self):+ result = extract_core_subject("best react tips")+ self.assertNotIn("tips", result)+++class TestNoiseWordsCompleteness(unittest.TestCase):+ """Verify NOISE_WORDS superset covers all platform sets."""++ def test_question_words_present(self):+ for w in ('who', 'why', 'when', 'where', 'does', 'should', 'could', 'would'):+ self.assertIn(w, NOISE_WORDS, f"Missing question word: {w}")++ def test_core_filler_present(self):+ for w in ('the', 'a', 'an', 'is', 'are', 'for', 'with', 'about'):+ self.assertIn(w, NOISE_WORDS)++ def test_research_meta_present(self):+ for w in ('best', 'top', 'latest', 'trending', 'popular'):+ self.assertIn(w, NOISE_WORDS)+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_relevance.py b/tests/test_relevance.pynew file mode 100644index 00000000..c641e764--- /dev/null+++ b/tests/test_relevance.py@@ -0,0 +1,122 @@+"""Tests for relevance.py — shared relevance scoring.++Migrated from test_youtube_relevance.py + new hashtag/synonym tests.+"""++import sys+import unittest+from pathlib import Path++sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))++from lib.relevance import STOPWORDS, SYNONYMS, token_overlap_relevance, tokenize+++class TestTokenize(unittest.TestCase):+ """Tests for tokenize()."""++ def test_removes_stopwords(self):+ tokens = tokenize("how to use the AI tools")+ self.assertNotIn("how", tokens)+ self.assertNotIn("to", tokens)+ self.assertNotIn("the", tokens)+ self.assertIn("ai", tokens)+ self.assertIn("tools", tokens)++ def test_lowercases(self):+ tokens = tokenize("Python REACT")+ self.assertIn("python", tokens)+ self.assertIn("react", tokens)++ def test_strips_punctuation(self):+ tokens = tokenize("hello, world!")+ self.assertIn("hello", tokens)+ self.assertIn("world", tokens)++ def test_drops_single_char(self):+ tokens = tokenize("a b c python")+ self.assertNotIn("a", tokens)+ self.assertNotIn("b", tokens)+ self.assertNotIn("c", tokens)+ self.assertIn("python", tokens)++ def test_expands_synonyms(self):+ tokens = tokenize("ai tools")+ self.assertIn("artificial", tokens)+ self.assertIn("intelligence", tokens)++ def test_expands_js_synonym(self):+ tokens = tokenize("js framework")+ self.assertIn("javascript", tokens)++ def test_expands_svelte(self):+ tokens = tokenize("svelte app")+ self.assertIn("sveltejs", tokens)++ def test_expands_vue(self):+ tokens = tokenize("vue components")+ self.assertIn("vuejs", tokens)+++class TestTokenOverlapRelevance(unittest.TestCase):+ """Tests for token_overlap_relevance()."""++ def test_high_relevance_exact_match(self):+ rel = token_overlap_relevance("claude code", "Claude Code tricks and tips")+ self.assertGreater(rel, 0.7)++ def test_low_relevance_no_match(self):+ rel = token_overlap_relevance("claude code tips", "Best AI tools for coding")+ self.assertLess(rel, 0.5)++ def test_empty_query_returns_neutral(self):+ rel = token_overlap_relevance("", "Some video title")+ self.assertEqual(rel, 0.5)++ def test_floor_at_0_1(self):+ rel = token_overlap_relevance("quantum physics", "cat dancing video")+ self.assertGreaterEqual(rel, 0.1)++ def test_full_match_returns_1(self):+ rel = token_overlap_relevance("python tutorial", "Python Tutorial for Beginners")+ self.assertEqual(rel, 1.0)++ def test_partial_match(self):+ rel = token_overlap_relevance("react native tutorial", "React Native Guide")+ self.assertGreater(rel, 0.3)+ self.assertLess(rel, 1.0)++ def test_synonym_boosts_relevance(self):+ # "js" should match "javascript" via synonym expansion+ rel_with_syn = token_overlap_relevance("js framework", "javascript framework comparison")+ rel_without = token_overlap_relevance("python framework", "javascript framework comparison")+ self.assertGreater(rel_with_syn, rel_without)++ def test_stopword_only_query(self):+ rel = token_overlap_relevance("the a is", "some content here")+ self.assertEqual(rel, 0.5)+++class TestHashtagRelevance(unittest.TestCase):+ """Tests for hashtag-aware relevance (TikTok/Instagram pattern)."""++ def test_hashtag_boost(self):+ rel_no_hash = token_overlap_relevance("claude code", "random video about stuff")+ rel_with_hash = token_overlap_relevance(+ "claude code", "random video about stuff", ["claudecode", "ai"]+ )+ self.assertGreater(rel_with_hash, rel_no_hash)++ def test_concatenated_hashtag_splitting(self):+ # "claudecode" should match "claude" from query via substring check+ rel = token_overlap_relevance("claude", "video", ["claudecode"])+ self.assertGreater(rel, 0.5)++ def test_none_hashtags_same_as_no_hashtags(self):+ rel1 = token_overlap_relevance("test query", "test content", None)+ rel2 = token_overlap_relevance("test query", "test content")+ self.assertEqual(rel1, rel2)+++if __name__ == "__main__":+ unittest.main()