Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

Add shared query.py and relevance.py modules

Jeffrey Sperlingd66758659703
diff --git a/scripts/lib/query.py b/scripts/lib/query.py
new file mode 100644
index 00000000..1d488751
--- /dev/null
+++ b/scripts/lib/query.py
@@ -0,0 +1,98 @@
+"""Shared query utilities for /last30days search modules.
+
+Consolidates duplicated _extract_core_subject() logic from bird_x, reddit,
+youtube_yt, tiktok, instagram, bluesky, and scrapecreators_x into one
+parameterized function. Each platform calls with its own overrides.
+"""
+
+from typing import FrozenSet, List, Optional, Set
+
+# Common multi-word prefixes stripped from all queries (identical across modules)
+PREFIXES = [
+ 'what are the best', 'what is the best', 'what are the latest',
+ 'what are people saying about', 'what do people think about',
+ 'how do i use', 'how to use', 'how to',
+ 'what are', 'what is', 'tips for', 'best practices for',
+]
+
+# Multi-word suffixes (used by bird_x)
+SUFFIXES = [
+ 'best practices', 'use cases', 'prompt techniques',
+ 'prompting techniques', 'prompting tips',
+]
+
+# Base noise words shared across most modules
+NOISE_WORDS = frozenset({
+ # Articles/prepositions/conjunctions
+ 'a', 'an', 'the', 'is', 'are', 'was', 'were', 'and', 'or',
+ 'of', 'in', 'on', 'for', 'with', 'about', 'to',
+ # Question words
+ 'how', 'what', 'which', 'who', 'why', 'when', 'where',
+ 'does', 'should', 'could', 'would',
+ # Research/meta descriptors
+ 'best', 'top', 'good', 'great', 'awesome', 'killer',
+ 'latest', 'new', 'news', 'update', 'updates',
+ 'trendiest', 'trending', 'hottest', 'hot', 'popular', 'viral',
+ 'practices', 'features', 'guide', 'tutorial',
+ 'recommendations', 'advice', 'review', 'reviews',
+ 'usecases', 'examples', 'comparison', 'versus', 'vs',
+ 'plugin', 'plugins', 'skill', 'skills', 'tool', 'tools',
+ # Prompting meta words
+ 'prompt', 'prompts', 'prompting', 'techniques', 'tips',
+ 'tricks', 'methods', 'strategies', 'approaches',
+ # Action words
+ 'using', 'uses', 'use',
+ # Misc filler
+ 'people', 'saying', 'think', 'said', 'lately',
+})
+
+
+def extract_core_subject(
+ topic: str,
+ *,
+ noise: Optional[FrozenSet[str]] = None,
+ max_words: Optional[int] = None,
+ strip_suffixes: bool = False,
+) -> str:
+ """Extract core subject from a verbose search query.
+
+ Strips common question/meta prefixes and noise words to produce a
+ compact search-friendly query. Platforms customize via parameters.
+
+ Args:
+ topic: Raw user query
+ noise: Override noise word set (default: NOISE_WORDS)
+ max_words: Cap result to N words (default: no cap)
+ strip_suffixes: Also strip trailing multi-word suffixes (bird_x uses this)
+
+ Returns:
+ Cleaned query string
+ """
+ text = topic.lower().strip()
+ if not text:
+ return text
+
+ # Phase 1: Strip multi-word prefixes (longest first, stop after first match)
+ for p in PREFIXES:
+ if text.startswith(p + ' '):
+ text = text[len(p):].strip()
+ break
+
+ # Phase 2: Strip multi-word suffixes (opt-in)
+ if strip_suffixes:
+ for s in SUFFIXES:
+ if text.endswith(' ' + s):
+ text = text[:-len(s)].strip()
+ break
+
+ # Phase 3: Filter individual noise words
+ noise_set = noise if noise is not None else NOISE_WORDS
+ words = text.split()
+ filtered = [w for w in words if w not in noise_set]
+
+ # Apply word cap if requested
+ if max_words is not None and filtered:
+ filtered = filtered[:max_words]
+
+ result = ' '.join(filtered) if filtered else text
+ return result.rstrip('?!.') if not max_words else (result or topic.lower().strip())
diff --git a/scripts/lib/relevance.py b/scripts/lib/relevance.py
new file mode 100644
index 00000000..f936b706
--- /dev/null
+++ b/scripts/lib/relevance.py
@@ -0,0 +1,95 @@
+"""Shared relevance scoring for /last30days search modules.
+
+Consolidates duplicated _tokenize, _compute_relevance, STOPWORDS, and SYNONYMS
+from youtube_yt, tiktok, instagram, and scrapecreators_x into one module.
+"""
+
+import re
+from typing import List, Optional, Set
+
+# Stopwords for relevance computation (common English words that dilute token overlap)
+STOPWORDS = frozenset({
+ 'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on',
+ 'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my',
+ 'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can',
+ 'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about',
+ 'all', 'just', 'get', 'has', 'have', 'was', 'will',
+})
+
+# Synonym groups for relevance scoring (bidirectional expansion)
+# Superset of all platform-specific synonym dicts
+SYNONYMS = {
+ 'hip': {'rap', 'hiphop'},
+ 'hop': {'rap', 'hiphop'},
+ 'rap': {'hip', 'hop', 'hiphop'},
+ 'hiphop': {'rap', 'hip', 'hop'},
+ 'js': {'javascript'},
+ 'javascript': {'js'},
+ 'ts': {'typescript'},
+ 'typescript': {'ts'},
+ 'ai': {'artificial', 'intelligence'},
+ 'ml': {'machine', 'learning'},
+ 'react': {'reactjs'},
+ 'reactjs': {'react'},
+ 'svelte': {'sveltejs'},
+ 'sveltejs': {'svelte'},
+ 'vue': {'vuejs'},
+ 'vuejs': {'vue'},
+}
+
+
+def tokenize(text: str) -> Set[str]:
+ """Lowercase, strip punctuation, remove stopwords, drop single-char tokens.
+
+ Expands tokens with synonyms for better cross-domain matching.
+ """
+ words = re.sub(r'[^\w\s]', ' ', text.lower()).split()
+ tokens = {w for w in words if w not in STOPWORDS and len(w) > 1}
+ expanded = set(tokens)
+ for t in tokens:
+ if t in SYNONYMS:
+ expanded.update(SYNONYMS[t])
+ return expanded
+
+
+def token_overlap_relevance(
+ query: str,
+ text: str,
+ hashtags: Optional[List[str]] = None,
+) -> float:
+ """Compute relevance as ratio of query tokens found in text.
+
+ Uses ratio overlap (intersection / query_length) so short queries
+ score higher when fully represented in the text. Floors at 0.1.
+
+ Args:
+ query: Search query
+ text: Content text to match against
+ hashtags: Optional list of hashtags (TikTok/Instagram). Concatenated
+ hashtags are split to match query tokens (e.g. "claudecode" matches "claude").
+
+ Returns:
+ Float between 0.1 and 1.0 (0.5 for empty queries)
+ """
+ q_tokens = tokenize(query)
+
+ # Combine text and hashtags for matching
+ combined = text
+ if hashtags:
+ combined = f"{text} {' '.join(hashtags)}"
+ t_tokens = tokenize(combined)
+
+ # Split concatenated hashtags (e.g., "claudecode" -> matches "claude", "code")
+ if hashtags:
+ for tag in hashtags:
+ tag_lower = tag.lower()
+ for qt in q_tokens:
+ if qt in tag_lower and qt != tag_lower:
+ t_tokens.add(qt)
+
+ if not q_tokens:
+ return 0.5 # Neutral fallback for empty/stopword-only queries
+
+ overlap = len(q_tokens & t_tokens)
+ ratio = overlap / len(q_tokens)
+ return max(0.1, min(1.0, ratio))
diff --git a/tests/test_query.py b/tests/test_query.py
new file mode 100644
index 00000000..a081f4ff
--- /dev/null
+++ b/tests/test_query.py
@@ -0,0 +1,130 @@
+"""Tests for query.py — shared query utilities."""
+
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
+
+from lib.query import NOISE_WORDS, extract_core_subject
+
+
+class TestExtractCoreSubject(unittest.TestCase):
+ """Tests for extract_core_subject() with default noise set."""
+
+ def test_strips_what_are_prefix(self):
+ self.assertEqual(extract_core_subject("what are the best AI tools"), "ai")
+
+ def test_strips_how_to_prefix(self):
+ self.assertEqual(extract_core_subject("how to use cursor IDE"), "cursor ide")
+
+ def test_strips_what_do_people_think(self):
+ result = extract_core_subject("what do people think about React Server Components")
+ self.assertEqual(result, "react server components")
+
+ def test_preserves_product_name(self):
+ self.assertEqual(extract_core_subject("cursor IDE"), "cursor ide")
+
+ def test_strips_trailing_punctuation(self):
+ result = extract_core_subject("what is Claude?")
+ self.assertFalse(result.endswith("?"))
+
+ def test_empty_string(self):
+ self.assertEqual(extract_core_subject(""), "")
+
+ def test_all_noise_returns_original(self):
+ # When all words are noise, fall back to original text
+ result = extract_core_subject("best latest new")
+ self.assertTrue(len(result) > 0)
+
+ def test_only_first_prefix_stripped(self):
+ # "how to" should match, stripping once, not recursively
+ result = extract_core_subject("how to use how to debug")
+ self.assertIn("debug", result)
+
+
+class TestMaxWords(unittest.TestCase):
+ """Tests for max_words parameter."""
+
+ def test_max_words_caps_output(self):
+ result = extract_core_subject(
+ "multi agent reinforcement learning framework",
+ max_words=5,
+ )
+ self.assertLessEqual(len(result.split()), 5)
+
+ def test_max_words_none_no_cap(self):
+ result = extract_core_subject("cursor IDE react native components")
+ # Without max_words, no cap applied
+ self.assertGreaterEqual(len(result.split()), 3)
+
+ def test_max_words_fallback_on_empty(self):
+ # All words filtered + max_words should fall back to original
+ result = extract_core_subject("best top latest", max_words=3)
+ self.assertTrue(len(result) > 0)
+
+
+class TestStripSuffixes(unittest.TestCase):
+ """Tests for strip_suffixes parameter."""
+
+ def test_strips_best_practices(self):
+ result = extract_core_subject(
+ "claude code best practices",
+ strip_suffixes=True,
+ )
+ self.assertNotIn("practices", result)
+
+ def test_strips_use_cases(self):
+ result = extract_core_subject(
+ "react hooks use cases",
+ strip_suffixes=True,
+ )
+ self.assertNotIn("cases", result)
+
+ def test_no_strip_without_flag(self):
+ result = extract_core_subject("claude code best practices")
+ # "best" and "practices" are noise words so they get filtered anyway
+ # but the suffix phase doesn't run
+ self.assertIn("claude", result)
+
+
+class TestCustomNoise(unittest.TestCase):
+ """Tests for noise override parameter."""
+
+ def test_custom_noise_keeps_tips(self):
+ # YouTube keeps tips/tricks/tutorial — pass a noise set without them
+ youtube_noise = frozenset({
+ 'best', 'top', 'good', 'great', 'awesome', 'killer',
+ 'latest', 'new', 'news', 'update', 'updates',
+ 'trending', 'hottest', 'popular', 'viral',
+ 'practices', 'features',
+ 'recommendations', 'advice',
+ 'prompt', 'prompts', 'prompting',
+ 'methods', 'strategies', 'approaches',
+ })
+ result = extract_core_subject("best react tips", noise=youtube_noise)
+ self.assertIn("tips", result)
+
+ def test_default_noise_removes_tips(self):
+ result = extract_core_subject("best react tips")
+ self.assertNotIn("tips", result)
+
+
+class TestNoiseWordsCompleteness(unittest.TestCase):
+ """Verify NOISE_WORDS superset covers all platform sets."""
+
+ def test_question_words_present(self):
+ for w in ('who', 'why', 'when', 'where', 'does', 'should', 'could', 'would'):
+ self.assertIn(w, NOISE_WORDS, f"Missing question word: {w}")
+
+ def test_core_filler_present(self):
+ for w in ('the', 'a', 'an', 'is', 'are', 'for', 'with', 'about'):
+ self.assertIn(w, NOISE_WORDS)
+
+ def test_research_meta_present(self):
+ for w in ('best', 'top', 'latest', 'trending', 'popular'):
+ self.assertIn(w, NOISE_WORDS)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_relevance.py b/tests/test_relevance.py
new file mode 100644
index 00000000..c641e764
--- /dev/null
+++ b/tests/test_relevance.py
@@ -0,0 +1,122 @@
+"""Tests for relevance.py — shared relevance scoring.
+
+Migrated from test_youtube_relevance.py + new hashtag/synonym tests.
+"""
+
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
+
+from lib.relevance import STOPWORDS, SYNONYMS, token_overlap_relevance, tokenize
+
+
+class TestTokenize(unittest.TestCase):
+ """Tests for tokenize()."""
+
+ def test_removes_stopwords(self):
+ tokens = tokenize("how to use the AI tools")
+ self.assertNotIn("how", tokens)
+ self.assertNotIn("to", tokens)
+ self.assertNotIn("the", tokens)
+ self.assertIn("ai", tokens)
+ self.assertIn("tools", tokens)
+
+ def test_lowercases(self):
+ tokens = tokenize("Python REACT")
+ self.assertIn("python", tokens)
+ self.assertIn("react", tokens)
+
+ def test_strips_punctuation(self):
+ tokens = tokenize("hello, world!")
+ self.assertIn("hello", tokens)
+ self.assertIn("world", tokens)
+
+ def test_drops_single_char(self):
+ tokens = tokenize("a b c python")
+ self.assertNotIn("a", tokens)
+ self.assertNotIn("b", tokens)
+ self.assertNotIn("c", tokens)
+ self.assertIn("python", tokens)
+
+ def test_expands_synonyms(self):
+ tokens = tokenize("ai tools")
+ self.assertIn("artificial", tokens)
+ self.assertIn("intelligence", tokens)
+
+ def test_expands_js_synonym(self):
+ tokens = tokenize("js framework")
+ self.assertIn("javascript", tokens)
+
+ def test_expands_svelte(self):
+ tokens = tokenize("svelte app")
+ self.assertIn("sveltejs", tokens)
+
+ def test_expands_vue(self):
+ tokens = tokenize("vue components")
+ self.assertIn("vuejs", tokens)
+
+
+class TestTokenOverlapRelevance(unittest.TestCase):
+ """Tests for token_overlap_relevance()."""
+
+ def test_high_relevance_exact_match(self):
+ rel = token_overlap_relevance("claude code", "Claude Code tricks and tips")
+ self.assertGreater(rel, 0.7)
+
+ def test_low_relevance_no_match(self):
+ rel = token_overlap_relevance("claude code tips", "Best AI tools for coding")
+ self.assertLess(rel, 0.5)
+
+ def test_empty_query_returns_neutral(self):
+ rel = token_overlap_relevance("", "Some video title")
+ self.assertEqual(rel, 0.5)
+
+ def test_floor_at_0_1(self):
+ rel = token_overlap_relevance("quantum physics", "cat dancing video")
+ self.assertGreaterEqual(rel, 0.1)
+
+ def test_full_match_returns_1(self):
+ rel = token_overlap_relevance("python tutorial", "Python Tutorial for Beginners")
+ self.assertEqual(rel, 1.0)
+
+ def test_partial_match(self):
+ rel = token_overlap_relevance("react native tutorial", "React Native Guide")
+ self.assertGreater(rel, 0.3)
+ self.assertLess(rel, 1.0)
+
+ def test_synonym_boosts_relevance(self):
+ # "js" should match "javascript" via synonym expansion
+ rel_with_syn = token_overlap_relevance("js framework", "javascript framework comparison")
+ rel_without = token_overlap_relevance("python framework", "javascript framework comparison")
+ self.assertGreater(rel_with_syn, rel_without)
+
+ def test_stopword_only_query(self):
+ rel = token_overlap_relevance("the a is", "some content here")
+ self.assertEqual(rel, 0.5)
+
+
+class TestHashtagRelevance(unittest.TestCase):
+ """Tests for hashtag-aware relevance (TikTok/Instagram pattern)."""
+
+ def test_hashtag_boost(self):
+ rel_no_hash = token_overlap_relevance("claude code", "random video about stuff")
+ rel_with_hash = token_overlap_relevance(
+ "claude code", "random video about stuff", ["claudecode", "ai"]
+ )
+ self.assertGreater(rel_with_hash, rel_no_hash)
+
+ def test_concatenated_hashtag_splitting(self):
+ # "claudecode" should match "claude" from query via substring check
+ rel = token_overlap_relevance("claude", "video", ["claudecode"])
+ self.assertGreater(rel, 0.5)
+
+ def test_none_hashtags_same_as_no_hashtags(self):
+ rel1 = token_overlap_relevance("test query", "test content", None)
+ rel2 = token_overlap_relevance("test query", "test content")
+ self.assertEqual(rel1, rel2)
+
+
+if __name__ == "__main__":
+ unittest.main()