160-tool MCP server for real-time global intelligence from free public APIs: markets, liquidity, prediction markets, SEC filings, conflict, military, aircraft-cohort early warning, migration and border flows, cyber, climate, news and more. Dated observation history, 29 research prompts, cited briefs, geofences, SSE dashboard. MIT, no paid API keys.
Python ai-toolsai-watchanthropicclaudecybersecuritydashboardgeofencinggeopoliticalgeospatialintelligencemarketsmcpmigrationmilitarymodel-context-protocolnuclearosintreal-timespace-weatherthreat-intelligence
"""News article clustering using Jaccard similarity.
Groups news articles by content similarity to reduce noise
and identify story clusters.
"""
import logging
import re
logger = logging.getLogger("world-intel-mcp.analysis.clustering")
_STOPWORDS = frozenset({
"the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
"have", "has", "had", "do", "does", "did", "will", "would", "could",
"should", "may", "might", "can", "shall", "in", "on", "at", "to",
"for", "of", "and", "or", "but", "not", "no", "nor", "with", "from",
"by", "it", "its", "that", "this", "these", "those", "he", "she",
"they", "we", "you", "i", "me", "my", "his", "her", "our", "your",
"their", "what", "which", "who", "whom", "how", "when", "where",
"why", "if", "then", "than", "so", "as", "up", "out", "about",
"into", "over", "after", "before", "between", "under", "again",
"also", "just", "more", "most", "other", "some", "such", "very",
"new", "said", "says", "one", "two", "first",
})
def _tokenize(text: str) -> set[str]:
"""Extract meaningful word tokens from text."""
words = re.findall(r'[a-z]{3,}', text.lower())
return {w for w in words if w not in _STOPWORDS}
def jaccard_similarity(set_a: set, set_b: set) -> float:
"""Compute Jaccard similarity between two sets."""
if not set_a or not set_b:
return 0.0
intersection = len(set_a & set_b)
union = len(set_a | set_b)
return intersection / union if union > 0 else 0.0
def cluster_articles(
articles: list[dict],
similarity_threshold: float = 0.3,
title_field: str = "title",
) -> list[dict]:
"""Cluster articles by title similarity.
Args:
articles: List of article dicts.
similarity_threshold: Minimum Jaccard similarity to merge.
title_field: Key containing the article title.
Returns:
List of cluster dicts with representative article and member count.
"""
if not articles:
return []
# Tokenize all titles
tokenized = []
for article in articles:
title = article.get(title_field, "") or ""
tokens = _tokenize(title)
tokenized.append(tokens)
# Greedy single-linkage clustering
assigned = [False] * len(articles)
clusters: list[dict] = []
for i in range(len(articles)):
if assigned[i]:
continue
cluster_members = [i]
assigned[i] = True
for j in range(i + 1, len(articles)):
if assigned[j]:
continue
# Check similarity against cluster representative (first member)
sim = jaccard_similarity(tokenized[i], tokenized[j])
if sim >= similarity_threshold:
cluster_members.append(j)
assigned[j] = True
# Build cluster output
representative = articles[cluster_members[0]]
clusters.append({
"representative": representative,
"member_count": len(cluster_members),
"member_indices": cluster_members,
})
# Sort by member count descending
clusters.sort(key=lambda c: c["member_count"], reverse=True)
return clusters
# ---------------------------------------------------------------------------
# MCP-facing async wrapper
# ---------------------------------------------------------------------------
async def fetch_news_clusters(
fetcher,
category: str | None = None,
limit: int = 100,
threshold: float = 0.25,
) -> dict:
"""Fetch recent news and cluster by topic similarity.
Args:
fetcher: HTTP fetcher instance.
category: Optional RSS feed category filter.
limit: Max news items to fetch.
threshold: Jaccard similarity threshold (0.0 - 1.0).
Returns:
Dict with clusters[], cluster_count, total_items, singleton_count,
source. Each cluster carries ``outlet_count``: distinct publishers
among ALL its members (sister feeds of one publisher count once,
see ``news.FEED_OUTLETS``). An RSS outage is passed through as
``degraded``/``reason``/``error``, never a quiet cluster-free day.
"""
import asyncio
from datetime import datetime, timezone
from ..sources import news
feed_data = await news.fetch_news_feed(fetcher, category=category, limit=limit)
items = feed_data.get("items", [])
# Pairwise Jaccard is quadratic in titles: off the event loop the
# collector and the MCP server share.
raw_clusters = await asyncio.to_thread(
cluster_articles, items, similarity_threshold=threshold
)
out_clusters = []
for c in raw_clusters:
if c["member_count"] <= 1:
continue
rep = c["representative"]
# Collect keywords from all member titles
all_tokens: set[str] = set()
member_items = []
for idx in c["member_indices"]:
art = items[idx]
all_tokens |= _tokenize(art.get("title", ""))
member_items.append({
"title": art.get("title", ""),
# fetch_news_feed names the outlet "feed_name"; the
# source_name/source fallbacks serve other article shapes.
"source": (
art.get("feed_name")
or art.get("source_name")
or art.get("source")
or ""
),
"link": art.get("link", ""),
})
# Top keywords by frequency
word_freq: dict[str, int] = {}
for idx in c["member_indices"]:
for w in _tokenize(items[idx].get("title", "")):
word_freq[w] = word_freq.get(w, 0) + 1
top_kw = sorted(word_freq, key=word_freq.get, reverse=True)[:8] # type: ignore[arg-type]
outlets = sorted(
{news.outlet_for(m["source"]) for m in member_items if m["source"]}
)
out_clusters.append({
"headline": rep.get("title", ""),
"size": c["member_count"],
"keywords": top_kw,
"sources": list({m["source"] for m in member_items if m["source"]}),
"outlets": outlets,
"outlet_count": len(outlets),
"items": member_items[:5],
})
singletons = sum(1 for c in raw_clusters if c["member_count"] == 1)
result = {
"clusters": out_clusters,
"cluster_count": len(out_clusters),
"total_items": len(items),
"singleton_count": singletons,
"threshold": threshold,
"source": "jaccard-clustering",
"timestamp": datetime.now(timezone.utc).isoformat(),
}
for key in (
"feeds_attempted", "feeds_ok", "feeds_failed", "feeds_stale",
"error", "degraded", "reason",
):
if key in feed_data:
result[key] = feed_data[key]
return result