Skip to content

160-tool MCP server for real-time global intelligence from free public APIs: markets, liquidity, prediction markets, SEC filings, conflict, military, aircraft-cohort early warning, migration and border flows, cyber, climate, news and more. Dated observation history, 29 research prompts, cited briefs, geofences, SSE dashboard. MIT, no paid API keys.

6.7 KB
Raw Blame
"""News article clustering using Jaccard similarity.

Groups news articles by content similarity to reduce noise
and identify story clusters.
"""

import logging
import re

logger = logging.getLogger("world-intel-mcp.analysis.clustering")

_STOPWORDS = frozenset({
    "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
    "have", "has", "had", "do", "does", "did", "will", "would", "could",
    "should", "may", "might", "can", "shall", "in", "on", "at", "to",
    "for", "of", "and", "or", "but", "not", "no", "nor", "with", "from",
    "by", "it", "its", "that", "this", "these", "those", "he", "she",
    "they", "we", "you", "i", "me", "my", "his", "her", "our", "your",
    "their", "what", "which", "who", "whom", "how", "when", "where",
    "why", "if", "then", "than", "so", "as", "up", "out", "about",
    "into", "over", "after", "before", "between", "under", "again",
    "also", "just", "more", "most", "other", "some", "such", "very",
    "new", "said", "says", "one", "two", "first",
})


def _tokenize(text: str) -> set[str]:
    """Extract meaningful word tokens from text."""
    words = re.findall(r'[a-z]{3,}', text.lower())
    return {w for w in words if w not in _STOPWORDS}


def jaccard_similarity(set_a: set, set_b: set) -> float:
    """Compute Jaccard similarity between two sets."""
    if not set_a or not set_b:
        return 0.0
    intersection = len(set_a & set_b)
    union = len(set_a | set_b)
    return intersection / union if union > 0 else 0.0


def cluster_articles(
    articles: list[dict],
    similarity_threshold: float = 0.3,
    title_field: str = "title",
) -> list[dict]:
    """Cluster articles by title similarity.

    Args:
        articles: List of article dicts.
        similarity_threshold: Minimum Jaccard similarity to merge.
        title_field: Key containing the article title.

    Returns:
        List of cluster dicts with representative article and member count.
    """
    if not articles:
        return []

    # Tokenize all titles
    tokenized = []
    for article in articles:
        title = article.get(title_field, "") or ""
        tokens = _tokenize(title)
        tokenized.append(tokens)

    # Greedy single-linkage clustering
    assigned = [False] * len(articles)
    clusters: list[dict] = []

    for i in range(len(articles)):
        if assigned[i]:
            continue

        cluster_members = [i]
        assigned[i] = True

        for j in range(i + 1, len(articles)):
            if assigned[j]:
                continue

            # Check similarity against cluster representative (first member)
            sim = jaccard_similarity(tokenized[i], tokenized[j])
            if sim >= similarity_threshold:
                cluster_members.append(j)
                assigned[j] = True

        # Build cluster output
        representative = articles[cluster_members[0]]
        clusters.append({
            "representative": representative,
            "member_count": len(cluster_members),
            "member_indices": cluster_members,
        })

    # Sort by member count descending
    clusters.sort(key=lambda c: c["member_count"], reverse=True)
    return clusters


# ---------------------------------------------------------------------------
# MCP-facing async wrapper
# ---------------------------------------------------------------------------

async def fetch_news_clusters(
    fetcher,
    category: str | None = None,
    limit: int = 100,
    threshold: float = 0.25,
) -> dict:
    """Fetch recent news and cluster by topic similarity.

    Args:
        fetcher: HTTP fetcher instance.
        category: Optional RSS feed category filter.
        limit: Max news items to fetch.
        threshold: Jaccard similarity threshold (0.0 - 1.0).

    Returns:
        Dict with clusters[], cluster_count, total_items, singleton_count,
        source. Each cluster carries ``outlet_count``: distinct publishers
        among ALL its members (sister feeds of one publisher count once,
        see ``news.FEED_OUTLETS``). An RSS outage is passed through as
        ``degraded``/``reason``/``error``, never a quiet cluster-free day.
    """
    import asyncio
    from datetime import datetime, timezone
    from ..sources import news

    feed_data = await news.fetch_news_feed(fetcher, category=category, limit=limit)
    items = feed_data.get("items", [])

    # Pairwise Jaccard is quadratic in titles: off the event loop the
    # collector and the MCP server share.
    raw_clusters = await asyncio.to_thread(
        cluster_articles, items, similarity_threshold=threshold
    )

    out_clusters = []
    for c in raw_clusters:
        if c["member_count"] <= 1:
            continue
        rep = c["representative"]
        # Collect keywords from all member titles
        all_tokens: set[str] = set()
        member_items = []
        for idx in c["member_indices"]:
            art = items[idx]
            all_tokens |= _tokenize(art.get("title", ""))
            member_items.append({
                "title": art.get("title", ""),
                # fetch_news_feed names the outlet "feed_name"; the
                # source_name/source fallbacks serve other article shapes.
                "source": (
                    art.get("feed_name")
                    or art.get("source_name")
                    or art.get("source")
                    or ""
                ),
                "link": art.get("link", ""),
            })

        # Top keywords by frequency
        word_freq: dict[str, int] = {}
        for idx in c["member_indices"]:
            for w in _tokenize(items[idx].get("title", "")):
                word_freq[w] = word_freq.get(w, 0) + 1
        top_kw = sorted(word_freq, key=word_freq.get, reverse=True)[:8]  # type: ignore[arg-type]

        outlets = sorted(
            {news.outlet_for(m["source"]) for m in member_items if m["source"]}
        )
        out_clusters.append({
            "headline": rep.get("title", ""),
            "size": c["member_count"],
            "keywords": top_kw,
            "sources": list({m["source"] for m in member_items if m["source"]}),
            "outlets": outlets,
            "outlet_count": len(outlets),
            "items": member_items[:5],
        })

    singletons = sum(1 for c in raw_clusters if c["member_count"] == 1)

    result = {
        "clusters": out_clusters,
        "cluster_count": len(out_clusters),
        "total_items": len(items),
        "singleton_count": singletons,
        "threshold": threshold,
        "source": "jaccard-clustering",
        "timestamp": datetime.now(timezone.utc).isoformat(),
    }
    for key in (
        "feeds_attempted", "feeds_ok", "feeds_failed", "feeds_stale",
        "error", "degraded", "reason",
    ):
        if key in feed_data:
            result[key] = feed_data[key]
    return result