Skip to content

A curated collection of practical AI projects implementing OCR systems, RAG, AI agents, and other AI use cases.

8.8 KB
Raw Blame
"""Near-duplicate detection for devable-research-agent.

Adapted from last30days dedupe.py.
Supports within-source dedup and cross-source linking.
"""

import re
from datetime import date as _date
from typing import List, Set, Tuple
from urllib.parse import urlparse

from .schema import TrackerItem

# Stopwords for token-based matching
STOPWORDS = frozenset({
    "the", "a", "an", "to", "for", "how", "is", "in", "of", "on",
    "and", "with", "from", "by", "at", "this", "that", "it", "my",
    "your", "i", "me", "we", "you", "what", "are", "do", "can",
    "its", "be", "or", "not", "no", "so", "if", "but", "about",
    "all", "just", "get", "has", "have", "was", "will", "new",
    "ai", "model", "release", "update", "now", "available",
})


def normalize_text(text: str) -> str:
    """Normalize text for comparison."""
    text = text.lower()
    text = re.sub(r"[^\w\s]", " ", text)
    text = re.sub(r"\s+", " ", text)
    return text.strip()


def get_ngrams(text: str, n: int = 3) -> Set[str]:
    """Get character n-grams from text."""
    text = normalize_text(text)
    if len(text) < n:
        return {text}
    return {text[i:i + n] for i in range(len(text) - n + 1)}


def jaccard_similarity(set1: Set[str], set2: Set[str]) -> float:
    """Compute Jaccard similarity between two sets."""
    if not set1 or not set2:
        return 0.0
    intersection = len(set1 & set2)
    union = len(set1 | set2)
    return intersection / union if union > 0 else 0.0


def get_item_text(item: TrackerItem) -> str:
    """Get comparable text from a TrackerItem."""
    if item.title:
        return item.title
    return item.raw_text[:150]


def _tokenize(text: str) -> Set[str]:
    """Tokenize text for token-level Jaccard."""
    words = re.sub(r"[^\w\s]", " ", text.lower()).split()
    return {w for w in words if w not in STOPWORDS and len(w) > 1}


def _token_jaccard(text_a: str, text_b: str) -> float:
    """Token-level Jaccard similarity."""
    tokens_a = _tokenize(text_a)
    tokens_b = _tokenize(text_b)
    if not tokens_a or not tokens_b:
        return 0.0
    intersection = len(tokens_a & tokens_b)
    union = len(tokens_a | tokens_b)
    return intersection / union if union else 0.0


def hybrid_similarity(text_a: str, text_b: str) -> float:
    """Hybrid similarity: max of char-trigram Jaccard and token Jaccard."""
    trigram_sim = jaccard_similarity(get_ngrams(text_a), get_ngrams(text_b))
    token_sim = _token_jaccard(text_a, text_b)
    return max(trigram_sim, token_sim)


def find_duplicates(
    items: List[TrackerItem],
    threshold: float = 0.7,
) -> List[Tuple[int, int]]:
    """Find near-duplicate pairs by Jaccard similarity.

    Args:
        items: List of items to check
        threshold: Similarity threshold (0-1)

    Returns:
        List of (i, j) index pairs where items are similar
    """
    duplicates = []
    ngrams = [get_ngrams(get_item_text(item)) for item in items]

    for i in range(len(items)):
        for j in range(i + 1, len(items)):
            similarity = jaccard_similarity(ngrams[i], ngrams[j])
            if similarity >= threshold:
                duplicates.append((i, j))

    return duplicates


def dedupe_items(
    items: List[TrackerItem],
    threshold: float = 0.7,
) -> List[TrackerItem]:
    """Remove near-duplicates, keeping highest-scored item.

    Args:
        items: List of items (should be pre-sorted by importance desc)
        threshold: Similarity threshold

    Returns:
        Deduplicated items
    """
    if len(items) <= 1:
        return items

    dup_pairs = find_duplicates(items, threshold)

    to_remove = set()
    for i, j in dup_pairs:
        if items[i].importance >= items[j].importance:
            to_remove.add(j)
        else:
            to_remove.add(i)

    return [item for idx, item in enumerate(items) if idx not in to_remove]


def dedupe_by_source(items: List[TrackerItem], threshold: float = 0.7) -> List[TrackerItem]:
    """Deduplicate items within each source separately.

    More efficient than global dedup for large item sets.

    Args:
        items: All collected items
        threshold: Similarity threshold

    Returns:
        Deduplicated items
    """
    by_source = {}
    for item in items:
        by_source.setdefault(item.source, []).append(item)

    result = []
    for source, source_items in by_source.items():
        source_items.sort(key=lambda x: x.importance, reverse=True)
        deduped = dedupe_items(source_items, threshold)
        result.extend(deduped)

    return result


def cross_source_link(
    items: List[TrackerItem],
    threshold: float = 0.40,
) -> List[TrackerItem]:
    """Link similar items across different sources.

    Uses hybrid similarity (char-trigram + token Jaccard) for cross-source
    matching. Adds bidirectional cross_refs when similarity exceeds threshold.

    Also considers entity matching: items about the same entity get a
    similarity boost for more accurate cross-referencing.

    Args:
        items: All items from all sources
        threshold: Similarity threshold for cross-linking

    Returns:
        Items with cross_refs populated
    """
    if len(items) <= 1:
        return items

    # Pre-compute normalized texts for cross-source comparison
    texts = []
    for item in items:
        texts.append(_normalize_for_cross_source(item))

    # Pre-compute URL domains for URL-based matching
    domains = []
    for item in items:
        domains.append(_extract_url_domain(item.source_url))

    # Pre-parse dates for proximity check
    parsed_dates = []
    for item in items:
        try:
            parsed_dates.append(_date.fromisoformat(item.date) if item.date else None)
        except ValueError:
            parsed_dates.append(None)

    for i in range(len(items)):
        for j in range(i + 1, len(items)):
            # Skip same-source comparisons
            if items[i].source == items[j].source:
                continue

            # Skip items with dates too far apart (>3 days)
            if parsed_dates[i] and parsed_dates[j]:
                if abs((parsed_dates[i] - parsed_dates[j]).days) > 3:
                    continue

            similarity = hybrid_similarity(texts[i], texts[j])

            # Entity match bonus: same entity increases likelihood of cross-ref
            same_entity = items[i].entity and items[i].entity == items[j].entity
            if same_entity:
                similarity += 0.25

            # URL domain bonus: items linking to same domain
            if domains[i] and domains[j] and domains[i] == domains[j]:
                similarity += 0.15

            # Use lower threshold for same-entity comparisons
            effective_threshold = 0.25 if same_entity else threshold

            if similarity >= effective_threshold:
                if items[j].id not in items[i].cross_refs:
                    items[i].cross_refs.append(items[j].id)
                if items[i].id not in items[j].cross_refs:
                    items[j].cross_refs.append(items[i].id)

    return items


def _normalize_for_cross_source(item: TrackerItem) -> str:
    """Normalize item text for cross-source comparison.

    Different sources use very different title formats:
    - HuggingFace: "nvidia/Gemma-4-31B-IT-NVFP4" (model IDs)
    - GitHub: "anthropics/claude-code v2.1.107" (repo + version)
    - HN/Reddit: Natural language titles

    This function normalizes them to improve cross-source matching.
    """
    text = get_item_text(item)

    if item.source == "huggingface":
        # Split org/model format, expand separators
        if "/" in text:
            text = text.split("/", 1)[-1]  # drop org prefix
        text = re.sub(r"[-_]", " ", text)

    elif item.source == "github":
        # Drop org prefix, remove version numbers
        if "/" in text:
            parts = text.split("/", 1)
            text = parts[-1]
        text = re.sub(r"\bv?\d+\.\d+\.\d+\b", "", text)  # remove semver
        text = re.sub(r"[-_]", " ", text)

    # Common cleanup for all sources
    text = re.sub(r"\bv?\d+(?:\.\d+)+\b", "", text)  # version numbers like v2.1, 2.1.107
    text = re.sub(r"\bv\d+\b", "", text)  # simple version tags like v2
    text = re.sub(r"\b\d{4,}\b", "", text)  # long numbers (dates, IDs)
    text = text.lower().strip()
    return text


def _extract_url_domain(url: str) -> str:
    """Extract domain from URL for cross-source matching."""
    if not url:
        return ""
    try:
        parsed = urlparse(url)
        domain = parsed.netloc or ""
        if domain.startswith("www."):
            domain = domain[4:]
        # Exclude generic platforms (they host many unrelated items)
        if domain in ("github.com", "news.ycombinator.com", "reddit.com",
                       "huggingface.co", "arxiv.org", "x.com", "twitter.com"):
            return ""
        return domain
    except Exception:
        return ""