Skip to content

A curated collection of practical AI projects implementing OCR systems, RAG, AI agents, and other AI use cases.

7.3 KB
Raw Blame
"""Hacker News collector for devable-research-agent (Algolia API, free).

Adapted from last30days hackernews.py.
"""

import html
import math
import re
import sys
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import Any, Dict, List, Optional
from urllib.parse import urlencode

from . import http
from .schema import TrackerItem, Engagement, CollectionResult, SOURCE_HACKERNEWS
from .util import log

ALGOLIA_SEARCH_URL = "https://hn.algolia.com/api/v1/search"
ALGOLIA_ITEM_URL = "https://hn.algolia.com/api/v1/items"

DEPTH_CONFIG = {"quick": 15, "default": 30, "deep": 60}
ENRICH_LIMITS = {"quick": 3, "default": 5, "deep": 10}

# Noise patterns to filter low-quality HN posts
NOISE_PATTERNS = [
    r"(?i)\bis\s+\w+\s+down\b",
    r"(?i)\bdown\b.*\bagain\b",
    r"(?i)\bregret\b.*\bai\b",
    r"(?i)\brant\b",
    r"(?i)^ask hn:.*\b(career|hire|interview|freelance)\b",
]

MIN_POINTS = 5


def _is_noise(title: str, points: int) -> bool:
    """Check if a story title matches noise patterns."""
    if points >= 100:
        return False
    for pat in NOISE_PATTERNS:
        if re.search(pat, title):
            return True
    return False


def _extract_domain(url: str) -> str:
    """Extract domain from URL for summary display."""
    if not url:
        return ""
    try:
        from urllib.parse import urlparse
        parsed = urlparse(url)
        domain = parsed.netloc or ""
        if domain.startswith("www."):
            domain = domain[4:]
        return domain
    except Exception:
        return ""


_log = lambda msg: log("HN", msg, tty_only=True)


def _date_to_unix(date_str: str) -> int:
    import datetime
    parts = date_str.split("-")
    dt = datetime.datetime(int(parts[0]), int(parts[1]), int(parts[2]), tzinfo=datetime.timezone.utc)
    return int(dt.timestamp())


def _unix_to_date(ts: int) -> str:
    import datetime
    dt = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone.utc)
    return dt.strftime("%Y-%m-%d")


def _strip_html(text: str) -> str:
    text = html.unescape(text)
    text = re.sub(r"<p>", "\n", text)
    text = re.sub(r"<[^>]+>", "", text)
    return text.strip()


def search(
    query: str,
    from_date: str,
    to_date: str,
    depth: str = "default",
) -> List[Dict[str, Any]]:
    """Search HN via Algolia API."""
    count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
    from_ts = _date_to_unix(from_date)
    to_ts = _date_to_unix(to_date) + 86400

    params = {
        "query": query,
        "tags": "story",
        "numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts},points>{MIN_POINTS}",
        "hitsPerPage": str(count),
    }
    url = f"{ALGOLIA_SEARCH_URL}?{urlencode(params)}"

    try:
        response = http.get(url, timeout=30)
    except Exception as e:
        _log(f"Search failed: {e}")
        return []

    return response.get("hits", [])


def _fetch_comments(object_id: str, max_comments: int = 5) -> List[str]:
    """Fetch top comments for a story."""
    url = f"{ALGOLIA_ITEM_URL}/{object_id}"
    try:
        data = http.get(url, timeout=15)
    except Exception:
        return []

    children = data.get("children", [])
    real_comments = [c for c in children if c.get("text") and c.get("author")]
    real_comments.sort(key=lambda c: c.get("points") or 0, reverse=True)

    insights = []
    for c in real_comments[:max_comments]:
        text = _strip_html(c.get("text", ""))
        first_sentence = text.split(". ")[0].split("\n")[0][:200]
        if first_sentence:
            insights.append(first_sentence)
    return insights


def collect(
    entities: Dict[str, List[str]],
    from_date: str,
    to_date: str,
    depth: str = "default",
) -> CollectionResult:
    """Collect HN stories for tracked entities.

    Args:
        entities: Dict mapping entity name -> list of search keywords
        from_date: Start date YYYY-MM-DD
        to_date: End date YYYY-MM-DD
        depth: Search depth

    Returns:
        CollectionResult
    """
    result = CollectionResult(source=SOURCE_HACKERNEWS)
    all_items = []
    enrich_limit = ENRICH_LIMITS.get(depth, ENRICH_LIMITS["default"])

    for entity_name, keywords in entities.items():
        result.entities_checked += 1
        entity_found = False

        for keyword in keywords:
            hits = search(keyword, from_date, to_date, depth)
            if not hits:
                continue

            for i, hit in enumerate(hits):
                object_id = hit.get("objectID", "")
                title = hit.get("title", "")
                points = hit.get("points") or 0
                num_comments_val = hit.get("num_comments") or 0
                created_at_i = hit.get("created_at_i")
                date_str = _unix_to_date(created_at_i) if created_at_i else None
                article_url = hit.get("url") or ""
                story_text = hit.get("story_text") or ""
                hn_url = f"https://news.ycombinator.com/item?id={object_id}"

                if _is_noise(title, points):
                    continue

                # Build meaningful summary
                if story_text:
                    # Text posts (Ask HN, Show HN): use story text
                    summary = _strip_html(story_text)[:300]
                elif article_url:
                    # Link posts: include domain for context
                    domain = _extract_domain(article_url)
                    summary = f"via {domain} ({points} pts, {num_comments_val} comments)" if domain else f"({points} pts, {num_comments_val} comments)"
                else:
                    summary = f"HN discussion ({points} pts, {num_comments_val} comments)"

                rank_score = max(0.3, 1.0 - (i * 0.02))
                engagement_boost = min(0.2, math.log1p(points) / 40)
                relevance = min(1.0, rank_score * 0.7 + engagement_boost + 0.1)

                all_items.append(TrackerItem(
                    id=f"HN-{object_id}",
                    title=title,
                    summary=summary,
                    entity=entity_name,
                    source=SOURCE_HACKERNEWS,
                    source_url=article_url or hn_url,
                    source_label=f"Hacker News",
                    date=date_str,
                    date_confidence="high" if date_str else "low",
                    raw_text=title,
                    engagement=Engagement(points=points, num_comments=num_comments_val),
                    relevance=round(relevance, 2),
                ))
                entity_found = True

        if entity_found:
            result.entities_with_updates += 1

    # Enrich top stories with comments
    top_items = sorted(all_items, key=lambda x: x.engagement.points, reverse=True)[:enrich_limit]
    with ThreadPoolExecutor(max_workers=5) as executor:
        futures = {}
        for item in top_items:
            oid = item.id.replace("HN-", "")
            futures[executor.submit(_fetch_comments, oid)] = item
        for future in as_completed(futures):
            item = futures[future]
            try:
                insights = future.result(timeout=15)
                if insights:
                    item.summary += "\nKey discussion points: " + "; ".join(insights[:3])
            except Exception:
                pass

    result.items = all_items
    _log(f"Collected {len(all_items)} HN stories from {result.entities_checked} entities")
    return result