A curated collection of practical AI projects implementing OCR systems, RAG, AI agents, and other AI use cases.
"""HuggingFace collector for devable-research-agent.
Monitors model/space/dataset updates via HuggingFace API.
New collector — no last30days equivalent.
"""
import re
import sys
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import Any, Dict, List, Optional
from urllib.parse import urlencode, quote
from . import http
from .schema import TrackerItem, Engagement, CollectionResult, SOURCE_HUGGINGFACE
from .util import log, parse_date
HF_API = "https://huggingface.co/api"
DEPTH_CONFIG = {"quick": 5, "default": 10, "deep": 20}
def _format_params(total: int) -> str:
"""Format parameter count to human-readable string (e.g. 31B, 600M)."""
if total >= 1_000_000_000:
val = total / 1_000_000_000
return f"{val:.0f}B" if val == int(val) else f"{val:.1f}B"
if total >= 1_000_000:
val = total / 1_000_000
return f"{val:.0f}M" if val == int(val) else f"{val:.1f}M"
if total >= 1_000:
val = total / 1_000
return f"{val:.0f}K" if val == int(val) else f"{val:.1f}K"
return str(total)
def _fetch_model_meta(model_id: str) -> Dict[str, str]:
"""Fetch rich metadata from HuggingFace model API.
Returns dict with keys: params, arch, license, base_model (all optional).
"""
url = f"{HF_API}/models/{quote(model_id, safe='/')}"
try:
data = http.get(url, timeout=15, retries=1)
except Exception:
return {}
if not isinstance(data, dict):
return {}
result: Dict[str, str] = {}
# Parameter count from safetensors
safetensors = data.get("safetensors")
if isinstance(safetensors, dict):
total = safetensors.get("total")
if isinstance(total, (int, float)) and total > 0:
result["params"] = _format_params(int(total))
# Architecture from config
config = data.get("config")
if isinstance(config, dict):
archs = config.get("architectures")
if isinstance(archs, list) and archs:
# "Gemma3ForConditionalGeneration" -> "Gemma3"
arch = archs[0]
arch = re.sub(r'(For\w+|LMHead\w*|Model)$', '', arch)
if arch:
result["arch"] = arch
if "arch" not in result:
model_type = config.get("model_type")
if model_type:
result["arch"] = model_type
# License and base_model from cardData
card = data.get("cardData")
if isinstance(card, dict):
lic = card.get("license")
if isinstance(lic, str) and lic:
result["license"] = lic
base = card.get("base_model")
if isinstance(base, str) and base:
result["base_model"] = base
elif isinstance(base, list) and base:
result["base_model"] = base[0] if isinstance(base[0], str) else ""
return result
def _fetch_model_description(model_id: str) -> str:
"""Fetch model card description from HuggingFace README.
Fetches the raw README.md and extracts the first meaningful paragraph
after the YAML frontmatter. Falls back to empty string on failure.
"""
url = f"https://huggingface.co/{model_id}/raw/main/README.md"
try:
text = http.get(url, timeout=10, retries=1, raw=True)
except Exception:
return ""
if not isinstance(text, str) or len(text) < 20:
return ""
# Strip YAML frontmatter
if text.startswith("---"):
end = text.find("---", 3)
if end > 0:
text = text[end + 3:]
# Find first substantial paragraph (skip blanks, headers, links, badges, emoji lines)
for line in text.split("\n"):
line = line.strip()
if not line:
continue
if line.startswith(("#", "|", "[", "!", "<", "-", "*", ">")):
continue
if line.startswith("```"):
break
# Skip lines starting with emoji (common promotional banners)
if re.match(r'^[\U0001F000-\U0001FFFF\u2600-\u27BF\u2B50]', line):
continue
# Skip short lines and markdown link-only lines
if len(line) < 50:
continue
if re.match(r'^\[.*\]\(.*\)$', line):
continue
# Found a real paragraph
return line[:300]
return ""
_log = lambda msg: log("HF", msg, tty_only=True)
_parse_date = parse_date
def get_author_models(
author: str,
from_date: str,
to_date: str,
limit: int = 10,
) -> List[Dict[str, Any]]:
"""Get recent models from an author/org on HuggingFace.
Args:
author: HuggingFace username or org (e.g. "meta-llama")
from_date: Start date YYYY-MM-DD
to_date: End date YYYY-MM-DD
limit: Max results
Returns:
List of model info dicts
"""
params = urlencode({
"author": author,
"sort": "createdAt",
"direction": "-1",
"limit": str(limit),
})
url = f"{HF_API}/models?{params}"
try:
response = http.get(url, timeout=30)
except Exception as e:
_log(f"Failed to fetch models for {author}: {e}")
return []
if not isinstance(response, list):
return []
models = []
for model in response:
created = _parse_date(model.get("createdAt"))
modified = _parse_date(model.get("lastModified"))
date = created or modified
if not date:
continue
if not (from_date <= date <= to_date):
continue
downloads = model.get("downloads", 0) or 0
likes = model.get("likes", 0) or 0
models.append({
"model_id": model.get("modelId", model.get("id", "")),
"pipeline_tag": model.get("pipeline_tag", ""),
"tags": model.get("tags", []),
"downloads": downloads,
"likes": likes,
"date": date,
"created": created,
"modified": modified,
"url": f"https://huggingface.co/{model.get('modelId', model.get('id', ''))}",
"private": model.get("private", False),
"library_name": model.get("library_name", ""),
})
_log(f"{author}: {len(models)} models in date range")
return models
def get_trending_models(
limit: int = 20,
) -> List[Dict[str, Any]]:
"""Get trending models from HuggingFace."""
params = urlencode({
"sort": "trending",
"direction": "-1",
"limit": str(limit),
})
url = f"{HF_API}/models?{params}"
try:
response = http.get(url, timeout=30)
except Exception as e:
_log(f"Failed to fetch trending models: {e}")
return []
if not isinstance(response, list):
return []
return response
def search_models(
query: str,
from_date: str,
to_date: str,
limit: int = 10,
) -> List[Dict[str, Any]]:
"""Search for models by keyword."""
params = urlencode({
"search": query,
"sort": "createdAt",
"direction": "-1",
"limit": str(limit),
})
url = f"{HF_API}/models?{params}"
try:
response = http.get(url, timeout=30)
except Exception as e:
_log(f"Model search '{query}' failed: {e}")
return []
if not isinstance(response, list):
return []
models = []
for model in response:
created = _parse_date(model.get("createdAt"))
modified = _parse_date(model.get("lastModified"))
date = created or modified
if date and (date < from_date or date > to_date):
continue
models.append(model)
return models
def collect(
entities: Dict[str, List[str]],
from_date: str,
to_date: str,
depth: str = "default",
) -> CollectionResult:
"""Collect HuggingFace model updates for tracked entities.
Args:
entities: Dict mapping entity name -> list of HF author/org names
e.g. {"Meta AI": ["meta-llama"], "Google": ["google"]}
from_date: Start date YYYY-MM-DD
to_date: End date YYYY-MM-DD
depth: Search depth
Returns:
CollectionResult
"""
result = CollectionResult(source=SOURCE_HUGGINGFACE)
all_items = []
limit = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
for entity_name, hf_authors in entities.items():
result.entities_checked += 1
entity_found = False
for author in hf_authors:
models = get_author_models(author, from_date, to_date, limit)
for model in models:
model_id = model.get("model_id", "")
pipeline = model.get("pipeline_tag", "")
downloads = model.get("downloads", 0)
likes = model.get("likes", 0)
tags = model.get("tags", [])
# Metadata fallback summary
tag_str = ", ".join(tags[:5]) if tags else ""
meta_parts = []
if pipeline:
meta_parts.append(f"Pipeline: {pipeline}")
if downloads:
meta_parts.append(f"{downloads:,} downloads")
if likes:
meta_parts.append(f"{likes} likes")
if tag_str:
meta_parts.append(f"Tags: {tag_str}")
fallback_summary = " | ".join(meta_parts) if meta_parts else model_id
all_items.append(TrackerItem(
id=f"HF-{model_id}",
title=model_id,
summary=fallback_summary,
entity=entity_name,
source=SOURCE_HUGGINGFACE,
source_url=model.get("url", f"https://huggingface.co/{model_id}"),
source_label=f"HuggingFace {author}",
date=model.get("date"),
date_confidence="high" if model.get("date") else "low",
raw_text=model_id,
engagement=Engagement(
likes=likes,
views=downloads,
),
relevance=min(1.0, 0.5 + min(0.5, (downloads / 10000 + likes / 100))),
))
entity_found = True
if entity_found:
result.entities_with_updates += 1
# Enrich summaries with model card descriptions + technical metadata (concurrent)
if all_items:
_log(f"Fetching model details for {len(all_items)} models...")
with ThreadPoolExecutor(max_workers=5) as executor:
desc_futures = {
executor.submit(_fetch_model_description, item.title): ("desc", item)
for item in all_items
}
meta_futures = {
executor.submit(_fetch_model_meta, item.title): ("meta", item)
for item in all_items
}
# Collect results keyed by item id
descriptions: Dict[str, str] = {}
metas: Dict[str, Dict[str, str]] = {}
for future in as_completed({**desc_futures, **meta_futures}):
if future in desc_futures:
_, item = desc_futures[future]
try:
descriptions[item.id] = future.result(timeout=20)
except Exception:
pass
else:
_, item = meta_futures[future]
try:
metas[item.id] = future.result(timeout=20)
except Exception:
pass
# Build enriched summaries
for item in all_items:
desc = descriptions.get(item.id, "")
model_meta = metas.get(item.id, {})
# Technical specs
specs = []
if model_meta.get("params"):
specs.append(f"{model_meta['params']} params")
if model_meta.get("arch"):
specs.append(f"{model_meta['arch']} architecture")
if model_meta.get("base_model"):
specs.append(f"based on {model_meta['base_model']}")
if model_meta.get("license"):
specs.append(model_meta["license"])
spec_str = ", ".join(specs)
# Engagement metrics
metrics = []
if item.engagement.views:
metrics.append(f"{item.engagement.views:,} downloads")
if item.engagement.likes:
metrics.append(f"{item.engagement.likes} likes")
metric_str = " | ".join(metrics)
# Assemble: description + specs + metrics
parts = []
if desc:
parts.append(desc[:200])
if spec_str:
parts.append(spec_str)
if metric_str:
parts.append(metric_str)
if parts:
item.summary = ". ".join(parts)
result.items = all_items
_log(f"Collected {len(all_items)} HF models from {result.entities_checked} entities")
return result