Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

18.5 KB
Raw Blame
"""Domain classifier using LLM for categorization."""

import json
from typing import Dict, List, Optional

from loguru import logger
from sqlalchemy.orm import Session

from ..config.llm_config import get_llm
from ..database.models import ResearchResource
from ..database.session_context import get_user_db_session
from ..security import url_authority_without_userinfo
from ..utilities.sql_utils import escape_like as _escape_like
from ..utilities.json_utils import extract_json, get_llm_response_text
from .models import DomainClassification
from ..utilities.llm_utils import invoke_llm_sync


# Predefined categories for domain classification
DOMAIN_CATEGORIES = {
    "Academic & Research": [
        "University/Education",
        "Scientific Journal",
        "Research Institution",
        "Academic Database",
    ],
    "News & Media": [
        "General News",
        "Tech News",
        "Business News",
        "Entertainment News",
        "Local/Regional News",
    ],
    "Reference & Documentation": [
        "Encyclopedia",
        "Technical Documentation",
        "API Documentation",
        "Tutorial/Guide",
        "Dictionary/Glossary",
    ],
    "Social & Community": [
        "Social Network",
        "Forum/Discussion",
        "Q&A Platform",  # Question-and-answer focused sites (StackOverflow, Quora)
        "Blog Platform",  # Structured publishing platforms (Medium, WordPress.com)
        "Personal Blog",  # Individual author blogs
    ],
    "Business & Commerce": [
        "E-commerce",
        "Corporate Website",
        "B2B Platform",
        "Financial Service",
        "Marketing/Advertising",
    ],
    "Technology": [
        "Software Development",
        "Cloud Service",
        "Open Source Project",
        "Tech Company",
        "Developer Tools",
    ],
    "Government & Organization": [
        "Government Agency",
        "Non-profit",
        "International Organization",
        "Think Tank",
        "Industry Association",
    ],
    "Entertainment & Lifestyle": [
        "Streaming Service",
        "Gaming",
        "Sports",
        "Arts & Culture",
        "Travel & Tourism",
    ],
    "Professional & Industry": [
        "Healthcare",
        "Legal",
        "Real Estate",
        "Manufacturing",
        "Energy & Utilities",
    ],
    "Other": ["Personal Website", "Miscellaneous", "Unknown"],
}


class DomainClassifier:
    """Classify domains using LLM with predefined categories."""

    def __init__(self, username: str, settings_snapshot: dict = None):
        """Initialize the domain classifier.

        Args:
            username: Username for database session
            settings_snapshot: Settings snapshot for LLM configuration
        """
        self.username = username
        self.settings_snapshot = settings_snapshot
        self.llm = None

    def _get_llm(self):
        """Get or initialize LLM instance."""
        if self.llm is None:
            # Thread the user so the LLM PEP enforces local-only policy: the
            # route builds settings_snapshot with a bare SettingsManager (no
            # `_username`), and classification prompts carry the user's own
            # resource titles, which must not fail open to a cloud LLM when
            # the user's default engine is a private retriever.
            self.llm = get_llm(
                settings_snapshot=self.settings_snapshot,
                username=self.username,
            )
        return self.llm

    def close(self) -> None:
        """Close the LLM client if one was created."""
        from ..utilities.resource_utils import safe_close

        safe_close(self.llm, "classifier LLM")

    def _get_domain_samples(
        self, domain: str, session: Session, limit: int = 5
    ) -> List[Dict]:
        """Get sample resources from a domain.

        Args:
            domain: Domain to get samples for
            session: Database session
            limit: Maximum number of samples

        Returns:
            List of resource samples
        """
        resources = (
            session.query(ResearchResource)
            .filter(
                ResearchResource.url.like(
                    f"%{_escape_like(domain)}%", escape="\\"
                )
            )
            .limit(limit)
            .all()
        )

        samples = []
        for resource in resources:
            samples.append(
                {
                    "title": resource.title or "Untitled",
                    "url": resource.url,
                    "preview": resource.content_preview[:200]
                    if resource.content_preview
                    else None,
                }
            )

        return samples

    def _build_classification_prompt(
        self, domain: str, samples: List[Dict]
    ) -> str:
        """Build prompt for LLM classification.

        This method uses actual content samples (titles, previews) from the domain
        rather than relying solely on domain name patterns, providing more
        accurate classification based on actual site content.

        Args:
            domain: Domain to classify
            samples: Sample resources from the domain (titles, URLs, content previews)

        Returns:
            Formatted prompt string
        """
        # Format categories for prompt
        categories_text = []
        for main_cat, subcats in DOMAIN_CATEGORIES.items():
            subcats_text = ", ".join(subcats)
            categories_text.append(f"{main_cat}: {subcats_text}")

        # Format samples
        samples_text = []
        for i, sample in enumerate(samples[:5], 1):
            samples_text.append(f"{i}. Title: {sample['title']}")
            if sample.get("preview"):
                samples_text.append(f"   Preview: {sample['preview'][:100]}...")

        return f"""Classify the following domain into one of the predefined categories.

Domain: {domain}

Sample content from this domain:
{chr(10).join(samples_text) if samples_text else "No samples available"}

Available Categories:
{chr(10).join(categories_text)}

Respond with a JSON object containing:
- "category": The main category (e.g., "News & Media")
- "subcategory": The specific subcategory (e.g., "Tech News")
- "confidence": A confidence score between 0 and 1
- "reasoning": A brief explanation (max 100 words) of why this classification was chosen

Focus on accuracy. If uncertain, use "Other" category with "Unknown" subcategory.

JSON Response:"""

    def classify_domain(
        self, domain: str, force_update: bool = False
    ) -> Optional[DomainClassification]:
        """Classify a single domain using LLM.

        Args:
            domain: Domain to classify
            force_update: If True, reclassify even if already exists

        Returns:
            DomainClassification object or None if failed
        """
        try:
            with get_user_db_session(self.username) as session:
                # Check if already classified
                existing = (
                    session.query(DomainClassification)
                    .filter_by(domain=domain)
                    .first()
                )

                if existing and not force_update:
                    logger.info(
                        f"Domain {domain} already classified as {existing.category}"
                    )
                    return existing

                # Get sample resources
                samples = self._get_domain_samples(domain, session)

                # Build prompt and get classification
                prompt = self._build_classification_prompt(domain, samples)
                llm = self._get_llm()

                response = invoke_llm_sync(llm, prompt)
                response_text = get_llm_response_text(response)

                result = extract_json(response_text, expected_type=dict)
                if result is None:
                    raise ValueError("Could not parse JSON from LLM response")  # noqa: TRY301 — inside db session; except logs and returns None

                # Create or update classification
                if existing:
                    existing.category = result.get("category", "Other")
                    existing.subcategory = result.get("subcategory", "Unknown")
                    existing.confidence = float(result.get("confidence", 0.5))
                    existing.reasoning = result.get("reasoning", "")
                    existing.sample_titles = json.dumps(
                        [s["title"] for s in samples]
                    )
                    existing.sample_count = len(samples)
                    classification = existing
                else:
                    classification = DomainClassification(
                        domain=domain,
                        category=result.get("category", "Other"),
                        subcategory=result.get("subcategory", "Unknown"),
                        confidence=float(result.get("confidence", 0.5)),
                        reasoning=result.get("reasoning", ""),
                        sample_titles=json.dumps([s["title"] for s in samples]),
                        sample_count=len(samples),
                    )
                    session.add(classification)

                session.commit()
                logger.info(
                    f"Classified {domain} as {classification.category}/{classification.subcategory} with confidence {classification.confidence}"
                )
                return classification

        except Exception as e:
            # Type name only: this frame holds the sample resource URLs,
            # which a traceback's locals or the exception text can echo.
            logger.warning(
                f"Error classifying domain {domain} ({type(e).__name__})"
            )
            return None

    def classify_all_domains(
        self, force_update: bool = False, progress_callback=None
    ) -> Dict:
        """Classify all unique domains in the database.

        Args:
            force_update: If True, reclassify all domains
            progress_callback: Optional callback function for progress updates

        Returns:
            Dictionary with classification results
        """
        results = {
            "total": 0,
            "classified": 0,
            "failed": 0,
            "skipped": 0,
            "domains": [],
        }

        try:
            with get_user_db_session(self.username) as session:
                # Get all unique domains
                resources = session.query(ResearchResource.url).distinct().all()
                domains = set()

                for (url,) in resources:
                    if url:
                        try:
                            # Authority minus userinfo (empty when that
                            # is ambiguous): a stored URL's user:pass@
                            # would otherwise land in the logs, the
                            # progress payload and the persisted
                            # DomainClassification key.
                            domain = url_authority_without_userinfo(url).lower()
                            if domain.startswith("www."):
                                domain = domain[4:]
                            if domain:
                                domains.add(domain)
                        except (ValueError, AttributeError):
                            logger.debug("Skipping malformed URL")
                            continue

                results["total"] = len(domains)
                logger.info(
                    f"Found {results['total']} unique domains to process"
                )

                # Classify each domain ONE BY ONE
                for i, domain in enumerate(sorted(domains), 1):
                    logger.info(
                        f"Processing domain {i}/{results['total']}: {domain}"
                    )

                    if progress_callback:
                        progress_callback(
                            {
                                "current": i,
                                "total": results["total"],
                                "domain": domain,
                                "percentage": (i / results["total"]) * 100,
                            }
                        )

                    try:
                        # Check if already classified
                        if not force_update:
                            existing = (
                                session.query(DomainClassification)
                                .filter_by(domain=domain)
                                .first()
                            )
                            if existing:
                                results["skipped"] += 1
                                results["domains"].append(
                                    {
                                        "domain": domain,
                                        "status": "skipped",
                                        "category": existing.category,
                                        "subcategory": existing.subcategory,
                                    }
                                )
                                logger.info(
                                    f"Domain {domain} already classified, skipping"
                                )
                                continue

                        # Classify this single domain
                        classification = self.classify_domain(
                            domain, force_update
                        )

                        if classification:
                            results["classified"] += 1
                            results["domains"].append(
                                {
                                    "domain": domain,
                                    "status": "classified",
                                    "category": classification.category,
                                    "subcategory": classification.subcategory,
                                    "confidence": classification.confidence,
                                }
                            )
                            logger.info(
                                f"Successfully classified {domain} as {classification.category}"
                            )
                        else:
                            results["failed"] += 1
                            results["domains"].append(
                                {"domain": domain, "status": "failed"}
                            )
                            logger.warning(
                                f"Failed to classify domain {domain}"
                            )

                    except Exception as e:
                        # Type name only: this frame holds the raw
                        # resource URLs, which a traceback can echo.
                        logger.warning(
                            f"Error classifying domain {domain} "
                            f"({type(e).__name__})"
                        )
                        results["failed"] += 1
                        results["domains"].append(
                            {
                                "domain": domain,
                                "status": "failed",
                                "error": "Classification failed",
                            }
                        )

                logger.info(
                    f"Classification complete: {results['classified']} classified, {results['skipped']} skipped, {results['failed']} failed"
                )
                return results

        except Exception as e:
            # Type name only: this frame holds the raw resource URLs.
            logger.warning(
                f"Error in classify_all_domains ({type(e).__name__})"
            )
            results["error"] = "Classification failed"
            return results

    def get_classification(self, domain: str) -> Optional[DomainClassification]:
        """Get existing classification for a domain.

        Args:
            domain: Domain to look up

        Returns:
            DomainClassification object or None if not found
        """
        try:
            with get_user_db_session(self.username) as session:
                return (
                    session.query(DomainClassification)
                    .filter_by(domain=domain)
                    .first()
                )
        except Exception:
            logger.exception(f"Error getting classification for {domain}")
            return None

    def get_all_classifications(self) -> List[DomainClassification]:
        """Get all domain classifications.

        Returns:
            List of all DomainClassification objects
        """
        try:
            with get_user_db_session(self.username) as session:
                return (
                    session.query(DomainClassification)
                    .order_by(
                        DomainClassification.category,
                        DomainClassification.domain,
                    )
                    .all()
                )
        except Exception:
            logger.exception("Error getting all classifications")
            return []

    def get_categories_summary(self) -> Dict:
        """Get summary of domain classifications by category.

        Returns:
            Dictionary with category counts and domains
        """
        try:
            with get_user_db_session(self.username) as session:
                classifications = session.query(DomainClassification).all()

                summary = {}
                for classification in classifications:
                    cat = classification.category
                    if cat not in summary:
                        summary[cat] = {
                            "count": 0,
                            "domains": [],
                            "subcategories": {},
                        }

                    summary[cat]["count"] += 1
                    summary[cat]["domains"].append(
                        {
                            "domain": classification.domain,
                            "subcategory": classification.subcategory,
                            "confidence": classification.confidence,
                        }
                    )

                    subcat = classification.subcategory
                    if subcat:
                        if subcat not in summary[cat]["subcategories"]:
                            summary[cat]["subcategories"][subcat] = 0
                        summary[cat]["subcategories"][subcat] += 1

                return summary

        except Exception:
            logger.exception("Error getting categories summary")
            return {}