Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

6.6 KB
Raw Blame
"""
Metrics and usage tracking models.
"""

from sqlalchemy import (
    JSON,
    Boolean,
    Column,
    Float,
    Index,
    Integer,
    String,
    Text,
)
from sqlalchemy_utc import UtcDateTime, utcnow

from .base import Base


class TokenUsage(Base):
    """
    Track token usage for LLM calls.
    """

    __tablename__ = "token_usage"

    id = Column(Integer, primary_key=True, autoincrement=True)
    research_id = Column(String(36), nullable=False, index=True)
    timestamp = Column(UtcDateTime, nullable=False, default=utcnow())

    # Model information
    model_provider = Column(String(100), nullable=False)
    model_name = Column(String(255), nullable=False)

    # Token counts
    prompt_tokens = Column(Integer, nullable=False)
    completion_tokens = Column(Integer, nullable=False)
    total_tokens = Column(Integer, nullable=False)

    # Cost tracking (in USD)
    prompt_cost = Column(Float, default=0.0)
    completion_cost = Column(Float, default=0.0)
    total_cost = Column(Float, default=0.0)

    # Context
    operation_type = Column(String(100))  # search, summarize, report, etc.
    operation_details = Column(JSON)
    research_mode = Column(String(50))  # standard, deep, expert, etc.

    # Enhanced metrics columns
    response_time_ms = Column(Integer)
    success_status = Column(String(50))
    error_type = Column(String(100))
    research_query = Column(Text)
    research_phase = Column(String(100))
    search_iteration = Column(Integer)
    search_engines_planned = Column(JSON)
    search_engine_selected = Column(String(100))
    calling_file = Column(String(255))
    calling_function = Column(String(255))
    call_stack = Column(JSON)

    # Context overflow detection columns
    context_limit = Column(Integer)  # The configured num_ctx or max tokens
    context_truncated = Column(
        Boolean, default=False
    )  # True if request was truncated due to context limit
    tokens_truncated = Column(Integer)  # Estimated tokens lost to truncation
    truncation_ratio = Column(Float)  # Percentage of prompt that was truncated

    # Raw Ollama response values for debugging
    ollama_prompt_eval_count = Column(
        Integer
    )  # Raw prompt_eval_count from Ollama
    ollama_eval_count = Column(Integer)  # Raw eval_count from Ollama
    ollama_total_duration = Column(
        Integer
    )  # Total time in nanoseconds (raw from Ollama API)
    ollama_load_duration = Column(
        Integer
    )  # Model load time in nanoseconds (raw from Ollama API)
    ollama_prompt_eval_duration = Column(
        Integer
    )  # Prompt eval time in nanoseconds (raw from Ollama API)
    ollama_eval_duration = Column(
        Integer
    )  # Generation time in nanoseconds (raw from Ollama API)

    # Compound indexes for query performance optimization
    __table_args__ = (
        Index("idx_token_research_timestamp", "research_id", "timestamp"),
        Index(
            "idx_token_model_timestamp",
            "model_provider",
            "model_name",
            "timestamp",
        ),
        Index("idx_token_operation_timestamp", "operation_type", "timestamp"),
        Index("idx_token_research_phase", "research_id", "research_phase"),
    )

    def __repr__(self):
        return f"<TokenUsage(model={self.model_name}, total_tokens={self.total_tokens}, cost=${self.total_cost:.4f})>"


class ModelUsage(Base):
    """
    Aggregate model usage statistics.
    """

    __tablename__ = "model_usage"

    id = Column(Integer, primary_key=True)
    model_provider = Column(String(100), nullable=False)
    model_name = Column(String(255), nullable=False)

    # Aggregate stats
    total_calls = Column(Integer, default=0)
    total_tokens = Column(Integer, default=0)
    total_cost = Column(Float, default=0.0)

    # Performance metrics
    avg_response_time_ms = Column(Float)
    error_count = Column(Integer, default=0)
    success_rate = Column(Float, default=100.0)

    # Time tracking
    first_used_at = Column(UtcDateTime, default=utcnow())
    last_used_at = Column(UtcDateTime, default=utcnow(), onupdate=utcnow())

    def __repr__(self):
        return f"<ModelUsage(model={self.model_name}, calls={self.total_calls}, cost=${self.total_cost:.2f})>"


class ResearchRating(Base):
    """
    User ratings for research results.
    """

    __tablename__ = "research_ratings"

    id = Column(Integer, primary_key=True)
    research_id = Column(String(36), nullable=False, unique=True, index=True)

    # Star rating (1-5)
    rating = Column(Integer, nullable=False)

    # Feedback categories
    accuracy = Column(Integer)  # 1-5
    completeness = Column(Integer)  # 1-5
    relevance = Column(Integer)  # 1-5
    readability = Column(Integer)  # 1-5

    # Written feedback
    feedback = Column(Text)

    # Timestamps
    created_at = Column(UtcDateTime, default=utcnow())
    updated_at = Column(UtcDateTime, default=utcnow(), onupdate=utcnow())

    def __repr__(self):
        return f"<ResearchRating(research_id={self.research_id}, rating={self.rating})>"


class SearchCall(Base):
    """
    Track individual search engine calls.
    """

    __tablename__ = "search_calls"

    id = Column(Integer, primary_key=True, autoincrement=True)
    research_id = Column(String(36), nullable=False, index=True)
    timestamp = Column(UtcDateTime, nullable=False, default=utcnow())

    # Search details
    search_engine = Column(String(100), nullable=False)
    query = Column(Text, nullable=False)
    num_results_requested = Column(Integer)
    num_results_returned = Column(Integer)

    # Performance
    response_time_ms = Column(Float)
    success = Column(Integer, default=1)  # 1 for success, 0 for failure
    error_message = Column(Text)

    # Rate limiting
    rate_limited = Column(Integer, default=0)  # 1 if rate limited
    wait_time_ms = Column(Float)

    # Research context
    research_mode = Column(String(50))  # standard, deep, expert, etc.
    research_query = Column(Text)
    research_phase = Column(String(100))
    search_iteration = Column(Integer)
    success_status = Column(String(50))
    error_type = Column(String(100))
    results_count = Column(Integer)

    # Compound indexes for query performance optimization
    __table_args__ = (
        Index("idx_search_research_timestamp", "research_id", "timestamp"),
        Index("idx_search_engine_timestamp", "search_engine", "timestamp"),
        Index("idx_search_success_timestamp", "success", "timestamp"),
        Index("idx_search_research_engine", "research_id", "search_engine"),
    )

    def __repr__(self):
        return f"<SearchCall(engine={self.search_engine}, query='{self.query[:50]}...', success={self.success})>"