Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

13.8 KB
Raw Blame
import logging
from datetime import date, datetime, time, timedelta
from typing import Any

from account_usage.models import PageUsage
from django.db.models import Count, QuerySet, Sum
from django.utils import timezone
from rest_framework.exceptions import APIException, ValidationError

from .constants import UsageKeys
from .models import Usage

# Try to import subscription plugin at module level
try:
    from pluggable_apps.subscription_v2.subscription_helper import (
        SubscriptionHelper,
    )

    SUBSCRIPTION_HELPER = SubscriptionHelper
except ImportError:
    SUBSCRIPTION_HELPER = None

logger = logging.getLogger(__name__)


class UsageHelper:
    @staticmethod
    def get_aggregated_pages_processed(
        run_id: str | None = None,
        run_ids: list[str] | None = None,
    ) -> int | None:
        """Retrieve aggregated pages processed for given run_id(s).

        Provide either a single run_id or a list of run_ids.

        Args:
            run_id: Single file execution ID.
            run_ids: List of file execution IDs.

        Returns:
            int | None: Total pages processed, or None if no records found.
        """
        if run_id:
            queryset = PageUsage.objects.filter(run_id=run_id)
        elif run_ids:
            queryset = PageUsage.objects.filter(run_id__in=run_ids)
        else:
            return None
        if not queryset.exists():
            return None
        result = queryset.aggregate(total_pages=Sum("pages_processed"))
        return result.get("total_pages")

    @staticmethod
    def get_aggregated_token_count(run_id: str) -> dict:
        """Retrieve aggregated token counts for the given run_id.

        Args:
            run_id (str): The identifier for the token usage.

        Returns:
            dict: A dictionary containing aggregated token counts
            for different token types.
                  Keys:
                    - 'embedding_tokens': Total embedding tokens.
                    - 'prompt_tokens': Total prompt tokens.
                    - 'completion_tokens': Total completion tokens.
                    - 'total_tokens': Total tokens.

        Raises:
            APIException: For unexpected errors during database operations.
        """
        try:
            # Aggregate the token counts for the given run_id
            usage_summary = Usage.objects.filter(run_id=run_id).aggregate(
                embedding_tokens=Sum(UsageKeys.EMBEDDING_TOKENS),
                prompt_tokens=Sum(UsageKeys.PROMPT_TOKENS),
                completion_tokens=Sum(UsageKeys.COMPLETION_TOKENS),
                total_tokens=Sum(UsageKeys.TOTAL_TOKENS),
                cost_in_dollars=Sum(UsageKeys.COST_IN_DOLLARS),
            )

            logger.debug(f"Token counts aggregated successfully for run_id: {run_id}")

            # Prepare the result dictionary with None as the default value
            result = {
                UsageKeys.EMBEDDING_TOKENS: usage_summary.get(UsageKeys.EMBEDDING_TOKENS),
                UsageKeys.PROMPT_TOKENS: usage_summary.get(UsageKeys.PROMPT_TOKENS),
                UsageKeys.COMPLETION_TOKENS: usage_summary.get(
                    UsageKeys.COMPLETION_TOKENS
                ),
                UsageKeys.TOTAL_TOKENS: usage_summary.get(UsageKeys.TOTAL_TOKENS),
                UsageKeys.COST_IN_DOLLARS: usage_summary.get(UsageKeys.COST_IN_DOLLARS),
            }
            return result
        except Usage.DoesNotExist:
            # Handle the case where no usage data is found for the given run_id
            logger.warning(f"Usage data not found for the specified run_id: {run_id}")
            return {}
        except Exception as e:
            # Handle any other exceptions that might occur during the execution
            logger.error(f"An unexpected error occurred for run_id {run_id}: {str(e)}")
            raise APIException("Error while aggregating token counts")

    @staticmethod
    def get_usage_by_model(run_id: str) -> dict[str, list[dict[str, Any]]]:
        """Get per-model usage breakdown matching prompt-service format.

        Groups usage data by (usage_type, llm_usage_reason, model_name) and
        returns cost arrays keyed as 'extraction_llm', 'challenge_llm',
        'embedding', etc. — matching the legacy prompt-service response.

        Args:
            run_id: The file_execution_id / run_id to query.

        Returns:
            Dict with keys like 'extraction_llm', 'embedding' mapping to
            lists of per-model cost entries. Empty dict on error.
        """
        try:
            rows = (
                Usage.objects.filter(run_id=run_id)
                .values("usage_type", "llm_usage_reason", "model_name")
                .annotate(
                    sum_input_tokens=Sum("prompt_tokens"),
                    sum_output_tokens=Sum("completion_tokens"),
                    sum_total_tokens=Sum("total_tokens"),
                    sum_embedding_tokens=Sum("embedding_tokens"),
                    sum_cost=Sum("cost_in_dollars"),
                )
            )
            result: dict[str, list[dict[str, Any]]] = {}
            for row in rows:
                usage_type = row["usage_type"]
                llm_reason = row["llm_usage_reason"]

                # Drop unlabeled LLM rows entirely.  Per the Usage model
                # contract (see usage_v2/models.py: llm_usage_reason
                # db_comment), an empty reason is only valid when
                # usage_type == "embedding".  An empty reason combined with
                # usage_type == "llm" is a producer-side bug (an LLM call
                # site forgot to pass llm_usage_reason in usage_kwargs).
                # Without this guard the row would surface in API
                # deployment responses as a malformed bare "llm" bucket
                # with no token breakdown.
                if usage_type == "llm" and not llm_reason:
                    logger.warning(
                        "Dropping unlabeled LLM usage row from per-model "
                        "breakdown: model_name=%s run_id=%s",
                        row["model_name"],
                        run_id,
                    )
                    continue

                cost_str = UsageHelper._format_float_positional(row["sum_cost"] or 0.0)

                key = usage_type
                item: dict[str, Any] = {
                    "model_name": row["model_name"],
                    "cost_in_dollars": cost_str,
                }
                if llm_reason:
                    key = f"{llm_reason}_{usage_type}"
                    item["input_tokens"] = row["sum_input_tokens"] or 0
                    item["output_tokens"] = row["sum_output_tokens"] or 0
                    item["total_tokens"] = row["sum_total_tokens"] or 0
                else:
                    item["embedding_tokens"] = row["sum_embedding_tokens"] or 0

                result.setdefault(key, []).append(item)
            return result
        except Exception as e:
            logger.error("Error querying per-model usage for run_id %s: %s", run_id, e)
            return {}

    @staticmethod
    def _format_float_positional(value: float, precision: int = 10) -> str:
        """Format float without scientific notation, stripping trailing zeros."""
        formatted: str = f"{value:.{precision}f}"
        return formatted.rstrip("0").rstrip(".") if "." in formatted else formatted

    @staticmethod
    def aggregate_usage_metrics(queryset: QuerySet) -> dict[str, Any]:
        """Aggregate usage metrics from a queryset of Usage objects.

        Args:
            queryset (QuerySet): A queryset of Usage objects.

        Returns:
            dict: A dictionary containing aggregated usage metrics.
        """
        return queryset.aggregate(
            total_prompt_tokens=Sum("prompt_tokens"),
            total_completion_tokens=Sum("completion_tokens"),
            total_tokens=Sum("total_tokens"),
            total_cost=Sum("cost_in_dollars"),
        )

    @staticmethod
    def format_usage_response(
        aggregated_data: dict[str, Any], start_date: datetime, end_date: datetime
    ) -> dict[str, Any]:
        """Format aggregated usage data into a structured response.

        Args:
            aggregated_data (dict): Aggregated usage metrics.
            start_date (datetime): Start date of the usage period.
            end_date (datetime): End date of the usage period.

        Returns:
            dict: Formatted response containing aggregated data and date range.
        """
        return {
            "aggregated_data": aggregated_data,
            "date_range": {"start_date": start_date, "end_date": end_date},
        }

    @staticmethod
    def _get_subscription_dates(
        organization_id: str,
    ) -> tuple[datetime | None, datetime | None]:
        """Attempt to get trial dates from subscription plugin.

        Assumes subscription plugin returns timezone-aware datetimes or dates
        that can be safely used without normalization (uniform storage).

        Args:
            organization_id: The organization identifier

        Returns:
            Tuple of (start_date, end_date) or (None, None) if not available
        """
        # Exit early if subscription plugin is not available
        if SUBSCRIPTION_HELPER is None:
            logger.debug("Subscription plugin not available, using fallback dates")
            return None, None

        # Plugin is available, attempt to get subscription data
        try:
            org_plans = SUBSCRIPTION_HELPER.get_subscription(organization_id)
        except AttributeError as e:
            logger.warning(f"Subscription plugin missing expected methods: {e}")
            return None, None

        if (
            not org_plans
            or not hasattr(org_plans, "start_date")
            or not hasattr(org_plans, "end_date")
        ):
            return None, None

        # If subscription returns dates, convert to datetime with proper times
        start_date = org_plans.start_date
        end_date = org_plans.end_date

        # Handle date objects by converting to start/end of day (assuming UTC)
        if isinstance(start_date, date) and not isinstance(start_date, datetime):
            start_date = datetime.combine(start_date, time.min).replace(
                tzinfo=timezone.utc
            )
        if isinstance(end_date, date) and not isinstance(end_date, datetime):
            end_date = datetime.combine(end_date, time.max).replace(tzinfo=timezone.utc)

        logger.info(f"Using subscription dates for org {organization_id}")
        return start_date, end_date

    @staticmethod
    def _calculate_usage_metrics(
        organization: Any, trial_start_date: datetime, trial_end_date: datetime
    ) -> tuple[dict, int]:
        """Calculate usage metrics for the trial period.

        Args:
            organization: The organization object
            trial_start_date: Start of trial period
            trial_end_date: End of trial period

        Returns:
            Tuple of (aggregated_data, documents_processed)
        """
        usage_queryset = Usage.objects.filter(
            organization=organization,
            created_at__gte=trial_start_date,
            created_at__lte=trial_end_date,
        )

        aggregated_data = usage_queryset.aggregate(
            total_cost=Sum("cost_in_dollars"),
            total_tokens=Sum("total_tokens"),
            unique_runs=Count("run_id", distinct=True),
            api_calls=Count("id"),
        )

        documents_processed = (
            usage_queryset.values("workflow_id", "execution_id").distinct().count()
        )

        return aggregated_data, documents_processed

    @staticmethod
    def get_trial_statistics(organization) -> dict[str, Any]:
        """Get comprehensive trial usage statistics for an organization.

        Args:
            organization: The organization object for which to retrieve trial statistics.
                         Must have 'organization_id' and 'created_at' attributes.

        Returns:
            dict: A dictionary containing comprehensive trial usage statistics with keys:
                - trial_start_date (str): ISO formatted trial start date
                - trial_end_date (str): ISO formatted trial end date
                - total_cost (float): Total cost in dollars during trial period
                - documents_processed (int): Number of unique document processing operations
                - api_calls (int): Total number of API calls made
                - etl_runs (int): Number of unique ETL pipeline runs
        """
        trial_start_date, trial_end_date = UsageHelper._get_subscription_dates(
            organization.organization_id
        )
        # Use fallback dates if needed (assuming uniform UTC storage)
        if not trial_start_date:
            trial_start_date = organization.created_at
        if not trial_end_date:
            # For end date, set to end of day (23:59:59.999999)
            end_of_trial_date = organization.created_at + timedelta(days=14)
            trial_end_date = end_of_trial_date.replace(
                hour=23, minute=59, second=59, microsecond=999999
            )
        # Validate trial window - guard against inverted date range
        if trial_end_date < trial_start_date:
            raise ValidationError("trial_end_date must be on or after trial_start_date")
        # Calculate usage metrics
        aggregated_data, documents_processed = UsageHelper._calculate_usage_metrics(
            organization, trial_start_date, trial_end_date
        )
        return {
            "trial_start_date": trial_start_date.isoformat(),
            "trial_end_date": trial_end_date.isoformat(),
            "total_cost": aggregated_data.get("total_cost", 0) or 0,
            "documents_processed": documents_processed,
            "api_calls": aggregated_data.get("api_calls", 0) or 0,
            "etl_runs": aggregated_data.get("unique_runs", 0) or 0,
        }