LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Cache layer for Dashboard Metrics.
Provides caching for computed metric aggregations to reduce database load.
Uses Redis through Django's cache framework.
Cache TTL Strategy:
- Current hour/day/month data: Configurable via DASHBOARD_CACHE_TTL_CURRENT_BUCKET (default 60s)
- Historical data: Configurable via DASHBOARD_CACHE_TTL_HISTORICAL (default 8 hours)
Bucket-based Caching:
- Time ranges are split into buckets (hourly/daily/monthly)
- Each bucket is cached independently with appropriate TTL
- MGET used to fetch multiple buckets in single Redis round-trip
- Partial cache hits allow reuse across overlapping queries
"""
import hashlib
import json
import logging
from collections.abc import Callable
from datetime import datetime, timedelta
from functools import wraps
from typing import Any
from django.conf import settings
from django.core.cache import cache
from django.utils import timezone
from rest_framework.response import Response
from utils.user_context import UserContext
logger = logging.getLogger(__name__)
# Try to get raw Redis client for MGET/pipeline operations
try:
from django_redis import get_redis_connection
REDIS_AVAILABLE = True
except ImportError:
REDIS_AVAILABLE = False
logger.warning("django_redis not available, bucket caching will use fallback")
# Cache TTL values from Django settings (configured in settings/base.py)
CACHE_TTL_CURRENT_BUCKET = settings.DASHBOARD_CACHE_TTL_CURRENT_BUCKET
CACHE_TTL_HISTORICAL = settings.DASHBOARD_CACHE_TTL_HISTORICAL
CACHE_TTL_OVERVIEW = settings.DASHBOARD_CACHE_TTL_OVERVIEW
CACHE_TTL_SUMMARY = settings.DASHBOARD_CACHE_TTL_SUMMARY
CACHE_TTL_SERIES = settings.DASHBOARD_CACHE_TTL_SERIES
CACHE_TTL_WORKFLOW_USAGE = settings.DASHBOARD_CACHE_TTL_WORKFLOW_USAGE
# Cache key prefix
CACHE_PREFIX = "dashboard_metrics"
def _build_cache_key(org_id: str, endpoint: str, params: dict[str, Any]) -> str:
"""Build a unique cache key from organization, endpoint, and parameters.
Args:
org_id: Organization identifier
endpoint: API endpoint name (e.g., 'summary', 'series')
params: Query parameters
Returns:
Unique cache key string
"""
# Sort params for consistent hashing
sorted_params = json.dumps(params, sort_keys=True, default=str)
params_hash = hashlib.sha256(sorted_params.encode()).hexdigest()[:12]
return f"{CACHE_PREFIX}:{org_id}:{endpoint}:{params_hash}"
def cache_metrics_response(
endpoint: str,
ttl: int | None = None,
) -> Callable:
"""Decorator to cache metrics API responses.
Args:
endpoint: Endpoint name for cache key
ttl: Cache TTL in seconds (defaults based on endpoint)
Returns:
Decorated function with caching
"""
# Default TTLs per endpoint
default_ttls = {
"overview": CACHE_TTL_OVERVIEW,
"summary": CACHE_TTL_SUMMARY,
"series": CACHE_TTL_SERIES,
"workflow_token_usage": CACHE_TTL_WORKFLOW_USAGE,
"recent_activity": CACHE_TTL_OVERVIEW,
}
def decorator(func: Callable) -> Callable:
@wraps(func)
def wrapper(self, request, *args, **kwargs):
# Get organization from context
org = UserContext.get_organization()
if not org:
# No caching without organization context
return func(self, request, *args, **kwargs)
org_id = str(org.id)
# Build cache key from query params (exclude refresh param)
params = {k: v for k, v in request.query_params.items() if k != "refresh"}
cache_key = _build_cache_key(org_id, endpoint, params)
# Skip cache if refresh=true requested
force_refresh = request.query_params.get("refresh", "").lower() == "true"
# Check cache (unless force refresh)
if not force_refresh:
cached = cache.get(cache_key)
if cached is not None:
logger.debug(f"Cache hit for {endpoint}: {cache_key}")
return Response(cached)
# Execute function and cache result
response = func(self, request, *args, **kwargs)
if response.status_code == 200:
cache_ttl = ttl or default_ttls.get(endpoint, CACHE_TTL_SUMMARY)
cache.set(cache_key, response.data, cache_ttl)
logger.debug(f"Cached {endpoint} for {cache_ttl}s: {cache_key}")
return response
return wrapper
return decorator
# =============================================================================
# Bucket-based Caching for Time-Range Queries
# =============================================================================
def _get_redis_client():
"""Get raw Redis client for MGET/pipeline operations.
Returns:
Redis client or None if not available
"""
if not REDIS_AVAILABLE:
return None
try:
return get_redis_connection("default")
except Exception:
logger.warning("Failed to get Redis connection")
return None
def generate_time_buckets(
start_date: datetime,
end_date: datetime,
granularity: str = "hourly",
) -> list[datetime]:
"""Generate time bucket timestamps for a date range.
Args:
start_date: Start of the range
end_date: End of the range
granularity: 'hourly', 'daily', or 'monthly'
Returns:
List of bucket start timestamps
"""
buckets = []
# Ensure timezone awareness
if start_date.tzinfo is None:
start_date = timezone.make_aware(start_date, timezone.utc)
if end_date.tzinfo is None:
end_date = timezone.make_aware(end_date, timezone.utc)
if granularity == "hourly":
# Truncate to hour
current = start_date.replace(minute=0, second=0, microsecond=0)
while current <= end_date:
buckets.append(current)
current = current + timedelta(hours=1)
elif granularity == "daily":
# Truncate to day
current = start_date.replace(hour=0, minute=0, second=0, microsecond=0)
while current <= end_date:
buckets.append(current)
current = current + timedelta(days=1)
elif granularity == "monthly":
# Truncate to first of month
current = start_date.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
while current <= end_date:
buckets.append(current)
# Move to next month
if current.month == 12:
current = current.replace(year=current.year + 1, month=1)
else:
current = current.replace(month=current.month + 1)
return buckets
def _build_bucket_key(
org_id: str,
granularity: str,
bucket_ts: datetime,
metric_name: str | None = None,
) -> str:
"""Build cache key for a specific time bucket.
Key format: metrics:bucket:<granularity>:<org_id>:<bucket_iso>[:metric_name]
Args:
org_id: Organization identifier
granularity: 'hourly', 'daily', or 'monthly'
bucket_ts: Bucket timestamp
metric_name: Optional metric name filter
Returns:
Cache key string
"""
bucket_str = (
bucket_ts.strftime("%Y-%m-%dT%H:00:00")
if granularity == "hourly"
else bucket_ts.strftime("%Y-%m-%d")
if granularity == "daily"
else bucket_ts.strftime("%Y-%m")
)
key = f"{CACHE_PREFIX}:bucket:{granularity}:{org_id}:{bucket_str}"
if metric_name:
key = f"{key}:{metric_name}"
return key
def _get_ttl_for_bucket(bucket_ts: datetime, granularity: str = "hourly") -> int:
"""Get appropriate TTL for a time bucket.
Current period uses short TTL (30s), historical uses long TTL (8h).
Args:
bucket_ts: Bucket timestamp
granularity: 'hourly', 'daily', or 'monthly'
Returns:
TTL in seconds
"""
now = timezone.now()
if granularity == "hourly":
current_bucket = now.replace(minute=0, second=0, microsecond=0)
elif granularity == "daily":
current_bucket = now.replace(hour=0, minute=0, second=0, microsecond=0)
else: # monthly
current_bucket = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
# Ensure bucket_ts is timezone aware for comparison
if bucket_ts.tzinfo is None:
bucket_ts = timezone.make_aware(bucket_ts, timezone.utc)
# Current bucket gets short TTL, historical gets long TTL
if bucket_ts >= current_bucket:
return CACHE_TTL_CURRENT_BUCKET
return CACHE_TTL_HISTORICAL
def mget_metrics_buckets(
org_id: str,
start_date: datetime,
end_date: datetime,
granularity: str = "hourly",
metric_name: str | None = None,
) -> tuple[dict[datetime, list[dict]], list[datetime]]:
"""Batch fetch cached metrics using Redis MGET.
Args:
org_id: Organization identifier
start_date: Start of date range
end_date: End of date range
granularity: 'hourly', 'daily', or 'monthly'
metric_name: Optional metric name filter
Returns:
Tuple of (cached_data dict, missing_buckets list)
- cached_data: {bucket_timestamp: [records]}
- missing_buckets: list of bucket timestamps not in cache
"""
buckets = generate_time_buckets(start_date, end_date, granularity)
if not buckets:
return {}, []
redis_client = _get_redis_client()
# Fallback to individual cache.get if Redis MGET not available
if not redis_client:
cached_data = {}
missing_buckets = []
for bucket_ts in buckets:
key = _build_bucket_key(org_id, granularity, bucket_ts, metric_name)
data = cache.get(key)
if data is not None:
cached_data[bucket_ts] = data
else:
missing_buckets.append(bucket_ts)
logger.debug(
f"Bucket cache (fallback): {len(cached_data)} hits, "
f"{len(missing_buckets)} misses for org {org_id}"
)
return cached_data, missing_buckets
# Use Redis MGET for batch retrieval
keys = [
_build_bucket_key(org_id, granularity, bucket_ts, metric_name)
for bucket_ts in buckets
]
try:
# MGET returns list of values (or None for missing keys)
values = redis_client.mget(keys)
cached_data = {}
missing_buckets = []
for bucket_ts, value in zip(buckets, values, strict=False):
if value is not None:
try:
cached_data[bucket_ts] = json.loads(value)
except (json.JSONDecodeError, TypeError):
missing_buckets.append(bucket_ts)
else:
missing_buckets.append(bucket_ts)
logger.debug(
f"Bucket cache (MGET): {len(cached_data)} hits, "
f"{len(missing_buckets)} misses for org {org_id}"
)
return cached_data, missing_buckets
except Exception:
logger.exception("MGET failed, falling back to no cache")
return {}, buckets
def mset_metrics_buckets(
org_id: str,
bucket_data: dict[datetime, list[dict]],
granularity: str = "hourly",
metric_name: str | None = None,
) -> int:
"""Batch save metrics buckets to cache using Redis pipeline.
Each bucket gets TTL based on whether it's current or historical.
Args:
org_id: Organization identifier
bucket_data: Dict of {bucket_timestamp: [records]}
granularity: 'hourly', 'daily', or 'monthly'
metric_name: Optional metric name filter
Returns:
Number of buckets saved
"""
if not bucket_data:
return 0
redis_client = _get_redis_client()
# Fallback to individual cache.set if Redis pipeline not available
if not redis_client:
saved = 0
for bucket_ts, data in bucket_data.items():
key = _build_bucket_key(org_id, granularity, bucket_ts, metric_name)
ttl = _get_ttl_for_bucket(bucket_ts, granularity)
try:
cache.set(key, data, ttl)
saved += 1
except Exception:
logger.warning(f"Failed to cache bucket {key}")
return saved
# Use Redis pipeline for batch save with different TTLs
try:
pipe = redis_client.pipeline()
for bucket_ts, data in bucket_data.items():
key = _build_bucket_key(org_id, granularity, bucket_ts, metric_name)
ttl = _get_ttl_for_bucket(bucket_ts, granularity)
serialized = json.dumps(data, default=str)
pipe.setex(key, ttl, serialized)
pipe.execute()
logger.debug(
f"Cached {len(bucket_data)} buckets for org {org_id} "
f"(granularity={granularity})"
)
return len(bucket_data)
except Exception:
logger.exception("Pipeline MSET failed")
return 0