Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

20.9 KB
Raw Blame
"""Offline feedback observability aggregation over persisted sessions."""

from __future__ import annotations

import json
from datetime import datetime
from pathlib import Path
from typing import Any


FEEDBACK_STATS_SORT_FIELDS = (
    "responses_total",
    "tracked_responses_total",
    "responses_with_feedback",
    "feedback_total",
    "thumb_up_total",
    "thumb_down_total",
    "outcomes_total",
    "positive_feedback_total",
    "negative_feedback_total",
    "reasked_total",
    "resolved_total",
    "follow_up_without_feedback_total",
    "feedback_coverage",
    "thumbs_up_rate",
    "thumbs_down_rate",
    "positive_feedback_rate",
    "negative_feedback_rate",
    "reask_rate",
    "one_turn_resolution_rate",
)

_SUMMARY_METRICS = (
    ("Sessions Scanned", "sessions_scanned", "count"),
    ("Responses", "responses_total", "count"),
    ("Tracked Responses", "tracked_responses_total", "count"),
    ("Responses With Feedback", "responses_with_feedback", "count"),
    ("Feedback Events", "feedback_total", "count"),
    ("Thumbs Up", "thumb_up_total", "count"),
    ("Thumbs Down", "thumb_down_total", "count"),
    ("Outcome Records", "outcomes_total", "count"),
    ("Positive Feedback Outcomes", "positive_feedback_total", "count"),
    ("Negative Feedback Outcomes", "negative_feedback_total", "count"),
    ("Reasked Outcomes", "reasked_total", "count"),
    ("Resolved Outcomes", "resolved_total", "count"),
    ("Follow-up Without Feedback", "follow_up_without_feedback_total", "count"),
    ("Feedback Coverage", "feedback_coverage", "percent"),
    ("Thumbs Up Rate", "thumbs_up_rate", "percent"),
    ("Thumbs Down Rate", "thumbs_down_rate", "percent"),
    ("Positive Feedback Rate", "positive_feedback_rate", "percent"),
    ("Negative Feedback Rate", "negative_feedback_rate", "percent"),
    ("Reask Rate", "reask_rate", "percent"),
    ("One-turn Resolution Rate", "one_turn_resolution_rate", "percent"),
)


def compute_feedback_stats(
    bot_data_path: Path,
    *,
    channel: str | None = None,
    session_key: str | None = None,
    updated_since: str | None = None,
    updated_until: str | None = None,
    include_sessions: bool = False,
) -> dict[str, Any]:
    """Aggregate minimal feedback observability metrics from session JSONL files."""
    sessions_dir = bot_data_path / "sessions"
    filters = {
        "channel": channel,
        "session_key": session_key,
        "updated_since": _parse_datetime(updated_since),
        "updated_until": _parse_datetime(updated_until),
    }
    totals: dict[str, Any] = {
        "sessions_scanned": 0,
        "responses_total": 0,
        "tracked_responses_total": 0,
        "responses_with_feedback": 0,
        "feedback_total": 0,
        "thumb_up_total": 0,
        "thumb_down_total": 0,
        "outcomes_total": 0,
        "positive_feedback_total": 0,
        "negative_feedback_total": 0,
        "reasked_total": 0,
        "resolved_total": 0,
        "follow_up_without_feedback_total": 0,
        "channels": {},
        "sessions": [] if include_sessions else None,
    }

    if not sessions_dir.exists():
        return _finalize_feedback_stats(totals)

    for session_path in sorted(sessions_dir.glob("*.jsonl")):
        session_metrics = _read_session_metrics(session_path, filters)
        if session_metrics is None:
            continue
        totals["sessions_scanned"] += 1
        _merge_feedback_stats(totals, session_metrics)

    return _finalize_feedback_stats(totals)


def format_feedback_stats_table(stats: dict[str, Any]) -> dict[str, Any]:
    """Return table-friendly rows for CLI rendering."""
    summary = stats.get("summary", {})
    summary_rows = [
        (label, _format_metric_value(summary.get(key, 0), value_type))
        for label, key, value_type in _SUMMARY_METRICS
    ]

    channel_rows = []
    for channel_key, channel_stats in stats.get("channels", {}).items():
        channel_rows.append(
            (
                channel_key,
                _format_metric_value(channel_stats.get("responses_total", 0), "count"),
                _format_metric_value(channel_stats.get("feedback_total", 0), "count"),
                _format_metric_value(channel_stats.get("feedback_coverage", 0.0), "percent"),
                _format_metric_value(channel_stats.get("thumbs_up_rate", 0.0), "percent"),
                _format_metric_value(channel_stats.get("thumbs_down_rate", 0.0), "percent"),
                _format_metric_value(channel_stats.get("one_turn_resolution_rate", 0.0), "percent"),
            )
        )

    session_rows = []
    for session_stats in stats.get("sessions", []):
        session_rows.append(
            (
                session_stats.get("session_key", ""),
                session_stats.get("channel", "unknown"),
                str(session_stats.get("updated_at") or ""),
                _format_metric_value(session_stats.get("responses_total", 0), "count"),
                _format_metric_value(session_stats.get("feedback_total", 0), "count"),
                _format_metric_value(session_stats.get("negative_feedback_total", 0), "count"),
                _format_metric_value(session_stats.get("reasked_total", 0), "count"),
                _format_metric_value(session_stats.get("one_turn_resolution_rate", 0.0), "percent"),
            )
        )

    return {
        "summary_rows": summary_rows,
        "channel_rows": channel_rows,
        "session_rows": session_rows,
    }


def format_feedback_stats_markdown(stats: dict[str, Any]) -> dict[str, str]:
    """Return markdown-friendly sections for console rendering."""
    table_data = format_feedback_stats_table(stats)

    summary_lines = [f"- **{label}:** {value}" for label, value in table_data["summary_rows"]]
    summary_markdown = "\n".join(summary_lines) if summary_lines else "No feedback stats available."

    channels_markdown = _format_markdown_table(
        ["Channel", "Responses", "Feedback", "Coverage", "Thumbs Up", "Thumbs Down", "Resolution"],
        table_data["channel_rows"],
        empty_message="No channel stats available.",
    )
    sessions_markdown = _format_markdown_table(
        [
            "Session",
            "Channel",
            "Updated At",
            "Responses",
            "Feedback",
            "Negative",
            "Reasked",
            "Resolution",
        ],
        table_data["session_rows"],
        empty_message="No session stats available.",
    )

    return {
        "summary_markdown": summary_markdown,
        "channels_markdown": channels_markdown,
        "sessions_markdown": sessions_markdown,
    }


def build_feedback_stats_display(
    bot_data_path: Path,
    *,
    channel: str | None = None,
    session_key: str | None = None,
    updated_since: str | None = None,
    updated_until: str | None = None,
    sort_by: str = "responses_total",
    top_n: int | None = None,
    include_sessions: bool = True,
    session_limit: int | None = None,
) -> dict[str, str]:
    """Compute, select, and format feedback stats for display surfaces."""
    stats = compute_feedback_stats(
        bot_data_path,
        channel=channel,
        session_key=session_key,
        updated_since=updated_since,
        updated_until=updated_until,
        include_sessions=include_sessions,
    )
    stats = select_feedback_stats(
        stats,
        sort_by=sort_by,
        top_n=top_n,
        session_limit=session_limit if include_sessions else None,
    )
    return format_feedback_stats_markdown(stats)


def select_feedback_stats(
    stats: dict[str, Any],
    *,
    sort_by: str = "responses_total",
    top_n: int | None = None,
    session_limit: int | None = None,
) -> dict[str, Any]:
    """Sort and optionally trim channel and session breakdowns for display."""
    validate_feedback_stats_sort_by(sort_by)
    channels = stats.get("channels", {})
    sessions = stats.get("sessions", [])

    sorted_channels = sorted(
        channels.items(),
        key=lambda item: _sort_value(item[1], sort_by),
        reverse=True,
    )
    if top_n is not None:
        sorted_channels = sorted_channels[:top_n]

    sorted_sessions = sorted(
        sessions,
        key=lambda item: _sort_value(item, "updated_at"),
        reverse=True,
    )
    if session_limit is not None:
        sorted_sessions = sorted_sessions[:session_limit]

    return {
        "summary": stats.get("summary", {}),
        "channels": dict(sorted_channels),
        "sessions": sorted_sessions,
    }


def validate_feedback_stats_sort_by(sort_by: str) -> str:
    """Validate the requested sort field for channel-level stats."""
    if sort_by not in FEEDBACK_STATS_SORT_FIELDS:
        allowed = ", ".join(FEEDBACK_STATS_SORT_FIELDS)
        raise ValueError(f"sort_by must be one of: {allowed}")
    return sort_by


def _read_session_metrics(session_path: Path, filters: dict[str, Any]) -> dict[str, Any] | None:
    try:
        with open(session_path, encoding="utf-8") as f:
            first_line = f.readline().strip()
            remaining_lines = f.readlines()
    except OSError:
        return None

    if not first_line:
        return None

    try:
        first_record = json.loads(first_line)
    except json.JSONDecodeError:
        return None

    if first_record.get("_type") != "metadata":
        return None

    session_key = first_record.get("session_key", "")
    metadata = first_record.get("metadata", {})
    channel = _channel_from_session_key(session_key)
    updated_at = _parse_datetime(first_record.get("updated_at"))

    if filters["channel"] and channel != filters["channel"]:
        return None
    if filters["session_key"] and session_key != filters["session_key"]:
        return None
    if filters["updated_since"] and (updated_at is None or updated_at < filters["updated_since"]):
        return None
    if filters["updated_until"] and (updated_at is None or updated_at > filters["updated_until"]):
        return None

    feedback_events = metadata.get("feedback_events", [])
    response_outcomes = metadata.get("response_outcomes", {})
    response_ids = _collect_response_ids(remaining_lines)

    feedback_by_response = {
        event.get("response_id")
        for event in feedback_events
        if isinstance(event, dict) and event.get("response_id")
    }
    tracked_response_ids = feedback_by_response | {
        response_id
        for response_id, payload in response_outcomes.items()
        if isinstance(response_id, str) and response_id and isinstance(payload, dict)
    }

    metrics: dict[str, Any] = {
        "session_key": session_key,
        "responses_total": len(response_ids),
        "tracked_responses_total": len(tracked_response_ids),
        "responses_with_feedback": len(feedback_by_response),
        "feedback_total": 0,
        "thumb_up_total": 0,
        "thumb_down_total": 0,
        "outcomes_total": 0,
        "positive_feedback_total": 0,
        "negative_feedback_total": 0,
        "reasked_total": 0,
        "resolved_total": 0,
        "follow_up_without_feedback_total": 0,
        "channel": channel,
        "updated_at": first_record.get("updated_at"),
    }

    for event in feedback_events:
        if not isinstance(event, dict):
            continue
        metrics["feedback_total"] += 1
        feedback_type = event.get("feedback_type")
        if feedback_type == "thumb_up":
            metrics["thumb_up_total"] += 1
        elif feedback_type == "thumb_down":
            metrics["thumb_down_total"] += 1

    for payload in response_outcomes.values():
        if not isinstance(payload, dict):
            continue
        metrics["outcomes_total"] += 1
        outcome_label = payload.get("outcome_label")
        if outcome_label == "positive_feedback":
            metrics["positive_feedback_total"] += 1
        elif outcome_label == "negative_feedback":
            metrics["negative_feedback_total"] += 1
        elif outcome_label == "reasked":
            metrics["reasked_total"] += 1
        elif outcome_label == "resolved":
            metrics["resolved_total"] += 1
        elif outcome_label == "follow_up_without_feedback":
            metrics["follow_up_without_feedback_total"] += 1

    return _finalize_session_metrics(metrics)


def _finalize_session_metrics(metrics: dict[str, Any]) -> dict[str, Any]:
    tracked_responses_total = metrics["tracked_responses_total"]
    metrics["feedback_coverage"] = _safe_ratio(
        metrics["responses_with_feedback"], tracked_responses_total
    )
    metrics["thumbs_up_rate"] = _safe_ratio(metrics["thumb_up_total"], metrics["feedback_total"])
    metrics["thumbs_down_rate"] = _safe_ratio(
        metrics["thumb_down_total"], metrics["feedback_total"]
    )
    metrics["positive_feedback_rate"] = _safe_ratio(
        metrics["positive_feedback_total"], tracked_responses_total
    )
    metrics["negative_feedback_rate"] = _safe_ratio(
        metrics["negative_feedback_total"], tracked_responses_total
    )
    metrics["reask_rate"] = _safe_ratio(metrics["reasked_total"], tracked_responses_total)
    metrics["one_turn_resolution_rate"] = _safe_ratio(
        metrics["resolved_total"] + metrics["positive_feedback_total"],
        tracked_responses_total,
    )

    return metrics


def _collect_response_ids(lines: list[str]) -> set[str]:
    response_ids: set[str] = set()
    for line in lines:
        line = line.strip()
        if not line:
            continue
        try:
            record = json.loads(line)
        except json.JSONDecodeError:
            continue
        if record.get("role") != "assistant":
            continue
        response_id = record.get("response_id")
        if isinstance(response_id, str) and response_id:
            response_ids.add(response_id)
    return response_ids


def _merge_feedback_stats(totals: dict[str, Any], session_metrics: dict[str, Any]) -> None:
    channel = session_metrics["channel"]
    channel_totals = totals["channels"].setdefault(
        channel,
        {
            "responses_total": 0,
            "tracked_responses_total": 0,
            "responses_with_feedback": 0,
            "feedback_total": 0,
            "thumb_up_total": 0,
            "thumb_down_total": 0,
            "outcomes_total": 0,
            "positive_feedback_total": 0,
            "negative_feedback_total": 0,
            "reasked_total": 0,
            "resolved_total": 0,
            "follow_up_without_feedback_total": 0,
        },
    )

    for key, value in session_metrics.items():
        if key in {
            "channel",
            "session_key",
            "updated_at",
            "feedback_coverage",
            "thumbs_up_rate",
            "thumbs_down_rate",
            "positive_feedback_rate",
            "negative_feedback_rate",
            "reask_rate",
            "one_turn_resolution_rate",
        }:
            continue
        totals[key] += value
        channel_totals[key] += value

    if totals["sessions"] is None:
        return

    totals["sessions"].append(
        {
            "session_key": session_metrics["session_key"],
            "channel": channel,
            "updated_at": session_metrics["updated_at"],
            "responses_total": session_metrics["responses_total"],
            "tracked_responses_total": session_metrics["tracked_responses_total"],
            "responses_with_feedback": session_metrics["responses_with_feedback"],
            "feedback_total": session_metrics["feedback_total"],
            "thumb_up_total": session_metrics["thumb_up_total"],
            "thumb_down_total": session_metrics["thumb_down_total"],
            "outcomes_total": session_metrics["outcomes_total"],
            "positive_feedback_total": session_metrics["positive_feedback_total"],
            "negative_feedback_total": session_metrics["negative_feedback_total"],
            "reasked_total": session_metrics["reasked_total"],
            "resolved_total": session_metrics["resolved_total"],
            "follow_up_without_feedback_total": session_metrics["follow_up_without_feedback_total"],
            "feedback_coverage": session_metrics["feedback_coverage"],
            "thumbs_up_rate": session_metrics["thumbs_up_rate"],
            "thumbs_down_rate": session_metrics["thumbs_down_rate"],
            "positive_feedback_rate": session_metrics["positive_feedback_rate"],
            "negative_feedback_rate": session_metrics["negative_feedback_rate"],
            "reask_rate": session_metrics["reask_rate"],
            "one_turn_resolution_rate": session_metrics["one_turn_resolution_rate"],
        }
    )


def _finalize_feedback_stats(totals: dict[str, Any]) -> dict[str, Any]:
    summary = {
        "sessions_scanned": totals["sessions_scanned"],
        "responses_total": totals["responses_total"],
        "tracked_responses_total": totals["tracked_responses_total"],
        "responses_with_feedback": totals["responses_with_feedback"],
        "feedback_total": totals["feedback_total"],
        "thumb_up_total": totals["thumb_up_total"],
        "thumb_down_total": totals["thumb_down_total"],
        "outcomes_total": totals["outcomes_total"],
        "positive_feedback_total": totals["positive_feedback_total"],
        "negative_feedback_total": totals["negative_feedback_total"],
        "reasked_total": totals["reasked_total"],
        "resolved_total": totals["resolved_total"],
        "follow_up_without_feedback_total": totals["follow_up_without_feedback_total"],
        "feedback_coverage": _safe_ratio(
            totals["responses_with_feedback"], totals["tracked_responses_total"]
        ),
        "thumbs_up_rate": _safe_ratio(totals["thumb_up_total"], totals["feedback_total"]),
        "thumbs_down_rate": _safe_ratio(totals["thumb_down_total"], totals["feedback_total"]),
        "positive_feedback_rate": _safe_ratio(
            totals["positive_feedback_total"], totals["tracked_responses_total"]
        ),
        "negative_feedback_rate": _safe_ratio(
            totals["negative_feedback_total"], totals["tracked_responses_total"]
        ),
        "reask_rate": _safe_ratio(totals["reasked_total"], totals["tracked_responses_total"]),
        "one_turn_resolution_rate": _safe_ratio(
            totals["resolved_total"] + totals["positive_feedback_total"],
            totals["tracked_responses_total"],
        ),
    }

    channels = {
        channel: {
            **channel_totals,
            "feedback_coverage": _safe_ratio(
                channel_totals["responses_with_feedback"], channel_totals["tracked_responses_total"]
            ),
            "thumbs_up_rate": _safe_ratio(
                channel_totals["thumb_up_total"], channel_totals["feedback_total"]
            ),
            "thumbs_down_rate": _safe_ratio(
                channel_totals["thumb_down_total"], channel_totals["feedback_total"]
            ),
            "positive_feedback_rate": _safe_ratio(
                channel_totals["positive_feedback_total"], channel_totals["tracked_responses_total"]
            ),
            "negative_feedback_rate": _safe_ratio(
                channel_totals["negative_feedback_total"], channel_totals["tracked_responses_total"]
            ),
            "reask_rate": _safe_ratio(
                channel_totals["reasked_total"], channel_totals["tracked_responses_total"]
            ),
            "one_turn_resolution_rate": _safe_ratio(
                channel_totals["resolved_total"] + channel_totals["positive_feedback_total"],
                channel_totals["tracked_responses_total"],
            ),
        }
        for channel, channel_totals in sorted(totals["channels"].items())
    }

    return {
        "summary": summary,
        "channels": channels,
        "sessions": sorted(totals["sessions"] or [], key=lambda item: item["session_key"]),
    }


def _channel_from_session_key(session_key: str) -> str:
    if not session_key or "__" not in session_key:
        return "unknown"
    parts = session_key.split("__")
    if len(parts) < 2:
        return "unknown"
    return f"{parts[0]}__{parts[1]}"


def _safe_ratio(numerator: int, denominator: int) -> float:
    if denominator <= 0:
        return 0.0
    return round(numerator / denominator, 4)


def _parse_datetime(value: str | None) -> datetime | None:
    if not value:
        return None
    normalized = value.replace("Z", "+00:00")
    return datetime.fromisoformat(normalized)


def _sort_value(payload: dict[str, Any], key: str) -> Any:
    value = payload.get(key)
    if key == "updated_at":
        parsed = _parse_datetime(value)
        return parsed or datetime.min
    if isinstance(value, (int, float)):
        return value
    return value or 0


def _format_metric_value(value: Any, value_type: str) -> str:
    if value_type == "percent":
        return f"{float(value) * 100:.1f}%"
    return str(value)


def _format_markdown_table(
    headers: list[str], rows: list[tuple[Any, ...]], *, empty_message: str
) -> str:
    if not rows:
        return empty_message

    header_row = "| " + " | ".join(headers) + " |"
    separator_row = "| " + " | ".join("---" for _ in headers) + " |"
    data_rows = ["| " + " | ".join(str(cell) for cell in row) + " |" for row in rows]
    return "\n".join([header_row, separator_row, *data_rows])