Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
24.9 KB
Raw Blame
"""Results files, readable summary and baseline diff for the retrieval evaluation harness.

A run produces one results dict (see README.md for the schema). This module
writes it as JSON, renders a Markdown summary, and compares two results dicts
question by question so a small change in the average cannot hide the
questions that broke.

Standard library only, so it can be unit-tested without the SDK or a stack.
"""

from __future__ import annotations

import json
from collections.abc import Mapping, Sequence
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit, urlunsplit

SCHEMA_VERSION = 1
EPSILON = 1e-9

# Run parameters that change scores on their own. A diff between two runs that
# disagree on any of them is not measuring the search change alone.
_COMPARABLE_RUN_FIELDS = (
    ("corpus_hash", "corpus"),
    ("queries_hash", "question file"),
    ("k_values", "k values"),
    ("fetch_limit", "fetch limit"),
    ("score_threshold", "score threshold"),
)


class ResultsError(ValueError):
    """A results file could not be read as harness output."""


def redact_url(url: str) -> str:
    """Drop credentials, query string and fragment so the URL is safe to store."""
    parts = urlsplit(url)
    host = parts.hostname or ""
    if ":" in host:
        # urlsplit strips the brackets from an IPv6 host; a URL needs them back.
        host = f"[{host}]"
    if parts.port:
        host = f"{host}:{parts.port}"
    return urlunsplit((parts.scheme, host, parts.path.rstrip("/"), "", ""))


def load_results(path: Path) -> dict[str, Any]:
    try:
        data = json.loads(path.read_text(encoding="utf-8"))
    except OSError as exc:
        raise ResultsError(f"cannot read results file {path}: {exc}") from exc
    except json.JSONDecodeError as exc:
        raise ResultsError(f"{path} is not valid JSON: {exc.msg}") from exc
    if not isinstance(data, dict) or not {"run", "metrics", "queries"} <= data.keys():
        raise ResultsError(f"{path} is not a retrieval evaluation results file")
    if data.get("schema_version") != SCHEMA_VERSION:
        raise ResultsError(
            f"{path} has schema version {data.get('schema_version')!r}; "
            f"this harness reads version {SCHEMA_VERSION}"
        )
    return data


def write_results(
    results: Mapping[str, Any],
    summary: str,
    output_dir: Path,
    started_at: datetime,
) -> list[Path]:
    """Write results and summary twice: as the latest run and as a timestamped copy."""
    stamp = started_at.astimezone(UTC).strftime("%Y%m%d_%H%M%S")
    output_dir.mkdir(parents=True, exist_ok=True)
    written: list[Path] = []
    for stem, suffix, content in (
        ("results", ".json", json.dumps(results, indent=2) + "\n"),
        ("summary", ".md", summary),
    ):
        for path in (output_dir / f"{stem}{suffix}", output_dir / f"{stem}_{stamp}{suffix}"):
            path.write_text(content, encoding="utf-8")
            written.append(path)
    return written


def compare_k(baseline_run: Mapping[str, Any], current_run: Mapping[str, Any]) -> int:
    """The cutoff to compare two runs on, which both runs must have evaluated.

    This is the current run's primary cutoff when the baseline has it too.
    Otherwise it is 5 if both runs have it, then the largest shared cutoff.
    Raises ``ResultsError`` when the runs share no cutoff, because recall and
    nDCG could then not be compared and a regression would go unnoticed.
    """
    base_k = set(baseline_run.get("k_values") or [baseline_run["primary_k"]])
    cur_k = set(current_run.get("k_values") or [current_run["primary_k"]])
    shared = base_k & cur_k
    if not shared:
        raise ResultsError(
            f"the runs share no k value (baseline {sorted(base_k)}, "
            f"current {sorted(cur_k)}), so recall@k and nDCG@k cannot be compared"
        )
    if current_run["primary_k"] in shared:
        return int(current_run["primary_k"])
    return 5 if 5 in shared else max(shared)


def _compare_metrics(k: int) -> list[str]:
    return [f"recall@{k}", "reciprocal_rank", f"ndcg@{k}"]


def _snippet_missing(query: Mapping[str, Any]) -> list[str]:
    """Expected files that were returned without the question's expected_text.

    Such a file keeps its rank but does not count as relevant, which would
    otherwise look like a score that dropped for no reason.
    """
    expected = set(query.get("expected_files", []))
    return [
        item["filename"]
        for item in query.get("ranked_files", [])
        if item["filename"] in expected and item.get("snippet_found") is False
    ]


def _mismatch_warnings(baseline: Mapping[str, Any], current: Mapping[str, Any]) -> list[str]:
    warnings: list[str] = []
    base_run, cur_run = baseline["run"], current["run"]
    for field, label in _COMPARABLE_RUN_FIELDS:
        base_value = base_run.get(field)
        cur_value = cur_run.get(field)
        if field == "queries_hash":
            base_value = base_run.get("retrieval_queries_hash", base_value)
            cur_value = cur_run.get("retrieval_queries_hash", cur_value)
        if base_value != cur_value:
            warnings.append(
                f"The {label} differs between the runs "
                f"(baseline {_short(base_run.get(field))}, current {_short(cur_run.get(field))})."
            )
    base_settings = baseline.get("settings") or {}
    cur_settings = current.get("settings") or {}
    if not base_settings or not cur_settings:
        if base_settings or cur_settings:
            warnings.append("Only one run recorded the stack settings, so they cannot be compared.")
    else:
        for key in sorted(base_settings.keys() | cur_settings.keys()):
            if base_settings.get(key) != cur_settings.get(key):
                warnings.append(
                    f"Setting '{key}' differs between the runs "
                    f"(baseline {base_settings.get(key)!r}, current {cur_settings.get(key)!r})."
                )
    return warnings


def _short(value: object) -> str:
    text = str(value)
    # Hashes are long; the first 12 characters are enough to tell two apart.
    return text[:12] if isinstance(value, str) and len(text) == 64 else text


def diff_results(
    baseline: Mapping[str, Any],
    current: Mapping[str, Any],
    epsilon: float = EPSILON,
) -> dict[str, Any]:
    """Compare two runs question by question.

    Questions are matched by id. Each is classified on recall@k at the cutoff
    from ``compare_k``, then reciprocal rank, then nDCG@k: the first of those
    that moved by more than ``epsilon`` decides whether the question improved
    or regressed.
    """
    k = compare_k(baseline["run"], current["run"])
    metrics = _compare_metrics(k)
    base_by_id = {q["id"]: q for q in baseline["queries"]}
    cur_by_id = {q["id"]: q for q in current["queries"]}

    regressed: list[dict[str, Any]] = []
    improved: list[dict[str, Any]] = []
    unchanged = 0
    for query_id, cur in cur_by_id.items():
        base = base_by_id.get(query_id)
        if base is None:
            continue
        deltas = {
            name: {
                "baseline": base["metrics"][name],
                "current": cur["metrics"][name],
                "delta": cur["metrics"][name] - base["metrics"][name],
            }
            for name in metrics
            if name in base["metrics"] and name in cur["metrics"]
        }
        direction = next(
            (d["delta"] for d in deltas.values() if abs(d["delta"]) > epsilon),
            0.0,
        )
        if direction == 0.0:
            unchanged += 1
            continue
        entry = {
            "id": query_id,
            "query": cur["query"],
            "metrics": deltas,
            "expected_ranks": {
                "baseline": base.get("expected_ranks", {}),
                "current": cur.get("expected_ranks", {}),
            },
            "snippet_missing": {
                "baseline": _snippet_missing(base),
                "current": _snippet_missing(cur),
            },
            "passage": {"baseline": _passage(base), "current": _passage(cur)},
        }
        (improved if direction > 0 else regressed).append(entry)

    added = [query_id for query_id in cur_by_id if query_id not in base_by_id]
    removed = [query_id for query_id in base_by_id if query_id not in cur_by_id]

    base_overall = baseline["metrics"].get("overall", {})
    cur_overall = current["metrics"].get("overall", {})
    aggregate = {
        name: {
            "baseline": base_overall[name],
            "current": cur_overall[name],
            "delta": cur_overall[name] - base_overall[name],
        }
        for name in cur_overall
        if name in base_overall
    }

    warnings = _mismatch_warnings(baseline, current)
    if k != current["run"]["primary_k"]:
        warnings.append(
            f"The baseline has no recall@{current['run']['primary_k']}, "
            f"so questions are compared on recall@{k}."
        )
    # Notes are informational. Unlike warnings, they do not make the file scores
    # incomparable, so the recall gate and baseline promotion ignore them.
    notes: list[str] = []
    if "passage_mrr" in cur_overall and "passage_mrr" not in base_overall:
        notes.append(
            "The baseline has no passage scores, so only file scores are compared. "
            "Rerun the baseline to compare passages."
        )
    answer_changes = []
    base_answers = any("answer" in q for q in baseline["queries"])
    cur_answers = any("answer" in q for q in current["queries"])
    if base_answers != cur_answers:
        notes.append("Only one run has answer scores; answer quality cannot be compared.")
    if base_answers and cur_answers:
        if baseline.get("answer_settings") != current.get("answer_settings"):
            notes.append(
                "Answer settings differ (agent, prompt, judge or retrieval limit); answer scores may not be comparable."
            )
        for query_id, cur in cur_by_id.items():
            base = base_by_id.get(query_id, {})
            before = (base.get("answer", {}).get("judgement") or {}).get("correctness")
            after = (cur.get("answer", {}).get("judgement") or {}).get("correctness")
            if before is not None and after is not None and abs(after - before) >= 1:
                answer_changes.append(
                    {
                        "id": query_id,
                        "query": cur["query"],
                        "baseline": before,
                        "current": after,
                        "delta": after - before,
                    }
                )
    if (base_answers or cur_answers) and baseline["run"].get("queries_hash") != current["run"].get(
        "queries_hash"
    ):
        notes.append("The full question file differs; answer references may have changed.")
    if added or removed:
        warnings.append(
            "The runs do not cover the same questions, so the aggregate deltas "
            "are not a like-for-like comparison."
        )

    return {
        "baseline_run_id": baseline["run"].get("run_id"),
        "current_run_id": current["run"].get("run_id"),
        "compare_metrics": metrics,
        "warnings": warnings,
        "notes": notes,
        "aggregate": aggregate,
        "answer_changes": answer_changes,
        "regressed": regressed,
        "improved": improved,
        "unchanged": unchanged,
        "added": added,
        "removed": removed,
    }


def recall_drop(diff: Mapping[str, Any]) -> float:
    """How far the average recall fell at the compared cutoff; 0 if it held or rose."""
    name = diff["compare_metrics"][0]
    return max(0.0, -diff["aggregate"][name]["delta"])


def recall_gate_failure(diff: Mapping[str, Any], max_drop: float) -> str | None:
    """Why the recall gate fails, or None when it passes.

    The gate fails when the runs are not like for like, because a score that
    held up between two different setups says nothing about the search.
    """
    if diff["warnings"]:
        return "the runs are not comparable, so recall cannot be checked: " + " ".join(
            diff["warnings"]
        )
    drop = recall_drop(diff)
    if drop > max_drop:
        return (
            f"average {diff['compare_metrics'][0]} fell by {drop:.3f}, "
            f"more than the allowed {max_drop:.3f}"
        )
    return None


def promotion_decision(
    baseline: Mapping[str, Any] | None,
    current: Mapping[str, Any],
    min_gain: float = 0.0,
) -> tuple[bool, str]:
    """Whether ``current`` should replace ``baseline`` as the reference.

    Yes when there is no baseline yet, or when average recall at the compared
    cutoff rose by more than ``min_gain``. A pair that is not like for like
    (different corpus, questions, cutoffs or stack settings) is never promoted,
    because the gain would not mean the search got better. A run that did not
    record the stack settings is never promoted either, since later runs could
    not tell whether they match it.
    """
    if not current.get("settings"):
        return False, "this run did not record the stack settings, so it cannot be a baseline"
    if baseline is None:
        return True, "no baseline yet"
    if not baseline.get("settings"):
        return (
            False,
            "the baseline did not record the stack settings, so the runs cannot be compared",
        )
    diff = diff_results(baseline, current)
    if diff["warnings"]:
        return False, "the runs are not comparable: " + " ".join(diff["warnings"])
    name = diff["compare_metrics"][0]
    gain = diff["aggregate"][name]["delta"]
    if gain > min_gain:
        return True, f"average {name} rose by {gain:.3f}"
    return False, f"average {name} changed by {gain:+.3f}, which is not above {min_gain:.3f}"


def _cell(text: object) -> str:
    return str(text).replace("|", "\\|").replace("\n", " ")


def _label(metric: str) -> str:
    if metric in ("mrr", "reciprocal_rank"):
        return "MRR" if metric == "mrr" else "Reciprocal rank"
    if metric in ("passage_mrr", "passage_rr"):
        return "Passage MRR" if metric == "passage_mrr" else "Passage reciprocal rank"
    answer_labels = {
        "answer_cited": "Cites expected file",
        "answer_has_phrase": "Contains answer phrase",
        "answer_correct": "Answer correctness",
        "answer_grounded": "Answer groundedness",
    }
    if metric in answer_labels:
        return answer_labels[metric]
    return metric.replace("ndcg", "nDCG").replace("passage_hit", "passage hit")


def _passage(query: Mapping[str, Any]) -> str:
    """Where the answer chunk ranked, for a question with an expected_text."""
    if "passage_rank" not in query:
        return "n/a"
    rank = query["passage_rank"]
    return "not returned" if rank is None else f"#{rank}"


def _ranks(ranks: Mapping[str, int | None], snippet_missing: Sequence[str] = ()) -> str:
    if not ranks:
        return "n/a"
    parts = []
    for name, rank in ranks.items():
        position = "not returned" if rank is None else f"#{rank}"
        note = " (snippet not found)" if name in snippet_missing else ""
        parts.append(f"{name}: {position}{note}")
    return ", ".join(parts)


def _changed_table(entries: Sequence[Mapping[str, Any]], metrics: Sequence[str]) -> list[str]:
    lines = [
        "| ID | Query | "
        + " | ".join(_label(m) for m in metrics)
        + " | Expected file ranks | Answer chunk rank |",
        "|---|---|" + "---|" * len(metrics) + "---|---|",
    ]
    for entry in entries:
        cells = []
        for name in metrics:
            delta = entry["metrics"].get(name)
            cells.append(
                f"{delta['baseline']:.3f} → {delta['current']:.3f}" if delta is not None else "n/a"
            )
        ranks = entry["expected_ranks"]
        missing = entry.get("snippet_missing", {})
        before = _ranks(ranks["baseline"], missing.get("baseline", ()))
        after = _ranks(ranks["current"], missing.get("current", ()))
        passage = entry.get("passage", {})
        lines.append(
            f"| {_cell(entry['id'])} | {_cell(entry['query'])} | "
            + " | ".join(cells)
            + f" | {_cell(before)} → {_cell(after)} "
            + f"| {passage.get('baseline', 'n/a')} → {passage.get('current', 'n/a')} |"
        )
    return lines


def render_diff(diff: Mapping[str, Any]) -> str:
    """Markdown section comparing a run with its baseline, regressions first."""
    metrics = diff["compare_metrics"]
    regressed, improved = diff["regressed"], diff["improved"]
    lines = [
        "## Compared with baseline",
        "",
        f"Baseline run `{diff['baseline_run_id']}`, current run `{diff['current_run_id']}`. "
        f"{len(regressed)} regressed, {len(improved)} improved, "
        f"{diff['unchanged']} unchanged, {len(diff['added'])} added, "
        f"{len(diff['removed'])} removed.",
        "",
    ]
    for warning in diff["warnings"]:
        lines.append(f"> **Warning:** {warning}")
    for note in diff.get("notes", ()):
        lines.append(f"> **Note:** {note}")
    if diff["warnings"] or diff.get("notes"):
        lines.append("")

    lines += [f"### Regressed ({len(regressed)})", ""]
    lines += _changed_table(regressed, metrics) if regressed else ["No question regressed."]
    lines += ["", f"### Improved ({len(improved)})", ""]
    lines += _changed_table(improved, metrics) if improved else ["No question improved."]

    if diff.get("answer_changes"):
        lines += [
            "",
            "### Answer correctness changes (0–2)",
            "",
            "| ID | Question | Baseline | Current |",
            "|---|---|---|---|",
        ]
        for entry in sorted(diff["answer_changes"], key=lambda item: item["delta"]):
            lines.append(
                f"| {_cell(entry['id'])} | {_cell(entry['query'])} | {entry['baseline']} | {entry['current']} |"
            )

    if diff["aggregate"]:
        lines += [
            "",
            "### Aggregate change",
            "",
            "| Metric | Baseline | Current | Change |",
            "|---|---|---|---|",
        ]
        for name, delta in diff["aggregate"].items():
            lines.append(
                f"| {_label(name)} | {delta['baseline']:.3f} | {delta['current']:.3f} "
                f"| {delta['delta']:+.3f} |"
            )
    for heading, ids in (
        ("Added questions", diff["added"]),
        ("Removed questions", diff["removed"]),
    ):
        if ids:
            lines += ["", f"### {heading} ({len(ids)})", ""]
            lines += [f"- {_cell(query_id)}" for query_id in ids]
    return "\n".join(lines) + "\n"


def _settings_line(results: Mapping[str, Any]) -> str:
    settings = results.get("settings")
    if not settings:
        return f"not recorded ({results.get('settings_note') or 'unavailable'})"
    return (
        f"embedding {settings.get('embedding_provider')}/{settings.get('embedding_model')}, "
        f"chunk size {settings.get('chunk_size')}, overlap {settings.get('chunk_overlap')}"
    )


def render_summary(results: Mapping[str, Any], diff: Mapping[str, Any] | None = None) -> str:
    """Markdown summary of one run, followed by the baseline comparison if given."""
    run = results["run"]
    k = run["primary_k"]
    overall = results["metrics"].get("overall", {})
    by_tag = results["metrics"].get("by_tag", {})
    headline = [f"recall@{k}", "mrr", f"ndcg@{k}"]

    lines = [
        "# OpenRAG retrieval evaluation",
        "",
        f"- **Run:** `{run['run_id']}` at {run['timestamp']}",
        f"- **Target:** {run['target_url']}",
        f"- **Commit:** {run.get('git_commit') or 'unknown'}",
        f"- **SDK version:** {run.get('sdk_version') or 'unknown'}",
        f"- **Dataset:** {run['corpus_files']} documents (corpus `{_short(run['corpus_hash'])}`), "
        f"{run['query_count']} questions (`{_short(run['queries_hash'])}`)",
        f"- **Search:** fetch limit {run['fetch_limit']}, score threshold {run['score_threshold']}",
        f"- **Settings:** {_settings_line(results)}",
        "",
        "## Overall",
        "",
        "| Metric | Score |",
        "|---|---|",
    ]
    lines += [
        f"| {_label(name)} | {value:.3f} |"
        for name, value in overall.items()
        if not name.startswith("answer_")
    ]
    if "passage_mrr" in overall:
        lines += [
            "",
            f"Passage scores cover the {run.get('passage_questions', 'n/a')} of "
            f"{run['query_count']} questions that have an expected_text. They rank the "
            "first chunk holding that text among all chunks returned.",
        ]

    if by_tag:
        if f"passage_hit@{k}" in overall:
            headline += [f"passage_hit@{k}", "passage_mrr"]
        lines += [
            "",
            "## By tag",
            "",
            "| Tag | Questions | " + " | ".join(_label(m) for m in headline) + " |",
            "|---|---|" + "---|" * len(headline),
        ]
        for tag, scores in by_tag.items():
            lines.append(
                f"| {_cell(tag)} | {scores['count']} | "
                # A tag whose questions all lack an expected_text has no passage scores.
                + " | ".join(f"{scores[m]:.3f}" if m in scores else "n/a" for m in headline)
                + " |"
            )

    missed = [q for q in results["queries"] if q["metrics"][f"recall@{k}"] < 1 - EPSILON]
    lines += ["", f"## Questions below full recall@{k} ({len(missed)})", ""]
    if missed:
        lines += [
            f"| ID | Query | recall@{k} | Expected file ranks | Snippet found "
            "| Answer chunk rank | Top files returned |",
            "|---|---|---|---|---|---|---|",
        ]
        for q in missed:
            snippet = {True: "yes", False: "no", None: "n/a"}[q.get("snippet_found")]
            top = ", ".join(item["filename"] for item in q["ranked_files"][:3]) or "none"
            lines.append(
                f"| {_cell(q['id'])} | {_cell(q['query'])} | {q['metrics'][f'recall@{k}']:.3f} "
                f"| {_cell(_ranks(q['expected_ranks'], _snippet_missing(q)))} | {snippet} "
                f"| {_passage(q)} | {_cell(top)} |"
            )
    else:
        lines.append(f"Every question reached full recall@{k}.")

    answer_rows = [q for q in results["queries"] if "answer" in q]
    if answer_rows:
        answer_names = [name for name in overall if name.startswith("answer_")]
        errors = [q for q in answer_rows if q["answer"].get("judge_error")]
        parse_errors = sum("could not be parsed" in q["answer"]["judge_error"] for q in errors)
        judged = sum(q["answer"].get("judgement") is not None for q in answer_rows)
        chat_failed = [q for q in answer_rows if q["answer"].get("chat_error")]
        lines += [
            "",
            "## Answer quality",
            "",
            f"{len(answer_rows) - len(chat_failed)} answers; {len(chat_failed)} chat errors; "
            f"{judged} usable judgements; {len(errors)} judge errors "
            f"({parse_errors} failed to parse). Questions without an answer or judgement "
            "are excluded from the averages.",
            "",
            "| Metric | Score |",
            "|---|---|",
        ]
        lines += [f"| {_label(name)} | {overall[name]:.3f} |" for name in answer_names]
        if by_tag:
            lines += [
                "",
                "### Answer quality by tag",
                "",
                "| Tag | " + " | ".join(_label(m) for m in answer_names) + " |",
                "|---|" + "---|" * len(answer_names),
            ]
            for tag, scores in by_tag.items():
                lines.append(
                    f"| {_cell(tag)} | "
                    + " | ".join(f"{scores[m]:.3f}" if m in scores else "n/a" for m in answer_names)
                    + " |"
                )
        wrong = [
            q for q in answer_rows if (q["answer"].get("judgement") or {}).get("correctness") == 0
        ]
        lines += ["", f"### Answers judged wrong ({len(wrong)})", ""]
        if wrong:
            lines += ["| Question | Reference | Answer excerpt | Reason |", "|---|---|---|---|"]
            for q in wrong:
                answer = q["answer"]
                lines.append(
                    "| "
                    + " | ".join(
                        _cell(v)
                        for v in (
                            q["query"],
                            q.get("expected_answer") or "n/a",
                            answer["text"][:240],
                            answer["judgement"]["reason"],
                        )
                    )
                    + " |"
                )
        else:
            lines.append("No answers judged wrong.")
        if chat_failed:
            lines += [
                "",
                f"### Chat errors ({len(chat_failed)})",
                "",
                "| ID | Error |",
                "|---|---|",
            ]
            lines += [
                f"| {_cell(q['id'])} | {_cell(q['answer']['chat_error'])} |" for q in chat_failed
            ]

    text = "\n".join(lines) + "\n"
    if diff is not None:
        text += "\n" + render_diff(diff)
    return text