Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: 2478 eval manual workflow(#2478) (#2554)

Sathvik Lakamsania75f10a264df
diff --git a/.github/workflows/eval-retrieval.yml b/.github/workflows/eval-retrieval.yml
new file mode 100644
index 000000000..c2f5a2649
--- /dev/null
+++ b/.github/workflows/eval-retrieval.yml
@@ -0,0 +1,274 @@
+name: Retrieval Eval
+
+# Manual only. Nothing here runs on a push or a pull request: someone starts it
+# from the Actions tab (or `gh workflow run eval-retrieval.yml`) when they want
+# to measure search quality. See scripts/eval/retrieval/README.md.
+#
+# Each run is compared with the tracked scripts/eval/retrieval/baseline.json.
+# That file changes only when a run asks for it (update_baseline), improved on
+# it, and ran on a branch other than the default one, so the new baseline
+# reaches main through a normal pull request.
+
+on:
+ workflow_dispatch:
+ inputs:
+ max_recall_drop:
+ description: >-
+ Fail the run if average recall at the compared cutoff falls by more than this
+ (0.02 = two points) below the tracked baseline. Leave empty to only report.
+ required: false
+ type: string
+ update_baseline:
+ description: >-
+ If this run improves on the tracked baseline, commit the new baseline.json to
+ the selected branch. Refused on the default branch.
+ required: false
+ type: boolean
+ default: false
+ min_gain:
+ description: >-
+ Average recall must rise by more than this to count as an improvement. Keeps
+ run-to-run noise from raising the baseline.
+ required: false
+ type: string
+ default: "0.025"
+
+permissions:
+ contents: read
+
+env:
+ PYTHON_VERSION: "3.13"
+ OPENRAG_VERSION: eval-${{ github.sha }}
+ LANGFLOW_SUPERUSER: "admin"
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+ AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
+ AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION || '2024-02-01' }}
+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ WATSONX_API_KEY: ${{ secrets.WATSONX_API_KEY }}
+ WATSONX_ENDPOINT: ${{ secrets.WATSONX_ENDPOINT }}
+ WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }}
+ OLLAMA_ENDPOINT: ${{ secrets.OLLAMA_ENDPOINT }}
+ # The embedding model decides the scores, so it is the same one test-ci.yml
+ # uses. Scores from different models are not comparable.
+ LLM_PROVIDER: ${{ secrets.LLM_PROVIDER || 'azure' }}
+ EMBEDDING_PROVIDER: ${{ secrets.EMBEDDING_PROVIDER || 'azure' }}
+ LLM_MODEL: ${{ secrets.LLM_MODEL || 'gpt-4.1' }}
+ EMBEDDING_MODEL: ${{ secrets.EMBEDDING_MODEL || 'text-embedding-3-small' }}
+ LANGFLOW_AUTO_LOGIN: "True"
+ LANGFLOW_NEW_USER_IS_ACTIVE: "True"
+ LANGFLOW_ENABLE_SUPERUSER_CLI: "True"
+ LANGFLOW_CHAT_FLOW_ID: ${{ vars.LANGFLOW_CHAT_FLOW_ID || '1098eea1-6649-4e1d-aed1-b77249fb8dd0' }}
+ LANGFLOW_INGEST_FLOW_ID: ${{ vars.LANGFLOW_INGEST_FLOW_ID || '5488df7c-b93f-4f87-a446-b67028bc0813' }}
+ NUDGES_FLOW_ID: ${{ vars.NUDGES_FLOW_ID || 'ebc01d31-1976-46ce-a385-b0240327226c' }}
+ LANGFLOW_URL_INGEST_FLOW_ID: ${{ vars.LANGFLOW_URL_INGEST_FLOW_ID || '72c3d17c-2dac-4a73-b48a-6518473d7830' }}
+ # No-auth mode, so the run can create its own API key. Startup ingest is
+ # off so default documents do not end up in the index being measured.
+ GOOGLE_OAUTH_CLIENT_ID: ""
+ GOOGLE_OAUTH_CLIENT_SECRET: ""
+ DISABLE_STARTUP_INGEST: "true"
+ # Single-node test cluster never reaches the multi-node readiness counts
+ OPENSEARCH_NODE_COUNT_CHECK_ENABLED: "false"
+
+jobs:
+ eval:
+ name: Retrieval eval
+ runs-on: ubuntu-latest
+ timeout-minutes: 90
+ # write is used only by the "Update the tracked baseline" step, which
+ # runs when update_baseline is set.
+ permissions:
+ contents: write
+ steps:
+ - name: Validate inputs
+ env:
+ MAX_RECALL_DROP: ${{ inputs.max_recall_drop }}
+ MIN_GAIN: ${{ inputs.min_gain || '0.025' }}
+ UPDATE_BASELINE: ${{ inputs.update_baseline }}
+ REF_NAME: ${{ github.ref_name }}
+ DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+ run: |
+ number='^[0-9]+(\.[0-9]+)?$'
+ if [ -n "$MAX_RECALL_DROP" ] && ! printf '%s' "$MAX_RECALL_DROP" | grep -Eq "$number"; then
+ echo "::error::max_recall_drop must be a non-negative number such as 0.02"
+ exit 1
+ fi
+ if ! printf '%s' "$MIN_GAIN" | grep -Eq "$number"; then
+ echo "::error::min_gain must be a non-negative number such as 0.025"
+ exit 1
+ fi
+ if [ "$UPDATE_BASELINE" = "true" ] && [ "$REF_NAME" = "$DEFAULT_BRANCH" ]; then
+ echo "::error::update_baseline is refused on ${DEFAULT_BRANCH}. Run it on a branch and merge the result in a pull request."
+ exit 1
+ fi
+
+ - name: Checkout
+ uses: actions/checkout@v7
+
+ # Throwaway passwords for the stack this run starts and then deletes, so
+ # no password is stored in the repository.
+ - name: Generate stack passwords
+ run: |
+ set -euo pipefail
+ for name in OPENSEARCH_PASSWORD LANGFLOW_SUPERUSER_PASSWORD; do
+ value="Ea1#$(openssl rand -hex 16)"
+ echo "::add-mask::${value}"
+ echo "${name}=${value}" >> "$GITHUB_ENV"
+ done
+
+ - name: Set up UV
+ uses: astral-sh/[email protected]
+ with:
+ version: latest
+ enable-cache: true
+
+ - name: Install Python
+ run: uv python install ${{ env.PYTHON_VERSION }}
+
+ - name: Check the corpus and questions
+ run: make eval-validate
+
+ - name: Build images from this ref
+ run: |
+ for image in opensearch backend frontend langflow; do
+ if [ "$image" = "opensearch" ]; then
+ dockerfile=Dockerfile
+ else
+ dockerfile="Dockerfile.${image}"
+ fi
+ docker build -f "$dockerfile" -t "langflowai/openrag-${image}:${OPENRAG_VERSION}" .
+ done
+
+ - name: Start the stack
+ env:
+ CI: "true"
+ run: |
+ chmod +x scripts/setup-e2e.sh
+ ./scripts/setup-e2e.sh
+
+ - name: Onboard the stack and create an API key
+ run: |
+ set -euo pipefail
+ base_url="http://localhost:3000"
+
+ payload="$(jq -n \
+ --arg llm_provider "$LLM_PROVIDER" \
+ --arg embedding_provider "$EMBEDDING_PROVIDER" \
+ --arg llm_model "$LLM_MODEL" \
+ --arg embedding_model "$EMBEDDING_MODEL" \
+ --arg azure_key "${AZURE_OPENAI_API_KEY:-}" \
+ --arg azure_base "${AZURE_OPENAI_ENDPOINT:-}" \
+ --arg azure_version "${AZURE_OPENAI_API_VERSION:-}" \
+ '{llm_provider: $llm_provider, embedding_provider: $embedding_provider,
+ llm_model: $llm_model, embedding_model: $embedding_model}
+ + (if $azure_key != "" and $azure_base != ""
+ then {provider_credentials: {azure: {api_key: $azure_key, api_base: $azure_base, api_version: $azure_version}}}
+ else {} end)')"
+
+ status="$(curl -sS --max-time 120 -o /tmp/onboarding.json -w '%{http_code}' \
+ -X POST "${base_url}/api/onboarding" -H 'Content-Type: application/json' -d "$payload")"
+ if [ "$status" != "200" ] && [ "$status" != "204" ]; then
+ echo "::error::Onboarding failed with HTTP ${status}"
+ exit 1
+ fi
+
+ api_key="$(curl -sS --max-time 60 --fail \
+ -X POST "${base_url}/api/keys" -H 'Content-Type: application/json' \
+ -d '{"name": "Retrieval Eval"}' | jq -r '.api_key')"
+ if [ -z "$api_key" ] || [ "$api_key" = "null" ]; then
+ echo "::error::Could not create an API key"
+ exit 1
+ fi
+ echo "::add-mask::${api_key}"
+ echo "OPENRAG_API_KEY=${api_key}" >> "$GITHUB_ENV"
+
+ - name: Run the retrieval eval
+ env:
+ OPENRAG_URL: http://localhost:3000
+ MAX_RECALL_DROP: ${{ inputs.max_recall_drop }}
+ run: |
+ set -euo pipefail
+ if [ -f scripts/eval/retrieval/baseline.json ]; then
+ args=""
+ if [ -n "$MAX_RECALL_DROP" ]; then
+ args="--max-recall-drop ${MAX_RECALL_DROP}"
+ fi
+ make eval-run BASELINE=baseline.json EVAL_ARGS="${args}"
+ else
+ echo "::warning::scripts/eval/retrieval/baseline.json does not exist yet, so nothing was compared. Run with update_baseline on a branch to create it."
+ make eval-run
+ fi
+
+ - name: Publish summary
+ if: ${{ always() }}
+ run: |
+ if [ -f scripts/eval/retrieval/results/summary.md ]; then
+ cat scripts/eval/retrieval/results/summary.md >> "$GITHUB_STEP_SUMMARY"
+ else
+ echo "The eval did not produce a summary." >> "$GITHUB_STEP_SUMMARY"
+ fi
+
+ - name: Update the tracked baseline
+ if: ${{ inputs.update_baseline }}
+ env:
+ MIN_GAIN: ${{ inputs.min_gain || '0.025' }}
+ REF_NAME: ${{ github.ref_name }}
+ run: |
+ # Exit status 3 means "not an improvement", which is not a failure, so
+ # read the status instead of letting the shell stop on it.
+ set +e
+ cd scripts/eval/retrieval
+ uv run run_eval.py promote results/results.json baseline.json --min-gain "$MIN_GAIN" > /tmp/promote.txt
+ status=$?
+ set -e
+ cat /tmp/promote.txt
+ {
+ echo "### Baseline"
+ cat /tmp/promote.txt
+ } >> "$GITHUB_STEP_SUMMARY"
+ if [ "$status" = "3" ]; then
+ exit 0
+ elif [ "$status" != "0" ]; then
+ exit "$status"
+ fi
+ cd "$GITHUB_WORKSPACE"
+ git config user.name "github-actions[bot]"
+ git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+ git add scripts/eval/retrieval/baseline.json
+ git commit -m "chore: update the retrieval eval baseline"
+ git push origin "HEAD:refs/heads/${REF_NAME}"
+
+ - name: Upload results
+ if: ${{ always() }}
+ uses: actions/upload-artifact@v7
+ with:
+ name: retrieval-eval-results
+ path: scripts/eval/retrieval/results/
+ if-no-files-found: ignore
+ retention-days: 90
+
+ - name: Collect service logs on failure
+ if: failure()
+ run: |
+ mkdir -p service-logs
+ docker logs openrag-opensearch > service-logs/opensearch.log 2>&1 || true
+ docker logs openrag-backend > service-logs/backend.log 2>&1 || true
+ docker logs openrag-frontend > service-logs/frontend.log 2>&1 || true
+ docker logs openrag-langflow > service-logs/langflow.log 2>&1 || true
+ cp ~/.openrag/tui/docling-serve.log service-logs/docling.log 2>&1 || true
+
+ - name: Upload service logs
+ if: failure()
+ uses: actions/upload-artifact@v7
+ with:
+ name: eval-service-logs
+ path: service-logs/
+ retention-days: 7
+
+ - name: Teardown infrastructure
+ if: ${{ always() }}
+ run: |
+ make docling-stop || true
+ docker rm -f openrag-backend-proxy 2>/dev/null || true
+ make clean || true
+ docker system prune -f || true
diff --git a/scripts/eval/retrieval/README.md b/scripts/eval/retrieval/README.md
index 7ce8db3c4..15a8aebf0 100644
--- a/scripts/eval/retrieval/README.md
+++ b/scripts/eval/retrieval/README.md
@@ -149,6 +149,9 @@ regressed first, then the ones that improved, then the change in each average.
For every changed question it shows the rank of each expected file before and
after.
+To make a run exit with status 1 when average recall falls too far, add
+`--max-recall-drop 0.02` (two points) next to `--baseline`. It is off by default.
+
To compare two results files without running anything:
```bash
@@ -182,9 +185,74 @@ To tell a real change from this noise:
question.
- For the steadiest numbers, use a stack that holds no other documents.
-No baseline is committed. Scores depend on the embedding model and settings of
-the stack that produced them, so create the baseline on the same stack you test
-against.
+Scores depend on the embedding model and settings of the stack that produced
+them, so create the baseline on the same stack you test against. The GitHub
+Actions workflow keeps a tracked baseline for its own stack; see
+[Run it from GitHub Actions](#run-it-from-github-actions).
+
+## Run it from GitHub Actions
+
+The **Retrieval Eval** workflow ([`eval-retrieval.yml`](../../../.github/workflows/eval-retrieval.yml))
+runs the harness on a fresh stack built from the branch you pick and compares
+it with the tracked `baseline.json` in this directory, which exists once the
+first run has created it. It runs only when someone
+starts it. It never runs on a push or a pull request, and it is not a required
+check.
+
+To start it, open the **Actions** tab, choose **Retrieval Eval**, click **Run
+workflow**, pick a branch and fill in the inputs. Or from the command line:
+
+```bash
+gh workflow run eval-retrieval.yml --ref my-branch
+```
+
+GitHub lists a manually started workflow only once its file is on the default
+branch, so this works after the workflow has been merged.
+
+| Input | Meaning |
+|---|---|
+| `max_recall_drop` | Fail the run if average recall at the compared cutoff falls by more than this below the baseline, for example `0.02` for two points. Leave it empty to report without failing. |
+| `update_baseline` | If the run improves on the baseline, commit the new `baseline.json` to the selected branch. Refused on the default branch. |
+| `min_gain` | Average recall must rise by more than this to count as an improvement. Default `0.025`. One of the 83 questions flipping moves average recall by about 0.012, so a lower value lets noise raise the baseline. |
+
+Each run writes the summary to the run's page and uploads `results.json` and
+`summary.md` as the `retrieval-eval-results` artifact, kept for 90 days.
+
+### The tracked baseline
+
+`baseline.json` is committed to the repository and is the reference for every
+run. It changes only through `update_baseline`:
+
+1. Run the workflow on your branch with `update_baseline` ticked.
+2. If average recall rose by more than `min_gain`, the workflow commits the new
+ `baseline.json` to your branch. Otherwise it leaves the file alone and says
+ why in the run summary.
+3. Merge the branch in a pull request. The reviewer sees the new scores in the
+ diff.
+
+The workflow never commits to the default branch, so a baseline change is
+always reviewed. A run is not promoted when it did not record the stack settings, or
+when it is not like for like with the baseline (a different corpus, question file, cutoffs or stack settings),
+because a higher number would not mean better search. To start over after such
+a change, for example a new embedding model, delete `baseline.json` and run
+with `update_baseline`. With no baseline, the first run becomes the baseline.
+
+The same logic is available locally:
+
+```bash
+uv run run_eval.py promote results/results.json baseline.json --min-gain 0.025
+```
+
+It exits with 0 when it wrote the baseline and 3 when it left it alone.
+
+The stack uses the embedding provider and model set by the repository's
+`EMBEDDING_PROVIDER` and `EMBEDDING_MODEL` secrets, the same as the end-to-end
+tests. Scores from different models are not comparable, and the comparison
+warns when the settings of the two runs differ.
+
+The harness gives the same stack slightly different scores from run to run (see
+[Repeatability](#repeatability)). Set `max_recall_drop` and `min_gain` above that
+noise, or runs will fail, or the baseline will creep up, on unchanged code.
## How questions are scored
diff --git a/scripts/eval/retrieval/report.py b/scripts/eval/retrieval/report.py
index 30f749463..952a703c9 100644
--- a/scripts/eval/retrieval/report.py
+++ b/scripts/eval/retrieval/report.py
@@ -249,6 +249,64 @@ def diff_results(
}
+def recall_drop(diff: Mapping[str, Any]) -> float:
+ """How far the average recall fell at the compared cutoff; 0 if it held or rose."""
+ name = diff["compare_metrics"][0]
+ return max(0.0, -diff["aggregate"][name]["delta"])
+
+
+def recall_gate_failure(diff: Mapping[str, Any], max_drop: float) -> str | None:
+ """Why the recall gate fails, or None when it passes.
+
+ The gate fails when the runs are not like for like, because a score that
+ held up between two different setups says nothing about the search.
+ """
+ if diff["warnings"]:
+ return "the runs are not comparable, so recall cannot be checked: " + " ".join(
+ diff["warnings"]
+ )
+ drop = recall_drop(diff)
+ if drop > max_drop:
+ return (
+ f"average {diff['compare_metrics'][0]} fell by {drop:.3f}, "
+ f"more than the allowed {max_drop:.3f}"
+ )
+ return None
+
+
+def promotion_decision(
+ baseline: Mapping[str, Any] | None,
+ current: Mapping[str, Any],
+ min_gain: float = 0.0,
+) -> tuple[bool, str]:
+ """Whether ``current`` should replace ``baseline`` as the reference.
+
+ Yes when there is no baseline yet, or when average recall at the compared
+ cutoff rose by more than ``min_gain``. A pair that is not like for like
+ (different corpus, questions, cutoffs or stack settings) is never promoted,
+ because the gain would not mean the search got better. A run that did not
+ record the stack settings is never promoted either, since later runs could
+ not tell whether they match it.
+ """
+ if not current.get("settings"):
+ return False, "this run did not record the stack settings, so it cannot be a baseline"
+ if baseline is None:
+ return True, "no baseline yet"
+ if not baseline.get("settings"):
+ return (
+ False,
+ "the baseline did not record the stack settings, so the runs cannot be compared",
+ )
+ diff = diff_results(baseline, current)
+ if diff["warnings"]:
+ return False, "the runs are not comparable: " + " ".join(diff["warnings"])
+ name = diff["compare_metrics"][0]
+ gain = diff["aggregate"][name]["delta"]
+ if gain > min_gain:
+ return True, f"average {name} rose by {gain:.3f}"
+ return False, f"average {name} changed by {gain:+.3f}, which is not above {min_gain:.3f}"
+
+
def _cell(text: object) -> str:
return str(text).replace("|", "\\|").replace("\n", " ")
diff --git a/scripts/eval/retrieval/run_eval.py b/scripts/eval/retrieval/run_eval.py
index 66e491ba4..145084de5 100644
--- a/scripts/eval/retrieval/run_eval.py
+++ b/scripts/eval/retrieval/run_eval.py
@@ -15,6 +15,7 @@ from __future__ import annotations
import argparse
import asyncio
import io
+import json
import os
import subprocess
import sys
@@ -65,6 +66,14 @@ def build_parser() -> argparse.ArgumentParser:
run.add_argument("--queries", type=Path, default=HERE / "queries.jsonl")
run.add_argument("--output-dir", type=Path, default=HERE / "results")
run.add_argument("--baseline", type=Path, help="results file to compare this run against")
+ run.add_argument(
+ "--max-recall-drop",
+ type=float,
+ help=(
+ "with --baseline, exit with status 1 when the average recall at the compared "
+ "cutoff falls by more than this (0.02 = two points). Off by default."
+ ),
+ )
run.add_argument(
"--k",
dest="k_values",
@@ -104,6 +113,18 @@ def build_parser() -> argparse.ArgumentParser:
compare = commands.add_parser("compare", help="diff two results files offline")
compare.add_argument("baseline", type=Path)
compare.add_argument("current", type=Path)
+
+ promote = commands.add_parser(
+ "promote", help="replace the baseline with a results file, only if it improves on it"
+ )
+ promote.add_argument("current", type=Path, help="results file of the new run")
+ promote.add_argument("baseline", type=Path, help="baseline file to update (may not exist yet)")
+ promote.add_argument(
+ "--min-gain",
+ type=float,
+ default=0.0,
+ help="average recall must rise by more than this (default: 0, any rise)",
+ )
return parser
@@ -391,6 +412,11 @@ async def run(args: argparse.Namespace) -> int:
print("Wrote:")
for path in written:
print(f" {path}")
+ if diff is not None and args.max_recall_drop is not None:
+ failure = report.recall_gate_failure(diff, args.max_recall_drop)
+ if failure:
+ print(f"error: {failure}", file=sys.stderr)
+ return 1
return 0
@@ -403,9 +429,32 @@ def compare(args: argparse.Namespace) -> int:
return 0
+NOT_PROMOTED = 3
+
+
+def promote(args: argparse.Namespace) -> int:
+ current = report.load_results(args.current)
+ baseline = report.load_results(args.baseline) if args.baseline.exists() else None
+ promoted, reason = report.promotion_decision(baseline, current, args.min_gain)
+ if not promoted:
+ print(f"Baseline not updated: {reason}")
+ return NOT_PROMOTED
+ args.baseline.parent.mkdir(parents=True, exist_ok=True)
+ args.baseline.write_text(json.dumps(current, indent=2) + "\n", encoding="utf-8")
+ print(f"Baseline updated ({reason}): {args.baseline}")
+ return 0
+
+
def main(argv: list[str] | None = None) -> int:
parser = build_parser()
args = parser.parse_args(argv)
+ if args.command == "promote" and args.min_gain < 0:
+ parser.error("--min-gain must not be negative")
+ if args.command == "run" and args.max_recall_drop is not None:
+ if args.max_recall_drop < 0:
+ parser.error("--max-recall-drop must not be negative")
+ if args.baseline is None:
+ parser.error("--max-recall-drop needs --baseline to compare against")
if args.command == "run" and args.fetch_limit < max(args.k_values):
# Fewer chunks than the largest cutoff cannot fill that many file
# ranks, so recall@k and nDCG@k at that cutoff would be meaningless.
@@ -416,6 +465,8 @@ def main(argv: list[str] | None = None) -> int:
try:
if args.command == "compare":
return compare(args)
+ if args.command == "promote":
+ return promote(args)
return asyncio.run(run(args))
except (dataset.DatasetError, report.ResultsError, EvalError) as exc:
print(f"error: {exc}", file=sys.stderr)
diff --git a/tests/unit/test_retrieval_eval.py b/tests/unit/test_retrieval_eval.py
index ed66a5a10..776e03188 100644
--- a/tests/unit/test_retrieval_eval.py
+++ b/tests/unit/test_retrieval_eval.py
@@ -606,3 +606,174 @@ def test_run_rejects_a_fetch_limit_below_the_largest_cutoff(fetch_limit: str) ->
assert completed.returncode == 2
assert "--fetch-limit must be at least the largest --k value (5)" in completed.stderr
+
+
+# --- recall-drop gate --------------------------------------------------------
+
+
+ ("current_recall", "expected_drop"),
+ [(0.0, 1.0), (1.0, 0.0)],
+)
+def test_recall_drop_reports_a_fall_and_ignores_a_gain(
+ current_recall: float, expected_drop: float
+) -> None:
+ baseline = make_results([make_query("q", 1.0, 1.0, 1.0, 1)])
+ current = make_results([make_query("q", current_recall, 1.0, 1.0, 1)])
+
+ assert report.recall_drop(report.diff_results(baseline, current)) == pytest.approx(
+ expected_drop
+ )
+
+ improved = make_results([make_query("q", 0.0, 0.0, 0.0, None)])
+ gain = make_results([make_query("q", 1.0, 1.0, 1.0, 1)])
+ assert report.recall_drop(report.diff_results(improved, gain)) == 0.0
+
+
+ ("extra_args", "message"),
+ [
+ (["--max-recall-drop", "0.02"], "--max-recall-drop needs --baseline"),
+ (["--max-recall-drop", "-0.1", "--baseline", "b.json"], "must not be negative"),
+ ],
+)
+def test_run_rejects_an_unusable_max_recall_drop(extra_args: list[str], message: str) -> None:
+ script = Path(__file__).resolve().parents[2] / "scripts/eval/retrieval/run_eval.py"
+ completed = subprocess.run(
+ [sys.executable, str(script), "run", "--validate-only", *extra_args],
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+
+ assert completed.returncode == 2
+ assert message in completed.stderr
+
+
+# --- baseline promotion ----------------------------------------------------
+
+
+def _results_with_recall(recall: float, **run_overrides: Any) -> dict[str, Any]:
+ return make_results([make_query("q", recall, recall, recall, 1)], **run_overrides)
+
+
+def test_promotion_accepts_the_first_baseline() -> None:
+ promoted, reason = report.promotion_decision(None, _results_with_recall(0.5))
+
+ assert promoted
+ assert "no baseline" in reason
+
+
+ ("baseline_recall", "current_recall", "min_gain", "expected"),
+ [
+ (0.5, 0.75, 0.0, True),
+ (0.5, 0.5, 0.0, False),
+ (0.75, 0.5, 0.0, False),
+ (0.5, 0.55, 0.1, False),
+ (0.5, 0.75, 0.1, True),
+ ],
+)
+def test_promotion_requires_a_gain_above_the_minimum(
+ baseline_recall: float, current_recall: float, min_gain: float, expected: bool
+) -> None:
+ promoted, _ = report.promotion_decision(
+ _results_with_recall(baseline_recall), _results_with_recall(current_recall), min_gain
+ )
+
+ assert promoted is expected
+
+
+def test_promotion_refuses_runs_that_are_not_comparable() -> None:
+ promoted, reason = report.promotion_decision(
+ _results_with_recall(0.5), _results_with_recall(1.0, corpus_hash="e" * 64)
+ )
+
+ assert not promoted
+ assert "not comparable" in reason
+
+
+def _run_cli(*args: str) -> subprocess.CompletedProcess[str]:
+ script = Path(__file__).resolve().parents[2] / "scripts/eval/retrieval/run_eval.py"
+ return subprocess.run(
+ [sys.executable, str(script), *args], capture_output=True, text=True, check=False
+ )
+
+
+def test_promote_writes_the_baseline_only_on_improvement(tmp_path: Path) -> None:
+ baseline = tmp_path / "baseline.json"
+ low, high = tmp_path / "low.json", tmp_path / "high.json"
+ low.write_text(json.dumps(_results_with_recall(0.5)), encoding="utf-8")
+ high.write_text(json.dumps(_results_with_recall(1.0)), encoding="utf-8")
+
+ first = _run_cli("promote", str(low), str(baseline))
+ assert first.returncode == 0
+ assert json.loads(baseline.read_text(encoding="utf-8")) == json.loads(low.read_text())
+
+ same = _run_cli("promote", str(low), str(baseline))
+ assert same.returncode == 3
+ assert "Baseline not updated" in same.stdout
+
+ better = _run_cli("promote", str(high), str(baseline))
+ assert better.returncode == 0
+ assert json.loads(baseline.read_text(encoding="utf-8")) == json.loads(high.read_text())
+
+ worse = _run_cli("promote", str(low), str(baseline))
+ assert worse.returncode == 3
+ assert json.loads(baseline.read_text(encoding="utf-8")) == json.loads(high.read_text())
+
+
+# --- review follow-ups -------------------------------------------------------
+
+
+def test_recall_gate_fails_when_the_runs_are_not_comparable() -> None:
+ baseline = _results_with_recall(0.5)
+ current = _results_with_recall(0.5, corpus_hash="e" * 64)
+
+ failure = report.recall_gate_failure(report.diff_results(baseline, current), 0.02)
+
+ assert failure is not None
+ assert "not comparable" in failure
+
+
+ ("current_recall", "fails"),
+ [(0.5, False), (0.49, False), (0.45, True), (1.0, False)],
+)
+def test_recall_gate_fails_only_on_a_drop_above_the_limit(
+ current_recall: float, fails: bool
+) -> None:
+ diff = report.diff_results(_results_with_recall(0.5), _results_with_recall(current_recall))
+
+ assert (report.recall_gate_failure(diff, 0.02) is not None) is fails
+
+
[email protected]("missing", ["current", "baseline"])
+def test_promotion_refuses_a_run_without_recorded_settings(missing: str) -> None:
+ baseline, current = _results_with_recall(0.5), _results_with_recall(1.0)
+ (current if missing == "current" else baseline)["settings"] = None
+
+ promoted, reason = report.promotion_decision(baseline, current)
+
+ assert not promoted
+ assert "settings" in reason
+
+
+def test_promotion_refuses_a_first_baseline_without_recorded_settings() -> None:
+ current = _results_with_recall(0.5)
+ current["settings"] = None
+
+ promoted, reason = report.promotion_decision(None, current)
+
+ assert not promoted
+ assert "settings" in reason
+
+
+def test_promote_rejects_a_negative_min_gain(tmp_path: Path) -> None:
+ results = tmp_path / "results.json"
+ results.write_text(json.dumps(_results_with_recall(0.5)), encoding="utf-8")
+
+ completed = _run_cli("promote", str(results), str(tmp_path / "b.json"), "--min-gain", "-0.1")
+
+ assert completed.returncode == 2
+ assert "--min-gain must not be negative" in completed.stderr