OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: 2478 eval manual workflow(#2478) (#2554)
Sathvik Lakamsania75f10a264df
diff --git a/.github/workflows/eval-retrieval.yml b/.github/workflows/eval-retrieval.ymlnew file mode 100644index 000000000..c2f5a2649--- /dev/null+++ b/.github/workflows/eval-retrieval.yml@@ -0,0 +1,274 @@+name: Retrieval Eval++# Manual only. Nothing here runs on a push or a pull request: someone starts it+# from the Actions tab (or `gh workflow run eval-retrieval.yml`) when they want+# to measure search quality. See scripts/eval/retrieval/README.md.+#+# Each run is compared with the tracked scripts/eval/retrieval/baseline.json.+# That file changes only when a run asks for it (update_baseline), improved on+# it, and ran on a branch other than the default one, so the new baseline+# reaches main through a normal pull request.++on:+ workflow_dispatch:+ inputs:+ max_recall_drop:+ description: >-+ Fail the run if average recall at the compared cutoff falls by more than this+ (0.02 = two points) below the tracked baseline. Leave empty to only report.+ required: false+ type: string+ update_baseline:+ description: >-+ If this run improves on the tracked baseline, commit the new baseline.json to+ the selected branch. Refused on the default branch.+ required: false+ type: boolean+ default: false+ min_gain:+ description: >-+ Average recall must rise by more than this to count as an improvement. Keeps+ run-to-run noise from raising the baseline.+ required: false+ type: string+ default: "0.025"++permissions:+ contents: read++env:+ PYTHON_VERSION: "3.13"+ OPENRAG_VERSION: eval-${{ github.sha }}+ LANGFLOW_SUPERUSER: "admin"+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}+ AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}+ AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}+ AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION || '2024-02-01' }}+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}+ WATSONX_API_KEY: ${{ secrets.WATSONX_API_KEY }}+ WATSONX_ENDPOINT: ${{ secrets.WATSONX_ENDPOINT }}+ WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }}+ OLLAMA_ENDPOINT: ${{ secrets.OLLAMA_ENDPOINT }}+ # The embedding model decides the scores, so it is the same one test-ci.yml+ # uses. Scores from different models are not comparable.+ LLM_PROVIDER: ${{ secrets.LLM_PROVIDER || 'azure' }}+ EMBEDDING_PROVIDER: ${{ secrets.EMBEDDING_PROVIDER || 'azure' }}+ LLM_MODEL: ${{ secrets.LLM_MODEL || 'gpt-4.1' }}+ EMBEDDING_MODEL: ${{ secrets.EMBEDDING_MODEL || 'text-embedding-3-small' }}+ LANGFLOW_AUTO_LOGIN: "True"+ LANGFLOW_NEW_USER_IS_ACTIVE: "True"+ LANGFLOW_ENABLE_SUPERUSER_CLI: "True"+ LANGFLOW_CHAT_FLOW_ID: ${{ vars.LANGFLOW_CHAT_FLOW_ID || '1098eea1-6649-4e1d-aed1-b77249fb8dd0' }}+ LANGFLOW_INGEST_FLOW_ID: ${{ vars.LANGFLOW_INGEST_FLOW_ID || '5488df7c-b93f-4f87-a446-b67028bc0813' }}+ NUDGES_FLOW_ID: ${{ vars.NUDGES_FLOW_ID || 'ebc01d31-1976-46ce-a385-b0240327226c' }}+ LANGFLOW_URL_INGEST_FLOW_ID: ${{ vars.LANGFLOW_URL_INGEST_FLOW_ID || '72c3d17c-2dac-4a73-b48a-6518473d7830' }}+ # No-auth mode, so the run can create its own API key. Startup ingest is+ # off so default documents do not end up in the index being measured.+ GOOGLE_OAUTH_CLIENT_ID: ""+ GOOGLE_OAUTH_CLIENT_SECRET: ""+ DISABLE_STARTUP_INGEST: "true"+ # Single-node test cluster never reaches the multi-node readiness counts+ OPENSEARCH_NODE_COUNT_CHECK_ENABLED: "false"++jobs:+ eval:+ name: Retrieval eval+ runs-on: ubuntu-latest+ timeout-minutes: 90+ # write is used only by the "Update the tracked baseline" step, which+ # runs when update_baseline is set.+ permissions:+ contents: write+ steps:+ - name: Validate inputs+ env:+ MAX_RECALL_DROP: ${{ inputs.max_recall_drop }}+ MIN_GAIN: ${{ inputs.min_gain || '0.025' }}+ UPDATE_BASELINE: ${{ inputs.update_baseline }}+ REF_NAME: ${{ github.ref_name }}+ DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}+ run: |+ number='^[0-9]+(\.[0-9]+)?$'+ if [ -n "$MAX_RECALL_DROP" ] && ! printf '%s' "$MAX_RECALL_DROP" | grep -Eq "$number"; then+ echo "::error::max_recall_drop must be a non-negative number such as 0.02"+ exit 1+ fi+ if ! printf '%s' "$MIN_GAIN" | grep -Eq "$number"; then+ echo "::error::min_gain must be a non-negative number such as 0.025"+ exit 1+ fi+ if [ "$UPDATE_BASELINE" = "true" ] && [ "$REF_NAME" = "$DEFAULT_BRANCH" ]; then+ echo "::error::update_baseline is refused on ${DEFAULT_BRANCH}. Run it on a branch and merge the result in a pull request."+ exit 1+ fi++ - name: Checkout+ uses: actions/checkout@v7++ # Throwaway passwords for the stack this run starts and then deletes, so+ # no password is stored in the repository.+ - name: Generate stack passwords+ run: |+ set -euo pipefail+ for name in OPENSEARCH_PASSWORD LANGFLOW_SUPERUSER_PASSWORD; do+ value="Ea1#$(openssl rand -hex 16)"+ echo "::add-mask::${value}"+ echo "${name}=${value}" >> "$GITHUB_ENV"+ done++ - name: Set up UV+ uses: astral-sh/[email protected]+ with:+ version: latest+ enable-cache: true++ - name: Install Python+ run: uv python install ${{ env.PYTHON_VERSION }}++ - name: Check the corpus and questions+ run: make eval-validate++ - name: Build images from this ref+ run: |+ for image in opensearch backend frontend langflow; do+ if [ "$image" = "opensearch" ]; then+ dockerfile=Dockerfile+ else+ dockerfile="Dockerfile.${image}"+ fi+ docker build -f "$dockerfile" -t "langflowai/openrag-${image}:${OPENRAG_VERSION}" .+ done++ - name: Start the stack+ env:+ CI: "true"+ run: |+ chmod +x scripts/setup-e2e.sh+ ./scripts/setup-e2e.sh++ - name: Onboard the stack and create an API key+ run: |+ set -euo pipefail+ base_url="http://localhost:3000"++ payload="$(jq -n \+ --arg llm_provider "$LLM_PROVIDER" \+ --arg embedding_provider "$EMBEDDING_PROVIDER" \+ --arg llm_model "$LLM_MODEL" \+ --arg embedding_model "$EMBEDDING_MODEL" \+ --arg azure_key "${AZURE_OPENAI_API_KEY:-}" \+ --arg azure_base "${AZURE_OPENAI_ENDPOINT:-}" \+ --arg azure_version "${AZURE_OPENAI_API_VERSION:-}" \+ '{llm_provider: $llm_provider, embedding_provider: $embedding_provider,+ llm_model: $llm_model, embedding_model: $embedding_model}+ + (if $azure_key != "" and $azure_base != ""+ then {provider_credentials: {azure: {api_key: $azure_key, api_base: $azure_base, api_version: $azure_version}}}+ else {} end)')"++ status="$(curl -sS --max-time 120 -o /tmp/onboarding.json -w '%{http_code}' \+ -X POST "${base_url}/api/onboarding" -H 'Content-Type: application/json' -d "$payload")"+ if [ "$status" != "200" ] && [ "$status" != "204" ]; then+ echo "::error::Onboarding failed with HTTP ${status}"+ exit 1+ fi++ api_key="$(curl -sS --max-time 60 --fail \+ -X POST "${base_url}/api/keys" -H 'Content-Type: application/json' \+ -d '{"name": "Retrieval Eval"}' | jq -r '.api_key')"+ if [ -z "$api_key" ] || [ "$api_key" = "null" ]; then+ echo "::error::Could not create an API key"+ exit 1+ fi+ echo "::add-mask::${api_key}"+ echo "OPENRAG_API_KEY=${api_key}" >> "$GITHUB_ENV"++ - name: Run the retrieval eval+ env:+ OPENRAG_URL: http://localhost:3000+ MAX_RECALL_DROP: ${{ inputs.max_recall_drop }}+ run: |+ set -euo pipefail+ if [ -f scripts/eval/retrieval/baseline.json ]; then+ args=""+ if [ -n "$MAX_RECALL_DROP" ]; then+ args="--max-recall-drop ${MAX_RECALL_DROP}"+ fi+ make eval-run BASELINE=baseline.json EVAL_ARGS="${args}"+ else+ echo "::warning::scripts/eval/retrieval/baseline.json does not exist yet, so nothing was compared. Run with update_baseline on a branch to create it."+ make eval-run+ fi++ - name: Publish summary+ if: ${{ always() }}+ run: |+ if [ -f scripts/eval/retrieval/results/summary.md ]; then+ cat scripts/eval/retrieval/results/summary.md >> "$GITHUB_STEP_SUMMARY"+ else+ echo "The eval did not produce a summary." >> "$GITHUB_STEP_SUMMARY"+ fi++ - name: Update the tracked baseline+ if: ${{ inputs.update_baseline }}+ env:+ MIN_GAIN: ${{ inputs.min_gain || '0.025' }}+ REF_NAME: ${{ github.ref_name }}+ run: |+ # Exit status 3 means "not an improvement", which is not a failure, so+ # read the status instead of letting the shell stop on it.+ set +e+ cd scripts/eval/retrieval+ uv run run_eval.py promote results/results.json baseline.json --min-gain "$MIN_GAIN" > /tmp/promote.txt+ status=$?+ set -e+ cat /tmp/promote.txt+ {+ echo "### Baseline"+ cat /tmp/promote.txt+ } >> "$GITHUB_STEP_SUMMARY"+ if [ "$status" = "3" ]; then+ exit 0+ elif [ "$status" != "0" ]; then+ exit "$status"+ fi+ cd "$GITHUB_WORKSPACE"+ git config user.name "github-actions[bot]"+ git config user.email "41898282+github-actions[bot]@users.noreply.github.com"+ git add scripts/eval/retrieval/baseline.json+ git commit -m "chore: update the retrieval eval baseline"+ git push origin "HEAD:refs/heads/${REF_NAME}"++ - name: Upload results+ if: ${{ always() }}+ uses: actions/upload-artifact@v7+ with:+ name: retrieval-eval-results+ path: scripts/eval/retrieval/results/+ if-no-files-found: ignore+ retention-days: 90++ - name: Collect service logs on failure+ if: failure()+ run: |+ mkdir -p service-logs+ docker logs openrag-opensearch > service-logs/opensearch.log 2>&1 || true+ docker logs openrag-backend > service-logs/backend.log 2>&1 || true+ docker logs openrag-frontend > service-logs/frontend.log 2>&1 || true+ docker logs openrag-langflow > service-logs/langflow.log 2>&1 || true+ cp ~/.openrag/tui/docling-serve.log service-logs/docling.log 2>&1 || true++ - name: Upload service logs+ if: failure()+ uses: actions/upload-artifact@v7+ with:+ name: eval-service-logs+ path: service-logs/+ retention-days: 7++ - name: Teardown infrastructure+ if: ${{ always() }}+ run: |+ make docling-stop || true+ docker rm -f openrag-backend-proxy 2>/dev/null || true+ make clean || true+ docker system prune -f || truediff --git a/scripts/eval/retrieval/README.md b/scripts/eval/retrieval/README.mdindex 7ce8db3c4..15a8aebf0 100644--- a/scripts/eval/retrieval/README.md+++ b/scripts/eval/retrieval/README.md@@ -149,6 +149,9 @@ regressed first, then the ones that improved, then the change in each average.For every changed question it shows the rank of each expected file before andafter.+To make a run exit with status 1 when average recall falls too far, add+`--max-recall-drop 0.02` (two points) next to `--baseline`. It is off by default.+To compare two results files without running anything:```bash@@ -182,9 +185,74 @@ To tell a real change from this noise:question.- For the steadiest numbers, use a stack that holds no other documents.-No baseline is committed. Scores depend on the embedding model and settings of-the stack that produced them, so create the baseline on the same stack you test-against.+Scores depend on the embedding model and settings of the stack that produced+them, so create the baseline on the same stack you test against. The GitHub+Actions workflow keeps a tracked baseline for its own stack; see+[Run it from GitHub Actions](#run-it-from-github-actions).++## Run it from GitHub Actions++The **Retrieval Eval** workflow ([`eval-retrieval.yml`](../../../.github/workflows/eval-retrieval.yml))+runs the harness on a fresh stack built from the branch you pick and compares+it with the tracked `baseline.json` in this directory, which exists once the+first run has created it. It runs only when someone+starts it. It never runs on a push or a pull request, and it is not a required+check.++To start it, open the **Actions** tab, choose **Retrieval Eval**, click **Run+workflow**, pick a branch and fill in the inputs. Or from the command line:++```bash+gh workflow run eval-retrieval.yml --ref my-branch+```++GitHub lists a manually started workflow only once its file is on the default+branch, so this works after the workflow has been merged.++| Input | Meaning |+|---|---|+| `max_recall_drop` | Fail the run if average recall at the compared cutoff falls by more than this below the baseline, for example `0.02` for two points. Leave it empty to report without failing. |+| `update_baseline` | If the run improves on the baseline, commit the new `baseline.json` to the selected branch. Refused on the default branch. |+| `min_gain` | Average recall must rise by more than this to count as an improvement. Default `0.025`. One of the 83 questions flipping moves average recall by about 0.012, so a lower value lets noise raise the baseline. |++Each run writes the summary to the run's page and uploads `results.json` and+`summary.md` as the `retrieval-eval-results` artifact, kept for 90 days.++### The tracked baseline++`baseline.json` is committed to the repository and is the reference for every+run. It changes only through `update_baseline`:++1. Run the workflow on your branch with `update_baseline` ticked.+2. If average recall rose by more than `min_gain`, the workflow commits the new+ `baseline.json` to your branch. Otherwise it leaves the file alone and says+ why in the run summary.+3. Merge the branch in a pull request. The reviewer sees the new scores in the+ diff.++The workflow never commits to the default branch, so a baseline change is+always reviewed. A run is not promoted when it did not record the stack settings, or+when it is not like for like with the baseline (a different corpus, question file, cutoffs or stack settings),+because a higher number would not mean better search. To start over after such+a change, for example a new embedding model, delete `baseline.json` and run+with `update_baseline`. With no baseline, the first run becomes the baseline.++The same logic is available locally:++```bash+uv run run_eval.py promote results/results.json baseline.json --min-gain 0.025+```++It exits with 0 when it wrote the baseline and 3 when it left it alone.++The stack uses the embedding provider and model set by the repository's+`EMBEDDING_PROVIDER` and `EMBEDDING_MODEL` secrets, the same as the end-to-end+tests. Scores from different models are not comparable, and the comparison+warns when the settings of the two runs differ.++The harness gives the same stack slightly different scores from run to run (see+[Repeatability](#repeatability)). Set `max_recall_drop` and `min_gain` above that+noise, or runs will fail, or the baseline will creep up, on unchanged code.## How questions are scoreddiff --git a/scripts/eval/retrieval/report.py b/scripts/eval/retrieval/report.pyindex 30f749463..952a703c9 100644--- a/scripts/eval/retrieval/report.py+++ b/scripts/eval/retrieval/report.py@@ -249,6 +249,64 @@ def diff_results(}+def recall_drop(diff: Mapping[str, Any]) -> float:+ """How far the average recall fell at the compared cutoff; 0 if it held or rose."""+ name = diff["compare_metrics"][0]+ return max(0.0, -diff["aggregate"][name]["delta"])+++def recall_gate_failure(diff: Mapping[str, Any], max_drop: float) -> str | None:+ """Why the recall gate fails, or None when it passes.++ The gate fails when the runs are not like for like, because a score that+ held up between two different setups says nothing about the search.+ """+ if diff["warnings"]:+ return "the runs are not comparable, so recall cannot be checked: " + " ".join(+ diff["warnings"]+ )+ drop = recall_drop(diff)+ if drop > max_drop:+ return (+ f"average {diff['compare_metrics'][0]} fell by {drop:.3f}, "+ f"more than the allowed {max_drop:.3f}"+ )+ return None+++def promotion_decision(+ baseline: Mapping[str, Any] | None,+ current: Mapping[str, Any],+ min_gain: float = 0.0,+) -> tuple[bool, str]:+ """Whether ``current`` should replace ``baseline`` as the reference.++ Yes when there is no baseline yet, or when average recall at the compared+ cutoff rose by more than ``min_gain``. A pair that is not like for like+ (different corpus, questions, cutoffs or stack settings) is never promoted,+ because the gain would not mean the search got better. A run that did not+ record the stack settings is never promoted either, since later runs could+ not tell whether they match it.+ """+ if not current.get("settings"):+ return False, "this run did not record the stack settings, so it cannot be a baseline"+ if baseline is None:+ return True, "no baseline yet"+ if not baseline.get("settings"):+ return (+ False,+ "the baseline did not record the stack settings, so the runs cannot be compared",+ )+ diff = diff_results(baseline, current)+ if diff["warnings"]:+ return False, "the runs are not comparable: " + " ".join(diff["warnings"])+ name = diff["compare_metrics"][0]+ gain = diff["aggregate"][name]["delta"]+ if gain > min_gain:+ return True, f"average {name} rose by {gain:.3f}"+ return False, f"average {name} changed by {gain:+.3f}, which is not above {min_gain:.3f}"++def _cell(text: object) -> str:return str(text).replace("|", "\\|").replace("\n", " ")diff --git a/scripts/eval/retrieval/run_eval.py b/scripts/eval/retrieval/run_eval.pyindex 66e491ba4..145084de5 100644--- a/scripts/eval/retrieval/run_eval.py+++ b/scripts/eval/retrieval/run_eval.py@@ -15,6 +15,7 @@ from __future__ import annotationsimport argparseimport asyncioimport io+import jsonimport osimport subprocessimport sys@@ -65,6 +66,14 @@ def build_parser() -> argparse.ArgumentParser:run.add_argument("--queries", type=Path, default=HERE / "queries.jsonl")run.add_argument("--output-dir", type=Path, default=HERE / "results")run.add_argument("--baseline", type=Path, help="results file to compare this run against")+ run.add_argument(+ "--max-recall-drop",+ type=float,+ help=(+ "with --baseline, exit with status 1 when the average recall at the compared "+ "cutoff falls by more than this (0.02 = two points). Off by default."+ ),+ )run.add_argument("--k",dest="k_values",@@ -104,6 +113,18 @@ def build_parser() -> argparse.ArgumentParser:compare = commands.add_parser("compare", help="diff two results files offline")compare.add_argument("baseline", type=Path)compare.add_argument("current", type=Path)++ promote = commands.add_parser(+ "promote", help="replace the baseline with a results file, only if it improves on it"+ )+ promote.add_argument("current", type=Path, help="results file of the new run")+ promote.add_argument("baseline", type=Path, help="baseline file to update (may not exist yet)")+ promote.add_argument(+ "--min-gain",+ type=float,+ default=0.0,+ help="average recall must rise by more than this (default: 0, any rise)",+ )return parser@@ -391,6 +412,11 @@ async def run(args: argparse.Namespace) -> int:print("Wrote:")for path in written:print(f" {path}")+ if diff is not None and args.max_recall_drop is not None:+ failure = report.recall_gate_failure(diff, args.max_recall_drop)+ if failure:+ print(f"error: {failure}", file=sys.stderr)+ return 1return 0@@ -403,9 +429,32 @@ def compare(args: argparse.Namespace) -> int:return 0+NOT_PROMOTED = 3+++def promote(args: argparse.Namespace) -> int:+ current = report.load_results(args.current)+ baseline = report.load_results(args.baseline) if args.baseline.exists() else None+ promoted, reason = report.promotion_decision(baseline, current, args.min_gain)+ if not promoted:+ print(f"Baseline not updated: {reason}")+ return NOT_PROMOTED+ args.baseline.parent.mkdir(parents=True, exist_ok=True)+ args.baseline.write_text(json.dumps(current, indent=2) + "\n", encoding="utf-8")+ print(f"Baseline updated ({reason}): {args.baseline}")+ return 0++def main(argv: list[str] | None = None) -> int:parser = build_parser()args = parser.parse_args(argv)+ if args.command == "promote" and args.min_gain < 0:+ parser.error("--min-gain must not be negative")+ if args.command == "run" and args.max_recall_drop is not None:+ if args.max_recall_drop < 0:+ parser.error("--max-recall-drop must not be negative")+ if args.baseline is None:+ parser.error("--max-recall-drop needs --baseline to compare against")if args.command == "run" and args.fetch_limit < max(args.k_values):# Fewer chunks than the largest cutoff cannot fill that many file# ranks, so recall@k and nDCG@k at that cutoff would be meaningless.@@ -416,6 +465,8 @@ def main(argv: list[str] | None = None) -> int:try:if args.command == "compare":return compare(args)+ if args.command == "promote":+ return promote(args)return asyncio.run(run(args))except (dataset.DatasetError, report.ResultsError, EvalError) as exc:print(f"error: {exc}", file=sys.stderr)diff --git a/tests/unit/test_retrieval_eval.py b/tests/unit/test_retrieval_eval.pyindex ed66a5a10..776e03188 100644--- a/tests/unit/test_retrieval_eval.py+++ b/tests/unit/test_retrieval_eval.py@@ -606,3 +606,174 @@ def test_run_rejects_a_fetch_limit_below_the_largest_cutoff(fetch_limit: str) ->assert completed.returncode == 2assert "--fetch-limit must be at least the largest --k value (5)" in completed.stderr+++# --- recall-drop gate --------------------------------------------------------+++ ("current_recall", "expected_drop"),+ [(0.0, 1.0), (1.0, 0.0)],+)+def test_recall_drop_reports_a_fall_and_ignores_a_gain(+ current_recall: float, expected_drop: float+) -> None:+ baseline = make_results([make_query("q", 1.0, 1.0, 1.0, 1)])+ current = make_results([make_query("q", current_recall, 1.0, 1.0, 1)])++ assert report.recall_drop(report.diff_results(baseline, current)) == pytest.approx(+ expected_drop+ )++ improved = make_results([make_query("q", 0.0, 0.0, 0.0, None)])+ gain = make_results([make_query("q", 1.0, 1.0, 1.0, 1)])+ assert report.recall_drop(report.diff_results(improved, gain)) == 0.0+++ ("extra_args", "message"),+ [+ (["--max-recall-drop", "0.02"], "--max-recall-drop needs --baseline"),+ (["--max-recall-drop", "-0.1", "--baseline", "b.json"], "must not be negative"),+ ],+)+def test_run_rejects_an_unusable_max_recall_drop(extra_args: list[str], message: str) -> None:+ script = Path(__file__).resolve().parents[2] / "scripts/eval/retrieval/run_eval.py"+ completed = subprocess.run(+ [sys.executable, str(script), "run", "--validate-only", *extra_args],+ capture_output=True,+ text=True,+ check=False,+ )++ assert completed.returncode == 2+ assert message in completed.stderr+++# --- baseline promotion ----------------------------------------------------+++def _results_with_recall(recall: float, **run_overrides: Any) -> dict[str, Any]:+ return make_results([make_query("q", recall, recall, recall, 1)], **run_overrides)+++def test_promotion_accepts_the_first_baseline() -> None:+ promoted, reason = report.promotion_decision(None, _results_with_recall(0.5))++ assert promoted+ assert "no baseline" in reason+++ ("baseline_recall", "current_recall", "min_gain", "expected"),+ [+ (0.5, 0.75, 0.0, True),+ (0.5, 0.5, 0.0, False),+ (0.75, 0.5, 0.0, False),+ (0.5, 0.55, 0.1, False),+ (0.5, 0.75, 0.1, True),+ ],+)+def test_promotion_requires_a_gain_above_the_minimum(+ baseline_recall: float, current_recall: float, min_gain: float, expected: bool+) -> None:+ promoted, _ = report.promotion_decision(+ _results_with_recall(baseline_recall), _results_with_recall(current_recall), min_gain+ )++ assert promoted is expected+++def test_promotion_refuses_runs_that_are_not_comparable() -> None:+ promoted, reason = report.promotion_decision(+ _results_with_recall(0.5), _results_with_recall(1.0, corpus_hash="e" * 64)+ )++ assert not promoted+ assert "not comparable" in reason+++def _run_cli(*args: str) -> subprocess.CompletedProcess[str]:+ script = Path(__file__).resolve().parents[2] / "scripts/eval/retrieval/run_eval.py"+ return subprocess.run(+ [sys.executable, str(script), *args], capture_output=True, text=True, check=False+ )+++def test_promote_writes_the_baseline_only_on_improvement(tmp_path: Path) -> None:+ baseline = tmp_path / "baseline.json"+ low, high = tmp_path / "low.json", tmp_path / "high.json"+ low.write_text(json.dumps(_results_with_recall(0.5)), encoding="utf-8")+ high.write_text(json.dumps(_results_with_recall(1.0)), encoding="utf-8")++ first = _run_cli("promote", str(low), str(baseline))+ assert first.returncode == 0+ assert json.loads(baseline.read_text(encoding="utf-8")) == json.loads(low.read_text())++ same = _run_cli("promote", str(low), str(baseline))+ assert same.returncode == 3+ assert "Baseline not updated" in same.stdout++ better = _run_cli("promote", str(high), str(baseline))+ assert better.returncode == 0+ assert json.loads(baseline.read_text(encoding="utf-8")) == json.loads(high.read_text())++ worse = _run_cli("promote", str(low), str(baseline))+ assert worse.returncode == 3+ assert json.loads(baseline.read_text(encoding="utf-8")) == json.loads(high.read_text())+++# --- review follow-ups -------------------------------------------------------+++def test_recall_gate_fails_when_the_runs_are_not_comparable() -> None:+ baseline = _results_with_recall(0.5)+ current = _results_with_recall(0.5, corpus_hash="e" * 64)++ failure = report.recall_gate_failure(report.diff_results(baseline, current), 0.02)++ assert failure is not None+ assert "not comparable" in failure+++ ("current_recall", "fails"),+ [(0.5, False), (0.49, False), (0.45, True), (1.0, False)],+)+def test_recall_gate_fails_only_on_a_drop_above_the_limit(+ current_recall: float, fails: bool+) -> None:+ diff = report.diff_results(_results_with_recall(0.5), _results_with_recall(current_recall))++ assert (report.recall_gate_failure(diff, 0.02) is not None) is fails++[email protected]("missing", ["current", "baseline"])+def test_promotion_refuses_a_run_without_recorded_settings(missing: str) -> None:+ baseline, current = _results_with_recall(0.5), _results_with_recall(1.0)+ (current if missing == "current" else baseline)["settings"] = None++ promoted, reason = report.promotion_decision(baseline, current)++ assert not promoted+ assert "settings" in reason+++def test_promotion_refuses_a_first_baseline_without_recorded_settings() -> None:+ current = _results_with_recall(0.5)+ current["settings"] = None++ promoted, reason = report.promotion_decision(None, current)++ assert not promoted+ assert "settings" in reason+++def test_promote_rejects_a_negative_min_gain(tmp_path: Path) -> None:+ results = tmp_path / "results.json"+ results.write_text(json.dumps(_results_with_recall(0.5)), encoding="utf-8")++ completed = _run_cli("promote", str(results), str(tmp_path / "b.json"), "--min-gain", "-0.1")++ assert completed.returncode == 2+ assert "--min-gain must not be negative" in completed.stderr