Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat(eval): check and clean up hosted environments for the retrieval eval (#2478) (#2579)

Nathan Bakareed857f2d394a
diff --git a/Makefile b/Makefile
index 276069cd5..5f1851fd5 100644
--- a/Makefile
+++ b/Makefile
@@ -115,7 +115,7 @@ endef
instana-agent-up instana-agent-down \
shell-backend shell-frontend install \
test test-unit test-integration test-ci test-ci-local test-ci-suite test-sdk test-os-jwt lint \
- eval-validate eval-run eval-baseline eval-compare \
+ eval-validate eval-check eval-cleanup eval-run eval-baseline eval-compare \
ci-build-images ci-save-images \
backend frontend docling docling-stop install-be install-fe build-be build-fe build-os build-lf logs-be logs-fe logs-lf logs-os \
shell-be shell-lf shell-os restart status health db-reset clear-os-data flow-upload setup factory-reset \
@@ -363,6 +363,8 @@ help_test: ## Show testing commands
@echo ''
@echo "$(PURPLE)Retrieval Evaluation:$(NC)"
@echo " $(PURPLE)make eval-validate$(NC) - Check the eval corpus and questions (no stack needed)"
+ @echo " $(PURPLE)make eval-check$(NC) - Check that OPENRAG_URL is ready for a run, without uploading anything"
+ @echo " $(PURPLE)make eval-cleanup$(NC) - List documents interrupted runs left behind (CONFIRM=1 to delete them)"
@echo " $(PURPLE)make eval-run$(NC) - Score search quality (requires running OpenRAG and OPENRAG_API_KEY)"
@echo " (BASELINE=results/baseline.json to compare with an earlier run)"
@echo " $(PURPLE)make eval-baseline$(NC) - Run the eval and save it as results/baseline.json"
@@ -1367,6 +1369,12 @@ CURRENT ?= $(OUTPUT_DIR)/results.json
eval-validate: ## Check the retrieval eval corpus and questions (no stack needed)
cd $(EVAL_DIR) && uv run run_eval.py run --validate-only $(EVAL_ARGS)
+eval-check: ## Check that the stack at OPENRAG_URL is ready for a retrieval eval run (uploads nothing)
+ cd $(EVAL_DIR) && uv run run_eval.py check $(EVAL_ARGS)
+
+eval-cleanup: ## List documents interrupted retrieval eval runs left behind; CONFIRM=1 to delete them
+ cd $(EVAL_DIR) && uv run run_eval.py cleanup $(if $(filter 1 true yes,$(CONFIRM)),--yes) $(EVAL_ARGS)
+
eval-run: ## Run the retrieval eval (requires running OpenRAG and OPENRAG_API_KEY); BASELINE=file to compare
cd $(EVAL_DIR) && uv run run_eval.py run --output-dir $(OUTPUT_DIR) $(if $(BASELINE),--baseline $(BASELINE)) $(EVAL_ARGS)
diff --git a/scripts/eval/retrieval/README.md b/scripts/eval/retrieval/README.md
index 20bd4b40c..a995fb2d7 100644
--- a/scripts/eval/retrieval/README.md
+++ b/scripts/eval/retrieval/README.md
@@ -41,6 +41,8 @@ Paths are relative to this directory.
| `make` target | Same as |
|---|---|
| `make eval-validate` | `uv run run_eval.py run --validate-only` |
+| `make eval-check` | `uv run run_eval.py check` |
+| `make eval-cleanup` | `uv run run_eval.py cleanup` (add `CONFIRM=1` for `--yes`) |
| `make eval-run` | `uv run run_eval.py run` |
| `make eval-run BASELINE=results/baseline.json` | `uv run run_eval.py run --baseline results/baseline.json` |
| `make eval-baseline` | `uv run run_eval.py run`, then copy the results to `results/baseline.json` |
@@ -62,7 +64,7 @@ The rest of this file explains each step.
|---|---|
| `corpus/` | 21 Markdown documents: the handbook of a fictional product, "Relaywise" |
| `queries.jsonl` | 83 hand-labeled questions, one JSON object per line |
-| `run_eval.py` | The command-line tool, with `run` and `compare` commands |
+| `run_eval.py` | The command-line tool, with `run`, `check`, `cleanup`, `compare` and `promote` commands |
| `dataset.py` | Loads and validates the corpus and questions |
| `metrics.py` | Turns ranked chunks into ranked files and scores them |
| `report.py` | Writes results, the summary and the baseline diff |
@@ -116,7 +118,8 @@ A run does the following:
Documents that were already in the stack are never searched, changed or
deleted. If a run is interrupted before cleanup, delete the leftover
-`eval-<run id>-*` files from the knowledge page.
+`eval-<run id>-*` files with `run_eval.py cleanup` (see
+[Clean up after an interrupted run](#clean-up-after-an-interrupted-run)).
Useful options:
@@ -133,6 +136,71 @@ Useful options:
Run `uv run run_eval.py run --help` for the full list, including timeouts.
+## Run against a hosted environment
+
+The harness runs from your machine against any stack it can reach, so a QA or
+staging environment needs no GitHub Actions. Point it at the environment and
+check it first:
+
+```bash
+cd scripts/eval/retrieval
+export OPENRAG_URL=https://openrag-qa.example.com
+export OPENRAG_API_KEY=orag_key_for_that_environment
+
+uv run run_eval.py check
+```
+
+`check` uploads nothing. It confirms that the stack answers and accepts the
+key, that it can read the embedding and chunk settings, and whether earlier
+runs left documents behind:
+
+```text
+Checking https://openrag-qa.example.com
+ ok dataset: 21 documents, 83 questions
+ ok search: the stack answered and accepted the API key
+ ok settings: watsonx ibm/granite-embedding-278m-multilingual, chunk size 1000, overlap 200
+ ok documents: no earlier run left documents behind
+Not checked: uploading and deleting documents (knowledge:upload, knowledge:delete:own) need a real run.
+Ready for a run.
+```
+
+It exits with status 1 when a run would fail or its results could not be
+compared, and says what to fix. A run needs the settings: without them its
+results cannot be compared with a baseline or become one.
+
+Keep a separate baseline for each environment, because scores from different
+embedding models or settings are not comparable. Write each environment's
+results to its own directory:
+
+```bash
+# Once, to record where the environment stands:
+make eval-baseline OUTPUT_DIR=results/qa
+
+# After each deploy to that environment:
+make eval-run OUTPUT_DIR=results/qa BASELINE=results/qa/baseline.json
+```
+
+The tracked `baseline.json` belongs to the GitHub workflow's stack. Do not
+compare a hosted environment with it.
+
+If the environment uses a certificate from a private CA, set `SSL_CERT_FILE` to
+the CA bundle before running.
+
+### Clean up after an interrupted run
+
+A run deletes its documents when it ends, even when it fails. If it is killed
+first, they stay in the stack. On a stack other people use, list them with:
+
+```bash
+uv run run_eval.py cleanup # or: make eval-cleanup
+uv run run_eval.py cleanup --yes # or: make eval-cleanup CONFIRM=1
+```
+
+`cleanup` only matches files named like an eval upload (`eval-<run id>-` plus a
+corpus filename), and only those the API key can see. It keeps any document
+indexed in the last 60 minutes, because it may belong to a run still in
+progress, possibly someone else's. Use `--min-age` to change that window.
+
## Compare before and after a change
```bash
diff --git a/scripts/eval/retrieval/dataset.py b/scripts/eval/retrieval/dataset.py
index eed2d750f..f63e0db23 100644
--- a/scripts/eval/retrieval/dataset.py
+++ b/scripts/eval/retrieval/dataset.py
@@ -11,7 +11,10 @@ from __future__ import annotations
import hashlib
import json
+import re
+from collections.abc import Collection, Iterable
from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
from pathlib import Path
REQUIRED_FIELDS = frozenset({"query", "expected_files"})
@@ -193,3 +196,53 @@ def hash_corpus(corpus_dir: Path, corpus_names: list[str]) -> str:
digest.update(hash_file(corpus_dir / name).encode("ascii"))
digest.update(b"\n")
return digest.hexdigest()
+
+
+# Every run uploads its documents as ``eval-<8 hex digits>-<corpus filename>``.
+RUN_ID_PATTERN = re.compile(r"^(eval-[0-9a-f]{8})-(.+)$")
+
+
+@dataclass(frozen=True)
+class Leftover:
+ """A document an earlier run uploaded and did not delete."""
+
+ filename: str
+ run_id: str
+ indexed_time: datetime | None # None when the stack's timestamp could not be read
+
+
+def _parse_time(value: str | None) -> datetime | None:
+ if not value:
+ return None
+ try:
+ parsed = datetime.fromisoformat(value)
+ except ValueError:
+ return None
+ return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
+
+
+def find_leftovers(
+ files: Iterable[tuple[str, str | None]], corpus_names: Collection[str]
+) -> list[Leftover]:
+ """Pick the eval uploads out of ``(filename, indexed_time)`` pairs from the stack.
+
+ A file counts only when its name is a run prefix followed by a corpus
+ filename, so a user document that merely starts with ``eval-`` is never
+ matched.
+ """
+ leftovers = []
+ for filename, indexed_time in files:
+ match = RUN_ID_PATTERN.match(filename)
+ if match and match.group(2) in corpus_names:
+ leftovers.append(Leftover(filename, match.group(1), _parse_time(indexed_time)))
+ return sorted(leftovers, key=lambda item: (item.run_id, item.filename))
+
+
+def old_enough(leftover: Leftover, now: datetime, min_age: timedelta) -> bool:
+ """Whether a leftover is old enough to delete safely.
+
+ A recent upload may belong to a run that is still going, possibly someone
+ else's on a shared stack, and a file without a readable timestamp cannot be
+ shown to be old, so both are kept.
+ """
+ return leftover.indexed_time is not None and now - leftover.indexed_time >= min_age
diff --git a/scripts/eval/retrieval/run_eval.py b/scripts/eval/retrieval/run_eval.py
index 5d5186fb4..7c584f300 100644
--- a/scripts/eval/retrieval/run_eval.py
+++ b/scripts/eval/retrieval/run_eval.py
@@ -2,12 +2,16 @@
"""Measure OpenRAG retrieval quality against a fixed corpus and labeled questions.
uv run run_eval.py run [--baseline results/baseline.json]
+ uv run run_eval.py check
+ uv run run_eval.py cleanup [--yes]
uv run run_eval.py compare <baseline.json> <current.json>
``run`` ingests the checked-in corpus under run-unique filenames, sends every
question through ``/v1/search``, scores the ranking by file, writes results and
-a summary, and deletes the documents it uploaded. ``compare`` diffs two results
-files offline. See README.md.
+a summary, and deletes the documents it uploaded. ``check`` confirms a stack is
+ready for a run without uploading anything, and ``cleanup`` removes documents
+that interrupted runs left behind. ``compare`` diffs two results files offline.
+See README.md.
"""
from __future__ import annotations
@@ -21,7 +25,7 @@ import subprocess
import sys
import time
import uuid
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
@@ -55,14 +59,26 @@ def build_parser() -> argparse.ArgumentParser:
)
commands = parser.add_subparsers(dest="command", required=True)
- run = commands.add_parser("run", help="ingest the corpus, run every question and score it")
- run.add_argument("--url", default=os.environ.get("OPENRAG_URL", DEFAULT_URL))
- run.add_argument(
+ # Options shared by every command that talks to a stack.
+ stack = argparse.ArgumentParser(add_help=False)
+ stack.add_argument(
+ "--url",
+ default=os.environ.get("OPENRAG_URL", DEFAULT_URL),
+ help=f"defaults to the OPENRAG_URL environment variable, then {DEFAULT_URL}",
+ )
+ stack.add_argument(
"--api-key",
default=os.environ.get("OPENRAG_API_KEY"),
help="defaults to the OPENRAG_API_KEY environment variable",
)
- run.add_argument("--corpus", type=Path, default=HERE / "corpus")
+ stack.add_argument(
+ "--request-timeout", type=float, default=60.0, help="seconds per HTTP request"
+ )
+ stack.add_argument("--corpus", type=Path, default=HERE / "corpus")
+
+ run = commands.add_parser(
+ "run", parents=[stack], help="ingest the corpus, run every question and score it"
+ )
run.add_argument("--queries", type=Path, default=HERE / "queries.jsonl")
run.add_argument("--output-dir", type=Path, default=HERE / "results")
run.add_argument("--baseline", type=Path, help="results file to compare this run against")
@@ -98,7 +114,6 @@ def build_parser() -> argparse.ArgumentParser:
default=120.0,
help="seconds to wait for an ingested file to become searchable",
)
- run.add_argument("--request-timeout", type=float, default=60.0, help="seconds per HTTP request")
run.add_argument(
"--keep-documents",
action="store_true",
@@ -110,6 +125,31 @@ def build_parser() -> argparse.ArgumentParser:
help="check the corpus and question file, then exit without contacting a stack",
)
+ check = commands.add_parser(
+ "check",
+ parents=[stack],
+ help="check that a stack is ready for a run, without uploading anything",
+ )
+ check.add_argument("--queries", type=Path, default=HERE / "queries.jsonl")
+
+ cleanup = commands.add_parser(
+ "cleanup",
+ parents=[stack],
+ help="list, and with --yes delete, documents that interrupted runs left behind",
+ )
+ cleanup.add_argument(
+ "--yes", action="store_true", help="delete the listed documents (default: list only)"
+ )
+ cleanup.add_argument(
+ "--min-age",
+ type=float,
+ default=60.0,
+ help=(
+ "only delete documents indexed at least this many minutes ago, so a run "
+ "still in progress is left alone (default: 60)"
+ ),
+ )
+
compare = commands.add_parser("compare", help="diff two results files offline")
compare.add_argument("baseline", type=Path)
compare.add_argument("current", type=Path)
@@ -429,6 +469,164 @@ async def run(args: argparse.Namespace) -> int:
return 0
+def _explain(exc: Exception, url: str) -> str:
+ """A reason for a failed request that says what to fix."""
+ status = getattr(exc, "status_code", None)
+ text = str(exc)
+ if status == 401:
+ return "the API key was rejected; create a new one under Settings → API Keys"
+ if status == 403:
+ return f"the API key lacks a permission this needs ({text})"
+ if "CERTIFICATE_VERIFY_FAILED" in text:
+ return (
+ f"the TLS certificate of {url} is not trusted; point SSL_CERT_FILE at the "
+ "CA bundle that signed it"
+ )
+ if type(exc).__name__ in ("ConnectError", "ConnectTimeout"):
+ return f"cannot reach {url}: {text or type(exc).__name__}"
+ return f"{type(exc).__name__}: {text}"
+
+
+async def _list_leftovers(client: Any, corpus_names: list[str]) -> list[dataset.Leftover]:
+ """Every document the stack lists for this key that an earlier run uploaded."""
+ files: list[tuple[str, str | None]] = []
+ after_key: str | None = None
+ while True:
+ page = await client.documents.list_files(search="eval-", page_size=500, after_key=after_key)
+ files += [(record.filename, record.indexed_time) for record in page.files]
+ if not page.files or not page.after_key:
+ break
+ after_key = json.dumps(page.after_key)
+ return dataset.find_leftovers(files, corpus_names)
+
+
+def _describe_runs(leftovers: list[dataset.Leftover]) -> str:
+ runs = sorted({item.run_id for item in leftovers})
+ noun = "run" if len(runs) == 1 else "runs"
+ return f"{len(leftovers)} documents from {len(runs)} earlier {noun}"
+
+
+async def check(args: argparse.Namespace) -> int:
+ """Check that a stack is ready for a run without uploading anything."""
+ corpus_names = dataset.list_corpus(args.corpus)
+ queries = dataset.load_queries(args.queries, corpus_names, args.corpus)
+ if not args.api_key:
+ raise EvalError("no API key: set OPENRAG_API_KEY or pass --api-key")
+
+ from openrag_sdk import OpenRAGClient, SearchFilters
+
+ target_url = report.redact_url(args.url)
+ print(f"Checking {target_url}")
+ print(f" ok dataset: {len(corpus_names)} documents, {len(queries)} questions")
+ failed = False
+ async with OpenRAGClient(
+ api_key=args.api_key, base_url=args.url, timeout=args.request_timeout
+ ) as client:
+ try:
+ # Scoped to a name no run uses, so nothing is returned.
+ await client.search.query(
+ "*", filters=SearchFilters(data_sources=["eval-check-0000"]), limit=1
+ )
+ except Exception as exc:
+ print(f" FAIL search: {_explain(exc, target_url)}")
+ return 1
+ print(" ok search: the stack answered and accepted the API key")
+
+ settings, settings_note = await _settings_snapshot(client)
+ if settings is None:
+ failed = True
+ print(
+ f" FAIL settings: {settings_note}. A run would still score, but its "
+ "results could not be compared with a baseline or become one; the key "
+ "needs providers:read"
+ )
+ else:
+ print(
+ f" ok settings: {settings.get('embedding_provider')} "
+ f"{settings.get('embedding_model')}, chunk size {settings.get('chunk_size')}, "
+ f"overlap {settings.get('chunk_overlap')}"
+ )
+
+ try:
+ leftovers = await _list_leftovers(client, corpus_names)
+ except Exception as exc:
+ failed = True
+ print(f" FAIL documents: cannot list documents: {_explain(exc, target_url)}")
+ else:
+ if leftovers:
+ print(
+ f" note documents: {_describe_runs(leftovers)} are still in the stack; "
+ "`run_eval.py cleanup` lists them"
+ )
+ else:
+ print(" ok documents: no earlier run left documents behind")
+
+ print(
+ "Not checked: uploading and deleting documents (knowledge:upload, "
+ "knowledge:delete:own) need a real run."
+ )
+ print("Not ready for a run." if failed else "Ready for a run.")
+ return 1 if failed else 0
+
+
+async def cleanup(args: argparse.Namespace) -> int:
+ """List the documents interrupted runs left behind and, with --yes, delete them."""
+ corpus_names = dataset.list_corpus(args.corpus)
+ if not args.api_key:
+ raise EvalError("no API key: set OPENRAG_API_KEY or pass --api-key")
+
+ from openrag_sdk import OpenRAGClient
+
+ target_url = report.redact_url(args.url)
+ min_age = timedelta(minutes=args.min_age)
+ now = datetime.now(UTC)
+ async with OpenRAGClient(
+ api_key=args.api_key, base_url=args.url, timeout=args.request_timeout
+ ) as client:
+ try:
+ leftovers = await _list_leftovers(client, corpus_names)
+ except Exception as exc:
+ raise EvalError(f"cannot list documents: {_explain(exc, target_url)}") from exc
+ if not leftovers:
+ print(f"No documents from earlier runs in {target_url}.")
+ return 0
+
+ to_delete = [item for item in leftovers if dataset.old_enough(item, now, min_age)]
+ kept = len(leftovers) - len(to_delete)
+ print(f"{_describe_runs(leftovers)} in {target_url}:")
+ for run_id in sorted({item.run_id for item in leftovers}):
+ files = [item for item in leftovers if item.run_id == run_id]
+ times = [item.indexed_time for item in files if item.indexed_time]
+ when = min(times).strftime("%Y-%m-%d %H:%M UTC") if times else "unknown time"
+ action = "delete" if all(item in to_delete for item in files) else "keep"
+ print(f" {run_id} {len(files)} documents, indexed {when} → {action}")
+ if kept:
+ print(
+ f"Keeping {kept} documents indexed less than {args.min_age:g} minutes ago or "
+ "at an unknown time, since their run may still be going. Lower --min-age "
+ "to include them."
+ )
+ if not to_delete:
+ return 0
+ if not args.yes:
+ print(f"Run again with --yes to delete {len(to_delete)} documents.")
+ return 0
+
+ failures = []
+ for item in to_delete:
+ try:
+ response = await client.documents.delete(filename=item.filename)
+ except Exception as exc:
+ failures.append(f"{item.filename}: {_explain(exc, target_url)}")
+ continue
+ if not response.success:
+ failures.append(f"{item.filename}: {response.error or 'not deleted'}")
+ print(f"Deleted {len(to_delete) - len(failures)} of {len(to_delete)} documents.")
+ for failure in failures:
+ print(f"warning: {failure}", file=sys.stderr)
+ return 1 if failures else 0
+
+
def compare(args: argparse.Namespace) -> int:
diff = report.diff_results(
report.load_results(args.baseline),
@@ -476,6 +674,12 @@ def main(argv: list[str] | None = None) -> int:
return compare(args)
if args.command == "promote":
return promote(args)
+ if args.command == "check":
+ return asyncio.run(check(args))
+ if args.command == "cleanup":
+ if args.min_age < 0:
+ parser.error("--min-age must not be negative")
+ return asyncio.run(cleanup(args))
return asyncio.run(run(args))
except (dataset.DatasetError, report.ResultsError, EvalError) as exc:
print(f"error: {exc}", file=sys.stderr)
diff --git a/tests/unit/test_retrieval_eval.py b/tests/unit/test_retrieval_eval.py
index 40886c090..9a3db11aa 100644
--- a/tests/unit/test_retrieval_eval.py
+++ b/tests/unit/test_retrieval_eval.py
@@ -11,7 +11,7 @@ import json
import math
import subprocess
import sys
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
@@ -881,3 +881,66 @@ def test_promote_rejects_a_negative_min_gain(tmp_path: Path) -> None:
assert completed.returncode == 2
assert "--min-gain must not be negative" in completed.stderr
+
+
+# --- hosted environments: leftovers and preflight --------------------------
+
+
+def test_find_leftovers_matches_only_run_uploads_of_corpus_files() -> None:
+ files = [
+ ("eval-abc12345-b.md", "2026-11-03T10:00:00Z"),
+ ("eval-abc12345-a.md", "2026-11-03T10:00:00"),
+ ("eval-0000ffff-c.md", None),
+ ("eval-abc12345-notes.md", "2026-11-03T10:00:00Z"), # not a corpus file
+ ("eval-plan-a.md", "2026-11-03T10:00:00Z"), # a user's document
+ ("EVAL-abc12345-a.md", "2026-11-03T10:00:00Z"),
+ ("my-eval-abc12345-a.md", "2026-11-03T10:00:00Z"),
+ ]
+
+ leftovers = dataset.find_leftovers(files, CORPUS)
+
+ assert [(item.run_id, item.filename) for item in leftovers] == [
+ ("eval-0000ffff", "eval-0000ffff-c.md"),
+ ("eval-abc12345", "eval-abc12345-a.md"),
+ ("eval-abc12345", "eval-abc12345-b.md"),
+ ]
+ # A timestamp without a zone is read as UTC; a missing one stays unknown.
+ assert leftovers[1].indexed_time == datetime(2026, 11, 3, 10, tzinfo=UTC)
+ assert leftovers[0].indexed_time is None
+
+
+ ("indexed_time", "expected"),
+ [
+ ("2026-11-03T09:00:00Z", True), # exactly the minimum age
+ ("2026-11-03T08:00:00+00:00", True),
+ ("2026-11-03T09:30:00Z", False), # may belong to a run still going
+ (None, False),
+ ("not a time", False),
+ ],
+)
+def test_old_enough_keeps_recent_and_undated_uploads(
+ indexed_time: str | None, expected: bool
+) -> None:
+ (leftover,) = dataset.find_leftovers([("eval-abc12345-a.md", indexed_time)], CORPUS)
+ now = datetime(2026, 11, 3, 10, tzinfo=UTC)
+
+ assert dataset.old_enough(leftover, now, timedelta(minutes=60)) is expected
+
+
+ ("argv", "message"),
+ [
+ (["check"], "no API key"),
+ (["cleanup"], "no API key"),
+ (["cleanup", "--min-age", "-5", "--api-key", "x"], "--min-age must not be negative"),
+ ],
+)
+def test_check_and_cleanup_reject_unusable_arguments_before_contacting_a_stack(
+ argv: list[str], message: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.delenv("OPENRAG_API_KEY", raising=False)
+ completed = _run_cli(*argv)
+
+ assert completed.returncode != 0
+ assert message in completed.stderr