Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: add retrieval evaluation harness (#2478) (#2498)

Nathan Bakaree4c4f0181b68
diff --git a/.github/workflows/lint-backend.yml b/.github/workflows/lint-backend.yml
index dbaa3a064..bb5152949 100644
--- a/.github/workflows/lint-backend.yml
+++ b/.github/workflows/lint-backend.yml
@@ -6,6 +6,7 @@ on:
- 'src/**/*.py'
- 'tests/**/*.py'
- 'scripts/ci/**/*.py'
+ - 'scripts/eval/**/*.py'
- 'pyproject.toml'
- 'uv.lock'
- '.github/workflows/lint-backend.yml'
diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml
index 9544e8806..4f2f75f77 100644
--- a/.github/workflows/test-backend.yml
+++ b/.github/workflows/test-backend.yml
@@ -15,6 +15,8 @@ on:
paths:
- 'src/**'
- 'tests/**'
+ # tests/unit/test_retrieval_eval.py covers the harness in this directory.
+ - 'scripts/eval/**'
- 'custom_components/**'
- 'flows/**'
- 'config/**'
diff --git a/Makefile b/Makefile
index ccb06aa7a..955976f1f 100644
--- a/Makefile
+++ b/Makefile
@@ -114,6 +114,7 @@ endef
instana-agent-up instana-agent-down \
shell-backend shell-frontend install \
test test-unit test-integration test-ci test-ci-local test-ci-suite test-sdk test-os-jwt lint \
+ eval-validate eval-run eval-baseline eval-compare \
ci-build-images ci-save-images \
backend frontend docling docling-stop install-be install-fe build-be build-fe build-os build-lf logs-be logs-fe logs-lf logs-os \
shell-be shell-lf shell-os restart status health db-reset clear-os-data flow-upload setup factory-reset \
@@ -345,6 +346,14 @@ help_test: ## Show testing commands
@echo " $(PURPLE)make test-sdk$(NC) - Run SDK integration tests"
@echo " (requires running OpenRAG at localhost:3000)"
@echo ''
+ @echo "$(PURPLE)Retrieval Evaluation:$(NC)"
+ @echo " $(PURPLE)make eval-validate$(NC) - Check the eval corpus and questions (no stack needed)"
+ @echo " $(PURPLE)make eval-run$(NC) - Score search quality (requires running OpenRAG and OPENRAG_API_KEY)"
+ @echo " (BASELINE=results/baseline.json to compare with an earlier run)"
+ @echo " $(PURPLE)make eval-baseline$(NC) - Run the eval and save it as results/baseline.json"
+ @echo " $(PURPLE)make eval-compare$(NC) - Compare two results files offline"
+ @echo " (BASELINE=... CURRENT=..., default baseline vs latest run)"
+ @echo ''
@echo "$(PURPLE)Diagnostic Tests:$(NC)"
@echo " $(PURPLE)make test-os-jwt$(NC) - Test JWT authentication against OpenSearch"
@echo " (requires running OpenSearch)"
@@ -1333,6 +1342,27 @@ test-sdk: ## Run SDK integration tests (requires running OpenRAG at localhost:30
cd sdks/typescript && npm install && npm run build && OPENRAG_URL=$(FRONTEND_LOCAL_URL) npm test
@echo "$(PURPLE)SDK tests complete.$(NC)"
+EVAL_DIR := scripts/eval/retrieval
+# Paths below are relative to EVAL_DIR. Override OUTPUT_DIR (not --output-dir in
+# EVAL_ARGS) to change where results land, so eval-baseline's copy step always
+# matches what run_eval.py actually wrote.
+OUTPUT_DIR ?= results
+CURRENT ?= $(OUTPUT_DIR)/results.json
+
+eval-validate: ## Check the retrieval eval corpus and questions (no stack needed)
+ cd $(EVAL_DIR) && uv run run_eval.py run --validate-only $(EVAL_ARGS)
+
+eval-run: ## Run the retrieval eval (requires running OpenRAG and OPENRAG_API_KEY); BASELINE=file to compare
+ cd $(EVAL_DIR) && uv run run_eval.py run --output-dir $(OUTPUT_DIR) $(if $(BASELINE),--baseline $(BASELINE)) $(EVAL_ARGS)
+
+eval-baseline: ## Run the retrieval eval and save it as $(OUTPUT_DIR)/baseline.json
+ $(if $(findstring --output-dir,$(EVAL_ARGS)),$(error Set OUTPUT_DIR instead of passing --output-dir in EVAL_ARGS))
+ cd $(EVAL_DIR) && uv run run_eval.py run --output-dir $(OUTPUT_DIR) $(EVAL_ARGS) && cp $(OUTPUT_DIR)/results.json $(OUTPUT_DIR)/baseline.json
+ @echo "$(PURPLE)Saved $(EVAL_DIR)/$(OUTPUT_DIR)/baseline.json$(NC)"
+
+eval-compare: ## Compare two retrieval eval results files offline (BASELINE=..., CURRENT=...)
+ cd $(EVAL_DIR) && uv run run_eval.py compare $(or $(BASELINE),$(OUTPUT_DIR)/baseline.json) $(CURRENT)
+
lint: ## Run linting checks
@echo "$(YELLOW)Running linting checks...$(NC)"
cd frontend && npm run lint
diff --git a/scripts/README.md b/scripts/README.md
index 487cce993..381eb50c0 100644
--- a/scripts/README.md
+++ b/scripts/README.md
@@ -133,6 +133,24 @@ uv run scripts/openrag_bulk.py bench arxiv summary
uv run scripts/openrag_bulk.py bench arxiv summary --detail latest
```
+## Retrieval evaluation (`eval/retrieval/`)
+
+`eval/retrieval/` is a harness that measures search quality. It ingests a fixed
+corpus, runs a set of labeled questions through `/v1/search`, and reports
+recall@k, MRR and nDCG@k, with a per-question comparison against an earlier
+run. Use it before and after a change to search, chunking or embeddings.
+
+```bash
+cd scripts/eval/retrieval
+export OPENRAG_URL=http://localhost:3000
+export OPENRAG_API_KEY=orag_your_api_key
+
+uv run run_eval.py run
+```
+
+See [`eval/retrieval/README.md`](eval/retrieval/README.md) for the baseline
+workflow, the output format and how to add questions.
+
## Sync default user roles (`sync_default_user_roles.py`)
Dev-only helper for RBAC in **OSS run mode** (`OPENRAG_RUN_MODE=oss`): updates
diff --git a/scripts/eval/retrieval/.gitignore b/scripts/eval/retrieval/.gitignore
new file mode 100644
index 000000000..6b4fdd1a6
--- /dev/null
+++ b/scripts/eval/retrieval/.gitignore
@@ -0,0 +1,4 @@
+.env
+.venv/
+__pycache__/
+results/
diff --git a/scripts/eval/retrieval/FRAMEWORK_COMPARISON.md b/scripts/eval/retrieval/FRAMEWORK_COMPARISON.md
new file mode 100644
index 000000000..354751867
--- /dev/null
+++ b/scripts/eval/retrieval/FRAMEWORK_COMPARISON.md
@@ -0,0 +1,239 @@
+# Retrieval eval: framework comparison
+
+Issue [#2478](https://github.com/langflow-ai/openrag/issues/2478) asked for a
+dependency-light harness that catches retrieval regressions. Before settling on
+the homegrown harness in this directory, we tried three alternatives,
+[RAGAS](https://docs.ragas.io), [DeepEval](https://deepeval.com) and
+[LangSmith](https://docs.smith.langchain.com), on the same inputs: this
+directory's 21-document `corpus/` and 83-question `queries.jsonl`. We also
+looked at two tools we did not build with, IBM watsonx.governance and Langfuse,
+because they matter for later phases.
+
+## How we tested
+
+- **Offline, all three frameworks.** Each one scored the same synthetic ranked
+ file lists and had to produce the three metrics this project needs:
+ file-level recall@k, MRR and nDCG@k. No LLM judge was used.
+- **Live, DeepEval only.** A DeepEval version of the full harness
+ (`deepeval==4.2.8`) ran against a local OpenRAG stack with watsonx
+ `ibm/granite-embedding-278m-multilingual` embeddings and 1000/200 chunking.
+ It ingested the corpus, ran all 83 questions through `/v1/search`, and
+ collected chat answers for 20 questions. See
+ [Live DeepEval run](#live-deepeval-run).
+
+The framework variants were throwaway spikes and are not part of this PR. Only
+this writeup and its findings are kept.
+
+## Summary
+
+| | Homegrown (this directory) | RAGAS | DeepEval | LangSmith |
+|---|---|---|---|---|
+| Unit tests | 29 | 4 | 7 offline, 32 in the live spike | 8 |
+| Tests passing | 29/29 | 4/4 | all | 8/8 |
+| New packages installed | 0 (`openrag-sdk` only, already required) | 95 | 68 | 25 |
+| Pulls in | nothing new | `langchain`, `langchain-community`, `langchain-openai`, `openai`, `datasets`, `pillow`, `networkx` | `openai`, `grpcio`, `posthog`, `opentelemetry-*` | its own small SDK |
+| Native recall@k, MRR, nDCG@k | Yes | No, hand-written | No, hand-written | No, hand-written |
+| Runs without a third-party API key | Yes | Yes, after pinning `langchain-community` | Yes, after turning off telemetry | No |
+| Fit for file-level ranked results | Built for it | Approximate: set-based text similarity, not rank-based | Poor: no ranked-result concept | Good: the evaluator signature maps cleanly |
+
+In the live DeepEval spike, `uv.lock` resolved 72 packages, against 13 for this
+harness. Both counts include `openrag-sdk` and its dependencies.
+
+None of the three frameworks computes the ranking metrics this project needs.
+Each one needed the same recall@k, MRR and nDCG@k code that already exists and
+is tested in `metrics.py`. The frameworks differ only in the tooling around that
+code, and each brings a cost with it.
+
+## RAGAS
+
+RAGAS evaluates RAG pipelines, mostly with LLM-judged metrics such as
+faithfulness, answer relevancy, and context precision and recall. It also has
+two metrics that need no LLM, `NonLLMContextRecall` and
+`NonLLMContextPrecisionWithReference`, which we used here.
+
+**Strengths**
+
+- The non-LLM metrics need no API key or network call. `ragas.evaluate()` does
+ not create a default LLM client when only `NonLLM*` metrics are selected.
+- `EvaluationDataset.to_pandas()` gives a dataframe for further analysis.
+- `NonLLMContextRecall` ran on all 83 questions (mean 0.717 on the synthetic
+ fixtures), so the non-LLM path works in practice.
+
+**Weaknesses**
+
+- No ranking metrics. Its context metrics compare sets of strings with
+ `rapidfuzz` and ignore rank. We wrote the same ranking code as the homegrown
+ harness and passed the results through RAGAS for the report.
+- The heaviest install: 95 packages, including the full LangChain stack, for
+ scoring that uses none of it.
+- `import ragas` fails with a current `langchain-community`. ragas 0.4.3
+ imports `langchain_community.chat_models.vertexai.ChatVertexAI` at import
+ time, and `langchain-community` 0.4 removed that module. We had to pin
+ `langchain-community==0.3.27`.
+- The non-LLM context metrics need `rapidfuzz`, which ragas does not install.
+
+## DeepEval
+
+DeepEval tests LLM output (correctness, faithfulness, relevancy) through
+`LLMTestCase` and mostly LLM-judged metrics, with a pytest-style
+`deepeval test run` command.
+
+**Strengths**
+
+- Custom deterministic metrics are clean to write. Subclassing `BaseMetric`
+ (`measure()`, `a_measure()`, `is_successful()`, `__name__`) plugged straight
+ into `evaluate()` and its report.
+- Telemetry (PostHog) turns off with `DEEPEVAL_TELEMETRY_OPT_OUT=YES`. Nothing
+ goes to Confident AI unless `CONFIDENT_API_KEY` is set. The offline 83-question
+ run took about 0.2 s with no network activity.
+- `FaithfulnessMetric` and `AnswerRelevancyMetric` come ready-made, with
+ prompts, verdict schemas, reasons and per-metric cost. This is the main reason
+ to consider DeepEval for answer quality.
+- `assert_test` with `deepeval test run` works as a CI gate with readable
+ failure messages.
+
+**Weaknesses**
+
+- No concept of a ranked result set or a set of expected documents.
+ `LLMTestCase` models one LLM input and output. Expected files had to go
+ through `metadata` or be packed into `expected_output` as a string. DeepEval
+ does no deduplication or ranking itself, so all of that is still custom code
+ that duplicates `metrics.py`.
+- 68 packages, including a hard dependency on `openai`, plus `grpcio` and
+ `opentelemetry-*`, for scoring that is plain arithmetic.
+- Built-in metrics need an LLM client even to be constructed. With no key,
+ `GPTModel` raises at construction.
+- `evaluate()` prints Confident AI banners and writes a `.deepeval/` directory
+ in the current directory.
+- Fast-moving API. Three releases (4.2.9 to 4.3.1) came out in the five days
+ before our run, and recent releases added new evaluation modes.
+
+### Live DeepEval run
+
+The spike ran the full retrieval eval twice against the local stack (83
+questions, fetch limit 50):
+
+| Run | recall@5 | MRR | nDCG@5 |
+|---|---|---|---|
+| Homegrown baseline (earlier stack state) | 0.904 | 0.809 | 0.808 |
+| DeepEval run 1 | 0.940 | 0.806 | 0.822 |
+| DeepEval run 2 | 0.928 | 0.836 | 0.840 |
+
+- **Same scores as this harness.** We re-scored every ranking from DeepEval
+ run 1 with this directory's `metrics.score_query`. All 83 questions matched.
+ The gap to the older baseline comes from the stack returning different
+ rankings, not from scoring. A homegrown run three days after the baseline
+ had already measured 0.928 / 0.814 / 0.823.
+- **Same noise.** 11 of 83 questions moved between the two unchanged runs, and
+ recall@5 moved by 0.012, about a third of a 0.03 gate margin. This matches
+ what the [README](README.md#repeatability) describes for this harness.
+- **Chat scoping works.** All 20 chat answers came back. Every source cited
+ (200 of 200) was one of the run's own files, so filters passed to
+ `/api/v1/chat` are honored.
+- **Answer quality was not scored.** No judge key was available, so faithfulness
+ and answer relevancy were not measured. Measured from the real prompts, a
+ gpt-4.1-mini judge would make about 140 calls and cost about $0.07 to $0.15
+ per 20-question run.
+- **Size.** About 1,355 lines of code, including tests and the answer-quality
+ commands. The ranking logic was still custom code, not DeepEval's.
+
+DeepEval reproduced this harness's numbers exactly, but it added no retrieval
+capability. Everything it measured, this harness already measures.
+
+## LangSmith
+
+LangSmith is a hosted service for experiment tracking and evaluation, with
+LLM-judged or custom code evaluators.
+
+**Strengths**
+
+- The lightest install of the three: 25 packages, about 33 MB, with no
+ `langchain` or `openai`.
+- The code-evaluator signature,
+ `(inputs, outputs, reference_outputs) -> {"key", "score"}`, fits file-level
+ ranked results without workarounds.
+- With a hosted account, it would provide experiment-to-experiment diffs in a
+ shared UI, per-example traces, dataset versioning and shareable experiment
+ links.
+
+**Weaknesses**
+
+- It does not work without a hosted account. `Client()` constructs without a
+ key, but `client.create_dataset(...)` fails with
+ `401 LangSmithAuthError: Invalid token`. `client.evaluate(...,
+ upload_results=False)` fails with the same error, because it looks the dataset
+ up on the hosted API before running. There is no offline mode, and every
+ headline feature needs the account.
+- Calling the evaluator functions directly reproduces the scores, but that is
+ plain Python, not LangSmith. Run-to-run comparison, its main draw for us, is
+ already covered by `report.py`.
+
+## Tools we did not build with
+
+### IBM watsonx.governance
+
+IBM's evaluation SDK (`ibm-watsonx-gov`, version 1.5.2 checked) is the closest
+match to how IBM evaluates RAG elsewhere, so it is the main candidate for
+phase 3.
+
+- **It has retrieval metrics.** Unlike the three frameworks above, it lists
+ hit rate, reciprocal rank, average precision, NDCG, retrieval precision and
+ context relevance, run through `MetricsEvaluator`. As documented, these are
+ computed from per-chunk relevance scores produced by a judge, not from
+ labeled expected files, so they answer a different question from this
+ harness. Confirm this before relying on them.
+- **It has answer metrics:** faithfulness, answer relevance and answer
+ similarity.
+- **Costs:** a proprietary IBM license, an IBM Cloud API key for every run, and
+ a judge model call per metric. It cannot run in CI without credentials.
+
+This harness's results files could be passed to it in phase 3 without changing
+the retrieval side.
+
+### Langfuse
+
+OpenRAG already sends traces to Langfuse when the `LANGFUSE_*` variables are
+set. Langfuse is open source, can be self-hosted, and has datasets and
+experiments with a UI. The issue lists it, with Arize Phoenix and Braintrust,
+for the customer-facing phase. It does not compute ranking metrics either, but
+it could store and display this harness's results without a new vendor.
+
+## Why we chose the homegrown harness
+
+1. **It is the only option with the metrics the issue asks for.** RAGAS,
+ DeepEval and LangSmith all needed recall@k, MRR and nDCG@k written by hand,
+ which is the code already in `metrics.py`. watsonx.governance's ranking
+ metrics are judged rather than label-based, need an IBM Cloud key, and fit
+ phase 3 rather than phase 1.
+2. **No new dependencies.** It needs only `openrag-sdk`, which the project
+ already has. RAGAS adds 95 packages, DeepEval 68 and LangSmith 25, and the
+ issue asks for something dependency-light.
+3. **It produces the same numbers.** The live DeepEval run matched this
+ harness's scores on all 83 questions, so switching would add size without
+ changing a result.
+4. **Its data model fits.** `Query`, `RankedFile` and `score_query()` model
+ file-level ranked scoring with an optional text check. Each framework needed
+ a workaround: expected files packed into strings, no ranked-result concept,
+ or no offline mode.
+5. **No third-party account, key or service.** A `run` still needs a reachable
+ OpenRAG stack and an OpenRAG API key, because it uploads the corpus and
+ queries `/v1/search`. Scoring, reporting and `compare` need no network at
+ all. Nothing goes to an outside vendor, which matters for running in CI in
+ phase 2. LangSmith cannot run without a hosted account, and RAGAS and
+ DeepEval avoid one only through careful metric choice and opt-out settings
+ that are off by default.
+6. **The reporting is already there.** `report.py` writes JSON results, a
+ Markdown summary and a question-by-question baseline diff, which covers what
+ the frameworks' reports offer without a hosted dashboard.
+
+## Where the other tools fit later
+
+The issue's phases:
+
+- **Phase 3, answer quality:** DeepEval, RAGAS and watsonx.governance are the
+ candidates, since LLM-judged answer metrics are what they are built for.
+ watsonx.governance fits IBM's existing evaluation practice. DeepEval had the
+ smoothest path in our spike.
+- **Phase 4, customer-facing reports:** Langfuse is already integrated with
+ OpenRAG. LangSmith is worth revisiting only if a hosted account becomes part
+ of the plan.
diff --git a/scripts/eval/retrieval/README.md b/scripts/eval/retrieval/README.md
new file mode 100644
index 000000000..7ce8db3c4
--- /dev/null
+++ b/scripts/eval/retrieval/README.md
@@ -0,0 +1,281 @@
+# Retrieval evaluation harness
+
+This harness measures how well OpenRAG search finds the right documents. It
+ingests a fixed set of documents, sends a fixed set of labeled questions
+through the public search API (`/v1/search`), and scores where the expected
+documents land in the results. Run it before and after a change to search,
+chunking or embeddings to see which questions got better and which got worse.
+
+It measures retrieval only. It does not judge chat answers.
+
+[FRAMEWORK_COMPARISON.md](FRAMEWORK_COMPARISON.md) explains why this harness is
+homegrown rather than built on RAGAS, DeepEval or LangSmith.
+
+## Quick start
+
+Run these from this directory against a running stack. You need an API key
+from **Settings → API Keys**.
+
+```bash
+cd scripts/eval/retrieval
+export OPENRAG_URL=http://localhost:3000
+export OPENRAG_API_KEY=orag_your_api_key
+
+# Check the corpus and questions. Needs no stack.
+uv run run_eval.py run --validate-only
+
+# Run every question and keep the results as a baseline.
+uv run run_eval.py run
+cp results/results.json results/baseline.json
+
+# After a change to search, chunking or embeddings, run again and compare.
+uv run run_eval.py run --baseline results/baseline.json
+
+# Read the summary of the latest run.
+cat results/summary.md
+```
+
+The same steps are available as `make` targets from the repository root.
+Paths are relative to this directory.
+
+| `make` target | Same as |
+|---|---|
+| `make eval-validate` | `uv run run_eval.py run --validate-only` |
+| `make eval-run` | `uv run run_eval.py run` |
+| `make eval-run BASELINE=results/baseline.json` | `uv run run_eval.py run --baseline results/baseline.json` |
+| `make eval-baseline` | `uv run run_eval.py run`, then copy the results to `results/baseline.json` |
+| `make eval-compare` | `uv run run_eval.py compare results/baseline.json results/results.json` |
+| `make eval-compare BASELINE=a.json CURRENT=b.json` | `uv run run_eval.py compare a.json b.json` |
+| `make eval-baseline OUTPUT_DIR=results/demo` | `uv run run_eval.py run --output-dir results/demo`, then copy the results to `results/demo/baseline.json` |
+
+Pass any other option with `EVAL_ARGS`, for example
+`make eval-run EVAL_ARGS="--keep-documents --fetch-limit 100"`. To write results
+somewhere else, set `OUTPUT_DIR` rather than passing `--output-dir` in
+`EVAL_ARGS`, so `eval-baseline` copies the file the run actually wrote.
+`make eval-baseline` stops with an error if `EVAL_ARGS` contains `--output-dir`.
+
+The rest of this file explains each step.
+
+## What is here
+
+| Path | Purpose |
+|---|---|
+| `corpus/` | 21 Markdown documents: the handbook of a fictional product, "Relaywise" |
+| `queries.jsonl` | 83 hand-labeled questions, one JSON object per line |
+| `run_eval.py` | The command-line tool, with `run` and `compare` commands |
+| `dataset.py` | Loads and validates the corpus and questions |
+| `metrics.py` | Turns ranked chunks into ranked files and scores them |
+| `report.py` | Writes results, the summary and the baseline diff |
+| `FRAMEWORK_COMPARISON.md` | How this harness compares with RAGAS, DeepEval, LangSmith and others |
+| `results/` | Output of each run (not committed) |
+
+The corpus is invented so that no model has seen it and no license applies. Its
+documents overlap on purpose. For example, API keys and signing secrets both
+have a "rotation" section, so the ranking has to tell them apart.
+
+## Prerequisites
+
+- A running OpenRAG stack that has finished onboarding, with working ingestion
+ and an embedding model.
+- Python 3.11 or later and [uv](https://docs.astral.sh/uv/).
+- An OpenRAG API key. When RBAC is enforced (`OPENRAG_RBAC_ENFORCE=true`), the
+ key needs these permissions:
+
+ | Permission | Used for |
+ |---|---|
+ | `knowledge:upload` | Ingesting the corpus |
+ | `knowledge:read:own` | Reading ingestion task status |
+ | `search:use` | Running the questions |
+ | `knowledge:delete:own` | Deleting the corpus when the run ends |
+ | `providers:read` | Recording the embedding and chunk settings (optional) |
+
+The harness depends only on `openrag-sdk`, which it takes from `sdks/python` in
+this checkout, so a run tests the SDK and API of the code you have checked out.
+
+## Run it
+
+```bash
+cd scripts/eval/retrieval
+export OPENRAG_URL=http://localhost:3000
+export OPENRAG_API_KEY=orag_your_api_key
+
+uv run run_eval.py run
+```
+
+A run does the following:
+
+1. Validates the corpus and the question file. A bad label stops the run
+ before anything is uploaded.
+2. Uploads each corpus file under a name unique to the run, such as
+ `eval-3f9a12cd-api-keys.md`, and waits until every file is searchable.
+3. Sends each question to `/v1/search`, limited to the files this run uploaded,
+ so other documents in the stack are never returned. They can still shift
+ the scores slightly; see [Repeatability](#repeatability).
+4. Prints a summary and writes it, with the full results, to `results/`.
+5. Deletes the files it uploaded.
+
+Documents that were already in the stack are never searched, changed or
+deleted. If a run is interrupted before cleanup, delete the leftover
+`eval-<run id>-*` files from the knowledge page.
+
+Useful options:
+
+| Option | Default | Purpose |
+|---|---|---|
+| `--baseline FILE` | none | Compare this run with an earlier results file |
+| `--output-dir DIR` | `results/` | Where to write results |
+| `--k 1,3,5,10` | `1,3,5,10` | Cutoffs to report |
+| `--fetch-limit N` | `50` | Chunks requested per question; must be at least the largest `--k` |
+| `--score-threshold X` | `0` | Minimum score passed to search |
+| `--concurrency N` | `4` | Parallel uploads |
+| `--keep-documents` | off | Leave the uploaded files in place for debugging |
+| `--validate-only` | off | Check the dataset and exit; needs no stack |
+
+Run `uv run run_eval.py run --help` for the full list, including timeouts.
+
+## Compare before and after a change
+
+```bash
+# 1. On main, or before your change:
+uv run run_eval.py run
+cp results/results.json results/baseline.json
+
+# 2. Apply the change, restart the stack, then:
+uv run run_eval.py run --baseline results/baseline.json
+```
+
+The summary then ends with a comparison that lists the questions that
+regressed first, then the ones that improved, then the change in each average.
+For every changed question it shows the rank of each expected file before and
+after.
+
+To compare two results files without running anything:
+
+```bash
+uv run run_eval.py compare results/baseline.json results/results.json
+```
+
+The comparison warns when the two runs differ in something other than the
+change under test: the corpus, the question file, the k values, the fetch
+limit, the score threshold, or the embedding and chunk settings of the stack.
+Scores from different embedding models or chunk sizes are not comparable unless
+that setting is what you are testing.
+
+### Repeatability
+
+Two runs against the same stack do not give exactly the same scores. Search
+combines vector similarity with keyword matching, and the keyword part weighs
+each word by how common it is across the whole index. That count includes
+documents outside the eval, and chunks that earlier runs deleted but OpenSearch
+has not yet purged. Each run therefore changes the index a little for the next.
+
+On a local stack with watsonx embeddings, three unchanged runs in a row gave
+the same recall@5 (0.904) and an MRR within 0.002, but up to 13 of the 83
+questions moved by one rank and were listed as regressed or improved. Halving
+the chunk size moved recall@5 by 0.036 and MRR by 0.070.
+
+To tell a real change from this noise:
+
+- Run the baseline twice and compare the two runs. Whatever that comparison
+ lists is the noise level of your stack.
+- Trust the averages and changes in recall@5 over a one-rank move on a single
+ question.
+- For the steadiest numbers, use a stack that holds no other documents.
+
+No baseline is committed. Scores depend on the embedding model and settings of
+the stack that produced them, so create the baseline on the same stack you test
+against.
+
+## How questions are scored
+
+Search returns chunks, but questions are labeled with files, because chunk
+boundaries move whenever chunking changes. For each question the harness:
+
+1. Requests `--fetch-limit` chunks.
+2. Keeps the first chunk of each file, which gives a ranked list of unique
+ files.
+3. Treats an expected file as relevant if it is in that list. If the question
+ has an `expected_text`, at least one returned chunk of that file must also
+ contain the text. The match ignores case and differences in whitespace.
+
+| Metric | Meaning |
+|---|---|
+| recall@k | Share of the expected files that appear in the top k files |
+| MRR | 1 divided by the rank of the first relevant file, averaged over questions; 0 when none is returned |
+| nDCG@k | Like recall@k, but a relevant file counts for more the nearer it is to the top |
+
+All three run from 0 to 1, and higher is better. Runs are compared on recall@5.
+If either run did not evaluate k=5, the comparison uses a cutoff both runs have
+and says so. Two runs with no cutoff in common cannot be compared.
+
+## Results files
+
+Each run writes four files to the output directory: `results.json` and
+`summary.md` for the latest run, and a timestamped copy of each.
+
+`results.json` has these parts:
+
+| Field | Contents |
+|---|---|
+| `schema_version` | Format version of the file |
+| `run` | Run ID, time, target URL, SDK version, git commit, hashes of the corpus and question file, k values, fetch limit and score threshold |
+| `settings` | Embedding provider and model, chunk size and overlap, as reported by the stack; `null` if they could not be read |
+| `metrics.overall` | Mean of each metric over all questions |
+| `metrics.by_tag` | The same means for each tag, with the number of questions |
+| `queries` | One record per question |
+
+Each record in `queries` holds the question, its labels, the ranked files with
+the score of their first chunk, the rank of each expected file
+(`expected_ranks`, `null` when a file was not returned), whether the expected
+text was found, and the metrics for that question.
+
+## Add or change a question
+
+Add a line to `queries.jsonl`:
+
+```json
+{"id": "key-rotate", "query": "How do I swap out an API key without causing downtime?", "expected_files": ["api-keys.md"], "expected_text": "overlap window", "tags": ["semantic"]}
+```
+
+| Field | Required | Meaning |
+|---|---|---|
+| `query` | Yes | The question, as a user would type it |
+| `expected_files` | Yes | Corpus filenames that answer it; list more than one only if each file answers it |
+| `id` | No | Stable name used to match the question between runs; defaults to the query text |
+| `expected_text` | No | A short phrase that appears in every expected file |
+| `tags` | No | Labels used to break the scores down |
+
+Guidelines:
+
+- Write the question in your own words. A question copied from the document
+ mostly tests keyword matching.
+- Give every question an `id`. Without one, rewording the question makes it
+ look like one question was removed and another added.
+- Keep `expected_text` to a few words from a single line of body text, not a
+ heading. A long phrase can be split across two chunks and then never match.
+- Use the existing tags where they fit: `semantic` for a paraphrased question,
+ `keyword` for one that uses the document's own terms, `exact-token` for an
+ identifier such as an error code or a setting name, and `multi-doc` when more
+ than one file is expected.
+
+To add a document, put a Markdown file in `corpus/` and add at least one
+question that expects it.
+
+Then check the dataset. This needs no stack:
+
+```bash
+uv run run_eval.py run --validate-only
+```
+
+Changing the corpus or the questions changes the scores, so create a new
+baseline afterwards. The comparison warns if you forget.
+
+## Tests
+
+The scoring, validation and comparison logic is covered by
+`tests/unit/test_retrieval_eval.py`, which needs neither the SDK nor a stack.
+From the repository root:
+
+```bash
+uv run pytest tests/unit/test_retrieval_eval.py
+```
diff --git a/scripts/eval/retrieval/corpus/api-keys.md b/scripts/eval/retrieval/corpus/api-keys.md
new file mode 100644
index 000000000..0df21d637
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/api-keys.md
@@ -0,0 +1,61 @@
+# API keys
+
+API keys authenticate requests your own code makes to Relaywise: sending events, managing destinations, reading delivery history. They are different from signing secrets, which Relaywise uses to sign the requests it sends to you.
+
+## Key format
+
+Every key starts with a prefix that tells you what kind of key it is:
+
+| Prefix | Kind | Typical use |
+|---|---|---|
+| `rwk_live_` | Live key | Production traffic |
+| `rwk_test_` | Test key | Development; events are accepted but never delivered outside the tunnel |
+| `rwk_ci_` | Automation key | Pipelines and infrastructure tooling; cannot send events |
+
+The full key is shown only once, at creation. Afterwards the console displays just the prefix and the last four characters.
+
+## Creating a key
+
+1. Open **Settings → API keys**.
+2. Select **Create key**.
+3. Name the key after the system that will use it, for example `checkout-backend`.
+4. Choose scopes. Grant only what that system needs.
+5. Optionally set an expiry date. Keys without an expiry stay valid until revoked.
+
+Only members with the Admin or Owner role can create keys. Developers can see the list of keys but not create or revoke them.
+
+## Scopes
+
+| Scope | Allows |
+|---|---|
+| `events:write` | Posting events to a source |
+| `events:read` | Listing events and reading delivery attempts |
+| `destinations:manage` | Creating, updating and deleting destinations and routes |
+| `replay:run` | Replaying events from history or from the dead-letter queue |
+| `workspace:admin` | Managing members, keys and settings |
+
+A request made with a key that lacks the required scope fails with error `RW-1403`.
+
+## Rotating a key
+
+Rotate a key on a schedule, and immediately if you suspect it has leaked. Rotation in Relaywise is designed so that you never need downtime:
+
+1. In **Settings → API keys**, open the key and select **Rotate**.
+2. Relaywise issues a replacement key with the same name and scopes. The old key keeps working during an overlap window, which is 24 hours by default.
+3. Deploy the replacement key to every system that used the old one.
+4. Check the **Last used** column. When the old key shows no recent use, select **Revoke now** to end the overlap early, or simply let the window run out.
+
+You can change the overlap window to anything between 1 hour and 7 days when you start the rotation. If the key has definitely leaked, choose **Revoke immediately** instead of rotating; that skips the overlap entirely and the old key stops working at once.
+
+From the command line the same operation is `rwctl keys rotate <key-name> --overlap 24h`.
+
+## Revoking a key
+
+Revoking is permanent. Requests that use a revoked key fail with error `RW-1401`. A revoked key cannot be restored; create a new one instead.
+
+## Good practice
+
+- Give each service its own key so that revoking one does not break the others.
+- Store keys in a secret manager, not in source control or in a build log.
+- Set an expiry on keys used by contractors and short-lived projects.
+- Review the **Last used** column every quarter and revoke keys that have been idle for more than 90 days. Relaywise flags these with an **Idle** badge.
diff --git a/scripts/eval/retrieval/corpus/audit-log.md b/scripts/eval/retrieval/corpus/audit-log.md
new file mode 100644
index 000000000..a7e4d3dfd
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/audit-log.md
@@ -0,0 +1,60 @@
+# Audit log
+
+The audit log is a record of who did what in a workspace. It answers questions such as "who revoked that key?", "when was this destination's URL changed?" and "has anyone looked at customer payloads this month?".
+
+The audit log is about actions taken by people and API keys. It is not a log of events or deliveries; those are in the event history.
+
+## What is recorded
+
+Each entry holds the time, the actor, the action, the thing that was acted on, the IP address the request came from, and whether the action succeeded. The actor is a member, an API key or Relaywise support.
+
+| Area | Example actions |
+|---|---|
+| Members | `member.invited`, `member.removed`, `member.role_changed` |
+| API keys | `key.created`, `key.rotated`, `key.revoked` |
+| Signing | `secret.revealed`, `secret.rolled` |
+| Configuration | `destination.created`, `destination.url_changed`, `route.deleted`, `transformation.activated` |
+| Data access | `payload.viewed`, `events.exported`, `events.deleted` |
+| Security | `sso.enabled`, `sso.enforced`, `login.failed`, `ip_allowlist.changed` |
+| Billing | `plan.changed`, `usage_cap.changed` |
+
+Entries describe changes without holding secrets. An entry for a rolled signing secret states that the roll happened; it does not contain the old or the new secret.
+
+## Who can read it
+
+Owners and Admins can read the audit log. Developers and Viewers cannot. On the Enterprise plan, a custom role can be given read access to the audit log and nothing else, which suits an auditor or a security reviewer.
+
+## Finding entries
+
+Open **Settings → Audit log**. Filter by actor, action, target or time range. Typing a destination ID into the search box lists everything that has happened to that destination.
+
+## How long entries are kept
+
+| Plan | Audit log kept for |
+|---|---|
+| Sandbox | 30 days |
+| Team | 1 year |
+| Scale | 1 year |
+| Enterprise | 7 years |
+
+This is separate from event retention. Shortening how long payloads are kept has no effect on the audit log.
+
+## Entries cannot be changed
+
+Nobody can edit or delete an audit log entry, including Owners and Relaywise staff. Each entry includes a hash of the entry before it, so removing or altering one would break the chain and be detectable. Use **Verify integrity** on the audit log page to check a time range.
+
+## Exporting
+
+Select **Export** to download the entries matching the current filter as CSV or JSON. One export can hold up to 500,000 entries.
+
+For continuous export, the Enterprise plan can stream audit entries to a Relaywise destination or to a storage bucket you own, typically within a minute of the action. Security teams use this to feed their own log analysis tools.
+
+## Actions by Relaywise support
+
+Support staff cannot see your payloads or change your configuration by default. If you ask them to investigate something, an Owner grants time-limited access under **Settings → Security → Support access**, for between 1 hour and 7 days. Everything support does during that time appears in the audit log with the actor shown as `support`, together with the ticket number.
+
+## Useful checks
+
+- **Monthly:** review `key.created` and `member.role_changed` for anything unexpected.
+- **After someone leaves:** search for their name to see which keys and secrets they created or revealed.
+- **After an incident:** filter by the time range and the affected destination to rebuild the sequence of changes.
diff --git a/scripts/eval/retrieval/corpus/billing-plans.md b/scripts/eval/retrieval/corpus/billing-plans.md
new file mode 100644
index 000000000..d765bf7e5
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/billing-plans.md
@@ -0,0 +1,56 @@
+# Plans and pricing
+
+Relaywise has four plans. They differ mainly in how many events are included each month, how fast you can send them, and how long history is kept.
+
+## Comparing the plans
+
+| | Sandbox | Team | Scale | Enterprise |
+|---|---|---|---|---|
+| Monthly price | Free | $49 | $399 | By contract |
+| Events included per month | 50,000 | 1 million | 20 million | By contract |
+| Price per extra 10,000 events | Not available | $0.60 | $0.25 | By contract |
+| Routes | 5 | 50 | 500 | Unlimited |
+| Members | 3 | 15 | Unlimited | Unlimited |
+| Single sign-on | No | No | Yes | Yes |
+| Custom roles | No | No | No | Yes |
+| Support | Community forum | Email, next working day | Email and chat, 4 hours | Named contact, 1 hour |
+
+## What counts as an event
+
+Each payload accepted by a source is one event, whatever number of destinations it is delivered to and however many times delivery is retried. Three things are easy to overlook:
+
+- Replayed events count again.
+- Events that match no route still count.
+- Events sent with a test key do not count.
+
+## Going over the included amount
+
+On the Team and Scale plans, events beyond the included amount are accepted and billed as overage at the per-10,000 rate shown above. Overage appears as a separate line on the next invoice.
+
+The Sandbox plan has no overage. When a Sandbox workspace reaches 50,000 events, further events are rejected until the next month begins.
+
+## Setting a spending cap
+
+If you would sooner lose events than receive an unexpected bill, set a hard cap under **Settings → Billing → Usage cap**. When usage reaches the cap, Relaywise rejects new events for the rest of the billing period. The cap can be set at the included amount or at any higher number.
+
+Relaywise emails the Owners when usage reaches 80 percent and 100 percent of the included amount. You can change these thresholds.
+
+## Paying yearly
+
+Team and Scale can be paid yearly. Yearly billing costs the same as ten months, so two months are free. The included events are still counted month by month; they do not pool across the year.
+
+## Upgrading
+
+An upgrade takes effect straight away. You are charged the difference for the remainder of the current billing period, and the higher limits apply at once. Only an Owner can change the plan.
+
+## Downgrading
+
+A downgrade takes effect at the end of the current billing period. Before it can go ahead, the workspace has to fit inside the lower plan's limits. For example, a workspace with 60 routes has to remove 10 before it can move to Team. Retention shortens when the downgrade takes effect, and history older than the new limit is deleted then.
+
+## Cancelling
+
+Cancelling a paid plan moves the workspace to Sandbox at the end of the billing period. Nothing is deleted immediately, but the Sandbox limits on routes, members and retention apply from that date.
+
+## Discounts
+
+Registered charities and accredited schools and universities get 50 percent off the Team and Scale plans. Open-source projects with a public repository can apply for a free Team plan. Both need a short application through support.
diff --git a/scripts/eval/retrieval/corpus/cli-reference.md b/scripts/eval/retrieval/corpus/cli-reference.md
new file mode 100644
index 000000000..81e60f8ab
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/cli-reference.md
@@ -0,0 +1,88 @@
+# Command-line tool reference
+
+`rwctl` is the Relaywise command-line tool. Anything you can do in the console you can also do with `rwctl`, which makes it the natural choice for scripts, pipelines and local development.
+
+## Installing
+
+`rwctl` is a single binary for Linux, macOS and Windows. Download it from the console under **Settings → Developer tools**, or install it with your system's package manager. Check the installation with:
+
+```bash
+rwctl version
+```
+
+To update, run `rwctl self-update`. The tool tells you when a newer release exists, but it never updates itself without being asked.
+
+## Signing in
+
+```bash
+rwctl login
+```
+
+This opens a browser window so that you can approve the session. It then stores a personal access token in your operating system's keychain. The token has the same permissions as your role in the workspace.
+
+In a pipeline, where no browser is available, set the `RELAYWISE_API_KEY` environment variable to an automation key instead. When the variable is set, `rwctl` uses it and ignores any stored login.
+
+## Profiles
+
+If you work with several workspaces, give each one a profile:
+
+```bash
+rwctl profile add staging --workspace acme-staging
+rwctl profile add production --workspace acme-prod
+rwctl profile use staging
+```
+
+Add `--profile production` to any command to run it against another profile without switching. The `RELAYWISE_PROFILE` environment variable does the same for a whole shell session.
+
+## Everyday commands
+
+| Command | What it does |
+|---|---|
+| `rwctl sources list` | List the sources in the workspace |
+| `rwctl destinations create --url <url>` | Create a destination |
+| `rwctl routes create --source <id> --destination <id>` | Connect a source to a destination |
+| `rwctl events send --source <id> --data @payload.json` | Send one event |
+| `rwctl events tail` | Show events as they arrive |
+| `rwctl events get <event-id>` | Show one event and all its delivery attempts |
+| `rwctl dlq list --destination <id>` | List events in a dead-letter queue |
+| `rwctl dlq replay --destination <id> --since 2h` | Replay failed events |
+| `rwctl keys rotate <key-name> --overlap 24h` | Rotate an API key |
+| `rwctl config set <key> <value>` | Change a setting |
+| `rwctl config explain <key>` | Show the value in force for a setting and where it comes from |
+| `rwctl config export > workspace.yaml` | Save the workspace configuration to a file |
+| `rwctl config import workspace.yaml` | Apply a configuration file |
+| `rwctl network egress-ips` | Print the addresses deliveries are sent from |
+
+## Developing on your own machine
+
+Your laptop cannot normally be reached from the internet, so Relaywise cannot deliver to it directly. The `listen` command opens a tunnel:
+
+```bash
+rwctl listen --forward-to localhost:4000/webhooks
+```
+
+While the command runs, events sent with a test key are forwarded to that local address. The command prints each delivery and the response your application gave. Press Ctrl+C to close the tunnel.
+
+Add `--events "order.*"` to forward only some event types. Add `--print-secret` to show the temporary signing secret for the tunnel, so that you can test signature verification locally.
+
+## Output formats
+
+Commands print a table by default. Add `--output json` for output a program can read, or `--output yaml`. Combine it with `--quiet` to suppress progress messages, which keeps the output safe to pipe into another program.
+
+## Exit codes
+
+| Code | Meaning |
+|---|---|
+| 0 | Success |
+| 1 | General error |
+| 2 | Wrong usage, such as an unknown flag or a missing argument |
+| 3 | Not signed in, or the credentials were rejected |
+| 4 | The thing you asked for does not exist |
+| 5 | Rate limited; try again later |
+| 6 | The operation was only partly completed, for example a bulk replay in which some events failed |
+
+Scripts should check for code 5 and wait before trying again.
+
+## Getting help
+
+Every command accepts `--help`. Add `--debug` to print the HTTP requests the tool makes, with credentials masked. Include that output when you contact support.
diff --git a/scripts/eval/retrieval/corpus/configuration-reference.md b/scripts/eval/retrieval/corpus/configuration-reference.md
new file mode 100644
index 000000000..319784b85
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/configuration-reference.md
@@ -0,0 +1,91 @@
+# Configuration reference
+
+This page lists every setting that can be changed on a workspace, a destination or a route. Each setting can be changed in the console, with `rwctl config set`, or through the management API.
+
+Settings are inherited. A value set on a destination overrides the workspace default, and a value set on a route overrides the destination. To see the value that is actually in force and where it came from, run `rwctl config explain <key> --destination <id>`.
+
+## Delivery settings
+
+Scope: workspace or destination.
+
+| Key | Default | Allowed values | Purpose |
+|---|---|---|---|
+| `delivery.max_attempts` | `8` | 1 to 20 | Total delivery attempts for one event |
+| `delivery.backoff` | `exponential` | `exponential`, `fixed` | How the wait between attempts grows |
+| `delivery.fixed_interval_seconds` | `300` | 10 to 86400 | Wait between attempts when the backoff is `fixed` |
+| `delivery.timeout_seconds` | `15` | 1 to 60 | How long to wait for the endpoint to respond |
+| `delivery.max_concurrency` | `25` | 1 to 500 | Deliveries in flight to one destination at the same time |
+| `delivery.rate_per_second` | unset | 1 to 5000 | Upper limit on deliveries per second to one destination |
+| `delivery.ordered` | `false` | `true`, `false` | Deliver events that share an ordering key strictly in order |
+| `delivery.ordering_key` | unset | A payload path | The field whose value groups events for ordered delivery |
+
+Lower `delivery.max_concurrency` if your endpoint cannot handle many requests in parallel. Set `delivery.rate_per_second` if the system behind the endpoint has a strict requests-per-second allowance of its own.
+
+## Signing settings
+
+Scope: destination.
+
+| Key | Default | Allowed values | Purpose |
+|---|---|---|---|
+| `signing.enabled` | `true` | `true`, `false` | Add signature headers to deliveries |
+| `signing.tolerance_seconds` | `300` | 60 to 900 | Age a timestamp may reach before the official libraries reject it |
+| `signing.header_prefix` | `Relaywise` | Letters and hyphens | Prefix used for the signature and timestamp header names |
+
+## Ingest settings
+
+Scope: workspace or source.
+
+| Key | Default | Allowed values | Purpose |
+|---|---|---|---|
+| `ingest.require_type` | `true` | `true`, `false` | Reject events that have no `type` field |
+| `ingest.dedupe_window_seconds` | `0` | 0 to 3600 | Discard events whose idempotency key was already seen within this window |
+| `ingest.allowed_ips` | unset | List of CIDR ranges | Accept events only from these source addresses |
+| `ingest.schema_validation` | `off` | `off`, `warn`, `reject` | Check payloads against the registered schema for their event type |
+
+With `ingest.schema_validation` set to `warn`, an event that does not match its schema is still accepted, and a warning is attached to it in the console. With `reject`, the event is refused.
+
+## Retention settings
+
+Scope: workspace.
+
+| Key | Default | Allowed values | Purpose |
+|---|---|---|---|
+| `retention.payload_days` | Plan maximum | 1 up to the plan maximum | Days that event bodies are kept |
+| `retention.metadata_days` | Plan maximum | 1 up to the plan maximum | Days that event metadata and attempt records are kept |
+| `retention.redact_fields` | unset | List of payload paths | Fields replaced with `[redacted]` before the event is stored |
+
+Redaction is applied to the stored copy only. The delivered payload is not changed.
+
+## Transformation settings
+
+Scope: route.
+
+| Key | Default | Allowed values | Purpose |
+|---|---|---|---|
+| `transform.timeout_ms` | `250` | 50 to 1000 | Time allowed for one run of the transformation |
+| `transform.memory_mb` | `32` | 16 to 128 | Memory available to one run |
+| `transform.on_error` | `dead_letter` | `dead_letter`, `pass_through`, `drop` | What happens to the event when the code fails |
+
+## Notification settings
+
+Scope: workspace.
+
+| Key | Default | Allowed values | Purpose |
+|---|---|---|---|
+| `notify.dlq_digest` | `true` | `true`, `false` | Send Admins a daily email about dead-letter queues that grew |
+| `notify.quota_thresholds` | `80,100` | List of percentages | Quota usage levels that trigger an email |
+| `notify.destination_disabled` | `true` | `true`, `false` | Email Admins when a destination is disabled automatically |
+
+## Console settings
+
+Scope: workspace.
+
+| Key | Default | Allowed values | Purpose |
+|---|---|---|---|
+| `console.session_hours` | `12` | 1 to 720 | Length of a console session; changing it needs the Enterprise plan |
+| `console.idle_timeout_minutes` | `60` | 5 to 240 | Idle time after which a session ends |
+| `console.payload_preview` | `true` | `true`, `false` | Show payload bodies in the event list |
+
+## Resetting a setting
+
+Run `rwctl config unset <key>` to remove an override, so that the setting falls back to the value inherited from the level above.
diff --git a/scripts/eval/retrieval/corpus/data-retention.md b/scripts/eval/retrieval/corpus/data-retention.md
new file mode 100644
index 000000000..107868646
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/data-retention.md
@@ -0,0 +1,58 @@
+# Data retention and deletion
+
+Relaywise stores the events you send so that you can inspect them, debug deliveries and replay them. This page explains what is stored, for how long, and how to remove it sooner.
+
+## What is stored
+
+For each event Relaywise keeps two kinds of record:
+
+- **Payload.** The body of the event, exactly as it was received, and the body after any transformation.
+- **Metadata.** Event ID, type, size, source, the time it was received, and one record per delivery attempt with its status code, duration and error code.
+
+The two are kept for different lengths of time. Metadata is small and useful for spotting trends, so it is kept longer. Payloads can contain personal data, so they are kept for a shorter time.
+
+## Retention by plan
+
+| Plan | Payloads | Metadata |
+|---|---|---|
+| Sandbox | 3 days | 7 days |
+| Team | 14 days | 30 days |
+| Scale | 30 days | 90 days |
+| Enterprise | 90 days | 400 days |
+
+These are maximums. You can choose shorter periods with the `retention.payload_days` and `retention.metadata_days` settings. Some teams set payload retention to a single day to keep as little personal data as possible.
+
+Once a payload has expired, the event still appears in the history with its metadata, but its body cannot be viewed and the event cannot be replayed. A replay attempt returns error `RW-6001`.
+
+## Storing less in the first place
+
+If a field should never be stored at all, add its path to `retention.redact_fields`. Relaywise replaces the value with `[redacted]` before the event is written to storage. The endpoint still receives the original value, because redaction changes only the stored copy.
+
+To remove a field from what is delivered as well, use a transformation.
+
+## Deleting events on request
+
+To delete particular events before they expire, for example to honour a request from a person to erase their data, use the deletion API or the console:
+
+- **One event.** Open the event and select **Delete payload**.
+- **By filter.** Under **Settings → Data → Delete events**, enter a time range and optionally a payload field and value, such as `data.customer_id` equal to `C-2291`.
+
+Deletion removes payloads at once from the primary store. Copies in encrypted backups age out within 35 days. Metadata is kept unless you also tick **Delete metadata**. Every deletion is written to the audit log with the filter that was used; the deleted content itself is not recorded.
+
+## Exporting before deletion
+
+If you need events for longer than your plan keeps them, export them. A scheduled export writes new events to a storage bucket that you own, every hour, as compressed JSON Lines files. Set it up under **Settings → Data → Export**. Exports are available on the Scale and Enterprise plans.
+
+For a single download, **Events → Export** produces a file of up to one million events matching the current filter.
+
+## When a workspace is closed
+
+When you delete a workspace, ingest stops immediately. All payloads and metadata are deleted within 7 days, and backups age out within 35 days after that. Invoices are kept for as long as accounting law requires.
+
+A closed workspace cannot be restored once the 7 days have passed. Within the 7 days, an Owner can ask support to reopen it.
+
+## What retention does not cover
+
+- The dead-letter queue has its own time limits, counted separately.
+- Audit log entries follow the audit log's retention, not the event retention.
+- Metric charts are built from aggregated numbers, not from stored events, and stay available longer.
diff --git a/scripts/eval/retrieval/corpus/dead-letter-queue.md b/scripts/eval/retrieval/corpus/dead-letter-queue.md
new file mode 100644
index 000000000..fc75d0c91
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/dead-letter-queue.md
@@ -0,0 +1,64 @@
+# Dead-letter queue and replay
+
+The dead-letter queue, or DLQ, holds events that Relaywise could not deliver. Nothing in it is lost: you can look at each event, see why it failed, and send it again once the cause is fixed.
+
+## How events end up in the queue
+
+An event moves to the DLQ of a destination when one of these happens:
+
+- Every delivery attempt allowed for that destination has failed.
+- The destination was disabled or paused when the event arrived.
+- A transformation on the route raised an error, so there was no payload to deliver.
+- The endpoint answered `410 Gone`.
+
+Each destination has its own queue. An event routed to three destinations can be delivered to two of them and sit in the queue of the third.
+
+## How long events are kept there
+
+| Plan | Time in the dead-letter queue |
+|---|---|
+| Sandbox | 3 days |
+| Team | 14 days |
+| Scale | 30 days |
+| Enterprise | 30 days, extendable to 90 by contract |
+
+When that time is up, the event is deleted from the queue and can no longer be replayed. This period is counted from the moment the event entered the queue, and it is separate from the retention of ordinary event history.
+
+## Inspecting a failed event
+
+Open **Destinations**, choose the destination, then the **Dead-letter queue** tab. For each event you can see the payload, every delivery attempt with its status code and response time, the first 2 KB of each response body, and the error code of the final attempt.
+
+Filter the list by error code, event type or time range to find a pattern. A queue full of timeouts points at a slow endpoint. A queue full of 401 responses usually means the signing secret on your side is out of date.
+
+## Replaying events
+
+Replaying sends an event again as a fresh delivery with a full retry budget.
+
+**One event.** Open the event and select **Replay**.
+
+**Many events.** Tick the events, or select everything that matches the current filter, then select **Replay selected**. Bulk replays run in the background at the destination's normal delivery rate, so they do not flood your endpoint.
+
+**From the command line.**
+
+```bash
+rwctl dlq replay --destination dst_4h7q2 --since 2h
+rwctl dlq replay --destination dst_4h7q2 --error RW-3001
+```
+
+**To a different destination.** Add `--to dst_9x1aa` to send the events somewhere other than where they first failed. This is useful when you have replaced an endpoint with a new one.
+
+Replayed deliveries carry the header `Relaywise-Replay: true` and keep the original event ID, so an endpoint that tracks event IDs will recognise anything it has already handled.
+
+Replaying needs the Developer role or higher, or an API key with the `replay:run` scope.
+
+## Purging
+
+If the events in a queue are no longer wanted, for example test traffic sent to a decommissioned endpoint, select **Purge queue**. Purging deletes every event in that destination's queue and cannot be undone. Only Admins and Owners can purge.
+
+## Getting told about failures
+
+A growing queue is often the first sign that an integration is broken. Create an alert rule on the `dlq.depth` metric to be told when a queue passes a size you choose. By default Relaywise also sends the workspace Admins a daily digest email listing destinations whose queues grew in the last 24 hours. Turn the digest off under **Settings → Notifications**.
+
+## Replay limits
+
+A single bulk replay can include at most 100,000 events. Larger recoveries have to be split by time range. Replayed events count towards your monthly event quota in the same way as new events.
diff --git a/scripts/eval/retrieval/corpus/delivery-retries.md b/scripts/eval/retrieval/corpus/delivery-retries.md
new file mode 100644
index 000000000..9a31d1998
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/delivery-retries.md
@@ -0,0 +1,64 @@
+# Delivery and retries
+
+Endpoints fail. They get deployed, overloaded, misconfigured or cut off by a network fault. Relaywise assumes this will happen and keeps trying to deliver an event until it succeeds or the retry budget runs out.
+
+## What counts as a successful delivery
+
+A delivery succeeds when your endpoint answers with any status code from 200 to 299 before the timeout. The response body is ignored, so return as little as possible and return it quickly. Do the slow work after you have acknowledged the event.
+
+Everything else is a failure: a 3xx redirect, a 4xx or 5xx status, a timeout, a refused connection, a TLS error or a DNS failure. Relaywise does not follow redirects.
+
+## The default retry schedule
+
+Each event gets up to 8 delivery attempts. The first attempt is made as soon as the event is accepted. If it fails, the waits before the following attempts are:
+
+| Attempt | Wait since the previous attempt |
+|---|---|
+| 2 | 30 seconds |
+| 3 | 2 minutes |
+| 4 | 10 minutes |
+| 5 | 1 hour |
+| 6 | 3 hours |
+| 7 | 8 hours |
+| 8 | 15 hours |
+
+Taken together the schedule covers a little over 27 hours. That is long enough to ride out an overnight outage without anyone having to step in.
+
+Each wait is adjusted by a random amount of up to 20 percent in either direction. Without this jitter, every event that failed during an outage would be retried at the same instant, and the burst could knock the recovering endpoint over again.
+
+## Changing the schedule
+
+Retry behaviour is set per destination, under **Settings → Delivery**, or with these configuration keys:
+
+- `delivery.max_attempts` sets the total number of attempts, from 1 to 20. The default is 8. A value of 1 disables retries.
+- `delivery.backoff` chooses between `exponential`, which is the schedule above, and `fixed`.
+- `delivery.fixed_interval_seconds` sets the wait when the backoff is `fixed`.
+- `delivery.timeout_seconds` sets how long Relaywise waits for a response. The default is 15 seconds and the maximum is 60.
+
+If you raise the number of attempts above 8, the extra attempts are spaced 15 hours apart.
+
+## Responses that change the schedule
+
+- **`Retry-After` header.** If a failed response includes this header, Relaywise waits at least that long before the next attempt, up to a ceiling of one hour.
+- **`410 Gone`.** Relaywise takes this to mean the endpoint has been removed on purpose. It stops retrying the event and pauses the destination so that you can check the URL.
+- **`429 Too Many Requests`.** Treated as a failure and retried, and it also counts towards the circuit breaker described below.
+
+## The circuit breaker
+
+If more than half of the deliveries to one destination fail within a ten-minute window, Relaywise marks the destination as **degraded**. While a destination is degraded, deliveries to it are slowed to one request per second so that a struggling endpoint has room to recover. Events are not dropped; they wait in the queue. The destination returns to normal after two minutes in which every delivery succeeds.
+
+## Automatic disabling
+
+A destination that has not accepted a single delivery for five days in a row is disabled, and Relaywise emails the workspace Admins. New events for a disabled destination are not attempted; they go straight to the dead-letter queue with error `RW-3410`. Re-enable the destination from its settings page once the endpoint is fixed.
+
+## Ordering
+
+Relaywise does not guarantee delivery order by default, because a retried event can arrive after newer ones. If your endpoint needs events for the same entity in order, turn on ordered delivery for the route and choose an ordering key, such as a customer ID. With ordered delivery, a failing event holds back later events that share its key until it succeeds or is exhausted.
+
+## Idempotency
+
+Because of retries, your endpoint can receive the same event more than once, for example when it processed the event but the response was lost. Every delivery carries the event ID in the `Relaywise-Event-Id` header. Store the IDs you have handled and skip repeats.
+
+## When the attempts run out
+
+An event that fails its final attempt is moved to the dead-letter queue, where it waits for you to inspect and replay it.
diff --git a/scripts/eval/retrieval/corpus/error-codes.md b/scripts/eval/retrieval/corpus/error-codes.md
new file mode 100644
index 000000000..9acfce579
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/error-codes.md
@@ -0,0 +1,75 @@
+# Error code reference
+
+Relaywise error codes have the form `RW-` followed by four digits. The first digit tells you which part of the system reported the problem. API responses include the code in the `error.code` field, and delivery attempts show it in the console next to the attempt.
+
+## 1xxx: authentication and authorisation
+
+**RW-1401: Invalid or revoked credentials.** The API key is missing, malformed, expired or revoked. Check that the `Authorization` header uses the `Bearer` scheme and that the key has not been revoked.
+
+**RW-1403: Missing scope.** The key is valid but is not allowed to do this. The response names the scope that is needed. Create a key with that scope, or add the scope to the existing key.
+
+**RW-1410: Wrong key type.** A test key was used against a live source, or an automation key was used to send events.
+
+**RW-1440: Session expired.** The console session has ended. Log in again.
+
+## 2xxx: configuration
+
+**RW-2001: Source not found.** The source ID in the URL does not exist in this workspace, or it has been deleted.
+
+**RW-2004: Insecure destination URL.** Destination URLs must use HTTPS. Plain HTTP is accepted only for `localhost` through the development tunnel.
+
+**RW-2010: Filter syntax error.** The route filter could not be parsed. The response gives the position of the first character that was not understood.
+
+**RW-2020: Route limit reached.** The workspace already has the maximum number of routes its plan allows.
+
+**RW-2031: Destination URL not allowed.** The URL resolves to a private or reserved network address. Relaywise delivers only to public addresses unless a private link is set up.
+
+## 3xxx: delivery
+
+**RW-3001: Delivery timeout.** The endpoint did not answer within the configured timeout. The attempt is retried.
+
+**RW-3002: TLS handshake failed.** The endpoint's certificate has expired, is self-signed, or does not match the hostname.
+
+**RW-3003: DNS resolution failed.** The hostname of the destination could not be resolved.
+
+**RW-3004: Connection refused.** Nothing was listening at the destination's address and port.
+
+**RW-3010: Non-success response.** The endpoint answered with a status code outside the 2xx range. The status code is recorded with the attempt.
+
+**RW-3410: Destination disabled.** The destination was disabled after failing continuously for five days, or was disabled by hand. Events go to the dead-letter queue without being attempted.
+
+## 4xxx: limits
+
+**RW-4130: Payload too large.** The event is bigger than 256 KB.
+
+**RW-4131: Batch too large.** The batch request is bigger than 5 MB or contains more than 500 events.
+
+**RW-4290: Rate limit exceeded.** The workspace is sending faster than its plan allows. Wait until the time given in the `Retry-After` header.
+
+**RW-4291: Monthly quota exhausted.** The workspace has used all the events included in its plan and has a hard cap, or is on the Sandbox plan, which has no overage.
+
+## 5xxx: transformations
+
+**RW-5001: Transformation runtime error.** The transformation code threw an exception. The message and line number are shown with the event.
+
+**RW-5002: Transformation timed out.** The code ran longer than the time allowed.
+
+**RW-5003: Transformation output too large.** The transformed payload is over the size limit.
+
+**RW-5004: Transformation returned an invalid value.** The function has to return an object, or `null` to drop the event.
+
+## 6xxx: replay
+
+**RW-6001: Event no longer available.** The event is older than the retention period and its payload has been deleted, so it cannot be replayed.
+
+**RW-6002: Replay already running.** A bulk replay for this destination is still in progress. Wait for it to finish or cancel it.
+
+## 7xxx: regions
+
+**RW-7001: Region mismatch.** The request was sent to an ingest hostname in a different region from the one the workspace lives in.
+
+## 9xxx: service errors
+
+**RW-9000: Internal error.** Something went wrong on the Relaywise side. The request is safe to retry. If the error continues, contact support and quote the request ID from the `Relaywise-Request-Id` response header.
+
+**RW-9003: Service temporarily unavailable.** Relaywise is shedding load or carrying out maintenance. Retry after the interval in the `Retry-After` header.
diff --git a/scripts/eval/retrieval/corpus/event-filtering.md b/scripts/eval/retrieval/corpus/event-filtering.md
new file mode 100644
index 000000000..ee3087121
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/event-filtering.md
@@ -0,0 +1,57 @@
+# Routing and event filters
+
+A route decides which events from a source go to which destinations. Without a filter, a route forwards everything. With a filter, it forwards only the events you describe, so each endpoint receives what it cares about and nothing else.
+
+## Writing a filter
+
+A filter is a single expression that is evaluated against each event. If it is true, the event is delivered to the route's destinations.
+
+```
+type == "order.created" && data.total > 100
+```
+
+Fields are addressed with dots. `type` is the event type. `data` is the payload body. `meta` holds information Relaywise adds, such as `meta.source` and `meta.received_at`.
+
+## Operators
+
+| Operator | Meaning | Example |
+|---|---|---|
+| `==`, `!=` | Equal, not equal | `data.currency == "EUR"` |
+| `>`, `>=`, `<`, `<=` | Numeric comparison | `data.items_count >= 3` |
+| `&&`, `\|\|`, `!` | And, or, not | `!(data.test == true)` |
+| `in` | Value is in a list | `data.country in ["DE", "FR", "NL"]` |
+| `matches` | Wildcard pattern | `type matches "invoice.*"` |
+| `exists` | Field is present | `exists(data.customer.email)` |
+| `contains` | Text or list contains a value | `data.tags contains "priority"` |
+
+In a `matches` pattern, `*` stands for any run of characters within one segment of a dotted name, and `**` stands for any number of segments. So `invoice.*` matches `invoice.paid` but not `invoice.line.added`, and `invoice.**` matches both.
+
+## Missing fields
+
+A comparison against a field that is not in the payload is false; it is not an error. This means `data.total > 100` quietly excludes events that have no `total`. Use `exists(...)` when the difference matters to you.
+
+## Trying a filter before you save it
+
+The route editor has a **Test** panel. It runs your expression against the 100 most recent events from the source and shows which ones would match. Use it every time. A filter with a typo in a field name matches nothing and produces no error, which is easy to miss.
+
+If the expression itself cannot be parsed, saving fails with error `RW-2010`, and the message points at the character where parsing stopped.
+
+## Sending one event to several places
+
+A route can have up to 20 destinations. Each of them gets its own delivery with its own retries, so a failure at one endpoint does not delay the others.
+
+One event can also match several routes. If two routes send the same event to the same destination, it is delivered twice. Relaywise does not merge duplicates across routes, because the routes may apply different transformations.
+
+## Events that match nothing
+
+An event that matches no route is marked **unrouted**. It is stored in the event history for the usual retention period, but nothing is delivered. Unrouted events still count towards the monthly quota. If you see many of them, either add a route for them or stop sending them.
+
+## Limits
+
+- A filter may be up to 4 KB long and contain up to 50 conditions.
+- Filters cannot call functions other than `exists`, and cannot read other events.
+- The number of routes in a workspace depends on the plan.
+
+## Filter or transformation?
+
+Use a filter to decide whether an event is delivered. Use a transformation to change what is delivered. A transformation can also drop an event by returning `null`, but a filter is cheaper, runs first, and makes the intent visible in the route list.
diff --git a/scripts/eval/retrieval/corpus/getting-started.md b/scripts/eval/retrieval/corpus/getting-started.md
new file mode 100644
index 000000000..889206126
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/getting-started.md
@@ -0,0 +1,57 @@
+# Getting started with Relaywise
+
+Relaywise is a hosted service that receives events from your applications and delivers them to HTTP endpoints you control. You send an event once; Relaywise takes care of queuing, signing, retrying and recording what happened to it.
+
+This page walks through the shortest path from a new account to a delivered event. It takes about ten minutes.
+
+## Core concepts
+
+- **Workspace.** The top-level container for everything you create. Billing, members and settings all belong to a workspace. Most companies need one workspace per environment, for example one for staging and one for production.
+- **Source.** Where events come from. A source has an ingest URL that your application posts events to.
+- **Destination.** An HTTP endpoint that should receive events. A destination has a URL, a signing secret and its own delivery settings.
+- **Route.** A rule that connects a source to one or more destinations, optionally with a filter and a transformation.
+- **Event.** A single JSON payload accepted by a source. One event can produce several deliveries, one per matching destination.
+
+## Step 1: Create a workspace
+
+Sign up at the Relaywise console and choose a workspace name. The name becomes part of your ingest hostname, so pick something short. You can rename a workspace later, but the ingest hostname keeps the original name.
+
+Every new workspace starts on the Sandbox plan, which is free and needs no payment card.
+
+## Step 2: Create a source
+
+Open **Sources** and select **New source**. Give it a name such as `orders-service`. Relaywise shows the ingest URL once the source is saved. The URL looks like this:
+
+```
+https://ingest.relaywise.example/v1/sources/src_8fk2m1/events
+```
+
+## Step 3: Create a destination
+
+Open **Destinations** and select **New destination**. Enter the HTTPS URL of the endpoint that should receive events. Plain HTTP URLs are rejected except for `localhost` when you use the development tunnel.
+
+When you save the destination, Relaywise generates a signing secret for it. Copy the secret now if you plan to verify signatures; you can also reveal it again later from the destination's settings.
+
+## Step 4: Connect them with a route
+
+Open **Routes**, select **New route**, then pick the source and the destination you just created. Leave the filter empty so that every event matches.
+
+## Step 5: Send a test event
+
+You need an API key to send events. Create one under **Settings → API keys** with the `events:write` scope, then post a payload:
+
+```bash
+curl -X POST https://ingest.relaywise.example/v1/sources/src_8fk2m1/events \
+ -H "Authorization: Bearer $RELAYWISE_API_KEY" \
+ -H "Content-Type: application/json" \
+ -d '{"type": "order.created", "order_id": "A-1042"}'
+```
+
+A successful request returns `202 Accepted` and an event ID that starts with `evt_`. Open **Events** in the console to watch the delivery. A green status means your endpoint answered with a 2xx response within the timeout.
+
+## Where to go next
+
+- Verify that deliveries really come from Relaywise by checking the signature header.
+- Decide how failed deliveries should be retried.
+- Invite your teammates and give each of them a role.
+- Install the command-line tool if you prefer working from a terminal.
diff --git a/scripts/eval/retrieval/corpus/invoices-and-payments.md b/scripts/eval/retrieval/corpus/invoices-and-payments.md
new file mode 100644
index 000000000..dfd9c7380
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/invoices-and-payments.md
@@ -0,0 +1,57 @@
+# Invoices and payments
+
+This page covers how Relaywise charges you, where to find invoices, and what happens when a payment does not go through. For what each plan costs and includes, see the plans page.
+
+## When you are charged
+
+The billing period starts on the day you first moved to a paid plan and repeats monthly or yearly from that date. On each renewal date Relaywise charges:
+
+- the plan fee for the coming period, in advance, and
+- any overage from the period that has just ended, in arrears.
+
+An invoice is issued on the renewal date and emailed to the billing contact.
+
+## Payment methods
+
+Paid plans can be paid by credit or debit card. Workspaces on a yearly Scale plan and all Enterprise workspaces can ask to pay by bank transfer against an invoice. Bank transfer invoices are due within 14 days of the invoice date.
+
+To change the card, open **Settings → Billing → Payment method**. The new card is used from the next charge. Only Owners can see or change payment details.
+
+## Finding invoices
+
+All invoices are listed under **Settings → Billing → Invoices**. Each one can be downloaded as a PDF. An invoice shows the plan fee, overage with the number of extra events, any discount, tax, and the total.
+
+## The billing contact
+
+Invoices and payment notices go to the billing contact, which is the first Owner unless you change it. Add a shared mailbox, such as your accounts department, under **Settings → Billing → Contacts**. You can list up to five addresses.
+
+## Tax
+
+Enter your company's tax registration number under **Settings → Billing → Tax details**. The number appears on future invoices. Depending on where your company is registered, a valid number may change how tax is applied. Invoices that have already been issued are not regenerated; ask support if you need a corrected copy.
+
+## When a payment fails
+
+Cards are declined for ordinary reasons: an expired card, a spending limit, a bank's fraud check. When a charge fails:
+
+1. Relaywise emails the billing contact and the Owners immediately.
+2. The charge is tried again 3 days, 5 days and 7 days after the first failure.
+3. During this time the workspace works normally.
+4. If the charge has still not succeeded 10 days after the first failure, the workspace is **suspended**.
+
+A suspended workspace rejects new events and makes no deliveries. Configuration and history stay in place. Paying the outstanding invoice lifts the suspension within a few minutes.
+
+If the invoice is still unpaid 30 days after suspension, the workspace is closed and its data is deleted.
+
+## Refunds
+
+Monthly plans are not refunded for a period that has started; cancel before the renewal date to avoid the next charge. Yearly plans can be refunded in full within 30 days of purchase. After 30 days a yearly plan runs to the end of its term.
+
+If you were charged overage because of a fault on the Relaywise side, such as a replay that ran twice, contact support and the extra events will be credited.
+
+## Credits
+
+Credits from a refund, a goodwill gesture or a service-level payout are held on the workspace and applied automatically to the next invoices until they are used up. The balance is shown at the top of the **Invoices** page. Credits cannot be paid out as cash.
+
+## Purchase orders
+
+Enterprise customers can add a purchase order number to their invoices. Send the number to your account contact before the renewal date.
diff --git a/scripts/eval/retrieval/corpus/monitoring-and-alerts.md b/scripts/eval/retrieval/corpus/monitoring-and-alerts.md
new file mode 100644
index 000000000..9bce49f72
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/monitoring-and-alerts.md
@@ -0,0 +1,72 @@
+# Monitoring and alerts
+
+Relaywise records how every source and destination is behaving and can tell you when something goes wrong, so that you do not learn about a broken integration from a customer.
+
+## The dashboard
+
+The **Overview** page shows the state of the workspace over the last 24 hours: events received, delivery success rate, the slowest destinations and the size of each dead-letter queue. Select any chart to narrow the view to one destination or change the time range.
+
+## Metrics
+
+| Metric | What it measures |
+|---|---|
+| `ingest.rate` | Events accepted per second |
+| `ingest.rejected` | Events refused, grouped by error code |
+| `delivery.success_rate` | Share of delivery attempts that got a 2xx response |
+| `delivery.latency_p50` | Median response time of the endpoint |
+| `delivery.latency_p95` | Response time that 95 percent of attempts stay under |
+| `delivery.attempts_per_event` | Average number of attempts needed for an event |
+| `dlq.depth` | Number of events waiting in a dead-letter queue |
+| `queue.age_seconds` | Age of the oldest event still waiting for delivery |
+| `quota.used_percent` | Share of the monthly included events used so far |
+
+Each metric can be split by destination, source or event type.
+
+Metrics are kept at one-minute resolution for 7 days and at one-hour resolution for 13 months. This is longer than events themselves are kept, because metrics hold only counts and timings, never payloads.
+
+## Alert rules
+
+An alert rule watches one metric and notifies you when it crosses a threshold.
+
+1. Open **Monitoring → Alerts** and select **New rule**.
+2. Choose a metric, and optionally limit the rule to one destination.
+3. Set the condition, for example `delivery.success_rate` below 95 percent.
+4. Set the evaluation window. The condition has to hold for the whole window before the alert fires. The shortest window is 5 minutes.
+5. Choose where the notification goes.
+
+Longer windows produce fewer false alarms. A destination that drops below 95 percent for one minute during a deployment is normal. One that stays there for 15 minutes is not.
+
+## Notification channels
+
+- **Email** to named members or any address.
+- **Chat webhook**, posting a message to a channel in your team chat tool.
+- **Paging integration**, opening an incident with your on-call service.
+- **A Relaywise destination**, delivering the alert as an event so that your own tooling can react to it.
+
+One rule can notify several channels.
+
+## Avoiding repeated notifications
+
+After a rule fires, it stays quiet for a cooldown period of 30 minutes even if the condition still holds. When the metric returns to normal, Relaywise sends a single **resolved** notification. The cooldown can be set per rule to anything from 5 minutes to 24 hours.
+
+To silence a rule during planned work, select **Mute** and choose a duration. A muted rule keeps evaluating and records what it would have sent.
+
+## Rules worth creating first
+
+| Rule | Why |
+|---|---|
+| `delivery.success_rate` below 95 percent for 15 minutes | An endpoint is failing |
+| `dlq.depth` above 100 | Events are being given up on |
+| `queue.age_seconds` above 600 | Deliveries are falling behind |
+| `quota.used_percent` above 80 | You are about to pay overage or reach a cap |
+| `delivery.latency_p95` above 5 seconds | An endpoint is slow and close to timing out |
+
+New workspaces on paid plans get the first two created automatically, with email notifications to the Admins.
+
+## Sending metrics to your own tools
+
+All of the metrics above can be read from the metrics API in a text format that most monitoring systems understand. Scrape it once a minute at most. Use an API key with the `events:read` scope.
+
+## Relaywise's own status
+
+Problems on the Relaywise side are published on the status page, which you can subscribe to by email. An incident there usually explains a sudden rise in `RW-9000` or `RW-9003` errors.
diff --git a/scripts/eval/retrieval/corpus/network-security.md b/scripts/eval/retrieval/corpus/network-security.md
new file mode 100644
index 000000000..39a9aea9a
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/network-security.md
@@ -0,0 +1,47 @@
+# Network security
+
+This page describes the controls that decide which machines may talk to Relaywise on your behalf, and how you can be sure that traffic reaching your endpoints really comes from Relaywise.
+
+## Encryption
+
+All traffic to and from Relaywise is encrypted in transit with TLS 1.2 or newer. Older protocol versions are refused. Stored event payloads, transformation secrets and backups are encrypted at rest with AES-256, using keys that are rotated every 90 days.
+
+On the Enterprise plan you can supply your own encryption key from a key management service that you control. If you withdraw the key, Relaywise can no longer read the stored payloads of that workspace.
+
+## Restricting who can send events
+
+By default any machine that holds a valid API key can send events. To narrow this, set an ingest allowlist under **Settings → Security → Ingest allowlist**, or with the `ingest.allowed_ips` setting. Enter one or more address ranges in CIDR notation, for example `203.0.113.0/24`.
+
+Once the allowlist has at least one entry, requests from any other address are refused before the API key is even checked. Add the new range first and remove the old one afterwards when your addresses change, or you will lock out your own senders.
+
+## Restricting who can use the console
+
+A separate console allowlist limits where members can log in from, which is useful for requiring the company network or VPN. It is set under **Settings → Security → Console allowlist**. Owners are exempt, so that a mistake in the list cannot lock everyone out.
+
+## Allowing Relaywise through your firewall
+
+Deliveries are sent from a fixed, published set of addresses for each region. If your endpoint is behind a firewall, allow those addresses and block everything else. Find them on the **Network** page or with `rwctl network egress-ips`.
+
+Do not rely on the firewall alone. Addresses prove where a request came from, not that it is genuine. Verify the signature on every delivery as well.
+
+## Mutual TLS
+
+For endpoints that require the caller to present a certificate, turn on mutual TLS for the destination. Relaywise then presents a client certificate with each delivery. Download the certificate authority bundle from the destination's **Security** tab and configure your server to trust it. Client certificates are renewed every 60 days, and the new certificate is signed by the same authority, so nothing has to change on your side.
+
+## Private connectivity
+
+On the Enterprise plan, deliveries can reach endpoints that have no public address through a private link between Relaywise and your cloud network. This is also the only way to deliver to private address ranges. Without it, a destination URL that resolves to a private address is rejected with error `RW-2031`.
+
+## Two-factor authentication
+
+Members who log in with a password can add a second factor, using an authenticator app or a hardware security key. Admins can make this compulsory for the whole workspace under **Settings → Security → Require two-factor**. Members without a second factor are then asked to set one up at their next login. Members who log in through single sign-on follow the identity provider's own rules instead.
+
+## Independent assurance
+
+Relaywise is audited every year by an independent firm against a widely used security standard for service providers. The current report is available to customers and serious prospects under a confidentiality agreement; request it from support. An external firm also carries out a penetration test once a year, and a summary of the findings is available on the same terms.
+
+A data processing agreement is included in the terms for every paid plan. No separate signature is needed, though a countersigned copy can be requested.
+
+## Reporting a vulnerability
+
+If you believe you have found a security weakness in Relaywise, email the security team at the address given on the security page of the website. Reports are acknowledged within two working days. Please allow time for a fix before publishing details.
diff --git a/scripts/eval/retrieval/corpus/rate-limits.md b/scripts/eval/retrieval/corpus/rate-limits.md
new file mode 100644
index 000000000..13512f294
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/rate-limits.md
@@ -0,0 +1,68 @@
+# Rate limits and size limits
+
+Relaywise limits how fast each workspace can send events and call the management API. The limits protect the service as a whole and stop a runaway loop in one of your systems from using up your monthly quota in an afternoon.
+
+## Ingest limits
+
+The ingest limit is counted in events per second, across all sources in the workspace.
+
+| Plan | Sustained ingest rate |
+|---|---|
+| Sandbox | 10 events per second |
+| Team | 200 events per second |
+| Scale | 2,000 events per second |
+| Enterprise | Set by contract |
+
+Short spikes are allowed. For up to 10 seconds a workspace may send at twice its sustained rate. After that, requests above the limit are rejected until the average falls back under it.
+
+## Management API limits
+
+Calls that read or change configuration, list events or start replays are limited to 600 requests per minute for each API key. This limit is the same on every plan.
+
+## What a rejected request looks like
+
+A request over the limit gets the status `429 Too Many Requests` and error code `RW-4290`. The event is not accepted, so your code has to send it again.
+
+Every response, whether accepted or rejected, includes these headers:
+
+```
+Relaywise-RateLimit-Limit: 200
+Relaywise-RateLimit-Remaining: 0
+Relaywise-RateLimit-Reset: 1767225612
+```
+
+`Reset` is the Unix time at which more capacity becomes available. A rejected response also carries a standard `Retry-After` header giving the same moment as a number of seconds.
+
+## Handling 429 responses well
+
+- Wait until the reset time before trying again. Retrying at once only prolongs the problem.
+- Add a small random delay so that many workers do not all resume together.
+- Put a queue in front of your sender if you produce events in bursts.
+- Use the batch endpoint when you have many events to send at once.
+
+The official client libraries do all of this for you. They retry a rate-limited request up to five times before giving up.
+
+## Batching
+
+The batch endpoint accepts up to 500 events in one request. A batch is one HTTP request, but every event inside it counts towards the events-per-second limit. Batching reduces connection overhead; it is not a way around the ingest limit.
+
+If part of a batch would go over the limit, the whole batch is rejected. Relaywise never accepts half of a batch.
+
+## Size limits
+
+| Item | Limit |
+|---|---|
+| One event payload | 256 KB |
+| One batch request | 5 MB |
+| Custom headers on a destination | 20 headers, 8 KB in total |
+| Event type name | 128 characters |
+
+A payload over 256 KB is rejected with error `RW-4130`. If you need to move something bigger, store it elsewhere and send a link to it in the event.
+
+## Monthly quota is a separate thing
+
+The rate limit controls how fast you send. The monthly quota controls how much you send in a billing period. Reaching the monthly quota produces a different error, `RW-4291`, and waiting a few seconds does not clear it. On paid plans events over the quota are accepted and billed as overage, unless you have set a hard cap.
+
+## Asking for a higher limit
+
+Workspaces on the Scale plan can request a temporary increase for a planned event such as a product launch or a data migration. Open a support ticket at least three working days ahead, and say what peak rate you expect and for how long.
diff --git a/scripts/eval/retrieval/corpus/regions.md b/scripts/eval/retrieval/corpus/regions.md
new file mode 100644
index 000000000..de98a3c53
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/regions.md
@@ -0,0 +1,50 @@
+# Regions and data residency
+
+A Relaywise workspace lives in one region. Events sent to that workspace are received, stored and delivered from that region, and are not copied to any other.
+
+## Available regions
+
+| Region code | Location | Ingest hostname |
+|---|---|---|
+| `us-east` | Virginia, United States | `ingest.us.relaywise.example` |
+| `eu-central` | Frankfurt, Germany | `ingest.eu.relaywise.example` |
+| `ap-southeast` | Singapore | `ingest.ap.relaywise.example` |
+
+Each region runs across three separate data centres. The loss of one data centre does not interrupt the service or lose accepted events.
+
+## Choosing a region
+
+You choose the region when you create the workspace. Two things should guide the choice:
+
+- **Where your data is allowed to be.** If your company or your customers require data to remain in the European Union, choose `eu-central`.
+- **Where your systems run.** Put the workspace close to the applications that send events and the endpoints that receive them. Distance adds delay to every request.
+
+If the two pull in different directions, the legal requirement wins.
+
+## What stays in the region
+
+Event payloads, event metadata, dead-letter queues, transformation code, logs produced by transformations and backups all stay in the workspace's region.
+
+A small amount of account information is held centrally in the United States so that login and billing work everywhere: member names and email addresses, the workspace name, plan and invoices. Event data is never part of this.
+
+## Changing region
+
+The region of an existing workspace cannot be changed. To move, create a new workspace in the target region, copy the configuration across with `rwctl config export` and `rwctl config import`, and point your senders at the new ingest hostname. Event history does not move.
+
+Enterprise customers can ask support for an assisted migration, which keeps destination IDs and signing secrets the same, so that receiving endpoints need no changes.
+
+## Sending to the wrong region
+
+Each ingest hostname accepts events only for workspaces in its own region. A request sent to the hostname of another region is rejected with error `RW-7001`. The general hostname `ingest.relaywise.example` redirects to the correct region, but it adds a network round trip, so use the regional hostname in production.
+
+## Where deliveries come from
+
+Deliveries leave from a fixed set of IP addresses in each region. If your endpoint sits behind a firewall, allow the addresses for your workspace's region. The current list is on the workspace's **Network** page and is printed by `rwctl network egress-ips`. Relaywise gives at least 30 days' notice before adding or removing an address.
+
+## Running in several regions
+
+A company that needs events handled in more than one region creates one workspace per region. Workspaces are independent. They do not share sources, destinations, quotas or history. Members can belong to several workspaces and switch between them in the console, and Enterprise contracts can pool the event allowance across workspaces.
+
+## Disaster recovery
+
+Backups are taken every hour and kept in the same region, in a different data centre from the primary copy. Relaywise aims to restore service within four hours of losing a whole region and to lose no more than one hour of event history. Events that had been accepted but not yet delivered are delivered once service resumes.
diff --git a/scripts/eval/retrieval/corpus/roles-and-permissions.md b/scripts/eval/retrieval/corpus/roles-and-permissions.md
new file mode 100644
index 000000000..3419f7729
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/roles-and-permissions.md
@@ -0,0 +1,55 @@
+# Roles and permissions
+
+Every member of a workspace has exactly one role. The role decides what that person can see and change in the console and through personal access tokens.
+
+## The four roles
+
+**Owner.** Full control, including billing, deleting the workspace and transferring ownership. A workspace must always have at least one Owner.
+
+**Admin.** Everything an Owner can do except billing, workspace deletion and ownership transfer. Admins manage members, API keys and security settings.
+
+**Developer.** Day-to-day work: creating and editing sources, destinations, routes, filters and transformations, and replaying events. Developers cannot manage members or keys.
+
+**Viewer.** Read-only access to configuration and delivery history. Viewers see event metadata but not payload bodies unless payload access is granted separately.
+
+## Permission matrix
+
+| Action | Owner | Admin | Developer | Viewer |
+|---|---|---|---|---|
+| View events and delivery attempts | Yes | Yes | Yes | Yes |
+| View event payload bodies | Yes | Yes | Yes | Only with payload access |
+| Create or edit destinations and routes | Yes | Yes | Yes | No |
+| Replay events | Yes | Yes | Yes | No |
+| Purge the dead-letter queue | Yes | Yes | No | No |
+| Create or revoke API keys | Yes | Yes | No | No |
+| Invite or remove members | Yes | Yes | No | No |
+| Change security settings | Yes | Yes | No | No |
+| Read the audit log | Yes | Yes | No | No |
+| Change plan or payment method | Yes | No | No | No |
+| Delete the workspace | Yes | No | No | No |
+
+## Inviting a member
+
+Admins and Owners invite people from **Settings → Members**. Enter an email address and pick a role. The invitation link is valid for 7 days. If it lapses, send it again from the same page; the original link stops working when a new one is issued.
+
+A person can belong to several workspaces and may have a different role in each.
+
+## Changing a role
+
+Open the member's row, choose a new role and confirm. The change takes effect on the member's next request; they do not need to log out. An Admin cannot change the role of an Owner.
+
+## Payload access
+
+Event bodies often contain personal data. By default Viewers see only metadata: event type, size, timestamps and delivery outcome. To let a Viewer read bodies, an Admin enables **Payload access** on that member. Each time someone opens a payload, the action is written to the audit log.
+
+## Removing a member
+
+Removing a member ends their sessions at once and revokes their personal access tokens. API keys they created keep working, because keys belong to the workspace and not to the person who made them. Review those keys separately if the departure was not amicable.
+
+## Transferring ownership
+
+Only an Owner can make someone else an Owner. To hand a workspace over, promote the new person first, confirm that they can log in, and then demote yourself. Relaywise refuses to remove the last Owner.
+
+## Custom roles
+
+The Enterprise plan adds custom roles. A custom role starts as a copy of one of the four built-in roles, and you then switch individual permissions on or off. A workspace can define up to 20 custom roles.
diff --git a/scripts/eval/retrieval/corpus/signing-secrets.md b/scripts/eval/retrieval/corpus/signing-secrets.md
new file mode 100644
index 000000000..9aa0b7fbf
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/signing-secrets.md
@@ -0,0 +1,52 @@
+# Signing secrets and signature verification
+
+Anyone who learns your endpoint URL can post data to it. To let you tell real deliveries from forged ones, Relaywise signs every request it sends with a secret that only you and Relaywise know. Your endpoint recomputes the signature and rejects requests where it does not match.
+
+Signing secrets are not API keys. An API key identifies you to Relaywise. A signing secret proves to you that a request came from Relaywise.
+
+## The signature header
+
+Each delivery carries two headers:
+
+```
+Relaywise-Timestamp: 1767225600
+Relaywise-Signature: v1=5f2b7c0e9a...
+```
+
+The signature is an HMAC-SHA256 digest, written in lowercase hexadecimal. The signed message is the timestamp, a full stop, and the exact raw request body:
+
+```
+<timestamp>.<raw body>
+```
+
+## Verifying a delivery
+
+1. Read the raw body as bytes, before any JSON parsing. Parsing and re-serialising changes whitespace and key order, and the signature will no longer match.
+2. Build the signed message from the timestamp header and the raw body.
+3. Compute the HMAC-SHA256 of that message using the destination's signing secret.
+4. Compare your result with the value after `v1=` using a constant-time comparison.
+5. Reject the request if the timestamp is more than five minutes old. This limits replay attacks, where someone captures a valid request and sends it again later.
+
+A request that fails verification should get a `401` response. Relaywise treats a 401 like any other failed delivery and retries it.
+
+## Where to find the secret
+
+Each destination has its own secret, which starts with `whsec_`. Open the destination, go to **Settings → Signing**, and select **Reveal**. Revealing a secret is recorded in the audit log.
+
+## Rolling a signing secret
+
+Change a signing secret when someone who knew it leaves the team, or when it may have been exposed.
+
+1. Open the destination's **Signing** settings and select **Roll secret**.
+2. Relaywise creates a new secret. For the next 72 hours every delivery carries two signatures in the header, one made with each secret, separated by a comma.
+3. Update your endpoint to use the new secret. Because both signatures are present, your endpoint keeps accepting deliveries whichever secret it holds during the changeover.
+4. After 72 hours the previous secret is discarded and deliveries carry a single signature again.
+
+The 72-hour dual-signature period is fixed and cannot be shortened. If the old secret is known to be compromised, select **Expire previous secret** to drop it at once.
+
+## Common mistakes
+
+- **Verifying the parsed body.** Frameworks that parse JSON automatically often discard the raw bytes. Configure the route to keep them.
+- **Clock drift.** If your server clock is wrong by more than five minutes, every delivery looks like a replay. Keep time synchronised.
+- **Using the wrong destination's secret.** Secrets are per destination, not per workspace.
+- **Comparing with `==`.** Ordinary string comparison leaks timing information. Use the constant-time function from your language's standard library.
diff --git a/scripts/eval/retrieval/corpus/single-sign-on.md b/scripts/eval/retrieval/corpus/single-sign-on.md
new file mode 100644
index 000000000..7bf6dbd23
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/single-sign-on.md
@@ -0,0 +1,48 @@
+# Single sign-on and user provisioning
+
+Single sign-on (SSO) lets your team log in to the Relaywise console through your company's identity provider instead of with a separate password. It is available on the Scale and Enterprise plans.
+
+## Supported protocols
+
+Relaywise supports SAML 2.0 and OpenID Connect. Both work with the common identity providers. If your provider supports both, prefer OpenID Connect: it is simpler to set up and its certificates rotate automatically.
+
+## Setting up SAML
+
+1. In the console, open **Settings → Security → Single sign-on** and select **Configure SAML**.
+2. Copy the two values Relaywise shows you into your identity provider: the Assertion Consumer Service URL and the Entity ID.
+3. In your identity provider, map the user's email address to the `NameID` attribute, in email format.
+4. Download the provider's metadata file and upload it to Relaywise.
+5. Select **Test connection**. Relaywise opens a login window and reports whether the response was valid.
+6. When the test passes, select **Enable**.
+
+## Verifying your domain
+
+Before SSO can be enforced, you must prove that you control your email domain. Relaywise gives you a TXT record to add to your DNS. Verification usually completes within an hour of the record being published. One workspace can verify several domains.
+
+## Enforcing SSO
+
+With SSO enabled but not enforced, members can use either SSO or their password. Turning on **Require SSO** removes the password option for everyone whose email address belongs to a verified domain.
+
+Owners keep the ability to log in with a password and a one-time code even when SSO is required. This is a deliberate break-glass path so that a broken identity provider cannot lock you out of your own workspace. Keep at least two Owners for this reason.
+
+## Session length
+
+A console session lasts 12 hours by default. After that the member is sent back to the identity provider. On the Enterprise plan you can set the session length anywhere between 1 hour and 30 days. Idle sessions end after 60 minutes regardless of the overall session length.
+
+## Automatic provisioning with SCIM
+
+SCIM keeps the member list in step with your identity provider, so you do not have to invite or remove people by hand.
+
+- When someone is assigned to the Relaywise application in the identity provider, they are created as a member.
+- When someone is unassigned or deactivated, their console access ends within a few minutes and their personal access tokens are revoked.
+- Group membership can set the member's role. Map each group to one role in **Settings → Security → Group mapping**.
+
+To enable it, generate a SCIM token under **Settings → Security → Provisioning** and paste it, together with the SCIM base URL, into your identity provider. The token is valid for one year, and Relaywise emails the Owners 30 days before it expires.
+
+Members created by SCIM default to the Viewer role unless a group mapping says otherwise.
+
+## Troubleshooting SSO
+
+- **"Response is not yet valid".** The clocks of the two systems disagree. Check the identity provider's time settings.
+- **Login loops back to the login page.** The email address in the response does not belong to a verified domain.
+- **A new colleague cannot log in.** They have not been assigned to the application in the identity provider.
diff --git a/scripts/eval/retrieval/corpus/transformations.md b/scripts/eval/retrieval/corpus/transformations.md
new file mode 100644
index 000000000..801f31300
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/transformations.md
@@ -0,0 +1,75 @@
+# Payload transformations
+
+A transformation is a small piece of JavaScript that runs on an event after it has passed the route's filter and before it is delivered. Use one when the receiving system expects a different shape from the one your application produces.
+
+## What transformations are for
+
+- Renaming or removing fields to match what the receiver expects.
+- Removing sensitive fields that a particular destination should not see.
+- Adding a computed field, such as a total or a formatted date.
+- Setting custom HTTP headers for one delivery.
+- Dropping events based on logic too involved for a filter.
+
+## Writing one
+
+A transformation exports a single function. It receives the event and a context object, and returns the payload to deliver.
+
+```javascript
+export default function transform(event, context) {
+ const { card_number, ...safe } = event.data;
+ return {
+ ...safe,
+ amount_formatted: (event.data.amount_cents / 100).toFixed(2),
+ forwarded_by: "relaywise",
+ };
+}
+```
+
+Return `null` to drop the event for this route. A dropped event is recorded as **filtered by transformation** and is not treated as a failure.
+
+Returning anything other than an object or `null` fails with error `RW-5004`.
+
+## The context object
+
+| Property | Contents |
+|---|---|
+| `context.headers` | Headers to send with the delivery. Add or change entries to set custom headers. |
+| `context.destination` | The ID and name of the destination being delivered to. |
+| `context.attempt` | The attempt number, starting at 1. |
+| `context.secrets` | Read-only values stored under the route's **Secrets** tab. |
+
+Put tokens and passwords in `context.secrets`; do not paste them into the code. Secrets are encrypted, never appear in logs, and are not included when a transformation is exported.
+
+## The sandbox
+
+Transformations run in an isolated sandbox with firm limits.
+
+- **No network access.** The code cannot make HTTP requests or open sockets.
+- **No file system and no clock tricks.** The code cannot read files, and timers are not available.
+- **Time.** Each run has 250 milliseconds by default, configurable up to one second. Going over fails with `RW-5002`.
+- **Memory.** Each run has 32 MB by default.
+- **Output size.** The returned payload must be within the normal event size limit, or the run fails with `RW-5003`.
+
+The transformation runs again on every delivery attempt, so it must give the same output for the same input. Do not rely on random numbers.
+
+## When the code fails
+
+If the function throws, the run fails with `RW-5001`. What happens next depends on the `transform.on_error` setting of the route:
+
+- `dead_letter`, the default, moves the event to the dead-letter queue without attempting delivery. Fix the code and replay.
+- `pass_through` delivers the original payload unchanged.
+- `drop` discards the event for this route.
+
+Choose `pass_through` only if the receiver can cope with both shapes.
+
+## Testing
+
+The editor has a **Run** panel. Paste a sample payload or pick a recent event, and the panel shows the output, the headers, anything written with `console.log`, and how long the run took. Log output is kept for 24 hours and capped at 4 KB per run.
+
+## Versions and rollback
+
+Every save creates a new version. The route always uses the version marked **active**. To undo a bad change, open **History**, choose an earlier version and select **Make active**. The switch applies to new deliveries within a few seconds. Relaywise keeps the last 50 versions of each transformation.
+
+## Sharing code between routes
+
+Save a transformation to the workspace library to use it on several routes. A change to a library transformation applies to every route that uses it, so test carefully, or copy it first if only one route needs the change.
diff --git a/scripts/eval/retrieval/corpus/troubleshooting.md b/scripts/eval/retrieval/corpus/troubleshooting.md
new file mode 100644
index 000000000..8f7dd52c6
--- /dev/null
+++ b/scripts/eval/retrieval/corpus/troubleshooting.md
@@ -0,0 +1,64 @@
+# Troubleshooting
+
+This page starts from what you can see going wrong and works back to a likely cause. Each section points to the page that covers the topic in full.
+
+## My endpoint is not receiving anything
+
+Work through the path an event takes.
+
+1. **Was the event accepted?** Look for it under **Events**. If it is not there, the problem is on the sending side: check the response your sender got when it posted the event.
+2. **Did it match a route?** An event marked **unrouted** was accepted but matched no route. Check the route's filter for a misspelled field name.
+3. **Is the destination enabled?** A destination that is paused or disabled receives nothing. Its status is shown at the top of its page.
+4. **Were attempts made?** Open the event and read the attempts. The status code or error on each attempt tells you what your endpoint did.
+
+## Deliveries fail with a timeout
+
+Your endpoint took longer than the timeout to answer. The usual cause is doing the real work before responding. Acknowledge the event first, then process it in the background. If the endpoint truly needs more time, raise the timeout on the destination.
+
+## Deliveries fail with a 401 or 403
+
+Your endpoint is turning Relaywise away. If you verify signatures, the secret your endpoint holds may be out of date, or the code may be checking a parsed body in place of the raw one. If your endpoint expects a token in a header, check the custom headers set on the destination.
+
+## Deliveries fail with a certificate error
+
+The certificate on your endpoint has expired, does not cover the hostname, or was issued by an authority that is not publicly trusted. Relaywise does not offer a way to skip certificate checks. Renew or replace the certificate.
+
+## Every signature check fails
+
+Three causes account for nearly all cases: the endpoint is using the secret of a different destination, the body was altered by a framework before the check, or the server's clock is more than a few minutes out.
+
+## The same event arrives more than once
+
+This is expected now and then. A delivery whose response was lost on the way back is retried even though your endpoint handled it. Record the event ID from the delivery headers and skip IDs you have seen. If duplicates are frequent, check whether two routes send the same event to the same destination.
+
+## Events arrive out of order
+
+Order is not guaranteed unless ordered delivery is switched on for the route. Either turn it on with a suitable ordering key, or make your endpoint compare a timestamp or version number in the payload before applying a change.
+
+## My sender is getting 429 responses
+
+You are sending faster than your plan allows. Slow down, wait for the time given in the response headers, and consider batching. If the response says the monthly quota is used up, waiting will not help: raise the cap or upgrade.
+
+## A destination was switched off by itself
+
+Relaywise disables a destination that has failed without a break for several days and emails the Admins. Fix the endpoint, re-enable the destination, then replay what collected in its dead-letter queue.
+
+## A replay says the event is no longer available
+
+The payload is older than the workspace's payload retention and has been deleted. The metadata may still be listed, but there is nothing left to send. Longer retention or a scheduled export prevents this in future.
+
+## A transformation worked in the editor but fails on real events
+
+Real events vary more than the sample you tested with. A field that is missing or null is the usual culprit. Open a failed event from the dead-letter queue, copy its payload into the editor, and run it there to see the exception.
+
+## I cannot log in
+
+If your workspace requires single sign-on, the password form will not accept you; use the **Log in with SSO** button. If single sign-on itself fails, ask a workspace Owner, who can still log in with a password. If you have lost your second factor, an Admin can reset it for you.
+
+## The command-line tool says I am not signed in
+
+A stored login lasts 30 days. Run `rwctl login` again. In a pipeline, check that the environment variable holding the key is set and that the key has not been revoked.
+
+## Contacting support
+
+When you write to support, include the workspace name, the event ID or destination ID, the time of the problem with its time zone, and the request ID from the response headers if you have one. These let support find the exact request without asking you for more.
diff --git a/scripts/eval/retrieval/dataset.py b/scripts/eval/retrieval/dataset.py
new file mode 100644
index 000000000..eed2d750f
--- /dev/null
+++ b/scripts/eval/retrieval/dataset.py
@@ -0,0 +1,195 @@
+"""Dataset loading and validation for the OpenRAG retrieval evaluation harness.
+
+The dataset is a directory of corpus files plus a JSONL file of labeled
+questions. Everything is validated before the harness touches a live stack, so
+a typo in a label fails fast instead of silently scoring as a miss.
+
+Standard library only, so it can be unit-tested without the SDK or a stack.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+REQUIRED_FIELDS = frozenset({"query", "expected_files"})
+OPTIONAL_FIELDS = frozenset({"id", "expected_text", "tags"})
+
+
+class DatasetError(ValueError):
+ """The corpus or the question file is invalid. The message lists every problem."""
+
+
+@dataclass(frozen=True)
+class Query:
+ id: str
+ query: str
+ expected_files: tuple[str, ...]
+ expected_text: str | None = None
+ tags: tuple[str, ...] = ()
+
+
+def list_corpus(corpus_dir: Path) -> list[str]:
+ """Sorted filenames of the corpus documents (top level, hidden files skipped)."""
+ if not corpus_dir.is_dir():
+ raise DatasetError(f"corpus directory not found: {corpus_dir}")
+ names = sorted(
+ path.name
+ for path in corpus_dir.iterdir()
+ if path.is_file() and not path.name.startswith(".")
+ )
+ if not names:
+ raise DatasetError(f"corpus directory is empty: {corpus_dir}")
+ return names
+
+
+def _is_string_list(value: object) -> bool:
+ return (
+ isinstance(value, list)
+ and bool(value)
+ and all(isinstance(item, str) and item.strip() for item in value)
+ )
+
+
+def _parse_row(row: object, corpus_names: frozenset[str]) -> tuple[Query | None, list[str]]:
+ if not isinstance(row, dict):
+ return None, ["row must be a JSON object"]
+
+ problems: list[str] = []
+ unknown = sorted(set(row) - REQUIRED_FIELDS - OPTIONAL_FIELDS)
+ if unknown:
+ problems.append(f"unknown field(s): {', '.join(unknown)}")
+
+ query = row.get("query")
+ if not isinstance(query, str) or not query.strip():
+ problems.append("'query' must be a non-empty string")
+
+ expected_files = row.get("expected_files")
+ if not _is_string_list(expected_files):
+ problems.append("'expected_files' must be a non-empty list of filenames")
+ else:
+ assert isinstance(expected_files, list)
+ if len(set(expected_files)) != len(expected_files):
+ problems.append("'expected_files' lists the same file twice")
+ for name in expected_files:
+ if name not in corpus_names:
+ problems.append(f"expected file not in corpus: {name}")
+
+ query_id = row.get("id", query)
+ if "id" in row and (not isinstance(query_id, str) or not query_id.strip()):
+ problems.append("'id' must be a non-empty string")
+
+ expected_text = row.get("expected_text")
+ if expected_text is not None and (
+ not isinstance(expected_text, str) or not expected_text.strip()
+ ):
+ problems.append("'expected_text' must be a non-empty string")
+
+ tags = row.get("tags", [])
+ if tags != [] and not _is_string_list(tags):
+ problems.append("'tags' must be a list of non-empty strings")
+
+ if problems:
+ return None, problems
+ return (
+ Query(
+ id=query_id.strip(),
+ query=query.strip(),
+ expected_files=tuple(expected_files),
+ expected_text=expected_text,
+ tags=tuple(tags),
+ ),
+ [],
+ )
+
+
+def _normalize(text: str) -> str:
+ # Same comparison the scorer uses: case-folded and single-spaced.
+ return " ".join(text.casefold().split())
+
+
+def _snippet_problems(query: Query, corpus_dir: Path, cache: dict[str, str]) -> list[str]:
+ """Expected files that do not contain the question's ``expected_text``.
+
+ A snippet that is not in the source document can never be found in one of
+ its chunks, so the question would score zero on every run.
+ """
+ if query.expected_text is None:
+ return []
+ needle = _normalize(query.expected_text)
+ problems = []
+ for name in query.expected_files:
+ if name not in cache:
+ cache[name] = _normalize((corpus_dir / name).read_text(encoding="utf-8"))
+ if needle not in cache[name]:
+ problems.append(f"'expected_text' {query.expected_text!r} does not appear in {name}")
+ return problems
+
+
+def load_queries(
+ path: Path,
+ corpus_names: list[str],
+ corpus_dir: Path | None = None,
+) -> list[Query]:
+ """Load and validate the question file.
+
+ Raises DatasetError naming the line of every malformed row, so one run of
+ the validator surfaces all problems. With ``corpus_dir``, also checks that
+ each ``expected_text`` appears in every file the question expects.
+ """
+ if not path.is_file():
+ raise DatasetError(f"question file not found: {path}")
+
+ known = frozenset(corpus_names)
+ queries: list[Query] = []
+ errors: list[str] = []
+ first_seen: dict[str, int] = {}
+ corpus_text: dict[str, str] = {}
+ for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
+ if not line.strip():
+ continue
+ try:
+ row = json.loads(line)
+ except json.JSONDecodeError as exc:
+ errors.append(f"{path.name}:{line_number}: invalid JSON ({exc.msg})")
+ continue
+ query, problems = _parse_row(row, known)
+ errors.extend(f"{path.name}:{line_number}: {problem}" for problem in problems)
+ if query is None:
+ continue
+ if query.id in first_seen:
+ errors.append(
+ f"{path.name}:{line_number}: duplicate id {query.id!r} "
+ f"(first used on line {first_seen[query.id]})"
+ )
+ continue
+ first_seen[query.id] = line_number
+ queries.append(query)
+ if corpus_dir is not None:
+ errors.extend(
+ f"{path.name}:{line_number}: {problem}"
+ for problem in _snippet_problems(query, corpus_dir, corpus_text)
+ )
+
+ if not queries and not errors:
+ errors.append(f"{path.name}: no questions found")
+ if errors:
+ raise DatasetError("\n".join(errors))
+ return queries
+
+
+def hash_file(path: Path) -> str:
+ return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def hash_corpus(corpus_dir: Path, corpus_names: list[str]) -> str:
+ """One digest over every corpus filename and its content, in sorted order."""
+ digest = hashlib.sha256()
+ for name in sorted(corpus_names):
+ digest.update(name.encode("utf-8"))
+ digest.update(b"\0")
+ digest.update(hash_file(corpus_dir / name).encode("ascii"))
+ digest.update(b"\n")
+ return digest.hexdigest()
diff --git a/scripts/eval/retrieval/metrics.py b/scripts/eval/retrieval/metrics.py
new file mode 100644
index 000000000..e6b6cd23d
--- /dev/null
+++ b/scripts/eval/retrieval/metrics.py
@@ -0,0 +1,173 @@
+"""File-level retrieval metrics for the OpenRAG retrieval evaluation harness.
+
+``/v1/search`` returns chunks, but questions are labeled by file because chunk
+boundaries move whenever chunking changes. This module turns a ranked chunk
+list into a ranked list of unique files and scores it with recall@k,
+reciprocal rank and nDCG@k.
+
+Standard library only, so it can be unit-tested without the SDK or a stack.
+"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Collection, Iterable, Mapping, Sequence
+from dataclasses import dataclass
+
+DEFAULT_K_VALUES: tuple[int, ...] = (1, 3, 5, 10)
+PREFERRED_PRIMARY_K = 5
+
+
+@dataclass(frozen=True)
+class Chunk:
+ """One search hit, as returned by the API."""
+
+ filename: str
+ text: str
+ score: float
+
+
+@dataclass(frozen=True)
+class RankedFile:
+ """One unique corpus file in the order its first chunk was returned."""
+
+ filename: str # canonical corpus filename, run prefix removed
+ rank: int # 1-based position among unique files
+ score: float # score of the file's first chunk
+ snippet_found: bool | None # None when the question has no expected_text
+
+
+def canonical_name(filename: str, run_prefix: str, corpus_names: Collection[str]) -> str | None:
+ """Map an uploaded filename back to its corpus filename.
+
+ Returns None for anything this run did not upload, so stray documents in
+ the stack never count for or against a question.
+ """
+ if not filename.startswith(run_prefix):
+ return None
+ name = filename[len(run_prefix) :]
+ return name if name in corpus_names else None
+
+
+def _normalize(text: str) -> str:
+ # Chunking can re-wrap lines, so compare on case-folded, single-spaced text.
+ return " ".join(text.casefold().split())
+
+
+def rank_files(
+ chunks: Iterable[Chunk],
+ run_prefix: str,
+ corpus_names: Collection[str],
+ expected_text: str | None = None,
+) -> list[RankedFile]:
+ """Deduplicate ranked chunks into ranked files, keeping first-occurrence order.
+
+ With ``expected_text``, each file also records whether any of its fetched
+ chunks contains the snippet (case- and whitespace-insensitive).
+ """
+ needle = _normalize(expected_text) if expected_text else None
+ order: list[str] = []
+ first_score: dict[str, float] = {}
+ found: dict[str, bool] = {}
+ for chunk in chunks:
+ name = canonical_name(chunk.filename, run_prefix, corpus_names)
+ if name is None:
+ continue
+ if name not in first_score:
+ order.append(name)
+ first_score[name] = chunk.score
+ found[name] = False
+ if needle is not None and not found[name] and needle in _normalize(chunk.text):
+ found[name] = True
+ return [
+ RankedFile(
+ filename=name,
+ rank=position,
+ score=first_score[name],
+ snippet_found=found[name] if needle is not None else None,
+ )
+ for position, name in enumerate(order, start=1)
+ ]
+
+
+def expected_ranks(
+ ranked: Sequence[RankedFile], expected_files: Iterable[str]
+) -> dict[str, int | None]:
+ """Rank of each expected file among the unique files, or None if it was not returned."""
+ by_name = {item.filename: item.rank for item in ranked}
+ return {name: by_name.get(name) for name in expected_files}
+
+
+def primary_k(k_values: Sequence[int]) -> int:
+ """The cutoff used to compare runs: 5 when it is evaluated, otherwise the largest k."""
+ return PREFERRED_PRIMARY_K if PREFERRED_PRIMARY_K in k_values else max(k_values)
+
+
+def score_query(
+ ranked: Sequence[RankedFile],
+ expected_files: Collection[str],
+ k_values: Sequence[int] = DEFAULT_K_VALUES,
+) -> dict[str, float]:
+ """Score one question with binary gains.
+
+ An expected file is relevant when it was returned and, if the question has
+ an ``expected_text``, one of its chunks contained the snippet.
+
+ - ``recall@k``: relevant files in the top k / number of expected files.
+ - ``reciprocal_rank``: 1 / rank of the first relevant file, 0 if none.
+ - ``ndcg@k``: DCG over the top k / the best DCG achievable with
+ min(expected files, k) relevant files at the top.
+ """
+ expected = set(expected_files)
+ if not expected:
+ raise ValueError("expected_files must not be empty")
+ relevant_ranks = sorted(
+ item.rank
+ for item in ranked
+ if item.filename in expected and item.snippet_found is not False
+ )
+
+ recall: dict[str, float] = {}
+ ndcg: dict[str, float] = {}
+ for k in k_values:
+ hits = [rank for rank in relevant_ranks if rank <= k]
+ recall[f"recall@{k}"] = len(hits) / len(expected)
+ dcg = sum(1.0 / math.log2(rank + 1) for rank in hits)
+ ideal = sum(1.0 / math.log2(i + 1) for i in range(1, min(len(expected), k) + 1))
+ ndcg[f"ndcg@{k}"] = dcg / ideal
+ reciprocal_rank = 1.0 / relevant_ranks[0] if relevant_ranks else 0.0
+ return {**recall, "reciprocal_rank": reciprocal_rank, **ndcg}
+
+
+def _mean_scores(rows: Sequence[Mapping[str, float]]) -> dict[str, float]:
+ means: dict[str, float] = {}
+ for key in rows[0]:
+ name = "mrr" if key == "reciprocal_rank" else key
+ means[name] = sum(row[key] for row in rows) / len(rows)
+ return means
+
+
+def aggregate(
+ per_query: Sequence[Mapping[str, float]],
+ tags: Sequence[Sequence[str]],
+) -> dict[str, object]:
+ """Mean of every metric over all questions, and again per tag.
+
+ ``per_query`` and ``tags`` are parallel: ``tags[i]`` holds the tags of the
+ question scored in ``per_query[i]``.
+ """
+ if len(per_query) != len(tags):
+ raise ValueError("per_query and tags must have the same length")
+ if not per_query:
+ return {"overall": {}, "by_tag": {}}
+
+ by_tag: dict[str, list[Mapping[str, float]]] = {}
+ for scores, query_tags in zip(per_query, tags, strict=True):
+ for tag in query_tags:
+ by_tag.setdefault(tag, []).append(scores)
+ return {
+ "overall": _mean_scores(per_query),
+ "by_tag": {
+ tag: {"count": len(rows), **_mean_scores(rows)} for tag, rows in sorted(by_tag.items())
+ },
+ }
diff --git a/scripts/eval/retrieval/pyproject.toml b/scripts/eval/retrieval/pyproject.toml
new file mode 100644
index 000000000..769d75d32
--- /dev/null
+++ b/scripts/eval/retrieval/pyproject.toml
@@ -0,0 +1,12 @@
+[project]
+name = "openrag-retrieval-eval"
+version = "0.1.0"
+description = "Retrieval-quality evaluation harness for OpenRAG search (/v1/search)"
+requires-python = ">=3.11"
+dependencies = ["openrag-sdk"]
+
+[tool.uv]
+package = false
+
+[tool.uv.sources]
+openrag-sdk = { path = "../../../sdks/python", editable = true }
diff --git a/scripts/eval/retrieval/queries.jsonl b/scripts/eval/retrieval/queries.jsonl
new file mode 100644
index 000000000..6e6d67f2a
--- /dev/null
+++ b/scripts/eval/retrieval/queries.jsonl
@@ -0,0 +1,83 @@
+{"id": "key-rotate", "query": "How do I swap out an API key without causing downtime?", "expected_files": ["api-keys.md"], "expected_text": "overlap window", "tags": ["semantic"]}
+{"id": "key-leaked", "query": "One of our keys was pushed to a public repo, what should we do right now?", "expected_files": ["api-keys.md"], "expected_text": "Revoke immediately", "tags": ["semantic"]}
+{"id": "key-scopes", "query": "What scopes can I give an API key?", "expected_files": ["api-keys.md"], "expected_text": "destinations:manage", "tags": ["keyword"]}
+{"id": "key-prefix-ci", "query": "What is a key starting with rwk_ci_ used for?", "expected_files": ["api-keys.md"], "expected_text": "Automation key", "tags": ["exact-token"]}
+{"id": "sig-verify", "query": "How can my server confirm that a webhook really came from Relaywise and was not forged?", "expected_files": ["signing-secrets.md"], "expected_text": "HMAC-SHA256", "tags": ["semantic"]}
+{"id": "sig-roll", "query": "How do I change the secret used to sign deliveries, and how long do the old and new ones both work?", "expected_files": ["signing-secrets.md"], "expected_text": "72 hours", "tags": ["semantic"]}
+{"id": "sig-max-age", "query": "How old can a signed request be before I should refuse it?", "expected_files": ["signing-secrets.md"], "expected_text": "five minutes", "tags": ["semantic"]}
+{"id": "sig-whsec", "query": "Where do I find the whsec_ value for a destination?", "expected_files": ["signing-secrets.md"], "expected_text": "Reveal", "tags": ["exact-token"]}
+{"id": "sso-setup", "query": "How do we let staff log in to the console with their company accounts?", "expected_files": ["single-sign-on.md"], "expected_text": "identity provider", "tags": ["semantic"]}
+{"id": "sso-lockout", "query": "If our identity provider goes down, can anyone still get into the console?", "expected_files": ["single-sign-on.md"], "expected_text": "break-glass", "tags": ["semantic"]}
+{"id": "sso-offboarding", "query": "How is someone's access removed automatically when they leave the company?", "expected_files": ["single-sign-on.md"], "expected_text": "SCIM", "tags": ["semantic"]}
+{"id": "session-length", "query": "How long before the console logs me out?", "expected_files": ["single-sign-on.md", "configuration-reference.md"], "tags": ["semantic", "multi-doc"]}
+{"id": "dlq-purge-who", "query": "Who is allowed to empty a dead-letter queue?", "expected_files": ["roles-and-permissions.md", "dead-letter-queue.md"], "tags": ["keyword", "multi-doc"]}
+{"id": "viewer-payloads", "query": "Can a read-only user see the contents of events?", "expected_files": ["roles-and-permissions.md"], "expected_text": "Payload access", "tags": ["semantic"]}
+{"id": "invite-expiry", "query": "How long does a team invitation link stay valid?", "expected_files": ["roles-and-permissions.md"], "expected_text": "valid for 7 days", "tags": ["semantic"]}
+{"id": "ownership-transfer", "query": "How do I hand a workspace over to a colleague?", "expected_files": ["roles-and-permissions.md"], "expected_text": "last Owner", "tags": ["semantic"]}
+{"id": "retry-schedule", "query": "How many times will a failed webhook be attempted, and over how long?", "expected_files": ["delivery-retries.md"], "expected_text": "27 hours", "tags": ["semantic"]}
+{"id": "cfg-max-attempts", "query": "What is the default for delivery.max_attempts?", "expected_files": ["delivery-retries.md", "configuration-reference.md"], "expected_text": "delivery.max_attempts", "tags": ["exact-token", "multi-doc"]}
+{"id": "circuit-breaker", "query": "What happens when most requests to one endpoint start failing?", "expected_files": ["delivery-retries.md"], "expected_text": "degraded", "tags": ["semantic"]}
+{"id": "gone-410", "query": "What does Relaywise do if my endpoint answers 410 Gone?", "expected_files": ["delivery-retries.md", "dead-letter-queue.md"], "expected_text": "410 Gone", "tags": ["exact-token", "multi-doc"]}
+{"id": "duplicate-events", "query": "Why did my endpoint get the same event twice?", "expected_files": ["delivery-retries.md", "troubleshooting.md"], "tags": ["semantic", "multi-doc"]}
+{"id": "ordered-delivery", "query": "Can I make sure events for one customer are delivered in sequence?", "expected_files": ["delivery-retries.md"], "expected_text": "ordering key", "tags": ["semantic"]}
+{"id": "destination-auto-disabled", "query": "Why was my destination turned off without anyone touching it?", "expected_files": ["delivery-retries.md", "troubleshooting.md"], "tags": ["semantic", "multi-doc"]}
+{"id": "dlq-what", "query": "Where do events go once every delivery attempt has been used up?", "expected_files": ["dead-letter-queue.md"], "tags": ["semantic"]}
+{"id": "dlq-retention-team", "query": "On the Team plan, how long are undeliverable events kept before they are removed?", "expected_files": ["dead-letter-queue.md"], "expected_text": "14 days", "tags": ["semantic"]}
+{"id": "dlq-replay-other-destination", "query": "How do I use rwctl dlq replay to resend failed events to a new endpoint?", "expected_files": ["dead-letter-queue.md"], "expected_text": "dst_9x1aa", "tags": ["exact-token"]}
+{"id": "replay-header", "query": "How can my endpoint tell that a delivery is a replay?", "expected_files": ["dead-letter-queue.md"], "expected_text": "Relaywise-Replay", "tags": ["semantic"]}
+{"id": "ingest-rate", "query": "How fast am I allowed to send events on each plan?", "expected_files": ["rate-limits.md"], "expected_text": "events per second", "tags": ["semantic"]}
+{"id": "err-4290", "query": "RW-4290", "expected_files": ["rate-limits.md", "error-codes.md"], "expected_text": "RW-4290", "tags": ["exact-token", "multi-doc"]}
+{"id": "payload-max-size", "query": "What is the largest event body I can send?", "expected_files": ["rate-limits.md"], "expected_text": "256 KB", "tags": ["semantic"]}
+{"id": "batch-size", "query": "How many events fit in one batch request?", "expected_files": ["rate-limits.md"], "expected_text": "500 events", "tags": ["keyword"]}
+{"id": "err-3002", "query": "What does error RW-3002 mean?", "expected_files": ["error-codes.md"], "expected_text": "TLS handshake failed", "tags": ["exact-token"]}
+{"id": "err-5004", "query": "RW-5004", "expected_files": ["error-codes.md", "transformations.md"], "expected_text": "RW-5004", "tags": ["exact-token", "multi-doc"]}
+{"id": "err-2031", "query": "Why was my destination URL rejected with RW-2031?", "expected_files": ["error-codes.md", "network-security.md"], "expected_text": "RW-2031", "tags": ["exact-token", "multi-doc"]}
+{"id": "err-7001", "query": "What causes RW-7001?", "expected_files": ["error-codes.md", "regions.md"], "expected_text": "RW-7001", "tags": ["exact-token", "multi-doc"]}
+{"id": "cfg-concurrency", "query": "Which setting limits how many requests hit my endpoint in parallel?", "expected_files": ["configuration-reference.md"], "expected_text": "delivery.max_concurrency", "tags": ["semantic"]}
+{"id": "cfg-dedupe-window", "query": "ingest.dedupe_window_seconds", "expected_files": ["configuration-reference.md"], "expected_text": "ingest.dedupe_window_seconds", "tags": ["exact-token"]}
+{"id": "cfg-explain", "query": "How do I find out which level a setting's value is coming from?", "expected_files": ["configuration-reference.md", "cli-reference.md"], "expected_text": "rwctl config explain", "tags": ["semantic", "multi-doc"]}
+{"id": "schema-validation", "query": "Can incoming payloads be checked against a schema and refused when they don't match?", "expected_files": ["configuration-reference.md"], "expected_text": "ingest.schema_validation", "tags": ["semantic"]}
+{"id": "filter-wildcard", "query": "How do I send every kind of invoice event, including nested types, to one endpoint?", "expected_files": ["event-filtering.md"], "expected_text": "invoice.**", "tags": ["semantic"]}
+{"id": "filter-missing-field", "query": "What happens when a filter refers to a field the event doesn't have?", "expected_files": ["event-filtering.md"], "expected_text": "it is not an error", "tags": ["semantic"]}
+{"id": "unrouted-events", "query": "What happens to an event that no route picks up?", "expected_files": ["event-filtering.md"], "expected_text": "unrouted", "tags": ["semantic"]}
+{"id": "route-fan-out", "query": "How many endpoints can one route deliver to?", "expected_files": ["event-filtering.md"], "expected_text": "up to 20 destinations", "tags": ["semantic"]}
+{"id": "strip-fields-before-delivery", "query": "How can I strip card numbers out of events before they are forwarded to a partner?", "expected_files": ["transformations.md"], "expected_text": "card_number", "tags": ["semantic"]}
+{"id": "transform-network", "query": "Can transformation code call an external API?", "expected_files": ["transformations.md"], "expected_text": "No network access", "tags": ["semantic"]}
+{"id": "transform-rollback", "query": "I shipped a broken transformation, how do I go back to the previous one?", "expected_files": ["transformations.md"], "expected_text": "Make active", "tags": ["semantic"]}
+{"id": "plan-scale-price", "query": "What does the Scale plan cost and how many events come with it?", "expected_files": ["billing-plans.md"], "expected_text": "20 million", "tags": ["keyword"]}
+{"id": "over-quota", "query": "What happens if we send more events in a month than our plan includes?", "expected_files": ["billing-plans.md"], "expected_text": "billed as overage", "tags": ["semantic"]}
+{"id": "spending-cap", "query": "How do I make sure we never get a surprise bill?", "expected_files": ["billing-plans.md"], "expected_text": "hard cap", "tags": ["semantic"]}
+{"id": "downgrade-timing", "query": "When does a plan downgrade kick in?", "expected_files": ["billing-plans.md"], "expected_text": "end of the current billing period", "tags": ["semantic"]}
+{"id": "nonprofit-discount", "query": "Is there a discount for nonprofits?", "expected_files": ["billing-plans.md"], "expected_text": "50 percent off", "tags": ["semantic"]}
+{"id": "card-declined", "query": "What happens if our card is declined?", "expected_files": ["invoices-and-payments.md"], "expected_text": "suspended", "tags": ["semantic"]}
+{"id": "invoice-download", "query": "Where can I download past invoices?", "expected_files": ["invoices-and-payments.md"], "expected_text": "PDF", "tags": ["keyword"]}
+{"id": "annual-refund", "query": "Can I get my money back on an annual subscription?", "expected_files": ["invoices-and-payments.md"], "expected_text": "30 days of purchase", "tags": ["semantic"]}
+{"id": "pay-by-wire", "query": "Can we pay by wire instead of by card?", "expected_files": ["invoices-and-payments.md"], "expected_text": "bank transfer", "tags": ["semantic"]}
+{"id": "payload-retention-scale", "query": "How long are event bodies stored on the Scale plan?", "expected_files": ["data-retention.md"], "expected_text": "These are maximums", "tags": ["semantic"]}
+{"id": "erasure-request", "query": "A customer asked us to erase their personal data, how do I remove their events?", "expected_files": ["data-retention.md"], "expected_text": "Delete events", "tags": ["semantic"]}
+{"id": "never-store-field", "query": "How do I keep a sensitive field from ever being saved?", "expected_files": ["data-retention.md", "configuration-reference.md"], "expected_text": "retention.redact_fields", "tags": ["semantic", "multi-doc"]}
+{"id": "export-to-own-storage", "query": "Can I keep events for longer than the plan allows by copying them to storage we own?", "expected_files": ["data-retention.md"], "expected_text": "scheduled export", "tags": ["semantic"]}
+{"id": "replay-event-gone", "query": "Replay fails saying the event isn't available any more, why?", "expected_files": ["data-retention.md", "troubleshooting.md"], "tags": ["semantic", "multi-doc"]}
+{"id": "eu-residency", "query": "We need our data to stay inside the EU, is that possible?", "expected_files": ["regions.md"], "expected_text": "eu-central", "tags": ["semantic"]}
+{"id": "move-region", "query": "Can I move an existing workspace to another region?", "expected_files": ["regions.md"], "expected_text": "cannot be changed", "tags": ["semantic"]}
+{"id": "firewall-ips", "query": "Which IP addresses do I need to let through our firewall so deliveries arrive?", "expected_files": ["regions.md", "network-security.md"], "expected_text": "rwctl network egress-ips", "tags": ["semantic", "multi-doc"]}
+{"id": "alert-on-failing-endpoint", "query": "How do I get notified when an endpoint starts failing?", "expected_files": ["monitoring-and-alerts.md"], "expected_text": "delivery.success_rate", "tags": ["semantic"]}
+{"id": "metric-dlq-depth", "query": "dlq.depth", "expected_files": ["monitoring-and-alerts.md", "dead-letter-queue.md"], "expected_text": "dlq.depth", "tags": ["exact-token", "multi-doc"]}
+{"id": "alert-noise", "query": "How do I stop an alert from notifying us over and over?", "expected_files": ["monitoring-and-alerts.md"], "expected_text": "cooldown", "tags": ["semantic"]}
+{"id": "metrics-history", "query": "How far back do the charts go?", "expected_files": ["monitoring-and-alerts.md"], "expected_text": "13 months", "tags": ["semantic"]}
+{"id": "audit-who-changed", "query": "How can I find out who changed a destination's URL?", "expected_files": ["audit-log.md"], "expected_text": "destination.url_changed", "tags": ["semantic"]}
+{"id": "audit-retention", "query": "How long is the record of admin actions kept?", "expected_files": ["audit-log.md"], "expected_text": "7 years", "tags": ["semantic"]}
+{"id": "audit-tamper", "query": "Could an admin cover their tracks by deleting log entries?", "expected_files": ["audit-log.md"], "expected_text": "hash", "tags": ["semantic"]}
+{"id": "support-access", "query": "Can Relaywise staff look at our payloads?", "expected_files": ["audit-log.md"], "expected_text": "Support access", "tags": ["semantic"]}
+{"id": "local-development", "query": "How do I receive webhooks on my laptop while developing?", "expected_files": ["cli-reference.md"], "expected_text": "rwctl listen", "tags": ["semantic"]}
+{"id": "cli-in-pipeline", "query": "How do I authenticate the command-line tool in a CI pipeline?", "expected_files": ["cli-reference.md"], "expected_text": "RELAYWISE_API_KEY", "tags": ["semantic"]}
+{"id": "cli-exit-code-5", "query": "What does exit code 5 from rwctl mean?", "expected_files": ["cli-reference.md"], "expected_text": "Rate limited", "tags": ["exact-token"]}
+{"id": "client-certificates", "query": "Our endpoint requires callers to present a client certificate, is that supported?", "expected_files": ["network-security.md"], "expected_text": "mutual TLS", "tags": ["semantic"]}
+{"id": "restrict-senders", "query": "Can I restrict which servers are allowed to send events to us?", "expected_files": ["network-security.md"], "expected_text": "ingest allowlist", "tags": ["semantic"]}
+{"id": "enforce-two-step", "query": "How do I force everyone to use two-step login?", "expected_files": ["network-security.md"], "expected_text": "Require two-factor", "tags": ["semantic"]}
+{"id": "encryption-at-rest", "query": "Is stored data encrypted?", "expected_files": ["network-security.md"], "expected_text": "AES-256", "tags": ["semantic"]}
+{"id": "nothing-arriving", "query": "My endpoint isn't getting any events, where do I start looking?", "expected_files": ["troubleshooting.md"], "expected_text": "Was the event accepted", "tags": ["semantic"]}
+{"id": "lost-authenticator", "query": "I lost my authenticator app, how do I get back in?", "expected_files": ["troubleshooting.md"], "expected_text": "second factor", "tags": ["semantic"]}
+{"id": "first-test-event", "query": "What is the quickest way to send a first test event?", "expected_files": ["getting-started.md"], "expected_text": "202 Accepted", "tags": ["semantic"]}
+{"id": "source-vs-destination", "query": "What is the difference between a source and a destination?", "expected_files": ["getting-started.md"], "expected_text": "Where events come from", "tags": ["keyword"]}
+{"id": "plain-http-destination", "query": "Can a destination use a plain http URL?", "expected_files": ["getting-started.md", "error-codes.md"], "expected_text": "localhost", "tags": ["keyword", "multi-doc"]}
diff --git a/scripts/eval/retrieval/report.py b/scripts/eval/retrieval/report.py
new file mode 100644
index 000000000..30f749463
--- /dev/null
+++ b/scripts/eval/retrieval/report.py
@@ -0,0 +1,415 @@
+"""Results files, readable summary and baseline diff for the retrieval evaluation harness.
+
+A run produces one results dict (see README.md for the schema). This module
+writes it as JSON, renders a Markdown summary, and compares two results dicts
+question by question so a small change in the average cannot hide the
+questions that broke.
+
+Standard library only, so it can be unit-tested without the SDK or a stack.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping, Sequence
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlsplit, urlunsplit
+
+SCHEMA_VERSION = 1
+EPSILON = 1e-9
+
+# Run parameters that change scores on their own. A diff between two runs that
+# disagree on any of them is not measuring the search change alone.
+_COMPARABLE_RUN_FIELDS = (
+ ("corpus_hash", "corpus"),
+ ("queries_hash", "question file"),
+ ("k_values", "k values"),
+ ("fetch_limit", "fetch limit"),
+ ("score_threshold", "score threshold"),
+)
+
+
+class ResultsError(ValueError):
+ """A results file could not be read as harness output."""
+
+
+def redact_url(url: str) -> str:
+ """Drop credentials, query string and fragment so the URL is safe to store."""
+ parts = urlsplit(url)
+ host = parts.hostname or ""
+ if ":" in host:
+ # urlsplit strips the brackets from an IPv6 host; a URL needs them back.
+ host = f"[{host}]"
+ if parts.port:
+ host = f"{host}:{parts.port}"
+ return urlunsplit((parts.scheme, host, parts.path.rstrip("/"), "", ""))
+
+
+def load_results(path: Path) -> dict[str, Any]:
+ try:
+ data = json.loads(path.read_text(encoding="utf-8"))
+ except OSError as exc:
+ raise ResultsError(f"cannot read results file {path}: {exc}") from exc
+ except json.JSONDecodeError as exc:
+ raise ResultsError(f"{path} is not valid JSON: {exc.msg}") from exc
+ if not isinstance(data, dict) or not {"run", "metrics", "queries"} <= data.keys():
+ raise ResultsError(f"{path} is not a retrieval evaluation results file")
+ if data.get("schema_version") != SCHEMA_VERSION:
+ raise ResultsError(
+ f"{path} has schema version {data.get('schema_version')!r}; "
+ f"this harness reads version {SCHEMA_VERSION}"
+ )
+ return data
+
+
+def write_results(
+ results: Mapping[str, Any],
+ summary: str,
+ output_dir: Path,
+ started_at: datetime,
+) -> list[Path]:
+ """Write results and summary twice: as the latest run and as a timestamped copy."""
+ stamp = started_at.astimezone(UTC).strftime("%Y%m%d_%H%M%S")
+ output_dir.mkdir(parents=True, exist_ok=True)
+ written: list[Path] = []
+ for stem, suffix, content in (
+ ("results", ".json", json.dumps(results, indent=2) + "\n"),
+ ("summary", ".md", summary),
+ ):
+ for path in (output_dir / f"{stem}{suffix}", output_dir / f"{stem}_{stamp}{suffix}"):
+ path.write_text(content, encoding="utf-8")
+ written.append(path)
+ return written
+
+
+def compare_k(baseline_run: Mapping[str, Any], current_run: Mapping[str, Any]) -> int:
+ """The cutoff to compare two runs on, which both runs must have evaluated.
+
+ This is the current run's primary cutoff when the baseline has it too.
+ Otherwise it is 5 if both runs have it, then the largest shared cutoff.
+ Raises ``ResultsError`` when the runs share no cutoff, because recall and
+ nDCG could then not be compared and a regression would go unnoticed.
+ """
+ base_k = set(baseline_run.get("k_values") or [baseline_run["primary_k"]])
+ cur_k = set(current_run.get("k_values") or [current_run["primary_k"]])
+ shared = base_k & cur_k
+ if not shared:
+ raise ResultsError(
+ f"the runs share no k value (baseline {sorted(base_k)}, "
+ f"current {sorted(cur_k)}), so recall@k and nDCG@k cannot be compared"
+ )
+ if current_run["primary_k"] in shared:
+ return int(current_run["primary_k"])
+ return 5 if 5 in shared else max(shared)
+
+
+def _compare_metrics(k: int) -> list[str]:
+ return [f"recall@{k}", "reciprocal_rank", f"ndcg@{k}"]
+
+
+def _snippet_missing(query: Mapping[str, Any]) -> list[str]:
+ """Expected files that were returned without the question's expected_text.
+
+ Such a file keeps its rank but does not count as relevant, which would
+ otherwise look like a score that dropped for no reason.
+ """
+ expected = set(query.get("expected_files", []))
+ return [
+ item["filename"]
+ for item in query.get("ranked_files", [])
+ if item["filename"] in expected and item.get("snippet_found") is False
+ ]
+
+
+def _mismatch_warnings(baseline: Mapping[str, Any], current: Mapping[str, Any]) -> list[str]:
+ warnings: list[str] = []
+ base_run, cur_run = baseline["run"], current["run"]
+ for field, label in _COMPARABLE_RUN_FIELDS:
+ if base_run.get(field) != cur_run.get(field):
+ warnings.append(
+ f"The {label} differs between the runs "
+ f"(baseline {_short(base_run.get(field))}, current {_short(cur_run.get(field))})."
+ )
+ base_settings = baseline.get("settings") or {}
+ cur_settings = current.get("settings") or {}
+ if not base_settings or not cur_settings:
+ if base_settings or cur_settings:
+ warnings.append("Only one run recorded the stack settings, so they cannot be compared.")
+ else:
+ for key in sorted(base_settings.keys() | cur_settings.keys()):
+ if base_settings.get(key) != cur_settings.get(key):
+ warnings.append(
+ f"Setting '{key}' differs between the runs "
+ f"(baseline {base_settings.get(key)!r}, current {cur_settings.get(key)!r})."
+ )
+ return warnings
+
+
+def _short(value: object) -> str:
+ text = str(value)
+ # Hashes are long; the first 12 characters are enough to tell two apart.
+ return text[:12] if isinstance(value, str) and len(text) == 64 else text
+
+
+def diff_results(
+ baseline: Mapping[str, Any],
+ current: Mapping[str, Any],
+ epsilon: float = EPSILON,
+) -> dict[str, Any]:
+ """Compare two runs question by question.
+
+ Questions are matched by id. Each is classified on recall@k at the cutoff
+ from ``compare_k``, then reciprocal rank, then nDCG@k: the first of those
+ that moved by more than ``epsilon`` decides whether the question improved
+ or regressed.
+ """
+ k = compare_k(baseline["run"], current["run"])
+ metrics = _compare_metrics(k)
+ base_by_id = {q["id"]: q for q in baseline["queries"]}
+ cur_by_id = {q["id"]: q for q in current["queries"]}
+
+ regressed: list[dict[str, Any]] = []
+ improved: list[dict[str, Any]] = []
+ unchanged = 0
+ for query_id, cur in cur_by_id.items():
+ base = base_by_id.get(query_id)
+ if base is None:
+ continue
+ deltas = {
+ name: {
+ "baseline": base["metrics"][name],
+ "current": cur["metrics"][name],
+ "delta": cur["metrics"][name] - base["metrics"][name],
+ }
+ for name in metrics
+ if name in base["metrics"] and name in cur["metrics"]
+ }
+ direction = next(
+ (d["delta"] for d in deltas.values() if abs(d["delta"]) > epsilon),
+ 0.0,
+ )
+ if direction == 0.0:
+ unchanged += 1
+ continue
+ entry = {
+ "id": query_id,
+ "query": cur["query"],
+ "metrics": deltas,
+ "expected_ranks": {
+ "baseline": base.get("expected_ranks", {}),
+ "current": cur.get("expected_ranks", {}),
+ },
+ "snippet_missing": {
+ "baseline": _snippet_missing(base),
+ "current": _snippet_missing(cur),
+ },
+ }
+ (improved if direction > 0 else regressed).append(entry)
+
+ added = [query_id for query_id in cur_by_id if query_id not in base_by_id]
+ removed = [query_id for query_id in base_by_id if query_id not in cur_by_id]
+
+ base_overall = baseline["metrics"].get("overall", {})
+ cur_overall = current["metrics"].get("overall", {})
+ aggregate = {
+ name: {
+ "baseline": base_overall[name],
+ "current": cur_overall[name],
+ "delta": cur_overall[name] - base_overall[name],
+ }
+ for name in cur_overall
+ if name in base_overall
+ }
+
+ warnings = _mismatch_warnings(baseline, current)
+ if k != current["run"]["primary_k"]:
+ warnings.append(
+ f"The baseline has no recall@{current['run']['primary_k']}, "
+ f"so questions are compared on recall@{k}."
+ )
+ if added or removed:
+ warnings.append(
+ "The runs do not cover the same questions, so the aggregate deltas "
+ "are not a like-for-like comparison."
+ )
+
+ return {
+ "baseline_run_id": baseline["run"].get("run_id"),
+ "current_run_id": current["run"].get("run_id"),
+ "compare_metrics": metrics,
+ "warnings": warnings,
+ "aggregate": aggregate,
+ "regressed": regressed,
+ "improved": improved,
+ "unchanged": unchanged,
+ "added": added,
+ "removed": removed,
+ }
+
+
+def _cell(text: object) -> str:
+ return str(text).replace("|", "\\|").replace("\n", " ")
+
+
+def _label(metric: str) -> str:
+ if metric in ("mrr", "reciprocal_rank"):
+ return "MRR" if metric == "mrr" else "Reciprocal rank"
+ return metric.replace("ndcg", "nDCG")
+
+
+def _ranks(ranks: Mapping[str, int | None], snippet_missing: Sequence[str] = ()) -> str:
+ if not ranks:
+ return "n/a"
+ parts = []
+ for name, rank in ranks.items():
+ position = "not returned" if rank is None else f"#{rank}"
+ note = " (snippet not found)" if name in snippet_missing else ""
+ parts.append(f"{name}: {position}{note}")
+ return ", ".join(parts)
+
+
+def _changed_table(entries: Sequence[Mapping[str, Any]], metrics: Sequence[str]) -> list[str]:
+ lines = [
+ "| ID | Query | " + " | ".join(_label(m) for m in metrics) + " | Expected file ranks |",
+ "|---|---|" + "---|" * len(metrics) + "---|",
+ ]
+ for entry in entries:
+ cells = []
+ for name in metrics:
+ delta = entry["metrics"].get(name)
+ cells.append(
+ f"{delta['baseline']:.3f} → {delta['current']:.3f}" if delta is not None else "n/a"
+ )
+ ranks = entry["expected_ranks"]
+ missing = entry.get("snippet_missing", {})
+ before = _ranks(ranks["baseline"], missing.get("baseline", ()))
+ after = _ranks(ranks["current"], missing.get("current", ()))
+ lines.append(
+ f"| {_cell(entry['id'])} | {_cell(entry['query'])} | "
+ + " | ".join(cells)
+ + f" | {_cell(before)} → {_cell(after)} |"
+ )
+ return lines
+
+
+def render_diff(diff: Mapping[str, Any]) -> str:
+ """Markdown section comparing a run with its baseline, regressions first."""
+ metrics = diff["compare_metrics"]
+ regressed, improved = diff["regressed"], diff["improved"]
+ lines = [
+ "## Compared with baseline",
+ "",
+ f"Baseline run `{diff['baseline_run_id']}`, current run `{diff['current_run_id']}`. "
+ f"{len(regressed)} regressed, {len(improved)} improved, "
+ f"{diff['unchanged']} unchanged, {len(diff['added'])} added, "
+ f"{len(diff['removed'])} removed.",
+ "",
+ ]
+ for warning in diff["warnings"]:
+ lines.append(f"> **Warning:** {warning}")
+ if diff["warnings"]:
+ lines.append("")
+
+ lines += [f"### Regressed ({len(regressed)})", ""]
+ lines += _changed_table(regressed, metrics) if regressed else ["No question regressed."]
+ lines += ["", f"### Improved ({len(improved)})", ""]
+ lines += _changed_table(improved, metrics) if improved else ["No question improved."]
+
+ if diff["aggregate"]:
+ lines += [
+ "",
+ "### Aggregate change",
+ "",
+ "| Metric | Baseline | Current | Change |",
+ "|---|---|---|---|",
+ ]
+ for name, delta in diff["aggregate"].items():
+ lines.append(
+ f"| {_label(name)} | {delta['baseline']:.3f} | {delta['current']:.3f} "
+ f"| {delta['delta']:+.3f} |"
+ )
+ for heading, ids in (
+ ("Added questions", diff["added"]),
+ ("Removed questions", diff["removed"]),
+ ):
+ if ids:
+ lines += ["", f"### {heading} ({len(ids)})", ""]
+ lines += [f"- {_cell(query_id)}" for query_id in ids]
+ return "\n".join(lines) + "\n"
+
+
+def _settings_line(results: Mapping[str, Any]) -> str:
+ settings = results.get("settings")
+ if not settings:
+ return f"not recorded ({results.get('settings_note') or 'unavailable'})"
+ return (
+ f"embedding {settings.get('embedding_provider')}/{settings.get('embedding_model')}, "
+ f"chunk size {settings.get('chunk_size')}, overlap {settings.get('chunk_overlap')}"
+ )
+
+
+def render_summary(results: Mapping[str, Any], diff: Mapping[str, Any] | None = None) -> str:
+ """Markdown summary of one run, followed by the baseline comparison if given."""
+ run = results["run"]
+ k = run["primary_k"]
+ overall = results["metrics"].get("overall", {})
+ by_tag = results["metrics"].get("by_tag", {})
+ headline = [f"recall@{k}", "mrr", f"ndcg@{k}"]
+
+ lines = [
+ "# OpenRAG retrieval evaluation",
+ "",
+ f"- **Run:** `{run['run_id']}` at {run['timestamp']}",
+ f"- **Target:** {run['target_url']}",
+ f"- **Commit:** {run.get('git_commit') or 'unknown'}",
+ f"- **SDK version:** {run.get('sdk_version') or 'unknown'}",
+ f"- **Dataset:** {run['corpus_files']} documents (corpus `{_short(run['corpus_hash'])}`), "
+ f"{run['query_count']} questions (`{_short(run['queries_hash'])}`)",
+ f"- **Search:** fetch limit {run['fetch_limit']}, score threshold {run['score_threshold']}",
+ f"- **Settings:** {_settings_line(results)}",
+ "",
+ "## Overall",
+ "",
+ "| Metric | Score |",
+ "|---|---|",
+ ]
+ lines += [f"| {_label(name)} | {value:.3f} |" for name, value in overall.items()]
+
+ if by_tag:
+ lines += [
+ "",
+ "## By tag",
+ "",
+ "| Tag | Questions | " + " | ".join(_label(m) for m in headline) + " |",
+ "|---|---|" + "---|" * len(headline),
+ ]
+ for tag, scores in by_tag.items():
+ lines.append(
+ f"| {_cell(tag)} | {scores['count']} | "
+ + " | ".join(f"{scores[m]:.3f}" for m in headline)
+ + " |"
+ )
+
+ missed = [q for q in results["queries"] if q["metrics"][f"recall@{k}"] < 1 - EPSILON]
+ lines += ["", f"## Questions below full recall@{k} ({len(missed)})", ""]
+ if missed:
+ lines += [
+ f"| ID | Query | recall@{k} | Expected file ranks | Snippet found | Top files returned |",
+ "|---|---|---|---|---|---|",
+ ]
+ for q in missed:
+ snippet = {True: "yes", False: "no", None: "n/a"}[q.get("snippet_found")]
+ top = ", ".join(item["filename"] for item in q["ranked_files"][:3]) or "none"
+ lines.append(
+ f"| {_cell(q['id'])} | {_cell(q['query'])} | {q['metrics'][f'recall@{k}']:.3f} "
+ f"| {_cell(_ranks(q['expected_ranks'], _snippet_missing(q)))} | {snippet} | {_cell(top)} |"
+ )
+ else:
+ lines.append(f"Every question reached full recall@{k}.")
+
+ text = "\n".join(lines) + "\n"
+ if diff is not None:
+ text += "\n" + render_diff(diff)
+ return text
diff --git a/scripts/eval/retrieval/run_eval.py b/scripts/eval/retrieval/run_eval.py
new file mode 100644
index 000000000..66e491ba4
--- /dev/null
+++ b/scripts/eval/retrieval/run_eval.py
@@ -0,0 +1,426 @@
+#!/usr/bin/env python3
+"""Measure OpenRAG retrieval quality against a fixed corpus and labeled questions.
+
+ uv run run_eval.py run [--baseline results/baseline.json]
+ uv run run_eval.py compare <baseline.json> <current.json>
+
+``run`` ingests the checked-in corpus under run-unique filenames, sends every
+question through ``/v1/search``, scores the ranking by file, writes results and
+a summary, and deletes the documents it uploaded. ``compare`` diffs two results
+files offline. See README.md.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import io
+import os
+import subprocess
+import sys
+import time
+import uuid
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import dataset
+import metrics
+import report
+
+HERE = Path(__file__).resolve().parent
+DEFAULT_URL = "http://localhost:3000"
+
+
+class EvalError(RuntimeError):
+ """The run cannot produce trustworthy scores."""
+
+
+def _parse_k_values(value: str) -> list[int]:
+ try:
+ k_values = sorted({int(part) for part in value.split(",") if part.strip()})
+ except ValueError as exc:
+ raise argparse.ArgumentTypeError(
+ f"not a comma-separated list of integers: {value}"
+ ) from exc
+ if not k_values or k_values[0] < 1:
+ raise argparse.ArgumentTypeError("k values must be positive integers")
+ return k_values
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(
+ description="Retrieval-quality evaluation harness for OpenRAG search.",
+ )
+ commands = parser.add_subparsers(dest="command", required=True)
+
+ run = commands.add_parser("run", help="ingest the corpus, run every question and score it")
+ run.add_argument("--url", default=os.environ.get("OPENRAG_URL", DEFAULT_URL))
+ run.add_argument(
+ "--api-key",
+ default=os.environ.get("OPENRAG_API_KEY"),
+ help="defaults to the OPENRAG_API_KEY environment variable",
+ )
+ run.add_argument("--corpus", type=Path, default=HERE / "corpus")
+ run.add_argument("--queries", type=Path, default=HERE / "queries.jsonl")
+ run.add_argument("--output-dir", type=Path, default=HERE / "results")
+ run.add_argument("--baseline", type=Path, help="results file to compare this run against")
+ run.add_argument(
+ "--k",
+ dest="k_values",
+ type=_parse_k_values,
+ default=list(metrics.DEFAULT_K_VALUES),
+ help="comma-separated cutoffs (default: 1,3,5,10)",
+ )
+ run.add_argument(
+ "--fetch-limit",
+ type=int,
+ default=50,
+ help="chunks requested per question before deduplicating to files (default: 50)",
+ )
+ run.add_argument("--score-threshold", type=float, default=0.0)
+ run.add_argument("--concurrency", type=int, default=4, help="parallel uploads (default: 4)")
+ run.add_argument(
+ "--ingest-timeout", type=float, default=300.0, help="seconds per ingestion task"
+ )
+ run.add_argument(
+ "--visibility-timeout",
+ type=float,
+ default=120.0,
+ help="seconds to wait for an ingested file to become searchable",
+ )
+ run.add_argument("--request-timeout", type=float, default=60.0, help="seconds per HTTP request")
+ run.add_argument(
+ "--keep-documents",
+ action="store_true",
+ help="leave the uploaded documents in the stack for debugging",
+ )
+ run.add_argument(
+ "--validate-only",
+ action="store_true",
+ help="check the corpus and question file, then exit without contacting a stack",
+ )
+
+ compare = commands.add_parser("compare", help="diff two results files offline")
+ compare.add_argument("baseline", type=Path)
+ compare.add_argument("current", type=Path)
+ return parser
+
+
+def _sdk_version() -> str:
+ try:
+ from importlib.metadata import version
+
+ return version("openrag-sdk")
+ except Exception:
+ return "unknown"
+
+
+def _git_commit() -> str | None:
+ try:
+ completed = subprocess.run(
+ ["git", "rev-parse", "HEAD"],
+ cwd=HERE,
+ capture_output=True,
+ text=True,
+ timeout=5,
+ check=True,
+ )
+ except (OSError, subprocess.SubprocessError):
+ return None
+ return completed.stdout.strip() or None
+
+
+def _file_failures(status: Any) -> list[str]:
+ """Reasons an ingestion task did not index its file, empty when it did.
+
+ The task counters are not enough: a file skipped as a duplicate of content
+ already in the stack is counted in ``successful_files`` but is not indexed
+ under the name this run uploaded.
+ """
+ problems: list[str] = []
+ if status.status != "completed":
+ problems.append(f"task status is {status.status!r}")
+ if status.failed_files > 0:
+ problems.append(f"{status.failed_files} file(s) failed")
+ if status.successful_files < 1:
+ problems.append("no file was ingested")
+ for name, entry in (status.files or {}).items():
+ if not isinstance(entry, dict):
+ continue
+ file_status = str(entry.get("status", "")).lower()
+ if file_status and file_status != "completed":
+ detail = entry.get("error") or (entry.get("result") or {}).get("reason") or ""
+ problems.append(f"{name}: {file_status} {detail}".strip())
+ return problems
+
+
+async def _ingest_corpus(
+ client: Any,
+ corpus_dir: Path,
+ manifest: dict[str, str],
+ run_id: str,
+ concurrency: int,
+ timeout: float,
+) -> None:
+ """Upload every corpus file under its run-unique name and require each to succeed."""
+ limit = asyncio.Semaphore(max(1, concurrency))
+ # Appended to each upload so its content hash is unique to this run. The
+ # stack skips content it already holds under another filename, which would
+ # otherwise happen whenever an earlier run's cleanup did not finish.
+ marker = f"\n\n<!-- openrag retrieval eval run {run_id} -->\n".encode()
+
+ async def upload(uploaded_name: str, canonical: str) -> str | None:
+ payload = (corpus_dir / canonical).read_bytes() + marker
+ async with limit:
+ try:
+ status = await client.documents.ingest(
+ file=io.BytesIO(payload),
+ filename=uploaded_name,
+ timeout=timeout,
+ )
+ except Exception as exc:
+ return f"{canonical}: {type(exc).__name__}: {exc}"
+ problems = _file_failures(status)
+ return f"{canonical}: {'; '.join(problems)}" if problems else None
+
+ outcomes = await asyncio.gather(
+ *(upload(uploaded, canonical) for uploaded, canonical in manifest.items())
+ )
+ failures = [outcome for outcome in outcomes if outcome]
+ if failures:
+ raise EvalError("ingestion failed:\n " + "\n ".join(failures))
+
+
+async def _wait_until_searchable(
+ client: Any,
+ manifest: dict[str, str],
+ timeout: float,
+) -> None:
+ """Poll until every uploaded file returns from search.
+
+ A completed task does not prove the chunks are indexed and visible, and a
+ question scored against a half-indexed corpus would look like a miss.
+ """
+ from openrag_sdk import SearchFilters
+
+ pending = set(manifest)
+ deadline = time.monotonic() + timeout
+ delay = 1.0
+ while True:
+ for uploaded_name in sorted(pending):
+ response = await client.search.query(
+ "*",
+ filters=SearchFilters(data_sources=[uploaded_name]),
+ limit=1,
+ )
+ if any(result.filename == uploaded_name for result in response.results):
+ pending.discard(uploaded_name)
+ if not pending:
+ return
+ if time.monotonic() >= deadline:
+ names = ", ".join(sorted(manifest[name] for name in pending))
+ raise EvalError(f"not searchable after {timeout:.0f}s: {names}")
+ await asyncio.sleep(delay)
+ delay = min(delay * 2, 8.0)
+
+
+async def _settings_snapshot(client: Any) -> tuple[dict[str, Any] | None, str | None]:
+ try:
+ settings = await client.settings.get()
+ except Exception as exc:
+ return None, f"could not read /v1/settings: {type(exc).__name__}: {exc}"
+ return settings.knowledge.model_dump(), None
+
+
+async def _run_queries(
+ client: Any,
+ queries: list[dataset.Query],
+ manifest: dict[str, str],
+ run_prefix: str,
+ args: argparse.Namespace,
+) -> list[dict[str, Any]]:
+ from openrag_sdk import SearchFilters
+
+ corpus_names = frozenset(manifest.values())
+ scope = SearchFilters(data_sources=sorted(manifest))
+ records: list[dict[str, Any]] = []
+ for query in queries:
+ response = await client.search.query(
+ query.query,
+ filters=scope,
+ limit=args.fetch_limit,
+ score_threshold=args.score_threshold,
+ )
+ chunks = [metrics.Chunk(r.filename, r.text, r.score) for r in response.results]
+ ranked = metrics.rank_files(chunks, run_prefix, corpus_names, query.expected_text)
+ snippet_found = None
+ if query.expected_text is not None:
+ snippet_found = any(
+ item.snippet_found for item in ranked if item.filename in query.expected_files
+ )
+ records.append(
+ {
+ "id": query.id,
+ "query": query.query,
+ "tags": list(query.tags),
+ "expected_files": list(query.expected_files),
+ "expected_text": query.expected_text,
+ "chunks_returned": len(chunks),
+ "unique_files_returned": len(ranked),
+ "ranked_files": [
+ {
+ "filename": item.filename,
+ "rank": item.rank,
+ "score": item.score,
+ "snippet_found": item.snippet_found,
+ }
+ for item in ranked
+ ],
+ "expected_ranks": metrics.expected_ranks(ranked, query.expected_files),
+ "snippet_found": snippet_found,
+ "metrics": metrics.score_query(ranked, query.expected_files, args.k_values),
+ }
+ )
+ return records
+
+
+async def _cleanup(client: Any, manifest: dict[str, str]) -> list[str]:
+ """Delete this run's documents by exact filename and confirm they are gone."""
+ from openrag_sdk import SearchFilters
+
+ warnings: list[str] = []
+ for uploaded_name in manifest:
+ try:
+ await client.documents.delete(filename=uploaded_name)
+ except Exception as exc:
+ warnings.append(f"could not delete {uploaded_name}: {type(exc).__name__}: {exc}")
+ try:
+ response = await client.search.query(
+ "*",
+ filters=SearchFilters(data_sources=sorted(manifest)),
+ limit=len(manifest),
+ )
+ except Exception as exc:
+ warnings.append(f"could not confirm cleanup: {type(exc).__name__}: {exc}")
+ else:
+ leftover = sorted({r.filename for r in response.results if r.filename in manifest})
+ if leftover:
+ warnings.append("still searchable after delete: " + ", ".join(leftover))
+ return warnings
+
+
+async def run(args: argparse.Namespace) -> int:
+ corpus_names = dataset.list_corpus(args.corpus)
+ queries = dataset.load_queries(args.queries, corpus_names, args.corpus)
+ if args.validate_only:
+ print(f"OK: {len(corpus_names)} documents, {len(queries)} questions")
+ return 0
+ # Read the baseline before the run so a bad path or an incomparable
+ # baseline fails in seconds.
+ baseline = report.load_results(args.baseline) if args.baseline else None
+ if baseline is not None:
+ report.compare_k(
+ baseline["run"],
+ {"k_values": args.k_values, "primary_k": metrics.primary_k(args.k_values)},
+ )
+ if not args.api_key:
+ raise EvalError("no API key: set OPENRAG_API_KEY or pass --api-key")
+
+ from openrag_sdk import OpenRAGClient
+
+ started_at = datetime.now(UTC)
+ started = time.monotonic()
+ run_id = f"eval-{uuid.uuid4().hex[:8]}"
+ run_prefix = f"{run_id}-"
+ manifest = {f"{run_prefix}{name}": name for name in corpus_names}
+
+ target_url = report.redact_url(args.url)
+ print(f"Run {run_id}: {len(corpus_names)} documents, {len(queries)} questions → {target_url}")
+ async with OpenRAGClient(
+ api_key=args.api_key, base_url=args.url, timeout=args.request_timeout
+ ) as client:
+ try:
+ print("Ingesting corpus...")
+ await _ingest_corpus(
+ client, args.corpus, manifest, run_id, args.concurrency, args.ingest_timeout
+ )
+ await _wait_until_searchable(client, manifest, args.visibility_timeout)
+ settings, settings_note = await _settings_snapshot(client)
+ print("Running questions...")
+ records = await _run_queries(client, queries, manifest, run_prefix, args)
+ finally:
+ if args.keep_documents:
+ print(f"Kept documents with prefix {run_prefix}")
+ else:
+ for warning in await _cleanup(client, manifest):
+ print(f"warning: cleanup: {warning}", file=sys.stderr)
+
+ results = {
+ "schema_version": report.SCHEMA_VERSION,
+ "run": {
+ "run_id": run_id,
+ "timestamp": started_at.strftime("%Y-%m-%dT%H:%M:%SZ"),
+ "duration_s": round(time.monotonic() - started, 1),
+ "target_url": target_url,
+ "sdk_version": _sdk_version(),
+ "git_commit": _git_commit(),
+ "corpus_files": len(corpus_names),
+ "corpus_hash": dataset.hash_corpus(args.corpus, corpus_names),
+ "query_count": len(queries),
+ "queries_hash": dataset.hash_file(args.queries),
+ "k_values": args.k_values,
+ "primary_k": metrics.primary_k(args.k_values),
+ "fetch_limit": args.fetch_limit,
+ "score_threshold": args.score_threshold,
+ },
+ "settings": settings,
+ "settings_note": settings_note,
+ "metrics": metrics.aggregate(
+ [record["metrics"] for record in records],
+ [record["tags"] for record in records],
+ ),
+ "queries": records,
+ }
+ diff = report.diff_results(baseline, results) if baseline is not None else None
+ summary = report.render_summary(results, diff)
+ written = report.write_results(results, summary, args.output_dir, started_at)
+
+ print()
+ print(summary)
+ print("Wrote:")
+ for path in written:
+ print(f" {path}")
+ return 0
+
+
+def compare(args: argparse.Namespace) -> int:
+ diff = report.diff_results(
+ report.load_results(args.baseline),
+ report.load_results(args.current),
+ )
+ print(report.render_diff(diff))
+ return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = build_parser()
+ args = parser.parse_args(argv)
+ if args.command == "run" and args.fetch_limit < max(args.k_values):
+ # Fewer chunks than the largest cutoff cannot fill that many file
+ # ranks, so recall@k and nDCG@k at that cutoff would be meaningless.
+ parser.error(
+ f"--fetch-limit must be at least the largest --k value "
+ f"({max(args.k_values)}), got {args.fetch_limit}"
+ )
+ try:
+ if args.command == "compare":
+ return compare(args)
+ return asyncio.run(run(args))
+ except (dataset.DatasetError, report.ResultsError, EvalError) as exc:
+ print(f"error: {exc}", file=sys.stderr)
+ return 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/eval/retrieval/uv.lock b/scripts/eval/retrieval/uv.lock
new file mode 100644
index 000000000..3543ea7f9
--- /dev/null
+++ b/scripts/eval/retrieval/uv.lock
@@ -0,0 +1,280 @@
+version = 1
+revision = 3
+requires-python = ">=3.11"
+
+[[package]]
+name = "annotated-types"
+version = "0.8.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" },
+]
+
+[[package]]
+name = "anyio"
+version = "4.15.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "idna" },
+ { name = "typing-extensions", marker = "python_full_version < '3.15'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a9/d2/f4d173e22df740bc37b1db102b386ba719b66e95b0f0d751f556b387e6d2/anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94", size = 276966, upload-time = "2026-09-05T10:42:39.44Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/12/b8/4bd346e22b28902df4d651910f5242c28d84e4a5c2435ca5c3f797ed7e2e/anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101", size = 132079, upload-time = "2026-09-05T10:42:37.923Z" },
+]
+
+[[package]]
+name = "certifi"
+version = "2026.7.22"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" },
+]
+
+[[package]]
+name = "h11"
+version = "0.16.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
+]
+
+[[package]]
+name = "httpcore"
+version = "1.0.9"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "certifi" },
+ { name = "h11" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" },
+]
+
+[[package]]
+name = "httpx"
+version = "0.28.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "anyio" },
+ { name = "certifi" },
+ { name = "httpcore" },
+ { name = "idna" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },
+]
+
+[[package]]
+name = "idna"
+version = "3.20"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" },
+]
+
+[[package]]
+name = "openrag-retrieval-eval"
+version = "0.1.0"
+source = { virtual = "." }
+dependencies = [
+ { name = "openrag-sdk" },
+]
+
+[package.metadata]
+requires-dist = [{ name = "openrag-sdk", editable = "../../../sdks/python" }]
+
+[[package]]
+name = "openrag-sdk"
+version = "0.4.0.dev0"
+source = { editable = "../../../sdks/python" }
+dependencies = [
+ { name = "httpx" },
+ { name = "idna" },
+ { name = "pydantic" },
+]
+
+[package.metadata]
+requires-dist = [
+ { name = "httpx", specifier = ">=0.25.0" },
+ { name = "idna", specifier = ">=3.15" },
+ { name = "pydantic", specifier = ">=2.0.0" },
+ { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0.0" },
+ { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.21.0" },
+]
+provides-extras = ["dev"]
+
+[[package]]
+name = "pydantic"
+version = "2.14.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "annotated-types" },
+ { name = "pydantic-core" },
+ { name = "typing-extensions" },
+ { name = "typing-inspection" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/6b/fb/6e44b63b26efea1cec48c26d8362313310202ef5ed6e7a52f1669e64e2cd/pydantic-2.14.0.tar.gz", hash = "sha256:8a51a7aaddd60f55566d1f07bdd87b92b463903f39a8f26b71a06314cd1548ae", size = 898613, upload-time = "2026-10-08T14:34:48.341Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/2d/eb/9146591cc819d040475bf7f2be786710c7f7eb8083693bf859728da2ca9c/pydantic-2.14.0-py3-none-any.whl", hash = "sha256:15fab1bea6f1dc5003b54fc2ecab230c1fd1dbade2acd4addc52d81e32416d4b", size = 487109, upload-time = "2026-10-08T14:34:46.864Z" },
+]
+
+[[package]]
+name = "pydantic-core"
+version = "2.50.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/e6/6d/196e8c819e0e934f35a1a33b3530396feadb0af4ca38fe9f995249e55794/pydantic_core-2.50.0.tar.gz", hash = "sha256:84d2d38f7d163c4dec292f379e9de1960c661795442aca6c90d706436cb3749e", size = 513355, upload-time = "2026-10-08T14:30:58.245Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/f1/99/114c2e71405da7610c884c64f8370f85b2fbb69bd427158a3f714515a803/pydantic_core-2.50.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:55684bc850059fc85ff8dc21a3e342b722d3178993c49ebdfdee8b698a432720", size = 2152583, upload-time = "2026-10-08T14:26:12.913Z" },
+ { url = "https://files.pythonhosted.org/packages/75/a7/fabcfea81a5c3525507af4347fa3a50e51036d5db6611a63e47379776e56/pydantic_core-2.50.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c3ae59461625518449f800acc4f874753ae96fcf993c47a27be07beb651371fd", size = 2045652, upload-time = "2026-10-08T14:26:14.38Z" },
+ { url = "https://files.pythonhosted.org/packages/f5/80/226019008a041da423dc59d34f980b2b6e60a71971d07a8f52e10fc4747a/pydantic_core-2.50.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7080928078ff56c07da393392f772054e93a39c8033dc9c04547bd949f9b614", size = 2050325, upload-time = "2026-10-08T14:26:16.081Z" },
+ { url = "https://files.pythonhosted.org/packages/b9/29/11c98cd9c49338f64aeb3d6e166c99b65d8949f00f285c5b66de75ade316/pydantic_core-2.50.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4b1bae96f41806b14dd8bb1568d39b5e33b5af7fb27f7be36c78b8fa84708917", size = 2189027, upload-time = "2026-10-08T14:26:17.581Z" },
+ { url = "https://files.pythonhosted.org/packages/cf/b7/af7d9c4de715cc13e947d1135b73f0c181bce2f01e9f1d9e3c428f9ee31c/pydantic_core-2.50.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8facaf0a16121ac82cc403a71399a061b74624c1ae2034eec9ed6169c5d16ecb", size = 2388815, upload-time = "2026-10-08T14:26:19.306Z" },
+ { url = "https://files.pythonhosted.org/packages/26/87/cd95dcd4d066180b2e22eff3dca01ac3af56607f9ecc43b7ba0748496f47/pydantic_core-2.50.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:610e9f483ac53c5cc59c2d3687224029b54eb494feec2d40a2bcfdd187635192", size = 2431821, upload-time = "2026-10-08T14:26:20.958Z" },
+ { url = "https://files.pythonhosted.org/packages/f0/14/7122b78e9915e7dd50f9dcec6041a4e32874907b41b9e0880db2f1c7ae1a/pydantic_core-2.50.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac223ea031905319a64d0df116979ae438b42d6265862a3d63c9f9808c2905a7", size = 2157622, upload-time = "2026-10-08T14:26:22.947Z" },
+ { url = "https://files.pythonhosted.org/packages/97/86/fcddead5bda54fc8efa0d80dbba5962e9a91193570e227404c7593707833/pydantic_core-2.50.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:75b451ec5e64a1d4b803317f34710131742ff2e42f4bf5403f597d0af857d7d8", size = 2254921, upload-time = "2026-10-08T14:26:24.607Z" },
+ { url = "https://files.pythonhosted.org/packages/6c/e8/041d1c3e3e1f203a656ce592f8c1548e1ef2b0939d19c5ce66432c30e4bd/pydantic_core-2.50.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7eee44c3f1f8acc220a743a5ed5588949a4f18b33df4cfbef2b1277470285401", size = 2268163, upload-time = "2026-10-08T14:26:26.207Z" },
+ { url = "https://files.pythonhosted.org/packages/8a/ac/f515994ef1351e883276e85af0a50d839ead9d0d22743fa2c6dfd552f7ce/pydantic_core-2.50.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f44107b5fdfecc03438feb165431652c8006650471326e53dd86d4c19124f5c5", size = 2302005, upload-time = "2026-10-08T14:26:27.808Z" },
+ { url = "https://files.pythonhosted.org/packages/04/6c/b970c4c87d92c4c974a5c5dc16bcf747f89b4e5729c5c45131267ef0421b/pydantic_core-2.50.0-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:718d05d4e078c7828f40ad3e310870fe4b837d93a32e7e8497a080c1f1040490", size = 2465387, upload-time = "2026-10-08T14:26:29.529Z" },
+ { url = "https://files.pythonhosted.org/packages/4a/84/c0a92f97f9de7f133dfe694d0e8ca033a1c90fdf692dc69dfee3bf8a2e7b/pydantic_core-2.50.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:031867348c98ab49c6d3a16ad39738369ea48da59900d76d71c1e64de71fe79b", size = 2478673, upload-time = "2026-10-08T14:26:31.898Z" },
+ { url = "https://files.pythonhosted.org/packages/f1/11/88e851761351cfef90382f175b07cede48edafc62a1509575dbc879934ef/pydantic_core-2.50.0-cp311-cp311-win32.whl", hash = "sha256:e41f9d1d9240e8e0d8a670ad3e66c0c00f0b1f7150a31bc6c445a4f87c1cb3ba", size = 2121424, upload-time = "2026-10-08T14:26:33.67Z" },
+ { url = "https://files.pythonhosted.org/packages/c4/1f/89184f13b9ebaf6d8710cd0d1cfb466d7dc829de23468ab471b543dd07eb/pydantic_core-2.50.0-cp311-cp311-win_amd64.whl", hash = "sha256:90874428bc6678b26434336c77931c9734ecebedf50132c179205e5a9908761d", size = 2176278, upload-time = "2026-10-08T14:26:35.388Z" },
+ { url = "https://files.pythonhosted.org/packages/ff/7a/5a630fff51218a860aa4e7699f781a7c9bf41c031987b613629c98f16618/pydantic_core-2.50.0-cp311-cp311-win_arm64.whl", hash = "sha256:b9be297ffe1015bfb2db4a23b6e1fec7e48361cf7c7b4f4f6bbdd008451b0a7b", size = 2183464, upload-time = "2026-10-08T14:26:37.485Z" },
+ { url = "https://files.pythonhosted.org/packages/e3/d8/e0fe374bc0082dfd337ca319505c9302ce383aeeeacc33d93383edd159e5/pydantic_core-2.50.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:79e8fc9c135ef628c45cb8aa5deef8d21de13d33bb4c59a859fa73d335ceb40a", size = 2222860, upload-time = "2026-10-08T14:26:39.575Z" },
+ { url = "https://files.pythonhosted.org/packages/ef/7d/0a2f829e3e1d809393faab907e3d9307245cd6043ebf54fad439f72f0003/pydantic_core-2.50.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0abe1b44d361b948404b6b2ed80be2583e0077340572e071afa6e0eda4e1de30", size = 2025752, upload-time = "2026-10-08T14:26:41.786Z" },
+ { url = "https://files.pythonhosted.org/packages/67/d4/e2808af12de4809dd6a4b532e3877baa5f1000df51bd8ebb8756450c6c92/pydantic_core-2.50.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae45853d25a23fba56681d2f9ed41f3e3f12f0a3b2393fefa08ff6406320a1f5", size = 2048432, upload-time = "2026-10-08T14:26:43.593Z" },
+ { url = "https://files.pythonhosted.org/packages/18/58/46ad42a321051585d89f4e563b115015332caf79ba4e96f2785715c7f52d/pydantic_core-2.50.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bd841dcf394ff261c26763a9d7be754176d4f1e6d26cb2a5d5331e91b6b56a5f", size = 2173565, upload-time = "2026-10-08T14:26:45.299Z" },
+ { url = "https://files.pythonhosted.org/packages/92/69/541206e657ecea865e5f6d49a4a0d8062586d05da482d9d7ed77fdcdfc86/pydantic_core-2.50.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2e97985641fe53ad7824d1b5bfeb7990a5ff788c4559ee44d7522642560ddc2", size = 2361974, upload-time = "2026-10-08T14:26:47.024Z" },
+ { url = "https://files.pythonhosted.org/packages/f6/c0/cf4850441d0d3d3736abe5e8b6fcb49a50166aa0651ed8444172f81f4b4f/pydantic_core-2.50.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7987866a2569396e6765c54d643fd6a7b3b234e89ee8d45a3729a7b4b2726145", size = 2440817, upload-time = "2026-10-08T14:26:48.937Z" },
+ { url = "https://files.pythonhosted.org/packages/ca/e2/2f793fa2f1b338aa03124efd4362956c84d7bf1b331dc86050bc0151e67d/pydantic_core-2.50.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f187030fc3d62c668feb0f09e92852e0eb414d7fcefc4748f2e67d245aade37e", size = 2160358, upload-time = "2026-10-08T14:26:50.74Z" },
+ { url = "https://files.pythonhosted.org/packages/db/00/35e314c08e721ec1f070408a4e41926b90347ecd2863c08c78daad5f8a16/pydantic_core-2.50.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:d187f43d1c5b844adc871c5b8c22b4aa12a116aaca4e9bd1521bc9ce479aae1f", size = 2231416, upload-time = "2026-10-08T14:26:52.733Z" },
+ { url = "https://files.pythonhosted.org/packages/0f/3a/6f7c36afe35a9eca24e5f74f33ca641ac8cf308aae4ac7af4e585d464a40/pydantic_core-2.50.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cb4fcaabb28cabf21396a9b816b9afe091f8c776805fdf03e2cbc606b64dfa7c", size = 2242496, upload-time = "2026-10-08T14:26:54.485Z" },
+ { url = "https://files.pythonhosted.org/packages/8d/af/77adf30285836c25f6a927e170df45f8dd4af713c89135b22abcbf3c6d67/pydantic_core-2.50.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1751e92d56fbb623b937d73985e500b1a2b1053e56f718a6c564660d99bb9cb0", size = 2279176, upload-time = "2026-10-08T14:26:56.432Z" },
+ { url = "https://files.pythonhosted.org/packages/08/37/4e2a05247f82b59337bf45e9bb14aed85e2d83aecd21e49ad6cedf91882b/pydantic_core-2.50.0-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:70df7ff903aea05383298715ea53551c8f27c8f70cbe54ba7f05606cd822e7e4", size = 2450488, upload-time = "2026-10-08T14:26:58.218Z" },
+ { url = "https://files.pythonhosted.org/packages/85/e6/75f25906212ecb94d69a0c8ed16f5c4ac312dc9961c2e93ff2a7fdf9a07a/pydantic_core-2.50.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:24302bf47319a64e5c5c7c29e971d2b7a20a190c64e59035a3df9582dec636fa", size = 2476664, upload-time = "2026-10-08T14:26:59.98Z" },
+ { url = "https://files.pythonhosted.org/packages/75/72/ebb97b3becd0c0f2c722f698d34dec951ab1aa6086edf535f8769b545e3e/pydantic_core-2.50.0-cp312-cp312-win32.whl", hash = "sha256:5dbf9f18c8af11db719e67633be0af556d7d765bee0ca9419bd706fe4b7ed9fe", size = 2102409, upload-time = "2026-10-08T14:27:01.846Z" },
+ { url = "https://files.pythonhosted.org/packages/3f/7b/5ebf3e62f5d0f6e3503ffdfffd7d0c2f8d45690afd37fa0cfdcb1786d7d6/pydantic_core-2.50.0-cp312-cp312-win_amd64.whl", hash = "sha256:1541c334af5d42cb9eb03862a9b4d2cfbfc670fd05172ec51f3ce02d704550f1", size = 2171536, upload-time = "2026-10-08T14:27:03.797Z" },
+ { url = "https://files.pythonhosted.org/packages/58/1c/879ee9d5b63c60e5a077bab74ad93a4a48476090ca9c97534ab4263d0bae/pydantic_core-2.50.0-cp312-cp312-win_arm64.whl", hash = "sha256:b1399f918aea8fb76ffa99b474c9b768accea1f079fde407ee538bec89f20fa7", size = 2162946, upload-time = "2026-10-08T14:27:05.876Z" },
+ { url = "https://files.pythonhosted.org/packages/81/25/f9a6958f73d92f66d620e3e1b091becf6b7a2ea89437118d397e2c6ca9ba/pydantic_core-2.50.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:049b0404792dcb942f1092bfdae5f819ef30445b0d174782e1909fbdd91bb48b", size = 2222904, upload-time = "2026-10-08T14:27:07.869Z" },
+ { url = "https://files.pythonhosted.org/packages/c5/41/7f299b2ecf0ddbec8c2a68057ed53d29458ea5b0850615570849c454dd89/pydantic_core-2.50.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8f16bc5bb12f4c581b0f40facc28dbf286db32d1bf4e7e4f4c4e0f7f4e34ecf9", size = 2024863, upload-time = "2026-10-08T14:27:09.78Z" },
+ { url = "https://files.pythonhosted.org/packages/89/db/a9852fa8780acca5dd81a21bb66d4b2fb41c39dc5672cec66c32cc7f13e9/pydantic_core-2.50.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1d084e92d0f4a096a5155b23d1ac603db8073ef98a3d1384badf1455e5ae742", size = 2047340, upload-time = "2026-10-08T14:27:11.85Z" },
+ { url = "https://files.pythonhosted.org/packages/90/82/cd174e776e71ebcb1d289a57cb3e89565123a84c5708865e12efa82313f9/pydantic_core-2.50.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f0af2d2f745ec00a6616c4dfba4477f9156ccfb45690f6ffa5fd34f42e871ed4", size = 2173461, upload-time = "2026-10-08T14:27:13.948Z" },
+ { url = "https://files.pythonhosted.org/packages/5a/5c/9c4b2aea09ec7b7af966e79631b1970f1adeb6a8ed001bc8bb64a2d1c23d/pydantic_core-2.50.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2f28a5a299d4cefd1066a6883d22600b9ae3606f881c3015d255784634647580", size = 2362629, upload-time = "2026-10-08T14:27:15.909Z" },
+ { url = "https://files.pythonhosted.org/packages/28/d9/93afd007b61c50425c3c8f0402425779341c8d498bd224e5788eaf419c4c/pydantic_core-2.50.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bdd70c2d9e73bca09fad54605dfeaae2b4e770b2800c65f5f5342901ed567b9f", size = 2440134, upload-time = "2026-10-08T14:27:17.716Z" },
+ { url = "https://files.pythonhosted.org/packages/c9/93/a4ef199535547aaa7c5d2cbf009a1d6a2b252885b14c6e70404e1c661716/pydantic_core-2.50.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e58acd43ac8d3905659c1d5309318dd576243e723dd7c3b5dd4f555479b77d4b", size = 2158480, upload-time = "2026-10-08T14:27:19.618Z" },
+ { url = "https://files.pythonhosted.org/packages/de/20/f216e028d3bfbd9f8c67f7a2f0f8db04240926f630e2e66a5a8ac45fe00d/pydantic_core-2.50.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:5a07c644047f5abc268b4c39f8cb5a30e08a3c349228cb70142c7d3ed87587c0", size = 2231837, upload-time = "2026-10-08T14:27:21.742Z" },
+ { url = "https://files.pythonhosted.org/packages/56/3f/57a6acf26e0acb82238ba4e25b35cbd6013b1d74eac99c4f358ff56bf6d6/pydantic_core-2.50.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8ad8dad549cd1645be591a50b4573995ee7e018f6619ddc2bc6ea44b2ad9f694", size = 2242240, upload-time = "2026-10-08T14:27:23.564Z" },
+ { url = "https://files.pythonhosted.org/packages/94/41/f5f4014b40f91db4479e87dc038ac48a460a60baadea7ff9c942bc64dc62/pydantic_core-2.50.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:1773001198030e16f946a7ff7760fc3ebd45b12150f66fd0f433780043dc13d8", size = 2277852, upload-time = "2026-10-08T14:27:25.517Z" },
+ { url = "https://files.pythonhosted.org/packages/f8/fa/485e4db093e10f29676fc696abf9d88f3626ee520e99cc488fb091222db9/pydantic_core-2.50.0-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:8ab9e74878172948e0426e3d27fba333fd6a1c3f9456d8e75643e8e6868ac1a1", size = 2450132, upload-time = "2026-10-08T14:27:27.493Z" },
+ { url = "https://files.pythonhosted.org/packages/d3/48/fd07063cfc65e498528cea59b27b979eb5c890a54dcf5bb19ab0a589516d/pydantic_core-2.50.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:e15bb1535f68a27f28e579ba3a8f74e7b05d350c45f5622b76a311d5a19ae48a", size = 2477011, upload-time = "2026-10-08T14:27:29.972Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/96/6f34c285dc87ed2fae5f1b53c51bb732eda21dc0c6c73d6a5d9dcc7033cd/pydantic_core-2.50.0-cp313-cp313-win32.whl", hash = "sha256:c21e6a6e4e6d32fb6acbc4f0fa69e8319cac0d65eeaa8298d757371cc2a9c687", size = 2102027, upload-time = "2026-10-08T14:27:32.497Z" },
+ { url = "https://files.pythonhosted.org/packages/5e/50/dbdb3ba6699d494e59db2f145aeb97990e28d1503bc9f7bdd57eb4c15677/pydantic_core-2.50.0-cp313-cp313-win_amd64.whl", hash = "sha256:4f96ccd9368ecbf6685d8f6981584ab72b4f0ffd20685e747737e7e340277b8d", size = 2169974, upload-time = "2026-10-08T14:27:34.692Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/ae/cfe0e52a9b45b5ba12b3a839db928669543948760a059a72f13cf0346c29/pydantic_core-2.50.0-cp313-cp313-win_arm64.whl", hash = "sha256:a2b88f9f9fa52e1c34938ff1a18ee7fbffe482df0bb43c087a9a60578d273d68", size = 2162504, upload-time = "2026-10-08T14:27:36.989Z" },
+ { url = "https://files.pythonhosted.org/packages/59/d9/6dd838672e5ccddf01556bfd1b4a6767e0c75abf9aaa4092cb87a56dbc4a/pydantic_core-2.50.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:62a93a9d206a3580c975c3c1f65a869cd063844d016c004f8d786a9309b3c591", size = 2224496, upload-time = "2026-10-08T14:27:39.281Z" },
+ { url = "https://files.pythonhosted.org/packages/b3/20/c57d2efcc63fb8fb6ae9ec2818baf30e2a68751e8d1dedefaf3dab2e45fd/pydantic_core-2.50.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5a8403eef4a66743e339102fb3cdd8c8b9016b8bd67924685893d062c88896f9", size = 2023860, upload-time = "2026-10-08T14:27:41.332Z" },
+ { url = "https://files.pythonhosted.org/packages/26/c8/f44ea3f1b00288f715e2320cbcd0f604104325370ef4bee42c5ed2ab076e/pydantic_core-2.50.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8354fbbe2abf0fb724b9303bef43bfd9b7a2779332813fa6d6559983954e7d8", size = 2049144, upload-time = "2026-10-08T14:27:43.695Z" },
+ { url = "https://files.pythonhosted.org/packages/74/5c/2f5cf84ceaf6d7c351737124aebc135f50ce48a9dd28d2b5283103face54/pydantic_core-2.50.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:00963fde61cf8880d9e7b5635a9830e0591edfa45168447fdc5b47635c0f6437", size = 2173456, upload-time = "2026-10-08T14:27:45.792Z" },
+ { url = "https://files.pythonhosted.org/packages/97/de/dc0bd815a328e62b72bb93b5f3c762939670d4119fd27405b454970833da/pydantic_core-2.50.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1bbd7da16d8b2912cc56c9d0b85c4998a6cbf39b220f81c0d8c397c64672ae0e", size = 2365204, upload-time = "2026-10-08T14:27:48.117Z" },
+ { url = "https://files.pythonhosted.org/packages/ca/a5/458c4a29f52fdecb16b192fb27f117359da9c9c58401ebfd956637822dca/pydantic_core-2.50.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ed557fa2744617eac3e34dd39b85037efbf23cd33f06851c00fdb8f18ad8f4c2", size = 2445007, upload-time = "2026-10-08T14:27:50.637Z" },
+ { url = "https://files.pythonhosted.org/packages/98/14/0c0e72e0663be91a456167a2f7ed49c65195b388e8f1e8b2bb5083b27458/pydantic_core-2.50.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59f816dc04e99627a5a6352ae51dfb30e603f2cb0b7009c91dc640c533def014", size = 2159911, upload-time = "2026-10-08T14:27:52.881Z" },
+ { url = "https://files.pythonhosted.org/packages/62/7f/64af6921e17ed04dda0e07ebbd394b29fb557d33618cb0f3f81ec59c0508/pydantic_core-2.50.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:779b6c74526596a86d38248dedaecfb7851bbaf319c234be042c57acddd2c8c4", size = 2235761, upload-time = "2026-10-08T14:27:55.194Z" },
+ { url = "https://files.pythonhosted.org/packages/82/11/b6ff9f7207af629094f4deac947d04be3812d5c95054010ad8f710a052f0/pydantic_core-2.50.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:21b62d45f327eb0802f842c132fda6d01a8376a8077922dc4dda69011c64d34a", size = 2243726, upload-time = "2026-10-08T14:27:57.336Z" },
+ { url = "https://files.pythonhosted.org/packages/8e/37/6f7101f3c746a1686987a32875ae2f7c19f518d783979cee1a8a32e0a9ef/pydantic_core-2.50.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:4f31af62efd1fd0257735b72e6716b32d4f207654adeabfe524d44baf1bb6bed", size = 2277158, upload-time = "2026-10-08T14:27:59.714Z" },
+ { url = "https://files.pythonhosted.org/packages/d2/74/67fe208f3ec7f5d8c47bf7cba70019a84b1cffad947ebc8caef14271b54f/pydantic_core-2.50.0-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:933f2d639eb81a3e1f145aec415453cc00983236629933f028d9507222583a2e", size = 2450059, upload-time = "2026-10-08T14:28:02.17Z" },
+ { url = "https://files.pythonhosted.org/packages/cd/10/4a9c56a69f5841bdd94ffff5876a12ad2733209d5b89cadb67638d69e7c7/pydantic_core-2.50.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:c05d035e72530f6b00297941b0162218601542b76870c3bf2756bd87f16fc538", size = 2479142, upload-time = "2026-10-08T14:28:04.494Z" },
+ { url = "https://files.pythonhosted.org/packages/bf/c1/30d36746051e42bb67df986ee874f787526edb99205f6b1583c33dd81202/pydantic_core-2.50.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:ab3f95f737fc1b258b8210308fc02ce1442cf5950cf6d30b09ad89ab9e8afebd", size = 1483929, upload-time = "2026-10-08T14:28:06.527Z" },
+ { url = "https://files.pythonhosted.org/packages/04/2c/c0e8949be4f99a74f02ae02d603bdd518c8a9eba72982a34d675134a792a/pydantic_core-2.50.0-cp314-cp314-win32.whl", hash = "sha256:f12d9690634414fc04b1a7072fdc35c34a9242232c1851fe4518383578bb09d4", size = 2104255, upload-time = "2026-10-08T14:28:08.729Z" },
+ { url = "https://files.pythonhosted.org/packages/ea/6f/a5a5baf99509c998e0f2335d66b815880c7d6c63231c8e4017b237d6837d/pydantic_core-2.50.0-cp314-cp314-win_amd64.whl", hash = "sha256:63263d64884688554fb025a3906c7b00573980cfee75f9afeb86239d577384bc", size = 2170939, upload-time = "2026-10-08T14:28:11.08Z" },
+ { url = "https://files.pythonhosted.org/packages/40/bc/c89b93b69d59cfe61c1e91b1cd3d87cc740e57327ecfef84ce34b7dcadac/pydantic_core-2.50.0-cp314-cp314-win_arm64.whl", hash = "sha256:753863dd4317ec8cc9eb3e6d9d01a8ef1a1726a8003b4354658d68a1ae05f9db", size = 2157035, upload-time = "2026-10-08T14:28:13.445Z" },
+ { url = "https://files.pythonhosted.org/packages/ff/5e/d0ad47a406c42db95876900f4bfc520313c28d07a0dd5080da4972220c27/pydantic_core-2.50.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:c4abc425789f8540e86ca4cfdbc8dc650433cc2ac7c6136fee9bcb29d5665a02", size = 2234007, upload-time = "2026-10-08T14:28:15.9Z" },
+ { url = "https://files.pythonhosted.org/packages/ae/f4/69a7ec8400c1e34e5e11ab967f850ffd8ca71bb3aa0a8db93df5912bb6f2/pydantic_core-2.50.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cf150693a51ca21e8288cd08a9de05e5ab331776dfcfd0537b14523338f0502a", size = 2000888, upload-time = "2026-10-08T14:28:17.983Z" },
+ { url = "https://files.pythonhosted.org/packages/d3/f1/860bb499f7cdf4bb453e8080af5bc182487f2f025d94baeee29c907bde6f/pydantic_core-2.50.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:158749408ee19682b8a2a7e7135cced7f41d6f2f9de96088b1b1609858a6a158", size = 2022761, upload-time = "2026-10-08T14:28:20.197Z" },
+ { url = "https://files.pythonhosted.org/packages/89/fd/f571420436e79b9f7cc8f8ec534dee1759c240b4dc086737608f14d05c93/pydantic_core-2.50.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dcbd1fe5083315243447c13f7252ae9fe9d128b1ae2857e3a916c609235dd863", size = 2192910, upload-time = "2026-10-08T14:28:22.502Z" },
+ { url = "https://files.pythonhosted.org/packages/cc/94/b47c4a01ea978a7a3d02551e3625f0344ae1352d67ea78011c2eca5e39e4/pydantic_core-2.50.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0fdeda6272d60b1f6fb63f6a3dade55e274af62e1514d24a6549a12150c385cc", size = 2390178, upload-time = "2026-10-08T14:28:24.861Z" },
+ { url = "https://files.pythonhosted.org/packages/77/61/d109f26b3ee4443cca220eaaa4f3fbcf78ff07aeb577becc37d9224db63f/pydantic_core-2.50.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8b16e164205a90b1050d2f5469f8a7829db7698ad198c69e6aab6cbfb5648b87", size = 2464453, upload-time = "2026-10-08T14:28:27.383Z" },
+ { url = "https://files.pythonhosted.org/packages/de/a4/7be2f608f6f8e7655155057a82a7e2f135ad9b37fcbadd34fdf11b1ade24/pydantic_core-2.50.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17a5ca9c197788a6424749a09a3dce824cb2c17b72f835f8a5e330935b973609", size = 2150682, upload-time = "2026-10-08T14:28:29.65Z" },
+ { url = "https://files.pythonhosted.org/packages/80/ef/a8d867f6d3981c232d0f3c2254458cdee8471de7bff3f7edd252cc00c30c/pydantic_core-2.50.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:01340f4fbb4f854b1f36a6fe9dcd2b26c8936aead4e4ad1205624ac025c875fc", size = 2245995, upload-time = "2026-10-08T14:28:31.934Z" },
+ { url = "https://files.pythonhosted.org/packages/9f/a6/7297a39c8814beab877be5ba4f594c2e166108cdd20c1f1c51fb71d5389e/pydantic_core-2.50.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:99e203d5c2a814facef78dcd993b0fb7a933124a3475a38979fc09cceae210b7", size = 2264090, upload-time = "2026-10-08T14:28:34.298Z" },
+ { url = "https://files.pythonhosted.org/packages/c3/65/634fc407eaf61abec0d015d6fd4fd8456c18c71358e984f60b0a05092d49/pydantic_core-2.50.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:2092ce156f92aff17e3345baba2b7c0c1701f32ba922c5de71bd6248fbe164e3", size = 2293708, upload-time = "2026-10-08T14:28:36.725Z" },
+ { url = "https://files.pythonhosted.org/packages/3a/04/b87fcf8062b8907c77769385a4340a814fe359660593dbbe4d8f3f7e9e61/pydantic_core-2.50.0-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:74cbb6cbd74445ca279668790e0c10eccac0428fd79fe061fce6c9e3982ad3fe", size = 2468464, upload-time = "2026-10-08T14:28:39.109Z" },
+ { url = "https://files.pythonhosted.org/packages/77/f4/d7aacec95f9e00dcbad84bee2fd0fc081ae292d380dc8a3268e4103de957/pydantic_core-2.50.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:42fff617cb0b08505d8123d71e6e7a8e54210d9007498f564855bd843ce984b1", size = 2511897, upload-time = "2026-10-08T14:28:41.681Z" },
+ { url = "https://files.pythonhosted.org/packages/a0/d8/0a755b0069f9a0f4c57f98fd05b524473952556406244649d632d25f8646/pydantic_core-2.50.0-cp314-cp314t-win32.whl", hash = "sha256:36f9ed6ae1069913e4f6e86d8233e119e83e00a20c54f88faf9c81292f2fecc0", size = 2098906, upload-time = "2026-10-08T14:28:44.218Z" },
+ { url = "https://files.pythonhosted.org/packages/27/4e/cd10a1fbd1ba1d730871e382ec2ac550487465c7c9be1a72b3f4f9181817/pydantic_core-2.50.0-cp314-cp314t-win_amd64.whl", hash = "sha256:980c81c2ec53ea9eb2227c14b3e6de35670b2de163b638f2a803e90a3bd5bbb0", size = 2148359, upload-time = "2026-10-08T14:28:46.47Z" },
+ { url = "https://files.pythonhosted.org/packages/b0/b1/a766eadfbc16b58d401f63dde3bf4e0943dcd2402011b6b8d9de2031ebe0/pydantic_core-2.50.0-cp314-cp314t-win_arm64.whl", hash = "sha256:dec0dafc116ac29a84d143fdbc3b83fdb5d4ed339276be2251154537ab30e14d", size = 2175241, upload-time = "2026-10-08T14:28:48.934Z" },
+ { url = "https://files.pythonhosted.org/packages/ae/d8/43d0e765a80d6fc7d87b5f66a9023777aad6da7e7d64f486a4c2be8158b7/pydantic_core-2.50.0-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:af2b808a79bb04075e87c81a5b6179365b93f9a851f29dafd67abff72085d0c8", size = 2224750, upload-time = "2026-10-08T14:28:51.227Z" },
+ { url = "https://files.pythonhosted.org/packages/9c/12/07e047c21ad90f184c7e8bfc6c9966b9ca51d4698e1a06460b65ca2ce105/pydantic_core-2.50.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2918547195ffb20118b829fdb8938e9dc92c9527fe6cbe58572594c96362880e", size = 2023936, upload-time = "2026-10-08T14:28:53.978Z" },
+ { url = "https://files.pythonhosted.org/packages/d8/a9/fb26fdd343ead65a699245c9d2150e84c47496a9b250859214a90cceecc9/pydantic_core-2.50.0-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0393763d66f6f61715488d074a2cefac04aeb3ee281e36fb4925dd44deaf9e17", size = 2048872, upload-time = "2026-10-08T14:28:56.342Z" },
+ { url = "https://files.pythonhosted.org/packages/f8/4d/9fc4ea28a8ecc1b85fe77a616f8df9a51696476aaa93c5b256269d025044/pydantic_core-2.50.0-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:62ec6568896e0abf258cbcd22c406c1c8bf27d16224b21bec8f75c4ae88a8173", size = 2173752, upload-time = "2026-10-08T14:28:58.805Z" },
+ { url = "https://files.pythonhosted.org/packages/70/7e/74b55196339413d787b283c2568cca4679b0645e48229421228acfa8839f/pydantic_core-2.50.0-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ec786cb9d597dd75d993f8c1273e31bb2114c9bc22f67fba611e654e8347701b", size = 2365284, upload-time = "2026-10-08T14:29:01.434Z" },
+ { url = "https://files.pythonhosted.org/packages/ba/0e/4ebc7a851f0ef63fcaba205739c171e50dd183f8d79dcf0099c3b46aa60b/pydantic_core-2.50.0-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3fce74add1099da1ea09473268950270071fd56773e2968604efb3ab1d240e02", size = 2445279, upload-time = "2026-10-08T14:29:04.034Z" },
+ { url = "https://files.pythonhosted.org/packages/07/09/03d3524fc7d4960840e32fd88c8cf487b5516daa7b6845ffc74274e0247a/pydantic_core-2.50.0-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:243088c95e23b12db9f2cd7661d584a3f00814e087489f40cde7f9feac56b694", size = 2160424, upload-time = "2026-10-08T14:29:06.628Z" },
+ { url = "https://files.pythonhosted.org/packages/f6/b9/7c530c84049089033dd746700bfbb51520ed9376bc90f3dbbcb319880acf/pydantic_core-2.50.0-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:42a56b0052ac11d9d0d87b1c94a1ce52e31914fd133f269585e0f63a4ed988f2", size = 2236109, upload-time = "2026-10-08T14:29:09.053Z" },
+ { url = "https://files.pythonhosted.org/packages/0a/08/70e07379ebc2538c22f8c372a37c009b96a3a1bd7921aee1dad28ee41be0/pydantic_core-2.50.0-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:350001f5573451150d919722ee095aa28bec037d6e23b86d7f04581a910fe924", size = 2244011, upload-time = "2026-10-08T14:29:11.549Z" },
+ { url = "https://files.pythonhosted.org/packages/66/49/294810baacae4b088bfac60edab06036951e5ae6e0c20e8f6cfef7d70f66/pydantic_core-2.50.0-cp315-cp315-musllinux_1_1_aarch64.whl", hash = "sha256:21e38a011783d8afc8b9d79928e273d06f349b52ae84edf63ec18ad07f077484", size = 2278392, upload-time = "2026-10-08T14:29:13.998Z" },
+ { url = "https://files.pythonhosted.org/packages/c3/0e/6990b812f124cf3564b36492799df7a66cd3cf5678004ffe65dda85b136b/pydantic_core-2.50.0-cp315-cp315-musllinux_1_1_armv7l.whl", hash = "sha256:588d309ce5c85448379556d72011b191c0414ee2e80d7c3f1ebe2ceb2d9027b1", size = 2450012, upload-time = "2026-10-08T14:29:16.524Z" },
+ { url = "https://files.pythonhosted.org/packages/a8/c4/f8f1a763550a077c51d2c69b7a32db56d1b54e61c613bc19e6c7fd2e7cb2/pydantic_core-2.50.0-cp315-cp315-musllinux_1_1_x86_64.whl", hash = "sha256:a8ee3965e01f10e4ff92ba1727626328ab7ba93bcd5674ff2b78ea1048ee0cea", size = 2479233, upload-time = "2026-10-08T14:29:19.111Z" },
+ { url = "https://files.pythonhosted.org/packages/9d/83/7a09baa1e0e4710be47b84aaf808e7b86247d7fa43251ac8934099113f8d/pydantic_core-2.50.0-cp315-cp315-win32.whl", hash = "sha256:c05b75ef3574c9ee4e05bbcf8513f7ccb155d426514be9efef5f6f53152d5f5c", size = 2104507, upload-time = "2026-10-08T14:29:21.694Z" },
+ { url = "https://files.pythonhosted.org/packages/ce/a9/f1cf61f747538834ea2c14c442264e4f29663c95294ad7933867633a1dea/pydantic_core-2.50.0-cp315-cp315-win_amd64.whl", hash = "sha256:8447678b49412294801c9ea15ae31ea3bae7da65425268c92d40e032c38eac6f", size = 2171379, upload-time = "2026-10-08T14:29:24.613Z" },
+ { url = "https://files.pythonhosted.org/packages/30/c8/9871261b760672cfa334498d94bc7dcbff2495345e639e806127f7087254/pydantic_core-2.50.0-cp315-cp315-win_arm64.whl", hash = "sha256:92016718bcf3e6f35a6bd986880191a8da7a35aa1f5b1e97544582ef938464cf", size = 2157182, upload-time = "2026-10-08T14:29:27.082Z" },
+ { url = "https://files.pythonhosted.org/packages/de/3c/5107269aee5ee7855fa370ff837998792dbf124b8545244c370bec251a66/pydantic_core-2.50.0-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:71061800a3c225e730f7997f8a7fad6e0d0dcbe36609cdc7e60576a099a2832b", size = 2234440, upload-time = "2026-10-08T14:29:29.705Z" },
+ { url = "https://files.pythonhosted.org/packages/fe/ac/cb179b0c404337ef517515539f9899164a6d3d43d5c79c2556165da211c1/pydantic_core-2.50.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bc87bd34239835c8d73171acd039e591cea0a2ca8615e6188044ad170a40fca1", size = 2001074, upload-time = "2026-10-08T14:29:32.327Z" },
+ { url = "https://files.pythonhosted.org/packages/b0/59/ef1714204f145e9497ab71e8bd82c5e40b5adf365d420be777d1e047cdaf/pydantic_core-2.50.0-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57c0e5b26d82bf31ab2b044781527b1b1a36e9ed400856b6aca5097eb1abb909", size = 2023016, upload-time = "2026-10-08T14:29:35.06Z" },
+ { url = "https://files.pythonhosted.org/packages/87/e9/08a3ade34b4af09a83740b62a4442b7623127d388fc21d4938798dd85a26/pydantic_core-2.50.0-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:41b9f2821f0a105f88cd54ad03fe1392ec600618ffa8610f7c5e466ecd98c531", size = 2193165, upload-time = "2026-10-08T14:29:37.721Z" },
+ { url = "https://files.pythonhosted.org/packages/2a/56/29aa3e540d3aca72139e5b7dc56395a940849d85da2b7a9ccfbff71d0928/pydantic_core-2.50.0-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:364f62d8024997536e57865cb1c8b36effad3249bf392a29a2379ea69db28238", size = 2390198, upload-time = "2026-10-08T14:29:40.616Z" },
+ { url = "https://files.pythonhosted.org/packages/76/63/a5cbdde0a2090c47002819d85317428e15b41d22e581cf38e9714b4903af/pydantic_core-2.50.0-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:955d7878130dfc124d6a5343e1b87d2933244fe14a4a0a3e98787e8e660a8eb4", size = 2464777, upload-time = "2026-10-08T14:29:43.29Z" },
+ { url = "https://files.pythonhosted.org/packages/c5/55/fff3a363b0dcf72fdd72311c676b1fa769f2c13342129c694b2eb9886580/pydantic_core-2.50.0-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf81432281af66ba3285b26b09c2d478d84730dc50ff90926c1bdcef54048a33", size = 2150389, upload-time = "2026-10-08T14:29:46.271Z" },
+ { url = "https://files.pythonhosted.org/packages/b7/f3/0062361fd2377185d322dc84cd05ec4b96da352195c89939c7b509664721/pydantic_core-2.50.0-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:34a4a0938eb30931baca56e55786c5a6871ac7aa891a38c8cabcdb7e49dab91b", size = 2246225, upload-time = "2026-10-08T14:29:49.173Z" },
+ { url = "https://files.pythonhosted.org/packages/f9/45/662e3870143d5f1632a02ceb9ce2d5168e88caf3da80e9b42f495ceb1284/pydantic_core-2.50.0-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f517a417cd02aa8fb05b603a0ac6d87b7b004c3ba4fcd279cad25cab7229043b", size = 2264404, upload-time = "2026-10-08T14:29:51.699Z" },
+ { url = "https://files.pythonhosted.org/packages/32/5e/8b14ebf111700362c6e519e40209d86f521da81ef171b2891af0bb5de311/pydantic_core-2.50.0-cp315-cp315t-musllinux_1_1_aarch64.whl", hash = "sha256:d01029d54ff1f45c195b1f7e6fbf6e58fb7e7e12cda9a6e639570d9c581decb2", size = 2294152, upload-time = "2026-10-08T14:29:54.349Z" },
+ { url = "https://files.pythonhosted.org/packages/55/29/85c486e0d25a8b523803032e97d7138edb7b7bc887fa8c6dd085e8824cdb/pydantic_core-2.50.0-cp315-cp315t-musllinux_1_1_armv7l.whl", hash = "sha256:d06dbbfe8da01a0574de27afc19915bb3e7dddfbd184bb97e958f94051d8531b", size = 2469160, upload-time = "2026-10-08T14:29:57.193Z" },
+ { url = "https://files.pythonhosted.org/packages/12/bf/c451db7567e92d6601c4d2f6bdbb945cf60104182e2b241d8fe0b6f01414/pydantic_core-2.50.0-cp315-cp315t-musllinux_1_1_x86_64.whl", hash = "sha256:5307a8bd49158b57a0ca4e10950d31085aa35d9007e934047043ccc6d28eea97", size = 2512204, upload-time = "2026-10-08T14:30:00.177Z" },
+ { url = "https://files.pythonhosted.org/packages/43/1e/5b93a2513ced099acb0dc4dd6684742d6564ade60a59abcbabd161624e07/pydantic_core-2.50.0-cp315-cp315t-win32.whl", hash = "sha256:c2b246fa7cbdf9918488d1542a82bbb928cf71bcba66905c24131981e759ff0b", size = 2099026, upload-time = "2026-10-08T14:30:03.154Z" },
+ { url = "https://files.pythonhosted.org/packages/37/f5/1b5967e31b025a23f2f627ffea1f5a874cff0adb72fa5f5af6bb36394ad9/pydantic_core-2.50.0-cp315-cp315t-win_amd64.whl", hash = "sha256:36c49d4e1769127461b609f110d91963790091ffcc2de401ac1d3b2f6a63bd54", size = 2148456, upload-time = "2026-10-08T14:30:05.971Z" },
+ { url = "https://files.pythonhosted.org/packages/48/15/213d6fe84816e8a5ed7b7539ecebb3f14c20739c2469f4e372f0f15bb042/pydantic_core-2.50.0-cp315-cp315t-win_arm64.whl", hash = "sha256:f3abcabb04503023e1053add24472e78878de0bfd5c8a93686be01f4032c363c", size = 2175368, upload-time = "2026-10-08T14:30:08.989Z" },
+ { url = "https://files.pythonhosted.org/packages/11/ff/8d126ca04417cbc76a81648c135296ecc2f36d07adfb0b3376638efd1dd0/pydantic_core-2.50.0-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:38b03c5439c6a7a952f56e3e6596dae44bc78c3fb6a69df46cabfb2f888e5f0c", size = 2162541, upload-time = "2026-10-08T14:30:12.075Z" },
+ { url = "https://files.pythonhosted.org/packages/b9/3f/e3f11c40950d0f37b3891c1ed015f04a4ef36c881d04ef1a5d1ac91d1c6f/pydantic_core-2.50.0-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:4faa766450ef44d9eabed5b65a280f63f903d1e1ed6d2e278961eb22345ec49f", size = 2075402, upload-time = "2026-10-08T14:30:14.968Z" },
+ { url = "https://files.pythonhosted.org/packages/42/f2/727d2b23ff6ca67c9c050c4ff4e5abd7b599d9ebcd819a74141d3b11c7c1/pydantic_core-2.50.0-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50920d66aaab60dbc6023c3035d55fbabfa58b6d205d0b39d47d820c4a7a13a8", size = 2108552, upload-time = "2026-10-08T14:30:17.774Z" },
+ { url = "https://files.pythonhosted.org/packages/4a/da/a4321058eea61f0504366668a09472a4c37514be30cce77967c35ab7ffaa/pydantic_core-2.50.0-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:586699b43066ca6038f96bd71d0eff0b6c292929e3d66ff63d5dc79d0f06d589", size = 2267745, upload-time = "2026-10-08T14:30:20.619Z" },
+ { url = "https://files.pythonhosted.org/packages/a9/c8/45e2cbac6379b8cec8cffd7e6aed50754fce13d93f0cff28bbbfe18c1d1c/pydantic_core-2.50.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:b916ff828d604d4311a5639b7b3da51eaea3923833ec3e300a5ee35eade99691", size = 2217049, upload-time = "2026-10-08T14:30:23.532Z" },
+ { url = "https://files.pythonhosted.org/packages/61/8c/b81c4139bff6305e7a98fa7845a99e05df1461d3c33d0fef417e02114ae7/pydantic_core-2.50.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:732efbb50977ab7cf18f01d4c255834aeb14cb2425459f7bff681a1ba3a4ffa1", size = 2039121, upload-time = "2026-10-08T14:30:26.423Z" },
+ { url = "https://files.pythonhosted.org/packages/0f/49/679293741809cf835290bccf80216aebfcf5d94cb84a34709f18b21abfd5/pydantic_core-2.50.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad9185366893714cd4514ae5e1a098227704fa395cb41a7855d75968ecedc826", size = 2109280, upload-time = "2026-10-08T14:30:29.216Z" },
+ { url = "https://files.pythonhosted.org/packages/f5/49/76f167aec9b9d77a82e38131539ca6c6fcf69d720965f76eda20b87b454d/pydantic_core-2.50.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0def1dc09802a790e4a1b3cbc4401f0b53c58f273ce3671df9867f7bdf1fbe20", size = 2269313, upload-time = "2026-10-08T14:30:32.096Z" },
+ { url = "https://files.pythonhosted.org/packages/1f/50/a6bd3c397b609aade207ffb78ce46f4ad4e29bb42a39007c87ee00e778f3/pydantic_core-2.50.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:68421ba548f6d86c7dededd70bfe530b4faf67eb811c53c70a1be69438d3ce99", size = 2154371, upload-time = "2026-10-08T14:30:34.965Z" },
+ { url = "https://files.pythonhosted.org/packages/11/3c/2d32a1f945b9809b65701d2b72315bdcd700f13c093df5e60baaa784bc6a/pydantic_core-2.50.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:23e0940b7d73f405d92e98b2a05b93b16db163be13b7478bb19e1db2e486ade6", size = 2087028, upload-time = "2026-10-08T14:30:37.814Z" },
+ { url = "https://files.pythonhosted.org/packages/aa/57/c2aea271192d8c4b81afeea9d61de8f5dc8686a1fcd1ace80e70a482fbce/pydantic_core-2.50.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee9913fa2b5bfa6111c11158cf481bb13544f8bd5d10382fb6ee2612b16fcf28", size = 2259611, upload-time = "2026-10-08T14:30:40.581Z" },
+ { url = "https://files.pythonhosted.org/packages/35/88/32128d5498b95722fe914a21a25f59edd7dc8917474f47924005682532a8/pydantic_core-2.50.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ad3532291deedfdfcad3cf5d351d0076d646c68a0c63869a1bebf87c22159600", size = 2268636, upload-time = "2026-10-08T14:30:43.511Z" },
+ { url = "https://files.pythonhosted.org/packages/8e/37/7dae31279e07aaa63dd4b17756f4ebd5647aed6bee5581ee3b3e85d82a0c/pydantic_core-2.50.0-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:9279f4be22bc3765dd9f61f5b4d38d9cb219d167bbd6f2e0ffa3368b1b71b4d1", size = 2305416, upload-time = "2026-10-08T14:30:46.386Z" },
+ { url = "https://files.pythonhosted.org/packages/10/2a/61e47defbd490618fcde2641f80d5c077f2961de62e88500edacb49297f9/pydantic_core-2.50.0-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:3b2f3e44af4c6512dd73557621385a71b18094c8be81d60e5a8e73a8ce96b7f4", size = 2466713, upload-time = "2026-10-08T14:30:49.214Z" },
+ { url = "https://files.pythonhosted.org/packages/61/57/81327752c007834d684b0b7d20ce00087206f512c7347a6236fd89ef0223/pydantic_core-2.50.0-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:593dd4f24abeb3ed90222f98bc6eefcac68cd5aa96fc7745f93862a802952bce", size = 2481971, upload-time = "2026-10-08T14:30:52.088Z" },
+ { url = "https://files.pythonhosted.org/packages/48/c3/e321d3a4b2737372cf775ffdd1a02428c7d2791ee5afc9f228793a0ee245/pydantic_core-2.50.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:b123f9d8702106f39dc3af63a031ca8b5279862a3747ec6c03ca49dbe78b71b9", size = 2337958, upload-time = "2026-10-08T14:30:55.045Z" },
+]
+
+[[package]]
+name = "typing-extensions"
+version = "4.16.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" },
+]
+
+[[package]]
+name = "typing-inspection"
+version = "0.4.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" },
+]
diff --git a/tests/unit/test_retrieval_eval.py b/tests/unit/test_retrieval_eval.py
new file mode 100644
index 000000000..ed66a5a10
--- /dev/null
+++ b/tests/unit/test_retrieval_eval.py
@@ -0,0 +1,608 @@
+"""Unit tests for the pure modules of the retrieval evaluation harness.
+
+The harness lives in scripts/eval/retrieval/. Its scoring, dataset and report
+modules use only the standard library, so they are tested here without the
+OpenRAG SDK or a running stack.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import subprocess
+import sys
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from scripts.eval.retrieval import dataset, metrics, report
+
+PREFIX = "eval-abc12345-"
+CORPUS = frozenset({"a.md", "b.md", "c.md", "d.md"})
+
+
+def chunk(name: str, text: str = "", score: float = 1.0, prefix: str = PREFIX) -> metrics.Chunk:
+ return metrics.Chunk(filename=f"{prefix}{name}", text=text, score=score)
+
+
+def ranked(*names: str) -> list[metrics.RankedFile]:
+ return metrics.rank_files([chunk(name) for name in names], PREFIX, CORPUS)
+
+
+# --- ranking ---------------------------------------------------------------
+
+
+def test_rank_files_deduplicates_in_first_occurrence_order() -> None:
+ chunks = [
+ chunk("b.md", score=0.9),
+ chunk("a.md", score=0.8),
+ chunk("b.md", score=0.7),
+ chunk("c.md", score=0.6),
+ chunk("a.md", score=0.5),
+ ]
+
+ result = metrics.rank_files(chunks, PREFIX, CORPUS)
+
+ assert [(item.filename, item.rank, item.score) for item in result] == [
+ ("b.md", 1, 0.9),
+ ("a.md", 2, 0.8),
+ ("c.md", 3, 0.6),
+ ]
+ assert all(item.snippet_found is None for item in result)
+
+
+def test_rank_files_ignores_files_this_run_did_not_upload() -> None:
+ chunks = [
+ metrics.Chunk("a.md", "", 0.9), # same name, no run prefix
+ chunk("a.md", prefix="eval-other999-"), # another run's copy
+ chunk("unknown.md"), # right prefix, not a corpus file
+ chunk("c.md", score=0.4),
+ ]
+
+ result = metrics.rank_files(chunks, PREFIX, CORPUS)
+
+ assert [(item.filename, item.rank) for item in result] == [("c.md", 1)]
+
+
+def test_canonical_name_strips_only_the_run_prefix() -> None:
+ assert metrics.canonical_name(f"{PREFIX}a.md", PREFIX, CORPUS) == "a.md"
+ assert metrics.canonical_name("a.md", PREFIX, CORPUS) is None
+ assert metrics.canonical_name(f"{PREFIX}{PREFIX}a.md", PREFIX, CORPUS) is None
+
+
+def test_expected_ranks_reports_missing_files_as_none() -> None:
+ assert metrics.expected_ranks(ranked("c.md", "a.md"), ["a.md", "b.md"]) == {
+ "a.md": 2,
+ "b.md": None,
+ }
+
+
+# --- metrics ---------------------------------------------------------------
+
+
+def test_score_query_single_expected_file_at_rank_two() -> None:
+ scores = metrics.score_query(ranked("b.md", "a.md", "c.md"), ["a.md"], [1, 3])
+
+ assert scores["recall@1"] == 0.0
+ assert scores["recall@3"] == 1.0
+ assert scores["reciprocal_rank"] == 0.5
+ assert scores["ndcg@1"] == 0.0
+ assert scores["ndcg@3"] == pytest.approx(1 / math.log2(3))
+
+
+def test_score_query_perfect_ranking_scores_one() -> None:
+ scores = metrics.score_query(ranked("a.md", "b.md", "c.md"), ["a.md", "b.md"], [1, 3, 5])
+
+ assert scores["recall@1"] == 0.5
+ assert scores["recall@3"] == 1.0
+ assert scores["reciprocal_rank"] == 1.0
+ # At k=1 only one of the two expected files can fit, so rank 1 is ideal.
+ assert scores["ndcg@1"] == 1.0
+ assert scores["ndcg@3"] == pytest.approx(1.0)
+ assert scores["ndcg@5"] == pytest.approx(1.0)
+
+
+def test_score_query_multi_file_partial_hit() -> None:
+ # Expected a.md and b.md; a.md comes back third, b.md not at all.
+ scores = metrics.score_query(ranked("c.md", "d.md", "a.md"), ["a.md", "b.md"], [3])
+
+ assert scores["recall@3"] == 0.5
+ assert scores["reciprocal_rank"] == pytest.approx(1 / 3)
+ ideal = 1 / math.log2(2) + 1 / math.log2(3)
+ assert scores["ndcg@3"] == pytest.approx((1 / math.log2(4)) / ideal)
+
+
+def test_score_query_no_hit_scores_zero() -> None:
+ scores = metrics.score_query(ranked("c.md", "d.md"), ["a.md"], [1, 5])
+
+ assert scores == {
+ "recall@1": 0.0,
+ "ndcg@1": 0.0,
+ "recall@5": 0.0,
+ "ndcg@5": 0.0,
+ "reciprocal_rank": 0.0,
+ }
+
+
+def test_score_query_handles_fewer_results_than_k() -> None:
+ scores = metrics.score_query(ranked("a.md"), ["a.md"], [10])
+
+ assert scores["recall@10"] == 1.0
+ assert scores["ndcg@10"] == 1.0
+ assert metrics.score_query([], ["a.md"], [10])["recall@10"] == 0.0
+
+
+def test_score_query_rejects_empty_expected_files() -> None:
+ with pytest.raises(ValueError):
+ metrics.score_query(ranked("a.md"), [], [5])
+
+
+def test_expected_text_gates_relevance_but_keeps_first_chunk_rank() -> None:
+ chunks = [
+ chunk("a.md", "Unrelated introduction."),
+ chunk("b.md", "Nothing useful here."),
+ chunk("a.md", "To ROTATE a key,\nopen the console."),
+ ]
+
+ result = metrics.rank_files(chunks, PREFIX, CORPUS, expected_text="rotate a key, open")
+
+ # The snippet is in a.md's second chunk, yet a.md keeps rank 1.
+ assert [(item.filename, item.rank, item.snippet_found) for item in result] == [
+ ("a.md", 1, True),
+ ("b.md", 2, False),
+ ]
+ assert metrics.score_query(result, ["a.md"], [1])["recall@1"] == 1.0
+
+
+def test_expected_file_without_the_snippet_is_not_relevant() -> None:
+ chunks = [chunk("a.md", "A chunk that never mentions it."), chunk("b.md", "rotate")]
+
+ result = metrics.rank_files(chunks, PREFIX, CORPUS, expected_text="rotate")
+ scores = metrics.score_query(result, ["a.md"], [5])
+
+ assert result[0].snippet_found is False
+ assert scores["recall@5"] == 0.0
+ assert scores["reciprocal_rank"] == 0.0
+
+
+def test_primary_k_prefers_five_then_largest() -> None:
+ assert metrics.primary_k([1, 3, 5, 10]) == 5
+ assert metrics.primary_k([1, 3]) == 3
+
+
+def test_aggregate_means_overall_and_per_tag() -> None:
+ per_query = [
+ {"recall@5": 1.0, "ndcg@5": 1.0, "reciprocal_rank": 1.0},
+ {"recall@5": 0.0, "ndcg@5": 0.0, "reciprocal_rank": 0.0},
+ {"recall@5": 0.5, "ndcg@5": 0.4, "reciprocal_rank": 0.5},
+ ]
+ tags = [["semantic"], ["semantic", "exact-token"], []]
+
+ result: Any = metrics.aggregate(per_query, tags)
+
+ assert result["overall"] == {
+ "recall@5": pytest.approx(0.5),
+ "ndcg@5": pytest.approx(1.4 / 3),
+ "mrr": pytest.approx(0.5),
+ }
+ assert list(result["by_tag"]) == ["exact-token", "semantic"]
+ assert result["by_tag"]["semantic"] == {
+ "count": 2,
+ "recall@5": 0.5,
+ "ndcg@5": 0.5,
+ "mrr": 0.5,
+ }
+ assert result["by_tag"]["exact-token"]["count"] == 1
+ assert metrics.aggregate([], []) == {"overall": {}, "by_tag": {}}
+
+
+# --- dataset ---------------------------------------------------------------
+
+
+def write_dataset(tmp_path: Path, rows: list[Any]) -> tuple[Path, Path]:
+ corpus = tmp_path / "corpus"
+ corpus.mkdir()
+ for name in ("a.md", "b.md"):
+ (corpus / name).write_text(f"# {name}\n", encoding="utf-8")
+ (corpus / ".DS_Store").write_text("", encoding="utf-8")
+ queries = tmp_path / "queries.jsonl"
+ queries.write_text(
+ "\n".join(row if isinstance(row, str) else json.dumps(row) for row in rows) + "\n",
+ encoding="utf-8",
+ )
+ return corpus, queries
+
+
+def test_load_queries_accepts_valid_rows_and_fills_defaults(tmp_path: Path) -> None:
+ corpus, queries_path = write_dataset(
+ tmp_path,
+ [
+ {"query": "how do I rotate a key", "expected_files": ["a.md"]},
+ "",
+ {
+ "id": "q-multi",
+ "query": " which docs cover billing ",
+ "expected_files": ["a.md", "b.md"],
+ "expected_text": "invoice",
+ "tags": ["multi-doc"],
+ },
+ ],
+ )
+ names = dataset.list_corpus(corpus)
+
+ queries = dataset.load_queries(queries_path, names)
+
+ assert names == ["a.md", "b.md"] # hidden files are not corpus documents
+ assert queries == [
+ dataset.Query(
+ id="how do I rotate a key", # falls back to the query text
+ query="how do I rotate a key",
+ expected_files=("a.md",),
+ ),
+ dataset.Query(
+ id="q-multi",
+ query="which docs cover billing",
+ expected_files=("a.md", "b.md"),
+ expected_text="invoice",
+ tags=("multi-doc",),
+ ),
+ ]
+
+
+def test_load_queries_reports_every_problem_with_its_line_number(tmp_path: Path) -> None:
+ corpus, queries_path = write_dataset(
+ tmp_path,
+ [
+ {"id": "q1", "query": "fine", "expected_files": ["a.md"]},
+ "{not json",
+ {"query": " ", "expected_files": ["a.md"]},
+ {"query": "no files", "expected_files": []},
+ {"query": "typo", "expected_files": ["missing.md"]},
+ {"id": "q1", "query": "same id", "expected_files": ["b.md"]},
+ {"query": "bad field", "expected_file": ["a.md"]},
+ ["not", "an", "object"],
+ {"query": "bad tags", "expected_files": ["a.md"], "tags": "semantic"},
+ {"query": "bad snippet", "expected_files": ["a.md"], "expected_text": ""},
+ ],
+ )
+
+ with pytest.raises(dataset.DatasetError) as excinfo:
+ dataset.load_queries(queries_path, dataset.list_corpus(corpus))
+
+ message = str(excinfo.value)
+ for expected in (
+ "queries.jsonl:2: invalid JSON",
+ "queries.jsonl:3: 'query' must be a non-empty string",
+ "queries.jsonl:4: 'expected_files' must be a non-empty list of filenames",
+ "queries.jsonl:5: expected file not in corpus: missing.md",
+ "queries.jsonl:6: duplicate id 'q1' (first used on line 1)",
+ "queries.jsonl:7: unknown field(s): expected_file",
+ "queries.jsonl:7: 'expected_files' must be a non-empty list of filenames",
+ "queries.jsonl:8: row must be a JSON object",
+ "queries.jsonl:9: 'tags' must be a list of non-empty strings",
+ "queries.jsonl:10: 'expected_text' must be a non-empty string",
+ ):
+ assert expected in message
+ assert "queries.jsonl:1:" not in message
+
+
+def test_load_queries_checks_expected_text_against_the_corpus(tmp_path: Path) -> None:
+ corpus, queries_path = write_dataset(
+ tmp_path,
+ [
+ {"query": "ok", "expected_files": ["a.md"], "expected_text": "# A.MD"},
+ {"query": "typo", "expected_files": ["a.md", "b.md"], "expected_text": "# a.md"},
+ ],
+ )
+ names = dataset.list_corpus(corpus)
+
+ # Without the corpus directory the snippets are not checked.
+ assert len(dataset.load_queries(queries_path, names)) == 2
+
+ with pytest.raises(dataset.DatasetError) as excinfo:
+ dataset.load_queries(queries_path, names, corpus)
+
+ assert str(excinfo.value) == (
+ "queries.jsonl:2: 'expected_text' '# a.md' does not appear in b.md"
+ )
+
+
+def test_load_queries_rejects_missing_or_empty_inputs(tmp_path: Path) -> None:
+ corpus, queries_path = write_dataset(tmp_path, [""])
+ names = dataset.list_corpus(corpus)
+
+ with pytest.raises(dataset.DatasetError, match="no questions found"):
+ dataset.load_queries(queries_path, names)
+ with pytest.raises(dataset.DatasetError, match="question file not found"):
+ dataset.load_queries(tmp_path / "nope.jsonl", names)
+ with pytest.raises(dataset.DatasetError, match="corpus directory not found"):
+ dataset.list_corpus(tmp_path / "nope")
+ empty = tmp_path / "empty"
+ empty.mkdir()
+ with pytest.raises(dataset.DatasetError, match="corpus directory is empty"):
+ dataset.list_corpus(empty)
+
+
+def test_hash_corpus_changes_with_content_and_with_filenames(tmp_path: Path) -> None:
+ corpus, _ = write_dataset(tmp_path, [""])
+ names = dataset.list_corpus(corpus)
+ original = dataset.hash_corpus(corpus, names)
+
+ assert dataset.hash_corpus(corpus, list(reversed(names))) == original
+
+ (corpus / "a.md").write_text("# a.md, edited\n", encoding="utf-8")
+ edited = dataset.hash_corpus(corpus, names)
+ assert edited != original
+
+ (corpus / "a.md").rename(corpus / "z.md")
+ assert dataset.hash_corpus(corpus, dataset.list_corpus(corpus)) != edited
+
+
+# --- report ----------------------------------------------------------------
+
+
+def make_query(
+ query_id: str, recall: float, rr: float, ndcg: float, rank: int | None
+) -> dict[str, Any]:
+ return {
+ "id": query_id,
+ "query": f"question {query_id}",
+ "tags": ["semantic"],
+ "expected_files": ["a.md"],
+ "expected_text": None,
+ "chunks_returned": 3,
+ "unique_files_returned": 2,
+ "ranked_files": [{"filename": "b.md", "rank": 1, "score": 0.9, "snippet_found": None}],
+ "expected_ranks": {"a.md": rank},
+ "snippet_found": None,
+ "metrics": {"recall@5": recall, "reciprocal_rank": rr, "ndcg@5": ndcg},
+ }
+
+
+def make_results(queries: list[dict[str, Any]], **run_overrides: Any) -> dict[str, Any]:
+ scores = [q["metrics"] for q in queries]
+ run = {
+ "run_id": "eval-11111111",
+ "timestamp": "2026-10-12T00:00:00Z",
+ "target_url": "http://localhost:3000",
+ "sdk_version": "0.4.0",
+ "git_commit": "abc123",
+ "corpus_files": 2,
+ "corpus_hash": "c" * 64,
+ "query_count": len(queries),
+ "queries_hash": "d" * 64,
+ "k_values": [1, 3, 5, 10],
+ "primary_k": 5,
+ "fetch_limit": 50,
+ "score_threshold": 0.0,
+ }
+ run.update(run_overrides)
+ return {
+ "schema_version": report.SCHEMA_VERSION,
+ "run": run,
+ "settings": {"embedding_model": "model-a", "chunk_size": 1000, "chunk_overlap": 200},
+ "settings_note": None,
+ "metrics": metrics.aggregate(scores, [q["tags"] for q in queries]),
+ "queries": queries,
+ }
+
+
+def test_diff_classifies_each_question_and_lists_added_and_removed() -> None:
+ baseline = make_results(
+ [
+ make_query("worse", 1.0, 1.0, 1.0, 1),
+ make_query("better", 0.0, 0.0, 0.0, None),
+ make_query("same", 1.0, 0.5, 0.63, 2),
+ make_query("rank-only", 1.0, 1.0, 1.0, 1),
+ make_query("gone", 1.0, 1.0, 1.0, 1),
+ ]
+ )
+ current = make_results(
+ [
+ make_query("worse", 0.0, 0.0, 0.0, None),
+ make_query("better", 1.0, 1.0, 1.0, 1),
+ make_query("same", 1.0, 0.5, 0.63, 2),
+ # Still inside the top 5, so recall holds; the lower rank decides.
+ make_query("rank-only", 1.0, 0.25, 0.43, 4),
+ make_query("new", 1.0, 1.0, 1.0, 1),
+ ],
+ run_id="eval-22222222",
+ )
+
+ diff = report.diff_results(baseline, current)
+
+ assert [entry["id"] for entry in diff["regressed"]] == ["worse", "rank-only"]
+ assert [entry["id"] for entry in diff["improved"]] == ["better"]
+ assert diff["unchanged"] == 1
+ assert diff["added"] == ["new"]
+ assert diff["removed"] == ["gone"]
+ worse = diff["regressed"][0]
+ assert worse["metrics"]["recall@5"] == {"baseline": 1.0, "current": 0.0, "delta": -1.0}
+ assert worse["expected_ranks"] == {"baseline": {"a.md": 1}, "current": {"a.md": None}}
+ assert diff["aggregate"]["recall@5"]["delta"] == pytest.approx(0.0)
+ assert any("do not cover the same questions" in w for w in diff["warnings"])
+
+
+def test_diff_ignores_changes_within_epsilon() -> None:
+ baseline = make_results([make_query("q", 1.0, 0.5, 0.63, 2)])
+ current = make_results([make_query("q", 1.0, 0.5 + 1e-12, 0.63, 2)])
+
+ diff = report.diff_results(baseline, current)
+
+ assert diff["regressed"] == []
+ assert diff["improved"] == []
+ assert diff["unchanged"] == 1
+ assert diff["warnings"] == []
+
+
+def test_diff_warns_when_runs_are_not_comparable() -> None:
+ baseline = make_results([make_query("q", 1.0, 1.0, 1.0, 1)])
+ current = make_results(
+ [make_query("q", 1.0, 1.0, 1.0, 1)],
+ corpus_hash="e" * 64,
+ queries_hash="f" * 64,
+ k_values=[1, 5],
+ fetch_limit=20,
+ score_threshold=0.2,
+ )
+ current["settings"]["chunk_size"] = 500
+
+ warnings = "\n".join(report.diff_results(baseline, current)["warnings"])
+
+ for expected in (
+ "The corpus differs",
+ "The question file differs",
+ "The k values differ",
+ "The fetch limit differs",
+ "The score threshold differs",
+ "Setting 'chunk_size' differs between the runs (baseline 1000, current 500)",
+ ):
+ assert expected in warnings
+ assert "embedding_model" not in warnings
+
+ current["settings"] = None
+ warnings = "\n".join(report.diff_results(baseline, current)["warnings"])
+ assert "Only one run recorded the stack settings" in warnings
+
+
+def test_summary_lists_regressions_before_improvements() -> None:
+ baseline = make_results(
+ [make_query("worse", 1.0, 1.0, 1.0, 1), make_query("better", 0.0, 0.0, 0.0, None)]
+ )
+ current = make_results(
+ [make_query("worse", 0.0, 0.0, 0.0, None), make_query("better", 1.0, 1.0, 1.0, 1)]
+ )
+
+ summary = report.render_summary(current, report.diff_results(baseline, current))
+
+ assert "| recall@5 | 0.500 |" in summary
+ assert "| semantic | 2 | 0.500 | 0.500 | 0.500 |" in summary
+ assert "## Questions below full recall@5 (1)" in summary
+ assert summary.index("### Regressed (1)") < summary.index("### Improved (1)")
+ assert "a.md: #1 → a.md: not returned" in summary
+ assert "1 regressed, 1 improved, 0 unchanged" in summary
+
+
+def test_diff_explains_a_drop_caused_by_a_missing_snippet() -> None:
+ # Same rank in both runs, but the current run no longer returns a chunk
+ # holding the expected text, so the file stops counting as relevant.
+ before = make_query("q", 1.0, 1.0, 1.0, 1)
+ before["ranked_files"] = [{"filename": "a.md", "rank": 1, "score": 0.9, "snippet_found": True}]
+ after = make_query("q", 0.0, 0.0, 0.0, 1)
+ after["ranked_files"] = [{"filename": "a.md", "rank": 1, "score": 0.9, "snippet_found": False}]
+
+ diff = report.diff_results(make_results([before]), make_results([after]))
+
+ assert diff["regressed"][0]["snippet_missing"] == {"baseline": [], "current": ["a.md"]}
+ assert "a.md: #1 → a.md: #1 (snippet not found)" in report.render_diff(diff)
+
+
+def test_render_diff_says_so_when_nothing_changed() -> None:
+ results = make_results([make_query("q", 1.0, 1.0, 1.0, 1)])
+
+ text = report.render_diff(report.diff_results(results, results))
+
+ assert "No question regressed." in text
+ assert "No question improved." in text
+
+
+def test_write_results_writes_latest_and_timestamped_copies(tmp_path: Path) -> None:
+ results = make_results([make_query("q", 1.0, 1.0, 1.0, 1)])
+ started_at = datetime(2026, 10, 12, 9, 30, 5, tzinfo=UTC)
+
+ written = report.write_results(results, "# summary\n", tmp_path / "results", started_at)
+
+ assert [path.name for path in written] == [
+ "results.json",
+ "results_20261012_093005.json",
+ "summary.md",
+ "summary_20261012_093005.md",
+ ]
+ assert report.load_results(written[0]) == results
+ assert written[1].read_text(encoding="utf-8") == written[0].read_text(encoding="utf-8")
+ assert written[3].read_text(encoding="utf-8") == "# summary\n"
+
+
+def test_load_results_rejects_files_that_are_not_harness_output(tmp_path: Path) -> None:
+ path = tmp_path / "results.json"
+
+ with pytest.raises(report.ResultsError, match="cannot read results file"):
+ report.load_results(path)
+
+ path.write_text("{not json", encoding="utf-8")
+ with pytest.raises(report.ResultsError, match="is not valid JSON"):
+ report.load_results(path)
+
+ path.write_text(json.dumps({"hello": "world"}), encoding="utf-8")
+ with pytest.raises(report.ResultsError, match="not a retrieval evaluation results file"):
+ report.load_results(path)
+
+ stale = make_results([make_query("q", 1.0, 1.0, 1.0, 1)])
+ stale["schema_version"] = 0
+ path.write_text(json.dumps(stale), encoding="utf-8")
+ with pytest.raises(report.ResultsError, match="schema version 0"):
+ report.load_results(path)
+
+
+ ("url", "expected"),
+ [
+ ("http://localhost:3000", "http://localhost:3000"),
+ ("http://[::1]:3000/", "http://[::1]:3000"),
+ ("http://user:secret@[2001:db8::1]/base", "http://[2001:db8::1]/base"),
+ (
+ "https://user:[email protected]:8443/base/?token=abc#frag",
+ "https://rag.example.com:8443/base",
+ ),
+ ],
+)
+def test_redact_url_drops_credentials_and_query(url: str, expected: str) -> None:
+ assert report.redact_url(url) == expected
+
+
+def test_diff_falls_back_to_a_cutoff_both_runs_have() -> None:
+ before = make_query("q", 1.0, 1.0, 1.0, 1)
+ before["metrics"].update({"recall@3": 1.0, "ndcg@3": 1.0})
+ after = make_query("q", 1.0, 1.0, 1.0, 1)
+ after["metrics"] = {"recall@3": 0.0, "recall@4": 1.0, "reciprocal_rank": 1.0, "ndcg@3": 0.0}
+
+ diff = report.diff_results(
+ make_results([before]), make_results([after], k_values=[3, 4], primary_k=4)
+ )
+
+ assert diff["compare_metrics"] == ["recall@3", "reciprocal_rank", "ndcg@3"]
+ assert [entry["id"] for entry in diff["regressed"]] == ["q"]
+ assert "compared on recall@3" in "\n".join(diff["warnings"])
+
+
+def test_diff_rejects_runs_without_a_shared_cutoff() -> None:
+ baseline = make_results([make_query("q", 1.0, 1.0, 1.0, 1)])
+ current = make_results([make_query("q", 1.0, 1.0, 1.0, 1)], k_values=[2, 4], primary_k=4)
+
+ with pytest.raises(report.ResultsError, match="share no k value"):
+ report.diff_results(baseline, current)
+
+
[email protected]("fetch_limit", ["0", "-1", "4"])
+def test_run_rejects_a_fetch_limit_below_the_largest_cutoff(fetch_limit: str) -> None:
+ script = Path(__file__).resolve().parents[2] / "scripts/eval/retrieval/run_eval.py"
+ completed = subprocess.run(
+ [
+ sys.executable,
+ str(script),
+ "run",
+ "--validate-only",
+ "--k",
+ "1,5",
+ "--fetch-limit",
+ fetch_limit,
+ ],
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+
+ assert completed.returncode == 2
+ assert "--fetch-limit must be at least the largest --k value (5)" in completed.stderr