Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix(2478): track eval baseline (#2582)

Sathvik Lakamsani23b1533fada1
diff --git a/.github/workflows/eval-retrieval.yml b/.github/workflows/eval-retrieval.yml
index c2f5a2649..5e3c7d694 100644
--- a/.github/workflows/eval-retrieval.yml
+++ b/.github/workflows/eval-retrieval.yml
@@ -234,7 +234,7 @@ jobs:
cd "$GITHUB_WORKSPACE"
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
- git add scripts/eval/retrieval/baseline.json
+ git add -f scripts/eval/retrieval/baseline.json
git commit -m "chore: update the retrieval eval baseline"
git push origin "HEAD:refs/heads/${REF_NAME}"
diff --git a/.gitignore b/.gitignore
index 5656f8589..05ccdf56a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,6 +24,7 @@ wheels/
!src/tui/_assets/flows/*.json
!src/tui/_assets/flows/components/*.json
!frontend/*.json
+!scripts/eval/retrieval/baseline.json
.DS_Store
logs
diff --git a/scripts/eval/retrieval/baseline.json b/scripts/eval/retrieval/baseline.json
new file mode 100644
index 000000000..219c6590f
--- /dev/null
+++ b/scripts/eval/retrieval/baseline.json
@@ -0,0 +1,12195 @@
+{
+ "schema_version": 1,
+ "run": {
+ "run_id": "eval-49debd2f",
+ "timestamp": "2026-11-05T18:03:57Z",
+ "duration_s": 85.9,
+ "target_url": "http://localhost:3000",
+ "sdk_version": "0.4.0.dev0",
+ "git_commit": "a7076dc551c4b940b63119b9c242665fa927f880",
+ "corpus_files": 21,
+ "corpus_hash": "9a346ec3274599771b53573f708fca382c23a27b19d102a84210377df06ab73a",
+ "query_count": 83,
+ "passage_questions": 77,
+ "queries_hash": "a76d3448549d17810642d5a5206a56f54b8e8f605b43def0d3791941ba9f31bf",
+ "k_values": [
+ 1,
+ 3,
+ 5,
+ 10
+ ],
+ "primary_k": 5,
+ "fetch_limit": 50,
+ "score_threshold": 0.0
+ },
+ "settings": {
+ "embedding_provider": "azure",
+ "embedding_model": "text-embedding-3-small",
+ "chunk_size": 1000,
+ "chunk_overlap": 200,
+ "table_structure": true,
+ "ocr": false,
+ "picture_descriptions": false
+ },
+ "settings_note": null,
+ "metrics": {
+ "overall": {
+ "recall@1": 0.6265060240963856,
+ "recall@3": 0.8493975903614458,
+ "recall@5": 0.9096385542168675,
+ "recall@10": 0.963855421686747,
+ "mrr": 0.7938898450946643,
+ "ndcg@1": 0.7108433734939759,
+ "ndcg@3": 0.7762517657398734,
+ "ndcg@5": 0.8030453462731483,
+ "ndcg@10": 0.8211616182694974,
+ "passage_hit@1": 0.5324675324675324,
+ "passage_hit@3": 0.7012987012987013,
+ "passage_hit@5": 0.7922077922077922,
+ "passage_hit@10": 0.8701298701298701,
+ "passage_mrr": 0.6384012790334507
+ },
+ "by_tag": {
+ "exact-token": {
+ "count": 13,
+ "passage_count": 13,
+ "recall@1": 0.7307692307692307,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "mrr": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 0.9876493521766443,
+ "ndcg@5": 0.9876493521766443,
+ "ndcg@10": 0.9876493521766443,
+ "passage_hit@1": 0.8461538461538461,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_mrr": 0.9230769230769231
+ },
+ "keyword": {
+ "count": 7,
+ "passage_count": 6,
+ "recall@1": 0.5714285714285714,
+ "recall@3": 0.9285714285714286,
+ "recall@5": 0.9285714285714286,
+ "recall@10": 1.0,
+ "mrr": 0.8333333333333333,
+ "ndcg@1": 0.7142857142857143,
+ "ndcg@3": 0.8205824209052738,
+ "ndcg@5": 0.8205824209052738,
+ "ndcg@10": 0.8459023190172805,
+ "passage_hit@1": 0.6666666666666666,
+ "passage_hit@3": 0.8333333333333334,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_mrr": 0.7638888888888888
+ },
+ "multi-doc": {
+ "count": 16,
+ "passage_count": 11,
+ "recall@1": 0.4375,
+ "recall@3": 0.71875,
+ "recall@5": 0.84375,
+ "recall@10": 0.9375,
+ "mrr": 0.903125,
+ "ndcg@1": 0.875,
+ "ndcg@3": 0.7440735963827292,
+ "ndcg@5": 0.8050524251877592,
+ "ndcg@10": 0.8413163264616205,
+ "passage_hit@1": 0.7272727272727273,
+ "passage_hit@3": 0.9090909090909091,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_mrr": 0.8257575757575758
+ },
+ "semantic": {
+ "count": 63,
+ "passage_count": 58,
+ "recall@1": 0.6111111111111112,
+ "recall@3": 0.8095238095238095,
+ "recall@5": 0.8888888888888888,
+ "recall@10": 0.9523809523809523,
+ "mrr": 0.7469765684051398,
+ "ndcg@1": 0.6507936507936508,
+ "ndcg@3": 0.7277044132027969,
+ "ndcg@5": 0.7630038923180636,
+ "ndcg@10": 0.7840580397611262,
+ "passage_hit@1": 0.4482758620689655,
+ "passage_hit@3": 0.6206896551724138,
+ "passage_hit@5": 0.7241379310344828,
+ "passage_hit@10": 0.8275862068965517,
+ "passage_mrr": 0.5616131922800408
+ }
+ }
+ },
+ "queries": [
+ {
+ "id": "key-rotate",
+ "query": "How do I swap out an API key without causing downtime?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "api-keys.md"
+ ],
+ "expected_text": "overlap window",
+ "chunks_returned": 50,
+ "unique_files_returned": 17,
+ "ranked_files": [
+ {
+ "filename": "api-keys.md",
+ "rank": 1,
+ "score": 4.485182,
+ "snippet_found": true
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 2,
+ "score": 4.1399813,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 3,
+ "score": 3.9088736,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 4,
+ "score": 3.9022071,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 5,
+ "score": 3.8871171,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 6,
+ "score": 3.7825556,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 7,
+ "score": 3.5549948,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 8,
+ "score": 3.3526278,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 9,
+ "score": 3.1453872,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 10,
+ "score": 3.0133064,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 11,
+ "score": 2.9531522,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 12,
+ "score": 2.9356976,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 13,
+ "score": 2.8147604,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 14,
+ "score": 2.790321,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 15,
+ "score": 2.6110516,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 16,
+ "score": 2.5505896,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 17,
+ "score": 2.3984993,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "api-keys.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 10,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 0.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.1
+ }
+ },
+ {
+ "id": "key-leaked",
+ "query": "One of our keys was pushed to a public repo, what should we do right now?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "api-keys.md"
+ ],
+ "expected_text": "Revoke immediately",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "api-keys.md",
+ "rank": 1,
+ "score": 4.5577087,
+ "snippet_found": true
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 2,
+ "score": 4.4239907,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 3,
+ "score": 3.8773596,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 4,
+ "score": 3.7586002,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 5,
+ "score": 3.7468228,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 6,
+ "score": 3.6312208,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 7,
+ "score": 3.373435,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 8,
+ "score": 3.268697,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 9,
+ "score": 3.2684705,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 10,
+ "score": 3.224593,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 11,
+ "score": 3.1721559,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 12,
+ "score": 3.1261327,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 13,
+ "score": 3.0540926,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 14,
+ "score": 3.023003,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 15,
+ "score": 2.9438276,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 16,
+ "score": 2.9110684,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 17,
+ "score": 2.8642788,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 18,
+ "score": 2.65653,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 19,
+ "score": 2.5132947,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "api-keys.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "key-scopes",
+ "query": "What scopes can I give an API key?",
+ "tags": [
+ "keyword"
+ ],
+ "expected_files": [
+ "api-keys.md"
+ ],
+ "expected_text": "destinations:manage",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "api-keys.md",
+ "rank": 1,
+ "score": 5.5974355,
+ "snippet_found": true
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 2,
+ "score": 5.0317492,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 3,
+ "score": 4.3466816,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 4,
+ "score": 3.7902362,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 5,
+ "score": 3.6450343,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 6,
+ "score": 3.5535414,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 7,
+ "score": 3.533179,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 8,
+ "score": 3.477205,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 9,
+ "score": 3.3471954,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 10,
+ "score": 3.334623,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 11,
+ "score": 3.179077,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 12,
+ "score": 3.147795,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 13,
+ "score": 3.0607226,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 14,
+ "score": 2.8921738,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 15,
+ "score": 2.697281,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 16,
+ "score": 2.5704966,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 17,
+ "score": 2.479469,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 18,
+ "score": 2.3684938,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 19,
+ "score": 2.3103514,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "api-keys.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "key-prefix-ci",
+ "query": "What is a key starting with rwk_ci_ used for?",
+ "tags": [
+ "exact-token"
+ ],
+ "expected_files": [
+ "api-keys.md"
+ ],
+ "expected_text": "Automation key",
+ "chunks_returned": 50,
+ "unique_files_returned": 20,
+ "ranked_files": [
+ {
+ "filename": "api-keys.md",
+ "rank": 1,
+ "score": 5.3789306,
+ "snippet_found": true
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 2,
+ "score": 4.635419,
+ "snippet_found": true
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 3,
+ "score": 3.814974,
+ "snippet_found": true
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 4,
+ "score": 3.5494313,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 5,
+ "score": 3.0389695,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 6,
+ "score": 2.91465,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 7,
+ "score": 2.8971639,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 8,
+ "score": 2.8961177,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 9,
+ "score": 2.875177,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 10,
+ "score": 2.8743634,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 11,
+ "score": 2.7834806,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 12,
+ "score": 2.7593927,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 13,
+ "score": 2.5931182,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 14,
+ "score": 2.5388367,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 15,
+ "score": 2.5265174,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 16,
+ "score": 2.4976048,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 17,
+ "score": 2.3593998,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 18,
+ "score": 2.2960348,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 19,
+ "score": 2.2928317,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 20,
+ "score": 2.171982,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "api-keys.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "sig-verify",
+ "query": "How can my server confirm that a webhook really came from Relaywise and was not forged?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "signing-secrets.md"
+ ],
+ "expected_text": "HMAC-SHA256",
+ "chunks_returned": 50,
+ "unique_files_returned": 20,
+ "ranked_files": [
+ {
+ "filename": "signing-secrets.md",
+ "rank": 1,
+ "score": 5.4620786,
+ "snippet_found": true
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 2,
+ "score": 5.1428657,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 3,
+ "score": 5.107025,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 4,
+ "score": 4.8653693,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 5,
+ "score": 4.459428,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 6,
+ "score": 4.323626,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 7,
+ "score": 4.092972,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 8,
+ "score": 4.032093,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 9,
+ "score": 3.9983268,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 10,
+ "score": 3.9548411,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 11,
+ "score": 3.8796291,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 12,
+ "score": 3.8189056,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 13,
+ "score": 3.8055673,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 14,
+ "score": 3.7612507,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 15,
+ "score": 3.748525,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 16,
+ "score": 3.7323694,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 17,
+ "score": 3.6424484,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 18,
+ "score": 3.5377412,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 19,
+ "score": 3.460004,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 20,
+ "score": 3.3449996,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "signing-secrets.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "sig-roll",
+ "query": "How do I change the secret used to sign deliveries, and how long do the old and new ones both work?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "signing-secrets.md"
+ ],
+ "expected_text": "72 hours",
+ "chunks_returned": 50,
+ "unique_files_returned": 20,
+ "ranked_files": [
+ {
+ "filename": "audit-log.md",
+ "rank": 1,
+ "score": 8.1549225,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 2,
+ "score": 7.6038017,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 3,
+ "score": 7.401187,
+ "snippet_found": true
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 4,
+ "score": 6.34908,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 5,
+ "score": 6.135564,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 6,
+ "score": 5.6227202,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 7,
+ "score": 5.285435,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 8,
+ "score": 5.0490365,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 9,
+ "score": 4.9172583,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 10,
+ "score": 4.916431,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 11,
+ "score": 4.8609548,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 12,
+ "score": 4.823242,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 13,
+ "score": 4.7858067,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 14,
+ "score": 4.7696395,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 15,
+ "score": 4.65145,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 16,
+ "score": 4.4600706,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 17,
+ "score": 4.159475,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 18,
+ "score": 3.9943762,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 19,
+ "score": 3.8904157,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 20,
+ "score": 3.7599213,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "signing-secrets.md": 3
+ },
+ "snippet_found": true,
+ "passage_rank": 3,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.3333333333333333,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.5,
+ "ndcg@5": 0.5,
+ "ndcg@10": 0.5,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.3333333333333333
+ }
+ },
+ {
+ "id": "sig-max-age",
+ "query": "How old can a signed request be before I should refuse it?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "signing-secrets.md"
+ ],
+ "expected_text": "five minutes",
+ "chunks_returned": 50,
+ "unique_files_returned": 21,
+ "ranked_files": [
+ {
+ "filename": "signing-secrets.md",
+ "rank": 1,
+ "score": 5.4995303,
+ "snippet_found": true
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 2,
+ "score": 5.3574557,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 3,
+ "score": 4.620363,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 4,
+ "score": 4.157878,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 5,
+ "score": 3.9991243,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 6,
+ "score": 3.7961216,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 7,
+ "score": 3.7712054,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 8,
+ "score": 3.6903996,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 9,
+ "score": 3.672656,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 10,
+ "score": 3.3198228,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 11,
+ "score": 3.3114727,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 12,
+ "score": 3.3018017,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 13,
+ "score": 3.2214346,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 14,
+ "score": 3.219668,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 15,
+ "score": 2.914604,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 16,
+ "score": 2.8591857,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 17,
+ "score": 2.8397179,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 18,
+ "score": 2.7246382,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 19,
+ "score": 2.6846876,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 20,
+ "score": 2.6837616,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 21,
+ "score": 2.5916595,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "signing-secrets.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "sig-whsec",
+ "query": "Where do I find the whsec_ value for a destination?",
+ "tags": [
+ "exact-token"
+ ],
+ "expected_files": [
+ "signing-secrets.md"
+ ],
+ "expected_text": "Reveal",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "signing-secrets.md",
+ "rank": 1,
+ "score": 6.0598135,
+ "snippet_found": true
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 2,
+ "score": 4.84408,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 3,
+ "score": 4.648713,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 4,
+ "score": 4.378025,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 5,
+ "score": 3.803798,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 6,
+ "score": 3.4391446,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 7,
+ "score": 3.3304148,
+ "snippet_found": true
+ },
+ {
+ "filename": "regions.md",
+ "rank": 8,
+ "score": 3.3192945,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 9,
+ "score": 2.9882765,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 10,
+ "score": 2.8431144,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 11,
+ "score": 2.8061593,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 12,
+ "score": 2.6899495,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 13,
+ "score": 2.6079462,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 14,
+ "score": 2.5567536,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 15,
+ "score": 2.555109,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 16,
+ "score": 2.4441526,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 17,
+ "score": 2.355443,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 18,
+ "score": 2.2208116,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 19,
+ "score": 2.1636539,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "signing-secrets.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "sso-setup",
+ "query": "How do we let staff log in to the console with their company accounts?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "single-sign-on.md"
+ ],
+ "expected_text": "identity provider",
+ "chunks_returned": 50,
+ "unique_files_returned": 20,
+ "ranked_files": [
+ {
+ "filename": "audit-log.md",
+ "rank": 1,
+ "score": 4.916114,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 2,
+ "score": 4.913085,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 3,
+ "score": 4.8817015,
+ "snippet_found": true
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 4,
+ "score": 4.7266417,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 5,
+ "score": 4.4032316,
+ "snippet_found": true
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 6,
+ "score": 4.382999,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 7,
+ "score": 4.0702477,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 8,
+ "score": 4.055292,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 9,
+ "score": 3.9624317,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 10,
+ "score": 3.7526836,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 11,
+ "score": 3.4365666,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 12,
+ "score": 3.0746837,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 13,
+ "score": 3.007635,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 14,
+ "score": 2.8946323,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 15,
+ "score": 2.8831072,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 16,
+ "score": 2.7998972,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 17,
+ "score": 2.716467,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 18,
+ "score": 2.7119622,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 19,
+ "score": 2.5842686,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 20,
+ "score": 2.5467687,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "single-sign-on.md": 5
+ },
+ "snippet_found": true,
+ "passage_rank": 6,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 0.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.2,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.0,
+ "ndcg@5": 0.38685280723454163,
+ "ndcg@10": 0.38685280723454163,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 0.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.16666666666666666
+ }
+ },
+ {
+ "id": "sso-lockout",
+ "query": "If our identity provider goes down, can anyone still get into the console?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "single-sign-on.md"
+ ],
+ "expected_text": "break-glass",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "single-sign-on.md",
+ "rank": 1,
+ "score": 7.240963,
+ "snippet_found": true
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 2,
+ "score": 3.965464,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 3,
+ "score": 3.9560423,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 4,
+ "score": 3.5496902,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 5,
+ "score": 3.2137072,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 6,
+ "score": 3.2129083,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 7,
+ "score": 3.2106323,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 8,
+ "score": 3.1655471,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 9,
+ "score": 3.0810347,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 10,
+ "score": 3.0189486,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 11,
+ "score": 2.8896675,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 12,
+ "score": 2.8698277,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 13,
+ "score": 2.6514459,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 14,
+ "score": 2.6178923,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 15,
+ "score": 2.6146202,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 16,
+ "score": 2.5582738,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 17,
+ "score": 2.5566497,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 18,
+ "score": 2.5499167,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 19,
+ "score": 2.1304684,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "single-sign-on.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 4,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.25
+ }
+ },
+ {
+ "id": "sso-offboarding",
+ "query": "How is someone's access removed automatically when they leave the company?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "single-sign-on.md"
+ ],
+ "expected_text": "SCIM",
+ "chunks_returned": 50,
+ "unique_files_returned": 20,
+ "ranked_files": [
+ {
+ "filename": "single-sign-on.md",
+ "rank": 1,
+ "score": 5.3100443,
+ "snippet_found": true
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 2,
+ "score": 4.564142,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 3,
+ "score": 4.112088,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 4,
+ "score": 4.080737,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 5,
+ "score": 3.9498796,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 6,
+ "score": 3.8103957,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 7,
+ "score": 3.748855,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 8,
+ "score": 3.6031716,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 9,
+ "score": 3.5195658,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 10,
+ "score": 3.4794078,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 11,
+ "score": 3.4275222,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 12,
+ "score": 3.338546,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 13,
+ "score": 3.278238,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 14,
+ "score": 3.2054431,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 15,
+ "score": 3.070385,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 16,
+ "score": 2.9049387,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 17,
+ "score": 2.4249506,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 18,
+ "score": 2.311882,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 19,
+ "score": 2.2950606,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 20,
+ "score": 2.2608783,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "single-sign-on.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "session-length",
+ "query": "How long before the console logs me out?",
+ "tags": [
+ "semantic",
+ "multi-doc"
+ ],
+ "expected_files": [
+ "single-sign-on.md",
+ "configuration-reference.md"
+ ],
+ "expected_text": null,
+ "chunks_returned": 50,
+ "unique_files_returned": 20,
+ "ranked_files": [
+ {
+ "filename": "network-security.md",
+ "rank": 1,
+ "score": 4.718949,
+ "snippet_found": null
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 2,
+ "score": 4.631812,
+ "snippet_found": null
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 3,
+ "score": 4.393179,
+ "snippet_found": null
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 4,
+ "score": 3.361267,
+ "snippet_found": null
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 5,
+ "score": 3.2693686,
+ "snippet_found": null
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 6,
+ "score": 3.2635205,
+ "snippet_found": null
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 7,
+ "score": 3.2346013,
+ "snippet_found": null
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 8,
+ "score": 3.1757994,
+ "snippet_found": null
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 9,
+ "score": 3.057241,
+ "snippet_found": null
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 10,
+ "score": 2.9434013,
+ "snippet_found": null
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 11,
+ "score": 2.8661394,
+ "snippet_found": null
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 12,
+ "score": 2.8531485,
+ "snippet_found": null
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 13,
+ "score": 2.7566361,
+ "snippet_found": null
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 14,
+ "score": 2.6207955,
+ "snippet_found": null
+ },
+ {
+ "filename": "regions.md",
+ "rank": 15,
+ "score": 2.4442549,
+ "snippet_found": null
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 16,
+ "score": 2.4371514,
+ "snippet_found": null
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 17,
+ "score": 2.3959312,
+ "snippet_found": null
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 18,
+ "score": 2.392912,
+ "snippet_found": null
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 19,
+ "score": 2.146281,
+ "snippet_found": null
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 20,
+ "score": 1.9371018,
+ "snippet_found": null
+ }
+ ],
+ "expected_ranks": {
+ "single-sign-on.md": 6,
+ "configuration-reference.md": 5
+ },
+ "snippet_found": null,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 0.0,
+ "recall@5": 0.5,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.2,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.0,
+ "ndcg@5": 0.23719771276929622,
+ "ndcg@10": 0.45560514958746035
+ }
+ },
+ {
+ "id": "dlq-purge-who",
+ "query": "Who is allowed to empty a dead-letter queue?",
+ "tags": [
+ "keyword",
+ "multi-doc"
+ ],
+ "expected_files": [
+ "roles-and-permissions.md",
+ "dead-letter-queue.md"
+ ],
+ "expected_text": null,
+ "chunks_returned": 50,
+ "unique_files_returned": 18,
+ "ranked_files": [
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 1,
+ "score": 5.936977,
+ "snippet_found": null
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 2,
+ "score": 5.469982,
+ "snippet_found": null
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 3,
+ "score": 4.6652865,
+ "snippet_found": null
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 4,
+ "score": 4.4582715,
+ "snippet_found": null
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 5,
+ "score": 4.1814966,
+ "snippet_found": null
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 6,
+ "score": 4.075127,
+ "snippet_found": null
+ },
+ {
+ "filename": "regions.md",
+ "rank": 7,
+ "score": 4.028639,
+ "snippet_found": null
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 8,
+ "score": 3.9006016,
+ "snippet_found": null
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 9,
+ "score": 3.8686628,
+ "snippet_found": null
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 10,
+ "score": 3.857781,
+ "snippet_found": null
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 11,
+ "score": 3.807433,
+ "snippet_found": null
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 12,
+ "score": 3.4361827,
+ "snippet_found": null
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 13,
+ "score": 2.9324346,
+ "snippet_found": null
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 14,
+ "score": 2.883803,
+ "snippet_found": null
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 15,
+ "score": 2.7358198,
+ "snippet_found": null
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 16,
+ "score": 2.616968,
+ "snippet_found": null
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 17,
+ "score": 2.392273,
+ "snippet_found": null
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 18,
+ "score": 2.3337693,
+ "snippet_found": null
+ }
+ ],
+ "expected_ranks": {
+ "roles-and-permissions.md": 10,
+ "dead-letter-queue.md": 1
+ },
+ "snippet_found": null,
+ "metrics": {
+ "recall@1": 0.5,
+ "recall@3": 0.5,
+ "recall@5": 0.5,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 0.6131471927654584,
+ "ndcg@5": 0.6131471927654584,
+ "ndcg@10": 0.7903864795495061
+ }
+ },
+ {
+ "id": "viewer-payloads",
+ "query": "Can a read-only user see the contents of events?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "roles-and-permissions.md"
+ ],
+ "expected_text": "Payload access",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 1,
+ "score": 4.9205885,
+ "snippet_found": true
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 2,
+ "score": 4.8499303,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 3,
+ "score": 4.828339,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 4,
+ "score": 4.6417646,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 5,
+ "score": 4.445083,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 6,
+ "score": 4.2635155,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 7,
+ "score": 3.988793,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 8,
+ "score": 3.8740609,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 9,
+ "score": 3.845269,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 10,
+ "score": 3.5629725,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 11,
+ "score": 3.4642334,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 12,
+ "score": 3.359555,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 13,
+ "score": 3.3335032,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 14,
+ "score": 3.3266916,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 15,
+ "score": 3.1584954,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 16,
+ "score": 3.0837433,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 17,
+ "score": 3.0490007,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 18,
+ "score": 2.998888,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 19,
+ "score": 2.7927902,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "roles-and-permissions.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "invite-expiry",
+ "query": "How long does a team invitation link stay valid?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "roles-and-permissions.md"
+ ],
+ "expected_text": "valid for 7 days",
+ "chunks_returned": 50,
+ "unique_files_returned": 21,
+ "ranked_files": [
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 1,
+ "score": 5.6239557,
+ "snippet_found": true
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 2,
+ "score": 4.895487,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 3,
+ "score": 4.1866374,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 4,
+ "score": 4.109663,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 5,
+ "score": 4.0150146,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 6,
+ "score": 3.691551,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 7,
+ "score": 3.5175457,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 8,
+ "score": 3.4534285,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 9,
+ "score": 3.2989433,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 10,
+ "score": 3.2002692,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 11,
+ "score": 2.9563212,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 12,
+ "score": 2.823382,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 13,
+ "score": 2.8055763,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 14,
+ "score": 2.7686763,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 15,
+ "score": 2.7495055,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 16,
+ "score": 2.7170486,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 17,
+ "score": 2.7069454,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 18,
+ "score": 2.6110837,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 19,
+ "score": 2.4631553,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 20,
+ "score": 2.0309317,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 21,
+ "score": 2.0022187,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "roles-and-permissions.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "ownership-transfer",
+ "query": "How do I hand a workspace over to a colleague?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "roles-and-permissions.md"
+ ],
+ "expected_text": "last Owner",
+ "chunks_returned": 50,
+ "unique_files_returned": 17,
+ "ranked_files": [
+ {
+ "filename": "troubleshooting.md",
+ "rank": 1,
+ "score": 4.235373,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 2,
+ "score": 4.2214937,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 3,
+ "score": 4.050256,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 4,
+ "score": 3.4798121,
+ "snippet_found": true
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 5,
+ "score": 3.3237166,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 6,
+ "score": 3.3111908,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 7,
+ "score": 3.0920124,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 8,
+ "score": 3.0649781,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 9,
+ "score": 2.9621043,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 10,
+ "score": 2.6565669,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 11,
+ "score": 2.6309865,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 12,
+ "score": 2.5766845,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 13,
+ "score": 2.551534,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 14,
+ "score": 2.524885,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 15,
+ "score": 2.5160358,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 16,
+ "score": 2.360861,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 17,
+ "score": 2.2255254,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "roles-and-permissions.md": 4
+ },
+ "snippet_found": true,
+ "passage_rank": 5,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 0.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.25,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.0,
+ "ndcg@5": 0.43067655807339306,
+ "ndcg@10": 0.43067655807339306,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.2
+ }
+ },
+ {
+ "id": "retry-schedule",
+ "query": "How many times will a failed webhook be attempted, and over how long?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "delivery-retries.md"
+ ],
+ "expected_text": "27 hours",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "delivery-retries.md",
+ "rank": 1,
+ "score": 5.953916,
+ "snippet_found": true
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 2,
+ "score": 5.9165845,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 3,
+ "score": 5.629542,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 4,
+ "score": 5.3382273,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 5,
+ "score": 5.2244463,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 6,
+ "score": 4.7950573,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 7,
+ "score": 4.6740932,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 8,
+ "score": 4.488102,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 9,
+ "score": 4.288582,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 10,
+ "score": 4.247769,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 11,
+ "score": 4.176223,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 12,
+ "score": 4.0876427,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 13,
+ "score": 3.7292528,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 14,
+ "score": 3.6605527,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 15,
+ "score": 3.4551523,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 16,
+ "score": 3.4263456,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 17,
+ "score": 3.2239738,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 18,
+ "score": 2.5783687,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 19,
+ "score": 2.4560182,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "delivery-retries.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 6,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 0.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.16666666666666666
+ }
+ },
+ {
+ "id": "cfg-max-attempts",
+ "query": "What is the default for delivery.max_attempts?",
+ "tags": [
+ "exact-token",
+ "multi-doc"
+ ],
+ "expected_files": [
+ "delivery-retries.md",
+ "configuration-reference.md"
+ ],
+ "expected_text": "delivery.max_attempts",
+ "chunks_returned": 50,
+ "unique_files_returned": 20,
+ "ranked_files": [
+ {
+ "filename": "delivery-retries.md",
+ "rank": 1,
+ "score": 4.100553,
+ "snippet_found": true
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 2,
+ "score": 3.990864,
+ "snippet_found": true
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 3,
+ "score": 2.4812531,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 4,
+ "score": 2.3984542,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 5,
+ "score": 2.262816,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 6,
+ "score": 2.2063332,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 7,
+ "score": 2.1881351,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 8,
+ "score": 2.1569738,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 9,
+ "score": 2.0785315,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 10,
+ "score": 2.0601966,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 11,
+ "score": 2.0537386,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 12,
+ "score": 2.0393662,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 13,
+ "score": 2.037712,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 14,
+ "score": 1.9448432,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 15,
+ "score": 1.8971537,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 16,
+ "score": 1.8890164,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 17,
+ "score": 1.8565283,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 18,
+ "score": 1.8449395,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 19,
+ "score": 1.8378721,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 20,
+ "score": 1.8362129,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "delivery-retries.md": 1,
+ "configuration-reference.md": 2
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 0.5,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "circuit-breaker",
+ "query": "What happens when most requests to one endpoint start failing?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "delivery-retries.md"
+ ],
+ "expected_text": "degraded",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 1,
+ "score": 5.0543203,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 2,
+ "score": 4.903905,
+ "snippet_found": true
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 3,
+ "score": 4.752765,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 4,
+ "score": 4.5258617,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 5,
+ "score": 4.1057053,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 6,
+ "score": 4.084237,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 7,
+ "score": 3.9673247,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 8,
+ "score": 3.947473,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 9,
+ "score": 3.8320982,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 10,
+ "score": 3.7889988,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 11,
+ "score": 3.705308,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 12,
+ "score": 3.6559482,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 13,
+ "score": 3.598694,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 14,
+ "score": 3.2583046,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 15,
+ "score": 3.1484778,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 16,
+ "score": 2.9071856,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 17,
+ "score": 2.8332481,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 18,
+ "score": 2.7670798,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 19,
+ "score": 2.5437899,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "delivery-retries.md": 2
+ },
+ "snippet_found": true,
+ "passage_rank": 2,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.5,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.6309297535714575,
+ "ndcg@5": 0.6309297535714575,
+ "ndcg@10": 0.6309297535714575,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.5
+ }
+ },
+ {
+ "id": "gone-410",
+ "query": "What does Relaywise do if my endpoint answers 410 Gone?",
+ "tags": [
+ "exact-token",
+ "multi-doc"
+ ],
+ "expected_files": [
+ "delivery-retries.md",
+ "dead-letter-queue.md"
+ ],
+ "expected_text": "410 Gone",
+ "chunks_returned": 50,
+ "unique_files_returned": 20,
+ "ranked_files": [
+ {
+ "filename": "delivery-retries.md",
+ "rank": 1,
+ "score": 5.6472993,
+ "snippet_found": true
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 2,
+ "score": 4.9761944,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 3,
+ "score": 4.925442,
+ "snippet_found": true
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 4,
+ "score": 4.794298,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 5,
+ "score": 4.408581,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 6,
+ "score": 4.156478,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 7,
+ "score": 4.053644,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 8,
+ "score": 3.842249,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 9,
+ "score": 3.769952,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 10,
+ "score": 3.6307647,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 11,
+ "score": 3.591357,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 12,
+ "score": 3.158509,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 13,
+ "score": 3.0485291,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 14,
+ "score": 2.950628,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 15,
+ "score": 2.7675254,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 16,
+ "score": 2.7243469,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 17,
+ "score": 2.7232513,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 18,
+ "score": 2.5460482,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 19,
+ "score": 2.5377195,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 20,
+ "score": 2.4653406,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "delivery-retries.md": 1,
+ "dead-letter-queue.md": 3
+ },
+ "snippet_found": true,
+ "passage_rank": 2,
+ "metrics": {
+ "recall@1": 0.5,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 0.9197207891481876,
+ "ndcg@5": 0.9197207891481876,
+ "ndcg@10": 0.9197207891481876,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.5
+ }
+ },
+ {
+ "id": "duplicate-events",
+ "query": "Why did my endpoint get the same event twice?",
+ "tags": [
+ "semantic",
+ "multi-doc"
+ ],
+ "expected_files": [
+ "delivery-retries.md",
+ "troubleshooting.md"
+ ],
+ "expected_text": null,
+ "chunks_returned": 50,
+ "unique_files_returned": 17,
+ "ranked_files": [
+ {
+ "filename": "troubleshooting.md",
+ "rank": 1,
+ "score": 6.117975,
+ "snippet_found": null
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 2,
+ "score": 5.701635,
+ "snippet_found": null
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 3,
+ "score": 4.079763,
+ "snippet_found": null
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 4,
+ "score": 3.9514606,
+ "snippet_found": null
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 5,
+ "score": 3.9501805,
+ "snippet_found": null
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 6,
+ "score": 3.8345904,
+ "snippet_found": null
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 7,
+ "score": 3.5684028,
+ "snippet_found": null
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 8,
+ "score": 3.4548547,
+ "snippet_found": null
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 9,
+ "score": 3.4442132,
+ "snippet_found": null
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 10,
+ "score": 3.2959082,
+ "snippet_found": null
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 11,
+ "score": 3.278033,
+ "snippet_found": null
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 12,
+ "score": 3.2729166,
+ "snippet_found": null
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 13,
+ "score": 3.2519,
+ "snippet_found": null
+ },
+ {
+ "filename": "regions.md",
+ "rank": 14,
+ "score": 3.0763547,
+ "snippet_found": null
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 15,
+ "score": 2.9944477,
+ "snippet_found": null
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 16,
+ "score": 2.795557,
+ "snippet_found": null
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 17,
+ "score": 2.4639845,
+ "snippet_found": null
+ }
+ ],
+ "expected_ranks": {
+ "delivery-retries.md": 5,
+ "troubleshooting.md": 1
+ },
+ "snippet_found": null,
+ "metrics": {
+ "recall@1": 0.5,
+ "recall@3": 0.5,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 0.6131471927654584,
+ "ndcg@5": 0.8503449055347546,
+ "ndcg@10": 0.8503449055347546
+ }
+ },
+ {
+ "id": "ordered-delivery",
+ "query": "Can I make sure events for one customer are delivered in sequence?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "delivery-retries.md"
+ ],
+ "expected_text": "ordering key",
+ "chunks_returned": 50,
+ "unique_files_returned": 18,
+ "ranked_files": [
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 1,
+ "score": 5.0055847,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 2,
+ "score": 4.8828163,
+ "snippet_found": true
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 3,
+ "score": 4.750148,
+ "snippet_found": true
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 4,
+ "score": 4.722822,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 5,
+ "score": 4.638679,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 6,
+ "score": 4.568696,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 7,
+ "score": 4.3880367,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 8,
+ "score": 4.3245573,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 9,
+ "score": 4.2667556,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 10,
+ "score": 4.13976,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 11,
+ "score": 4.1162443,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 12,
+ "score": 4.1093607,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 13,
+ "score": 4.1089215,
+ "snippet_found": true
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 14,
+ "score": 4.0251784,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 15,
+ "score": 3.751133,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 16,
+ "score": 3.4920778,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 17,
+ "score": 3.3043363,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 18,
+ "score": 3.2784362,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "delivery-retries.md": 3
+ },
+ "snippet_found": true,
+ "passage_rank": 3,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.3333333333333333,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.5,
+ "ndcg@5": 0.5,
+ "ndcg@10": 0.5,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.3333333333333333
+ }
+ },
+ {
+ "id": "destination-auto-disabled",
+ "query": "Why was my destination turned off without anyone touching it?",
+ "tags": [
+ "semantic",
+ "multi-doc"
+ ],
+ "expected_files": [
+ "delivery-retries.md",
+ "troubleshooting.md"
+ ],
+ "expected_text": null,
+ "chunks_returned": 50,
+ "unique_files_returned": 18,
+ "ranked_files": [
+ {
+ "filename": "troubleshooting.md",
+ "rank": 1,
+ "score": 5.720254,
+ "snippet_found": null
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 2,
+ "score": 4.6855354,
+ "snippet_found": null
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 3,
+ "score": 4.230507,
+ "snippet_found": null
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 4,
+ "score": 4.065977,
+ "snippet_found": null
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 5,
+ "score": 3.9990711,
+ "snippet_found": null
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 6,
+ "score": 3.6033638,
+ "snippet_found": null
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 7,
+ "score": 3.3606205,
+ "snippet_found": null
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 8,
+ "score": 3.3185124,
+ "snippet_found": null
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 9,
+ "score": 3.0583892,
+ "snippet_found": null
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 10,
+ "score": 2.7916946,
+ "snippet_found": null
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 11,
+ "score": 2.7792022,
+ "snippet_found": null
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 12,
+ "score": 2.6849184,
+ "snippet_found": null
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 13,
+ "score": 2.385123,
+ "snippet_found": null
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 14,
+ "score": 2.2865496,
+ "snippet_found": null
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 15,
+ "score": 2.131452,
+ "snippet_found": null
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 16,
+ "score": 2.1214397,
+ "snippet_found": null
+ },
+ {
+ "filename": "regions.md",
+ "rank": 17,
+ "score": 2.0443506,
+ "snippet_found": null
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 18,
+ "score": 1.9859804,
+ "snippet_found": null
+ }
+ ],
+ "expected_ranks": {
+ "delivery-retries.md": 5,
+ "troubleshooting.md": 1
+ },
+ "snippet_found": null,
+ "metrics": {
+ "recall@1": 0.5,
+ "recall@3": 0.5,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 0.6131471927654584,
+ "ndcg@5": 0.8503449055347546,
+ "ndcg@10": 0.8503449055347546
+ }
+ },
+ {
+ "id": "dlq-what",
+ "query": "Where do events go once every delivery attempt has been used up?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "dead-letter-queue.md"
+ ],
+ "expected_text": null,
+ "chunks_returned": 50,
+ "unique_files_returned": 18,
+ "ranked_files": [
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 1,
+ "score": 7.40966,
+ "snippet_found": null
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 2,
+ "score": 6.4829597,
+ "snippet_found": null
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 3,
+ "score": 6.2448034,
+ "snippet_found": null
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 4,
+ "score": 5.8351073,
+ "snippet_found": null
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 5,
+ "score": 5.679214,
+ "snippet_found": null
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 6,
+ "score": 5.4656425,
+ "snippet_found": null
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 7,
+ "score": 5.4533553,
+ "snippet_found": null
+ },
+ {
+ "filename": "regions.md",
+ "rank": 8,
+ "score": 5.332749,
+ "snippet_found": null
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 9,
+ "score": 4.9943905,
+ "snippet_found": null
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 10,
+ "score": 4.968727,
+ "snippet_found": null
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 11,
+ "score": 4.9405565,
+ "snippet_found": null
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 12,
+ "score": 4.714102,
+ "snippet_found": null
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 13,
+ "score": 4.632801,
+ "snippet_found": null
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 14,
+ "score": 4.599147,
+ "snippet_found": null
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 15,
+ "score": 4.52356,
+ "snippet_found": null
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 16,
+ "score": 4.458351,
+ "snippet_found": null
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 17,
+ "score": 4.3109264,
+ "snippet_found": null
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 18,
+ "score": 3.9744198,
+ "snippet_found": null
+ }
+ ],
+ "expected_ranks": {
+ "dead-letter-queue.md": 1
+ },
+ "snippet_found": null,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0
+ }
+ },
+ {
+ "id": "dlq-retention-team",
+ "query": "On the Team plan, how long are undeliverable events kept before they are removed?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "dead-letter-queue.md"
+ ],
+ "expected_text": "14 days",
+ "chunks_returned": 50,
+ "unique_files_returned": 18,
+ "ranked_files": [
+ {
+ "filename": "data-retention.md",
+ "rank": 1,
+ "score": 7.8197613,
+ "snippet_found": true
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 2,
+ "score": 7.3345337,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 3,
+ "score": 7.332274,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 4,
+ "score": 6.697984,
+ "snippet_found": true
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 5,
+ "score": 5.410752,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 6,
+ "score": 5.0231695,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 7,
+ "score": 4.924449,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 8,
+ "score": 4.6471853,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 9,
+ "score": 4.3576174,
+ "snippet_found": true
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 10,
+ "score": 4.2551737,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 11,
+ "score": 4.2262077,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 12,
+ "score": 4.0809183,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 13,
+ "score": 3.9892564,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 14,
+ "score": 3.9197059,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 15,
+ "score": 3.8503838,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 16,
+ "score": 3.7696798,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 17,
+ "score": 3.5610816,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 18,
+ "score": 3.4350631,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "dead-letter-queue.md": 4
+ },
+ "snippet_found": true,
+ "passage_rank": 4,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 0.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.25,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.0,
+ "ndcg@5": 0.43067655807339306,
+ "ndcg@10": 0.43067655807339306,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.25
+ }
+ },
+ {
+ "id": "dlq-replay-other-destination",
+ "query": "How do I use rwctl dlq replay to resend failed events to a new endpoint?",
+ "tags": [
+ "exact-token"
+ ],
+ "expected_files": [
+ "dead-letter-queue.md"
+ ],
+ "expected_text": "dst_9x1aa",
+ "chunks_returned": 50,
+ "unique_files_returned": 17,
+ "ranked_files": [
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 1,
+ "score": 8.870468,
+ "snippet_found": true
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 2,
+ "score": 6.881461,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 3,
+ "score": 5.751782,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 4,
+ "score": 5.043284,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 5,
+ "score": 4.7557015,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 6,
+ "score": 4.6869016,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 7,
+ "score": 4.359108,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 8,
+ "score": 4.1611156,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 9,
+ "score": 4.0299006,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 10,
+ "score": 3.9948087,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 11,
+ "score": 3.844854,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 12,
+ "score": 3.5940793,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 13,
+ "score": 3.408236,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 14,
+ "score": 3.3590999,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 15,
+ "score": 3.3468688,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 16,
+ "score": 3.2550454,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 17,
+ "score": 3.1481986,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "dead-letter-queue.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 2,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.5
+ }
+ },
+ {
+ "id": "replay-header",
+ "query": "How can my endpoint tell that a delivery is a replay?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "dead-letter-queue.md"
+ ],
+ "expected_text": "Relaywise-Replay",
+ "chunks_returned": 50,
+ "unique_files_returned": 18,
+ "ranked_files": [
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 1,
+ "score": 5.8395176,
+ "snippet_found": true
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 2,
+ "score": 5.7776,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 3,
+ "score": 4.659917,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 4,
+ "score": 4.5585823,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 5,
+ "score": 4.535796,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 6,
+ "score": 4.4854813,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 7,
+ "score": 4.418273,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 8,
+ "score": 4.399255,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 9,
+ "score": 4.092454,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 10,
+ "score": 3.7910876,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 11,
+ "score": 3.6927078,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 12,
+ "score": 3.5761201,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 13,
+ "score": 3.4670155,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 14,
+ "score": 3.373952,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 15,
+ "score": 3.302986,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 16,
+ "score": 3.2588396,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 17,
+ "score": 3.145359,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 18,
+ "score": 3.0399675,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "dead-letter-queue.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 10,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 0.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.1
+ }
+ },
+ {
+ "id": "ingest-rate",
+ "query": "How fast am I allowed to send events on each plan?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "rate-limits.md"
+ ],
+ "expected_text": "events per second",
+ "chunks_returned": 50,
+ "unique_files_returned": 18,
+ "ranked_files": [
+ {
+ "filename": "rate-limits.md",
+ "rank": 1,
+ "score": 6.559215,
+ "snippet_found": true
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 2,
+ "score": 6.377366,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 3,
+ "score": 6.3756666,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 4,
+ "score": 5.5507555,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 5,
+ "score": 4.7628126,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 6,
+ "score": 4.5803623,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 7,
+ "score": 4.3698053,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 8,
+ "score": 4.1771226,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 9,
+ "score": 4.1309533,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 10,
+ "score": 4.102806,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 11,
+ "score": 3.8547676,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 12,
+ "score": 3.605217,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 13,
+ "score": 3.5623853,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 14,
+ "score": 3.453558,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 15,
+ "score": 3.3093355,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 16,
+ "score": 3.2472394,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 17,
+ "score": 3.2431312,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 18,
+ "score": 3.1259098,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "rate-limits.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "err-4290",
+ "query": "RW-4290",
+ "tags": [
+ "exact-token",
+ "multi-doc"
+ ],
+ "expected_files": [
+ "rate-limits.md",
+ "error-codes.md"
+ ],
+ "expected_text": "RW-4290",
+ "chunks_returned": 8,
+ "unique_files_returned": 2,
+ "ranked_files": [
+ {
+ "filename": "error-codes.md",
+ "rank": 1,
+ "score": 5.104968,
+ "snippet_found": true
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 2,
+ "score": 4.61705,
+ "snippet_found": true
+ }
+ ],
+ "expected_ranks": {
+ "rate-limits.md": 2,
+ "error-codes.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 0.5,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "payload-max-size",
+ "query": "What is the largest event body I can send?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "rate-limits.md"
+ ],
+ "expected_text": "256 KB",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "troubleshooting.md",
+ "rank": 1,
+ "score": 5.1728625,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 2,
+ "score": 4.963256,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 3,
+ "score": 4.2940288,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 4,
+ "score": 3.8478184,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 5,
+ "score": 3.7793765,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 6,
+ "score": 3.7538767,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 7,
+ "score": 3.7113836,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 8,
+ "score": 3.647768,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 9,
+ "score": 3.63453,
+ "snippet_found": true
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 10,
+ "score": 3.572057,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 11,
+ "score": 3.561708,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 12,
+ "score": 3.4531813,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 13,
+ "score": 3.378756,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 14,
+ "score": 3.3345623,
+ "snippet_found": true
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 15,
+ "score": 3.2072341,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 16,
+ "score": 3.0846722,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 17,
+ "score": 2.9861572,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 18,
+ "score": 2.791448,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 19,
+ "score": 2.7770343,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "rate-limits.md": 9
+ },
+ "snippet_found": true,
+ "passage_rank": 14,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 0.0,
+ "recall@5": 0.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.1111111111111111,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.0,
+ "ndcg@5": 0.0,
+ "ndcg@10": 0.3010299956639812,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 0.0,
+ "passage_hit@10": 0.0,
+ "passage_rr": 0.07142857142857142
+ }
+ },
+ {
+ "id": "batch-size",
+ "query": "How many events fit in one batch request?",
+ "tags": [
+ "keyword"
+ ],
+ "expected_files": [
+ "rate-limits.md"
+ ],
+ "expected_text": "500 events",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "rate-limits.md",
+ "rank": 1,
+ "score": 6.440647,
+ "snippet_found": true
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 2,
+ "score": 4.471184,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 3,
+ "score": 4.471135,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 4,
+ "score": 4.4559145,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 5,
+ "score": 4.4337754,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 6,
+ "score": 4.210376,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 7,
+ "score": 4.1672397,
+ "snippet_found": true
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 8,
+ "score": 3.9377017,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 9,
+ "score": 3.857754,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 10,
+ "score": 3.7983751,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 11,
+ "score": 3.741257,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 12,
+ "score": 3.216155,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 13,
+ "score": 3.2107887,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 14,
+ "score": 3.144632,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 15,
+ "score": 2.9834402,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 16,
+ "score": 2.9600403,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 17,
+ "score": 2.867922,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 18,
+ "score": 2.8529837,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 19,
+ "score": 2.5421212,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "rate-limits.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "err-3002",
+ "query": "What does error RW-3002 mean?",
+ "tags": [
+ "exact-token"
+ ],
+ "expected_files": [
+ "error-codes.md"
+ ],
+ "expected_text": "TLS handshake failed",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "error-codes.md",
+ "rank": 1,
+ "score": 7.1374054,
+ "snippet_found": true
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 2,
+ "score": 4.6004024,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 3,
+ "score": 3.5720081,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 4,
+ "score": 3.5708163,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 5,
+ "score": 3.373531,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 6,
+ "score": 3.3219764,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 7,
+ "score": 3.270485,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 8,
+ "score": 3.0856066,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 9,
+ "score": 2.9129028,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 10,
+ "score": 2.8195572,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 11,
+ "score": 2.7496767,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 12,
+ "score": 2.6049972,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 13,
+ "score": 2.339275,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 14,
+ "score": 2.285453,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 15,
+ "score": 2.267887,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 16,
+ "score": 2.1199887,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 17,
+ "score": 1.9273345,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 18,
+ "score": 1.7775464,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 19,
+ "score": 1.6508594,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "error-codes.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "err-5004",
+ "query": "RW-5004",
+ "tags": [
+ "exact-token",
+ "multi-doc"
+ ],
+ "expected_files": [
+ "error-codes.md",
+ "transformations.md"
+ ],
+ "expected_text": "RW-5004",
+ "chunks_returned": 8,
+ "unique_files_returned": 2,
+ "ranked_files": [
+ {
+ "filename": "error-codes.md",
+ "rank": 1,
+ "score": 7.083034,
+ "snippet_found": true
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 2,
+ "score": 4.1704693,
+ "snippet_found": true
+ }
+ ],
+ "expected_ranks": {
+ "error-codes.md": 1,
+ "transformations.md": 2
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 0.5,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "err-2031",
+ "query": "Why was my destination URL rejected with RW-2031?",
+ "tags": [
+ "exact-token",
+ "multi-doc"
+ ],
+ "expected_files": [
+ "error-codes.md",
+ "network-security.md"
+ ],
+ "expected_text": "RW-2031",
+ "chunks_returned": 50,
+ "unique_files_returned": 20,
+ "ranked_files": [
+ {
+ "filename": "error-codes.md",
+ "rank": 1,
+ "score": 7.7312093,
+ "snippet_found": true
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 2,
+ "score": 5.75021,
+ "snippet_found": true
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 3,
+ "score": 4.6584964,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 4,
+ "score": 4.269464,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 5,
+ "score": 4.031956,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 6,
+ "score": 3.6961586,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 7,
+ "score": 3.5220945,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 8,
+ "score": 3.4944685,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 9,
+ "score": 3.3147504,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 10,
+ "score": 3.3085356,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 11,
+ "score": 3.0455709,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 12,
+ "score": 3.013853,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 13,
+ "score": 3.0125816,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 14,
+ "score": 3.006339,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 15,
+ "score": 2.862323,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 16,
+ "score": 2.804049,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 17,
+ "score": 2.4442608,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 18,
+ "score": 2.4357595,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 19,
+ "score": 2.4208562,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 20,
+ "score": 2.021038,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "error-codes.md": 1,
+ "network-security.md": 2
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 0.5,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "err-7001",
+ "query": "What causes RW-7001?",
+ "tags": [
+ "exact-token",
+ "multi-doc"
+ ],
+ "expected_files": [
+ "error-codes.md",
+ "regions.md"
+ ],
+ "expected_text": "RW-7001",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "error-codes.md",
+ "rank": 1,
+ "score": 4.55962,
+ "snippet_found": true
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 2,
+ "score": 3.2900271,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 3,
+ "score": 3.231194,
+ "snippet_found": true
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 4,
+ "score": 3.0057564,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 5,
+ "score": 2.8333864,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 6,
+ "score": 2.6037674,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 7,
+ "score": 2.4505215,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 8,
+ "score": 2.2581377,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 9,
+ "score": 2.2200274,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 10,
+ "score": 2.1641731,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 11,
+ "score": 2.1146564,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 12,
+ "score": 2.0174422,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 13,
+ "score": 1.6311694,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 14,
+ "score": 1.4668834,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 15,
+ "score": 1.456635,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 16,
+ "score": 1.4466288,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 17,
+ "score": 1.4390881,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 18,
+ "score": 1.4314262,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 19,
+ "score": 1.4135953,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "error-codes.md": 1,
+ "regions.md": 3
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 0.5,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 0.9197207891481876,
+ "ndcg@5": 0.9197207891481876,
+ "ndcg@10": 0.9197207891481876,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "cfg-concurrency",
+ "query": "Which setting limits how many requests hit my endpoint in parallel?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "configuration-reference.md"
+ ],
+ "expected_text": "delivery.max_concurrency",
+ "chunks_returned": 50,
+ "unique_files_returned": 18,
+ "ranked_files": [
+ {
+ "filename": "configuration-reference.md",
+ "rank": 1,
+ "score": 6.5323057,
+ "snippet_found": true
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 2,
+ "score": 5.4767127,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 3,
+ "score": 4.882698,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 4,
+ "score": 4.4162087,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 5,
+ "score": 4.319521,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 6,
+ "score": 3.912937,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 7,
+ "score": 3.9068096,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 8,
+ "score": 3.5763764,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 9,
+ "score": 3.4460037,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 10,
+ "score": 3.442761,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 11,
+ "score": 3.3418725,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 12,
+ "score": 3.2560182,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 13,
+ "score": 3.2276452,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 14,
+ "score": 3.1950235,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 15,
+ "score": 3.0526667,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 16,
+ "score": 2.9480395,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 17,
+ "score": 2.910755,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 18,
+ "score": 2.5600786,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "configuration-reference.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "cfg-dedupe-window",
+ "query": "ingest.dedupe_window_seconds",
+ "tags": [
+ "exact-token"
+ ],
+ "expected_files": [
+ "configuration-reference.md"
+ ],
+ "expected_text": "ingest.dedupe_window_seconds",
+ "chunks_returned": 8,
+ "unique_files_returned": 1,
+ "ranked_files": [
+ {
+ "filename": "configuration-reference.md",
+ "rank": 1,
+ "score": 3.756914,
+ "snippet_found": true
+ }
+ ],
+ "expected_ranks": {
+ "configuration-reference.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "cfg-explain",
+ "query": "How do I find out which level a setting's value is coming from?",
+ "tags": [
+ "semantic",
+ "multi-doc"
+ ],
+ "expected_files": [
+ "configuration-reference.md",
+ "cli-reference.md"
+ ],
+ "expected_text": "rwctl config explain",
+ "chunks_returned": 50,
+ "unique_files_returned": 21,
+ "ranked_files": [
+ {
+ "filename": "configuration-reference.md",
+ "rank": 1,
+ "score": 5.1910233,
+ "snippet_found": true
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 2,
+ "score": 5.1194305,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 3,
+ "score": 4.7757354,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 4,
+ "score": 4.3601513,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 5,
+ "score": 4.301612,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 6,
+ "score": 4.265447,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 7,
+ "score": 3.9866903,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 8,
+ "score": 3.753189,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 9,
+ "score": 3.693151,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 10,
+ "score": 3.283983,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 11,
+ "score": 3.2680726,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 12,
+ "score": 3.261208,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 13,
+ "score": 3.0610666,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 14,
+ "score": 2.967939,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 15,
+ "score": 2.8458738,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 16,
+ "score": 2.8134208,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 17,
+ "score": 2.7762527,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 18,
+ "score": 2.6981857,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 19,
+ "score": 2.6218233,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 20,
+ "score": 2.5113726,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 21,
+ "score": 2.4949827,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "configuration-reference.md": 1,
+ "cli-reference.md": 8
+ },
+ "snippet_found": true,
+ "passage_rank": 3,
+ "metrics": {
+ "recall@1": 0.5,
+ "recall@3": 0.5,
+ "recall@5": 0.5,
+ "recall@10": 0.5,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 0.6131471927654584,
+ "ndcg@5": 0.6131471927654584,
+ "ndcg@10": 0.6131471927654584,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.3333333333333333
+ }
+ },
+ {
+ "id": "schema-validation",
+ "query": "Can incoming payloads be checked against a schema and refused when they don't match?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "configuration-reference.md"
+ ],
+ "expected_text": "ingest.schema_validation",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "configuration-reference.md",
+ "rank": 1,
+ "score": 7.2857604,
+ "snippet_found": true
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 2,
+ "score": 4.1829777,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 3,
+ "score": 3.9681184,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 4,
+ "score": 3.9628394,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 5,
+ "score": 3.8958888,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 6,
+ "score": 3.8805716,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 7,
+ "score": 3.7766762,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 8,
+ "score": 3.7272916,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 9,
+ "score": 3.6178882,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 10,
+ "score": 3.5861323,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 11,
+ "score": 3.5555618,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 12,
+ "score": 3.5408092,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 13,
+ "score": 3.2773092,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 14,
+ "score": 3.2318847,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 15,
+ "score": 3.1293461,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 16,
+ "score": 3.0882106,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 17,
+ "score": 2.9718976,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 18,
+ "score": 2.8200388,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 19,
+ "score": 2.7177334,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "configuration-reference.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "filter-wildcard",
+ "query": "How do I send every kind of invoice event, including nested types, to one endpoint?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "event-filtering.md"
+ ],
+ "expected_text": "invoice.**",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 1,
+ "score": 6.599655,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 2,
+ "score": 5.760765,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 3,
+ "score": 5.569148,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 4,
+ "score": 5.527859,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 5,
+ "score": 5.472649,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 6,
+ "score": 5.421536,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 7,
+ "score": 5.261847,
+ "snippet_found": true
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 8,
+ "score": 5.023038,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 9,
+ "score": 4.937884,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 10,
+ "score": 4.873948,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 11,
+ "score": 4.553656,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 12,
+ "score": 4.50159,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 13,
+ "score": 4.4150305,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 14,
+ "score": 4.278399,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 15,
+ "score": 3.8844829,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 16,
+ "score": 3.8552125,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 17,
+ "score": 3.8130789,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 18,
+ "score": 3.5139594,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 19,
+ "score": 3.3147485,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "event-filtering.md": 7
+ },
+ "snippet_found": true,
+ "passage_rank": 43,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 0.0,
+ "recall@5": 0.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.14285714285714285,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.0,
+ "ndcg@5": 0.0,
+ "ndcg@10": 0.3333333333333333,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 0.0,
+ "passage_hit@10": 0.0,
+ "passage_rr": 0.023255813953488372
+ }
+ },
+ {
+ "id": "filter-missing-field",
+ "query": "What happens when a filter refers to a field the event doesn't have?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "event-filtering.md"
+ ],
+ "expected_text": "it is not an error",
+ "chunks_returned": 50,
+ "unique_files_returned": 17,
+ "ranked_files": [
+ {
+ "filename": "event-filtering.md",
+ "rank": 1,
+ "score": 6.3981233,
+ "snippet_found": true
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 2,
+ "score": 6.132348,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 3,
+ "score": 5.273807,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 4,
+ "score": 4.9314632,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 5,
+ "score": 4.871355,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 6,
+ "score": 4.5469966,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 7,
+ "score": 4.467554,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 8,
+ "score": 4.247962,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 9,
+ "score": 3.781559,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 10,
+ "score": 3.587966,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 11,
+ "score": 3.5403461,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 12,
+ "score": 3.4803455,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 13,
+ "score": 3.4259849,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 14,
+ "score": 3.240863,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 15,
+ "score": 3.217833,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 16,
+ "score": 3.142176,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 17,
+ "score": 3.012438,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "event-filtering.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "unrouted-events",
+ "query": "What happens to an event that no route picks up?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "event-filtering.md"
+ ],
+ "expected_text": "unrouted",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "event-filtering.md",
+ "rank": 1,
+ "score": 6.7026176,
+ "snippet_found": true
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 2,
+ "score": 6.576373,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 3,
+ "score": 6.105928,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 4,
+ "score": 5.184962,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 5,
+ "score": 4.8381486,
+ "snippet_found": true
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 6,
+ "score": 4.526141,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 7,
+ "score": 4.250954,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 8,
+ "score": 4.041794,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 9,
+ "score": 4.0392456,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 10,
+ "score": 3.9502084,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 11,
+ "score": 3.8146112,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 12,
+ "score": 3.8039114,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 13,
+ "score": 3.4500139,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 14,
+ "score": 3.4208,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 15,
+ "score": 3.2320712,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 16,
+ "score": 3.1246333,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 17,
+ "score": 3.1182313,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 18,
+ "score": 3.030007,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 19,
+ "score": 2.9305377,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "event-filtering.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "route-fan-out",
+ "query": "How many endpoints can one route deliver to?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "event-filtering.md"
+ ],
+ "expected_text": "up to 20 destinations",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 1,
+ "score": 6.023015,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 2,
+ "score": 5.745181,
+ "snippet_found": true
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 3,
+ "score": 5.1195517,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 4,
+ "score": 4.844228,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 5,
+ "score": 4.83197,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 6,
+ "score": 4.238139,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 7,
+ "score": 4.2260113,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 8,
+ "score": 4.079329,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 9,
+ "score": 3.8717012,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 10,
+ "score": 3.8605428,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 11,
+ "score": 3.7901826,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 12,
+ "score": 3.5144677,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 13,
+ "score": 3.5060587,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 14,
+ "score": 3.367303,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 15,
+ "score": 3.3218155,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 16,
+ "score": 3.022322,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 17,
+ "score": 2.9369514,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 18,
+ "score": 2.922502,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 19,
+ "score": 2.7017128,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "event-filtering.md": 2
+ },
+ "snippet_found": true,
+ "passage_rank": 2,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.5,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.6309297535714575,
+ "ndcg@5": 0.6309297535714575,
+ "ndcg@10": 0.6309297535714575,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.5
+ }
+ },
+ {
+ "id": "strip-fields-before-delivery",
+ "query": "How can I strip card numbers out of events before they are forwarded to a partner?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "transformations.md"
+ ],
+ "expected_text": "card_number",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 1,
+ "score": 5.678779,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 2,
+ "score": 4.841915,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 3,
+ "score": 4.5332537,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 4,
+ "score": 4.3789315,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 5,
+ "score": 4.374852,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 6,
+ "score": 4.303219,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 7,
+ "score": 4.297907,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 8,
+ "score": 4.2757826,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 9,
+ "score": 4.0928054,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 10,
+ "score": 4.013118,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 11,
+ "score": 3.958882,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 12,
+ "score": 3.8992069,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 13,
+ "score": 3.830959,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 14,
+ "score": 3.8095558,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 15,
+ "score": 3.6439047,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 16,
+ "score": 3.4828746,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 17,
+ "score": 3.4731426,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 18,
+ "score": 3.339641,
+ "snippet_found": true
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 19,
+ "score": 3.257053,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "transformations.md": 18
+ },
+ "snippet_found": true,
+ "passage_rank": 44,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 0.0,
+ "recall@5": 0.0,
+ "recall@10": 0.0,
+ "reciprocal_rank": 0.05555555555555555,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.0,
+ "ndcg@5": 0.0,
+ "ndcg@10": 0.0,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 0.0,
+ "passage_hit@10": 0.0,
+ "passage_rr": 0.022727272727272728
+ }
+ },
+ {
+ "id": "transform-network",
+ "query": "Can transformation code call an external API?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "transformations.md"
+ ],
+ "expected_text": "No network access",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "error-codes.md",
+ "rank": 1,
+ "score": 4.3027563,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 2,
+ "score": 3.7211428,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 3,
+ "score": 3.5431662,
+ "snippet_found": true
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 4,
+ "score": 3.4637728,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 5,
+ "score": 3.4438903,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 6,
+ "score": 3.3542087,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 7,
+ "score": 3.1996362,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 8,
+ "score": 3.1790874,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 9,
+ "score": 3.1691957,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 10,
+ "score": 2.7622945,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 11,
+ "score": 2.7219043,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 12,
+ "score": 2.7049937,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 13,
+ "score": 2.6479259,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 14,
+ "score": 2.5942392,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 15,
+ "score": 2.5000453,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 16,
+ "score": 2.412778,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 17,
+ "score": 2.2223032,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 18,
+ "score": 2.120767,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 19,
+ "score": 2.0920844,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "transformations.md": 3
+ },
+ "snippet_found": true,
+ "passage_rank": 3,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.3333333333333333,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.5,
+ "ndcg@5": 0.5,
+ "ndcg@10": 0.5,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.3333333333333333
+ }
+ },
+ {
+ "id": "transform-rollback",
+ "query": "I shipped a broken transformation, how do I go back to the previous one?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "transformations.md"
+ ],
+ "expected_text": "Make active",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "troubleshooting.md",
+ "rank": 1,
+ "score": 5.23054,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 2,
+ "score": 4.2707195,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 3,
+ "score": 4.0321445,
+ "snippet_found": true
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 4,
+ "score": 3.5525649,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 5,
+ "score": 3.4241276,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 6,
+ "score": 3.4105673,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 7,
+ "score": 3.4089398,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 8,
+ "score": 3.4029415,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 9,
+ "score": 3.3298483,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 10,
+ "score": 3.2877781,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 11,
+ "score": 3.2613475,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 12,
+ "score": 3.2458348,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 13,
+ "score": 3.2236872,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 14,
+ "score": 3.2179403,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 15,
+ "score": 3.1666298,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 16,
+ "score": 2.8349867,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 17,
+ "score": 2.7176538,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 18,
+ "score": 2.5744977,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 19,
+ "score": 2.480778,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "transformations.md": 3
+ },
+ "snippet_found": true,
+ "passage_rank": 23,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.3333333333333333,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.5,
+ "ndcg@5": 0.5,
+ "ndcg@10": 0.5,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 0.0,
+ "passage_hit@10": 0.0,
+ "passage_rr": 0.043478260869565216
+ }
+ },
+ {
+ "id": "plan-scale-price",
+ "query": "What does the Scale plan cost and how many events come with it?",
+ "tags": [
+ "keyword"
+ ],
+ "expected_files": [
+ "billing-plans.md"
+ ],
+ "expected_text": "20 million",
+ "chunks_returned": 50,
+ "unique_files_returned": 18,
+ "ranked_files": [
+ {
+ "filename": "data-retention.md",
+ "rank": 1,
+ "score": 6.165932,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 2,
+ "score": 5.6396055,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 3,
+ "score": 5.595186,
+ "snippet_found": true
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 4,
+ "score": 4.96609,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 5,
+ "score": 4.8011065,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 6,
+ "score": 4.7937565,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 7,
+ "score": 4.7090893,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 8,
+ "score": 4.6951523,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 9,
+ "score": 4.6432734,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 10,
+ "score": 4.542534,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 11,
+ "score": 3.9930236,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 12,
+ "score": 3.9314911,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 13,
+ "score": 3.7961717,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 14,
+ "score": 3.7279844,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 15,
+ "score": 3.7183979,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 16,
+ "score": 3.5648823,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 17,
+ "score": 3.5081198,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 18,
+ "score": 3.369672,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "billing-plans.md": 3
+ },
+ "snippet_found": true,
+ "passage_rank": 3,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.3333333333333333,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.5,
+ "ndcg@5": 0.5,
+ "ndcg@10": 0.5,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.3333333333333333
+ }
+ },
+ {
+ "id": "over-quota",
+ "query": "What happens if we send more events in a month than our plan includes?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "billing-plans.md"
+ ],
+ "expected_text": "billed as overage",
+ "chunks_returned": 50,
+ "unique_files_returned": 18,
+ "ranked_files": [
+ {
+ "filename": "billing-plans.md",
+ "rank": 1,
+ "score": 6.611384,
+ "snippet_found": true
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 2,
+ "score": 5.573196,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 3,
+ "score": 5.069791,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 4,
+ "score": 4.896354,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 5,
+ "score": 4.8308477,
+ "snippet_found": true
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 6,
+ "score": 4.6927853,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 7,
+ "score": 4.615057,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 8,
+ "score": 4.568859,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 9,
+ "score": 4.510277,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 10,
+ "score": 4.4583926,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 11,
+ "score": 4.3350053,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 12,
+ "score": 4.3333607,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 13,
+ "score": 4.2085896,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 14,
+ "score": 3.864415,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 15,
+ "score": 3.6676457,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 16,
+ "score": 3.6663413,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 17,
+ "score": 3.626868,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 18,
+ "score": 3.5489278,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "billing-plans.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 3,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.3333333333333333
+ }
+ },
+ {
+ "id": "spending-cap",
+ "query": "How do I make sure we never get a surprise bill?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "billing-plans.md"
+ ],
+ "expected_text": "hard cap",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "cli-reference.md",
+ "rank": 1,
+ "score": 4.176035,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 2,
+ "score": 3.7238424,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 3,
+ "score": 3.373507,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 4,
+ "score": 3.3062384,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 5,
+ "score": 3.1803827,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 6,
+ "score": 2.8217483,
+ "snippet_found": true
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 7,
+ "score": 2.8194714,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 8,
+ "score": 2.7950668,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 9,
+ "score": 2.7918391,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 10,
+ "score": 2.7697334,
+ "snippet_found": true
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 11,
+ "score": 2.3493757,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 12,
+ "score": 2.2208354,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 13,
+ "score": 2.1967921,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 14,
+ "score": 2.0774598,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 15,
+ "score": 2.0550961,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 16,
+ "score": 2.0002537,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 17,
+ "score": 1.9782275,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 18,
+ "score": 1.8935627,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 19,
+ "score": 1.8432963,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "billing-plans.md": 10
+ },
+ "snippet_found": true,
+ "passage_rank": 11,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 0.0,
+ "recall@5": 0.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.1,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.0,
+ "ndcg@5": 0.0,
+ "ndcg@10": 0.2890648263178879,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 0.0,
+ "passage_hit@10": 0.0,
+ "passage_rr": 0.09090909090909091
+ }
+ },
+ {
+ "id": "downgrade-timing",
+ "query": "When does a plan downgrade kick in?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "billing-plans.md"
+ ],
+ "expected_text": "end of the current billing period",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "billing-plans.md",
+ "rank": 1,
+ "score": 4.5011973,
+ "snippet_found": true
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 2,
+ "score": 3.9330368,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 3,
+ "score": 3.6819746,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 4,
+ "score": 3.3807042,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 5,
+ "score": 3.145202,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 6,
+ "score": 2.9044037,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 7,
+ "score": 2.6842203,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 8,
+ "score": 2.652769,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 9,
+ "score": 2.559432,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 10,
+ "score": 2.5516438,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 11,
+ "score": 2.5177975,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 12,
+ "score": 2.4889445,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 13,
+ "score": 2.4745228,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 14,
+ "score": 2.397379,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 15,
+ "score": 2.3737328,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 16,
+ "score": 2.3688858,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 17,
+ "score": 2.2727196,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 18,
+ "score": 2.255967,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 19,
+ "score": 2.1137109,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "billing-plans.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "nonprofit-discount",
+ "query": "Is there a discount for nonprofits?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "billing-plans.md"
+ ],
+ "expected_text": "50 percent off",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 1,
+ "score": 3.192665,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 2,
+ "score": 3.1329155,
+ "snippet_found": true
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 3,
+ "score": 2.7976594,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 4,
+ "score": 2.500914,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 5,
+ "score": 2.431467,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 6,
+ "score": 2.3387825,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 7,
+ "score": 2.1462817,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 8,
+ "score": 2.138791,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 9,
+ "score": 2.0935802,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 10,
+ "score": 1.9763438,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 11,
+ "score": 1.9572263,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 12,
+ "score": 1.9540287,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 13,
+ "score": 1.8847063,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 14,
+ "score": 1.8813934,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 15,
+ "score": 1.8805516,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 16,
+ "score": 1.8092102,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 17,
+ "score": 1.799421,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 18,
+ "score": 1.7747855,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 19,
+ "score": 1.7663033,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "billing-plans.md": 2
+ },
+ "snippet_found": true,
+ "passage_rank": 2,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.5,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.6309297535714575,
+ "ndcg@5": 0.6309297535714575,
+ "ndcg@10": 0.6309297535714575,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.5
+ }
+ },
+ {
+ "id": "card-declined",
+ "query": "What happens if our card is declined?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "invoices-and-payments.md"
+ ],
+ "expected_text": "suspended",
+ "chunks_returned": 50,
+ "unique_files_returned": 18,
+ "ranked_files": [
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 1,
+ "score": 4.5424986,
+ "snippet_found": true
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 2,
+ "score": 3.1390238,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 3,
+ "score": 2.9242349,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 4,
+ "score": 2.853714,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 5,
+ "score": 2.7698298,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 6,
+ "score": 2.6912606,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 7,
+ "score": 2.6068764,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 8,
+ "score": 2.4017735,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 9,
+ "score": 2.3534894,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 10,
+ "score": 2.3194094,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 11,
+ "score": 2.2925973,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 12,
+ "score": 2.2087455,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 13,
+ "score": 2.170509,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 14,
+ "score": 2.1145196,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 15,
+ "score": 2.0395908,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 16,
+ "score": 1.8723415,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 17,
+ "score": 1.8695025,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 18,
+ "score": 1.8511734,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "invoices-and-payments.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "invoice-download",
+ "query": "Where can I download past invoices?",
+ "tags": [
+ "keyword"
+ ],
+ "expected_files": [
+ "invoices-and-payments.md"
+ ],
+ "expected_text": "PDF",
+ "chunks_returned": 50,
+ "unique_files_returned": 20,
+ "ranked_files": [
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 1,
+ "score": 5.479479,
+ "snippet_found": true
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 2,
+ "score": 3.3870986,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 3,
+ "score": 3.3651319,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 4,
+ "score": 3.1572304,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 5,
+ "score": 2.869647,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 6,
+ "score": 2.8024464,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 7,
+ "score": 2.6480045,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 8,
+ "score": 2.558901,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 9,
+ "score": 2.514241,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 10,
+ "score": 2.4659653,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 11,
+ "score": 2.4333763,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 12,
+ "score": 2.4234314,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 13,
+ "score": 2.3397257,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 14,
+ "score": 2.3184114,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 15,
+ "score": 2.2858236,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 16,
+ "score": 2.261516,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 17,
+ "score": 2.2397711,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 18,
+ "score": 1.9229008,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 19,
+ "score": 1.9125963,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 20,
+ "score": 1.8740361,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "invoices-and-payments.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "annual-refund",
+ "query": "Can I get my money back on an annual subscription?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "invoices-and-payments.md"
+ ],
+ "expected_text": "30 days of purchase",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "troubleshooting.md",
+ "rank": 1,
+ "score": 4.030309,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 2,
+ "score": 3.4916003,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 3,
+ "score": 3.1594803,
+ "snippet_found": true
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 4,
+ "score": 3.083531,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 5,
+ "score": 2.862523,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 6,
+ "score": 2.7023883,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 7,
+ "score": 2.6665974,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 8,
+ "score": 2.5352874,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 9,
+ "score": 2.4236565,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 10,
+ "score": 2.3765519,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 11,
+ "score": 2.3002856,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 12,
+ "score": 2.2416017,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 13,
+ "score": 2.06355,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 14,
+ "score": 1.9668009,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 15,
+ "score": 1.899621,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 16,
+ "score": 1.8948866,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 17,
+ "score": 1.8911364,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 18,
+ "score": 1.875198,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 19,
+ "score": 1.7308294,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "invoices-and-payments.md": 3
+ },
+ "snippet_found": true,
+ "passage_rank": 39,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.3333333333333333,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.5,
+ "ndcg@5": 0.5,
+ "ndcg@10": 0.5,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 0.0,
+ "passage_hit@10": 0.0,
+ "passage_rr": 0.02564102564102564
+ }
+ },
+ {
+ "id": "pay-by-wire",
+ "query": "Can we pay by wire instead of by card?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "invoices-and-payments.md"
+ ],
+ "expected_text": "bank transfer",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 1,
+ "score": 5.018153,
+ "snippet_found": true
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 2,
+ "score": 3.2159462,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 3,
+ "score": 3.201638,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 4,
+ "score": 3.147363,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 5,
+ "score": 2.6815596,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 6,
+ "score": 2.6559532,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 7,
+ "score": 2.5634036,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 8,
+ "score": 2.364353,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 9,
+ "score": 2.3472905,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 10,
+ "score": 2.2389264,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 11,
+ "score": 2.2166698,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 12,
+ "score": 2.180029,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 13,
+ "score": 2.1364298,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 14,
+ "score": 2.1318357,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 15,
+ "score": 2.0653362,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 16,
+ "score": 1.992836,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 17,
+ "score": 1.948524,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 18,
+ "score": 1.9307742,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 19,
+ "score": 1.7839562,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "invoices-and-payments.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "payload-retention-scale",
+ "query": "How long are event bodies stored on the Scale plan?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "data-retention.md"
+ ],
+ "expected_text": "These are maximums",
+ "chunks_returned": 50,
+ "unique_files_returned": 18,
+ "ranked_files": [
+ {
+ "filename": "data-retention.md",
+ "rank": 1,
+ "score": 7.134757,
+ "snippet_found": true
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 2,
+ "score": 5.646868,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 3,
+ "score": 5.632083,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 4,
+ "score": 5.5159082,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 5,
+ "score": 5.322894,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 6,
+ "score": 4.917618,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 7,
+ "score": 4.646406,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 8,
+ "score": 4.593455,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 9,
+ "score": 4.1423883,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 10,
+ "score": 4.102504,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 11,
+ "score": 3.5780063,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 12,
+ "score": 3.4474115,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 13,
+ "score": 3.1836896,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 14,
+ "score": 3.1513236,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 15,
+ "score": 3.037868,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 16,
+ "score": 2.9461775,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 17,
+ "score": 2.7285311,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 18,
+ "score": 2.6023915,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "data-retention.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 14,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 0.0,
+ "passage_hit@10": 0.0,
+ "passage_rr": 0.07142857142857142
+ }
+ },
+ {
+ "id": "erasure-request",
+ "query": "A customer asked us to erase their personal data, how do I remove their events?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "data-retention.md"
+ ],
+ "expected_text": "Delete events",
+ "chunks_returned": 50,
+ "unique_files_returned": 18,
+ "ranked_files": [
+ {
+ "filename": "data-retention.md",
+ "rank": 1,
+ "score": 7.6901236,
+ "snippet_found": true
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 2,
+ "score": 6.390702,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 3,
+ "score": 5.357597,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 4,
+ "score": 4.2579355,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 5,
+ "score": 4.2515087,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 6,
+ "score": 4.1859593,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 7,
+ "score": 4.102975,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 8,
+ "score": 3.948564,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 9,
+ "score": 3.9145777,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 10,
+ "score": 3.8292284,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 11,
+ "score": 3.7746987,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 12,
+ "score": 3.7034235,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 13,
+ "score": 3.6457257,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 14,
+ "score": 3.4510388,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 15,
+ "score": 3.381026,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 16,
+ "score": 3.1673608,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 17,
+ "score": 3.1267197,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 18,
+ "score": 2.711932,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "data-retention.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "never-store-field",
+ "query": "How do I keep a sensitive field from ever being saved?",
+ "tags": [
+ "semantic",
+ "multi-doc"
+ ],
+ "expected_files": [
+ "data-retention.md",
+ "configuration-reference.md"
+ ],
+ "expected_text": "retention.redact_fields",
+ "chunks_returned": 50,
+ "unique_files_returned": 20,
+ "ranked_files": [
+ {
+ "filename": "transformations.md",
+ "rank": 1,
+ "score": 4.269417,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 2,
+ "score": 4.132926,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 3,
+ "score": 4.115541,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 4,
+ "score": 3.9203172,
+ "snippet_found": true
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 5,
+ "score": 3.7807415,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 6,
+ "score": 3.7652113,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 7,
+ "score": 3.6132348,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 8,
+ "score": 3.4858923,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 9,
+ "score": 3.1855412,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 10,
+ "score": 3.0955997,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 11,
+ "score": 3.075557,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 12,
+ "score": 2.8341694,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 13,
+ "score": 2.6905017,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 14,
+ "score": 2.600214,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 15,
+ "score": 2.5657752,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 16,
+ "score": 2.553431,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 17,
+ "score": 2.3961258,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 18,
+ "score": 2.3078952,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 19,
+ "score": 2.2757673,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 20,
+ "score": 2.2442474,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "data-retention.md": 4,
+ "configuration-reference.md": 17
+ },
+ "snippet_found": true,
+ "passage_rank": 4,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 0.0,
+ "recall@5": 0.5,
+ "recall@10": 0.5,
+ "reciprocal_rank": 0.25,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.0,
+ "ndcg@5": 0.2640681225725909,
+ "ndcg@10": 0.2640681225725909,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.25
+ }
+ },
+ {
+ "id": "export-to-own-storage",
+ "query": "Can I keep events for longer than the plan allows by copying them to storage we own?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "data-retention.md"
+ ],
+ "expected_text": "scheduled export",
+ "chunks_returned": 50,
+ "unique_files_returned": 20,
+ "ranked_files": [
+ {
+ "filename": "data-retention.md",
+ "rank": 1,
+ "score": 7.004419,
+ "snippet_found": true
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 2,
+ "score": 5.72364,
+ "snippet_found": true
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 3,
+ "score": 5.051481,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 4,
+ "score": 4.8311186,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 5,
+ "score": 4.7930336,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 6,
+ "score": 4.558029,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 7,
+ "score": 4.422286,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 8,
+ "score": 4.2385845,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 9,
+ "score": 4.049222,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 10,
+ "score": 4.0140843,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 11,
+ "score": 3.8202207,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 12,
+ "score": 3.7686496,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 13,
+ "score": 3.7210777,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 14,
+ "score": 3.7047937,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 15,
+ "score": 3.7034209,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 16,
+ "score": 3.6294847,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 17,
+ "score": 3.6183372,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 18,
+ "score": 3.476426,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 19,
+ "score": 3.2785215,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 20,
+ "score": 3.2107797,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "data-retention.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "replay-event-gone",
+ "query": "Replay fails saying the event isn't available any more, why?",
+ "tags": [
+ "semantic",
+ "multi-doc"
+ ],
+ "expected_files": [
+ "data-retention.md",
+ "troubleshooting.md"
+ ],
+ "expected_text": null,
+ "chunks_returned": 50,
+ "unique_files_returned": 18,
+ "ranked_files": [
+ {
+ "filename": "troubleshooting.md",
+ "rank": 1,
+ "score": 5.111964,
+ "snippet_found": null
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 2,
+ "score": 4.6264553,
+ "snippet_found": null
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 3,
+ "score": 4.382534,
+ "snippet_found": null
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 4,
+ "score": 4.204933,
+ "snippet_found": null
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 5,
+ "score": 4.008878,
+ "snippet_found": null
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 6,
+ "score": 4.000078,
+ "snippet_found": null
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 7,
+ "score": 3.8352492,
+ "snippet_found": null
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 8,
+ "score": 3.6457763,
+ "snippet_found": null
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 9,
+ "score": 3.5607314,
+ "snippet_found": null
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 10,
+ "score": 3.5086422,
+ "snippet_found": null
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 11,
+ "score": 3.5036902,
+ "snippet_found": null
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 12,
+ "score": 3.4338822,
+ "snippet_found": null
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 13,
+ "score": 3.0842357,
+ "snippet_found": null
+ },
+ {
+ "filename": "regions.md",
+ "rank": 14,
+ "score": 3.065458,
+ "snippet_found": null
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 15,
+ "score": 2.9612713,
+ "snippet_found": null
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 16,
+ "score": 2.9603195,
+ "snippet_found": null
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 17,
+ "score": 2.8865094,
+ "snippet_found": null
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 18,
+ "score": 2.8189383,
+ "snippet_found": null
+ }
+ ],
+ "expected_ranks": {
+ "data-retention.md": 9,
+ "troubleshooting.md": 1
+ },
+ "snippet_found": null,
+ "metrics": {
+ "recall@1": 0.5,
+ "recall@3": 0.5,
+ "recall@5": 0.5,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 0.6131471927654584,
+ "ndcg@5": 0.6131471927654584,
+ "ndcg@10": 0.7977228895450266
+ }
+ },
+ {
+ "id": "eu-residency",
+ "query": "We need our data to stay inside the EU, is that possible?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "regions.md"
+ ],
+ "expected_text": "eu-central",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "regions.md",
+ "rank": 1,
+ "score": 5.606211,
+ "snippet_found": true
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 2,
+ "score": 3.7901576,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 3,
+ "score": 3.3422961,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 4,
+ "score": 3.0132427,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 5,
+ "score": 2.9385717,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 6,
+ "score": 2.8419957,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 7,
+ "score": 2.7693527,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 8,
+ "score": 2.7328167,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 9,
+ "score": 2.4419742,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 10,
+ "score": 2.4058669,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 11,
+ "score": 2.3926773,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 12,
+ "score": 2.3721557,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 13,
+ "score": 2.3270187,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 14,
+ "score": 2.2621386,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 15,
+ "score": 2.2538724,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 16,
+ "score": 2.1800587,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 17,
+ "score": 2.1209936,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 18,
+ "score": 2.111018,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 19,
+ "score": 1.9036365,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "regions.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "move-region",
+ "query": "Can I move an existing workspace to another region?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "regions.md"
+ ],
+ "expected_text": "cannot be changed",
+ "chunks_returned": 50,
+ "unique_files_returned": 20,
+ "ranked_files": [
+ {
+ "filename": "regions.md",
+ "rank": 1,
+ "score": 7.4162574,
+ "snippet_found": true
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 2,
+ "score": 4.9218082,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 3,
+ "score": 4.279954,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 4,
+ "score": 3.787121,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 5,
+ "score": 3.5782912,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 6,
+ "score": 3.4162853,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 7,
+ "score": 3.3293962,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 8,
+ "score": 3.0762732,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 9,
+ "score": 3.0027943,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 10,
+ "score": 2.9621854,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 11,
+ "score": 2.8567605,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 12,
+ "score": 2.7653418,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 13,
+ "score": 2.742486,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 14,
+ "score": 2.7398138,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 15,
+ "score": 2.7382836,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 16,
+ "score": 2.6832218,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 17,
+ "score": 2.652599,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 18,
+ "score": 2.6192703,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 19,
+ "score": 2.2097597,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 20,
+ "score": 2.1313481,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "regions.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "firewall-ips",
+ "query": "Which IP addresses do I need to let through our firewall so deliveries arrive?",
+ "tags": [
+ "semantic",
+ "multi-doc"
+ ],
+ "expected_files": [
+ "regions.md",
+ "network-security.md"
+ ],
+ "expected_text": "rwctl network egress-ips",
+ "chunks_returned": 50,
+ "unique_files_returned": 21,
+ "ranked_files": [
+ {
+ "filename": "regions.md",
+ "rank": 1,
+ "score": 7.6239576,
+ "snippet_found": true
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 2,
+ "score": 7.329286,
+ "snippet_found": true
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 3,
+ "score": 4.260872,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 4,
+ "score": 3.736021,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 5,
+ "score": 3.5849426,
+ "snippet_found": true
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 6,
+ "score": 3.4546218,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 7,
+ "score": 3.4361572,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 8,
+ "score": 3.4263103,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 9,
+ "score": 3.3875096,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 10,
+ "score": 3.3755388,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 11,
+ "score": 3.2378645,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 12,
+ "score": 3.1179535,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 13,
+ "score": 3.0980186,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 14,
+ "score": 3.062451,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 15,
+ "score": 3.0315938,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 16,
+ "score": 2.801557,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 17,
+ "score": 2.7423558,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 18,
+ "score": 2.728594,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 19,
+ "score": 2.5501351,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 20,
+ "score": 2.5037901,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 21,
+ "score": 2.3320217,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "regions.md": 1,
+ "network-security.md": 2
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 0.5,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "alert-on-failing-endpoint",
+ "query": "How do I get notified when an endpoint starts failing?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "monitoring-and-alerts.md"
+ ],
+ "expected_text": "delivery.success_rate",
+ "chunks_returned": 50,
+ "unique_files_returned": 18,
+ "ranked_files": [
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 1,
+ "score": 4.7551217,
+ "snippet_found": true
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 2,
+ "score": 4.4837484,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 3,
+ "score": 3.97679,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 4,
+ "score": 3.7504718,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 5,
+ "score": 3.7289722,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 6,
+ "score": 3.680981,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 7,
+ "score": 3.4579203,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 8,
+ "score": 3.360943,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 9,
+ "score": 3.2502306,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 10,
+ "score": 3.0607262,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 11,
+ "score": 3.031942,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 12,
+ "score": 2.939257,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 13,
+ "score": 2.9016738,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 14,
+ "score": 2.7974367,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 15,
+ "score": 2.660655,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 16,
+ "score": 2.4786308,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 17,
+ "score": 2.3564715,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 18,
+ "score": 2.2467048,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "monitoring-and-alerts.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "metric-dlq-depth",
+ "query": "dlq.depth",
+ "tags": [
+ "exact-token",
+ "multi-doc"
+ ],
+ "expected_files": [
+ "monitoring-and-alerts.md",
+ "dead-letter-queue.md"
+ ],
+ "expected_text": "dlq.depth",
+ "chunks_returned": 8,
+ "unique_files_returned": 2,
+ "ranked_files": [
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 1,
+ "score": 3.1213505,
+ "snippet_found": true
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 2,
+ "score": 2.994694,
+ "snippet_found": true
+ }
+ ],
+ "expected_ranks": {
+ "monitoring-and-alerts.md": 2,
+ "dead-letter-queue.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 0.5,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "alert-noise",
+ "query": "How do I stop an alert from notifying us over and over?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "monitoring-and-alerts.md"
+ ],
+ "expected_text": "cooldown",
+ "chunks_returned": 50,
+ "unique_files_returned": 21,
+ "ranked_files": [
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 1,
+ "score": 5.3605433,
+ "snippet_found": true
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 2,
+ "score": 4.4922824,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 3,
+ "score": 3.8867335,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 4,
+ "score": 3.680799,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 5,
+ "score": 3.500159,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 6,
+ "score": 3.3970802,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 7,
+ "score": 3.2065027,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 8,
+ "score": 3.1697018,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 9,
+ "score": 3.005413,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 10,
+ "score": 2.8459678,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 11,
+ "score": 2.7075675,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 12,
+ "score": 2.6995163,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 13,
+ "score": 2.6899762,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 14,
+ "score": 2.6840582,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 15,
+ "score": 2.664556,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 16,
+ "score": 2.4973686,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 17,
+ "score": 2.4592614,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 18,
+ "score": 2.4189043,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 19,
+ "score": 2.3941422,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 20,
+ "score": 2.3926606,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 21,
+ "score": 2.2645342,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "monitoring-and-alerts.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 17,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 0.0,
+ "passage_hit@10": 0.0,
+ "passage_rr": 0.058823529411764705
+ }
+ },
+ {
+ "id": "metrics-history",
+ "query": "How far back do the charts go?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "monitoring-and-alerts.md"
+ ],
+ "expected_text": "13 months",
+ "chunks_returned": 50,
+ "unique_files_returned": 20,
+ "ranked_files": [
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 1,
+ "score": 3.7359376,
+ "snippet_found": true
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 2,
+ "score": 3.1819804,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 3,
+ "score": 2.9821012,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 4,
+ "score": 2.8422427,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 5,
+ "score": 2.8005157,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 6,
+ "score": 2.783891,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 7,
+ "score": 2.782668,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 8,
+ "score": 2.712003,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 9,
+ "score": 2.498731,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 10,
+ "score": 2.2379708,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 11,
+ "score": 2.218436,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 12,
+ "score": 2.206134,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 13,
+ "score": 2.102532,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 14,
+ "score": 2.0636818,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 15,
+ "score": 2.0426698,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 16,
+ "score": 2.0245354,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 17,
+ "score": 1.9765356,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 18,
+ "score": 1.8800985,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 19,
+ "score": 1.7948415,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 20,
+ "score": 1.7783153,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "monitoring-and-alerts.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 14,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 0.0,
+ "passage_hit@10": 0.0,
+ "passage_rr": 0.07142857142857142
+ }
+ },
+ {
+ "id": "audit-who-changed",
+ "query": "How can I find out who changed a destination's URL?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "audit-log.md"
+ ],
+ "expected_text": "destination.url_changed",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 1,
+ "score": 4.8638945,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 2,
+ "score": 4.4864283,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 3,
+ "score": 4.4840837,
+ "snippet_found": true
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 4,
+ "score": 4.211418,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 5,
+ "score": 4.071476,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 6,
+ "score": 4.007015,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 7,
+ "score": 3.974052,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 8,
+ "score": 3.8140574,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 9,
+ "score": 3.7421665,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 10,
+ "score": 3.5506918,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 11,
+ "score": 3.4370897,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 12,
+ "score": 3.4214826,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 13,
+ "score": 3.28876,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 14,
+ "score": 3.161325,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 15,
+ "score": 3.1208224,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 16,
+ "score": 2.7832265,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 17,
+ "score": 2.6834779,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 18,
+ "score": 2.5932093,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 19,
+ "score": 2.4568262,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "audit-log.md": 3
+ },
+ "snippet_found": true,
+ "passage_rank": 4,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.3333333333333333,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.5,
+ "ndcg@5": 0.5,
+ "ndcg@10": 0.5,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.25
+ }
+ },
+ {
+ "id": "audit-retention",
+ "query": "How long is the record of admin actions kept?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "audit-log.md"
+ ],
+ "expected_text": "7 years",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "audit-log.md",
+ "rank": 1,
+ "score": 6.7694893,
+ "snippet_found": true
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 2,
+ "score": 5.902327,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 3,
+ "score": 4.3268433,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 4,
+ "score": 4.1397276,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 5,
+ "score": 4.104513,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 6,
+ "score": 4.062165,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 7,
+ "score": 4.053916,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 8,
+ "score": 3.2497015,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 9,
+ "score": 3.210817,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 10,
+ "score": 3.150314,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 11,
+ "score": 3.0505908,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 12,
+ "score": 2.9865234,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 13,
+ "score": 2.8661394,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 14,
+ "score": 2.650631,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 15,
+ "score": 2.542187,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 16,
+ "score": 2.2279477,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 17,
+ "score": 2.0776277,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 18,
+ "score": 1.9082611,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 19,
+ "score": 1.9064578,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "audit-log.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "audit-tamper",
+ "query": "Could an admin cover their tracks by deleting log entries?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "audit-log.md"
+ ],
+ "expected_text": "hash",
+ "chunks_returned": 50,
+ "unique_files_returned": 18,
+ "ranked_files": [
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 1,
+ "score": 5.3748984,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 2,
+ "score": 5.285913,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 3,
+ "score": 5.285846,
+ "snippet_found": true
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 4,
+ "score": 4.522437,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 5,
+ "score": 3.9469934,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 6,
+ "score": 3.3514473,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 7,
+ "score": 3.1368482,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 8,
+ "score": 3.126635,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 9,
+ "score": 2.75564,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 10,
+ "score": 2.7482202,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 11,
+ "score": 2.715976,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 12,
+ "score": 2.5803761,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 13,
+ "score": 2.460242,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 14,
+ "score": 2.3662038,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 15,
+ "score": 2.3606703,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 16,
+ "score": 2.197319,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 17,
+ "score": 2.1412895,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 18,
+ "score": 2.0583878,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "audit-log.md": 3
+ },
+ "snippet_found": true,
+ "passage_rank": 4,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.3333333333333333,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.5,
+ "ndcg@5": 0.5,
+ "ndcg@10": 0.5,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.25
+ }
+ },
+ {
+ "id": "support-access",
+ "query": "Can Relaywise staff look at our payloads?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "audit-log.md"
+ ],
+ "expected_text": "Support access",
+ "chunks_returned": 50,
+ "unique_files_returned": 20,
+ "ranked_files": [
+ {
+ "filename": "audit-log.md",
+ "rank": 1,
+ "score": 4.0565033,
+ "snippet_found": true
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 2,
+ "score": 3.8535635,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 3,
+ "score": 3.7648094,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 4,
+ "score": 3.761508,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 5,
+ "score": 3.7174695,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 6,
+ "score": 3.6284103,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 7,
+ "score": 3.3209238,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 8,
+ "score": 3.0343482,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 9,
+ "score": 2.9823442,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 10,
+ "score": 2.81755,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 11,
+ "score": 2.5708063,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 12,
+ "score": 2.5682056,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 13,
+ "score": 2.5657067,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 14,
+ "score": 2.5156288,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 15,
+ "score": 2.5056129,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 16,
+ "score": 2.4120002,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 17,
+ "score": 2.404485,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 18,
+ "score": 2.3355746,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 19,
+ "score": 2.2381802,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 20,
+ "score": 2.1149168,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "audit-log.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 3,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.3333333333333333
+ }
+ },
+ {
+ "id": "local-development",
+ "query": "How do I receive webhooks on my laptop while developing?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "cli-reference.md"
+ ],
+ "expected_text": "rwctl listen",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "cli-reference.md",
+ "rank": 1,
+ "score": 5.920323,
+ "snippet_found": true
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 2,
+ "score": 3.614477,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 3,
+ "score": 3.504744,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 4,
+ "score": 3.2636237,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 5,
+ "score": 3.0580826,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 6,
+ "score": 3.0453985,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 7,
+ "score": 2.8457193,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 8,
+ "score": 2.6562397,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 9,
+ "score": 2.5232592,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 10,
+ "score": 2.3091094,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 11,
+ "score": 2.298862,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 12,
+ "score": 2.2715812,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 13,
+ "score": 2.1830935,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 14,
+ "score": 1.984105,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 15,
+ "score": 1.9079537,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 16,
+ "score": 1.9055259,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 17,
+ "score": 1.8890364,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 18,
+ "score": 1.8846616,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 19,
+ "score": 1.8736366,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "cli-reference.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "cli-in-pipeline",
+ "query": "How do I authenticate the command-line tool in a CI pipeline?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "cli-reference.md"
+ ],
+ "expected_text": "RELAYWISE_API_KEY",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "cli-reference.md",
+ "rank": 1,
+ "score": 6.744873,
+ "snippet_found": true
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 2,
+ "score": 6.4332047,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 3,
+ "score": 4.639228,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 4,
+ "score": 4.624231,
+ "snippet_found": true
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 5,
+ "score": 4.0451546,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 6,
+ "score": 3.0904324,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 7,
+ "score": 2.9595504,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 8,
+ "score": 2.9179506,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 9,
+ "score": 2.8892794,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 10,
+ "score": 2.8273864,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 11,
+ "score": 2.6481714,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 12,
+ "score": 2.6116657,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 13,
+ "score": 2.4750228,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 14,
+ "score": 2.3840961,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 15,
+ "score": 2.3526254,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 16,
+ "score": 2.3453667,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 17,
+ "score": 2.329196,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 18,
+ "score": 2.3105922,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 19,
+ "score": 2.0088358,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "cli-reference.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 6,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 0.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.16666666666666666
+ }
+ },
+ {
+ "id": "cli-exit-code-5",
+ "query": "What does exit code 5 from rwctl mean?",
+ "tags": [
+ "exact-token"
+ ],
+ "expected_files": [
+ "cli-reference.md"
+ ],
+ "expected_text": "Rate limited",
+ "chunks_returned": 50,
+ "unique_files_returned": 21,
+ "ranked_files": [
+ {
+ "filename": "cli-reference.md",
+ "rank": 1,
+ "score": 6.5858765,
+ "snippet_found": true
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 2,
+ "score": 4.121351,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 3,
+ "score": 3.7905457,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 4,
+ "score": 3.750536,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 5,
+ "score": 3.116803,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 6,
+ "score": 3.0487542,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 7,
+ "score": 2.8950555,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 8,
+ "score": 2.7257786,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 9,
+ "score": 2.7019591,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 10,
+ "score": 2.624712,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 11,
+ "score": 2.6233444,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 12,
+ "score": 2.5043364,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 13,
+ "score": 2.4628139,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 14,
+ "score": 2.430686,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 15,
+ "score": 2.3871965,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 16,
+ "score": 2.359127,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 17,
+ "score": 2.3341298,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 18,
+ "score": 2.2257278,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 19,
+ "score": 2.1425583,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 20,
+ "score": 2.125866,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 21,
+ "score": 2.0921934,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "cli-reference.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "client-certificates",
+ "query": "Our endpoint requires callers to present a client certificate, is that supported?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "network-security.md"
+ ],
+ "expected_text": "mutual TLS",
+ "chunks_returned": 50,
+ "unique_files_returned": 18,
+ "ranked_files": [
+ {
+ "filename": "network-security.md",
+ "rank": 1,
+ "score": 9.485439,
+ "snippet_found": true
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 2,
+ "score": 4.8805037,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 3,
+ "score": 4.1398563,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 4,
+ "score": 3.9929626,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 5,
+ "score": 3.6816976,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 6,
+ "score": 3.5573359,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 7,
+ "score": 3.1600873,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 8,
+ "score": 3.1363287,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 9,
+ "score": 3.1233253,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 10,
+ "score": 3.0414858,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 11,
+ "score": 2.9609323,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 12,
+ "score": 2.8957937,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 13,
+ "score": 2.7488675,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 14,
+ "score": 2.703147,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 15,
+ "score": 2.6985838,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 16,
+ "score": 2.6221676,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 17,
+ "score": 2.2819054,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 18,
+ "score": 2.1841009,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "network-security.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "restrict-senders",
+ "query": "Can I restrict which servers are allowed to send events to us?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "network-security.md"
+ ],
+ "expected_text": "ingest allowlist",
+ "chunks_returned": 50,
+ "unique_files_returned": 17,
+ "ranked_files": [
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 1,
+ "score": 4.716089,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 2,
+ "score": 4.5632486,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 3,
+ "score": 4.4465055,
+ "snippet_found": true
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 4,
+ "score": 4.4074636,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 5,
+ "score": 4.3561697,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 6,
+ "score": 4.2924013,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 7,
+ "score": 4.2620754,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 8,
+ "score": 4.0919485,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 9,
+ "score": 4.0235558,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 10,
+ "score": 3.9080744,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 11,
+ "score": 3.8798714,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 12,
+ "score": 3.6969318,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 13,
+ "score": 3.6393707,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 14,
+ "score": 3.5710607,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 15,
+ "score": 3.4468415,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 16,
+ "score": 3.0487945,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 17,
+ "score": 2.857802,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "network-security.md": 3
+ },
+ "snippet_found": true,
+ "passage_rank": 3,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.3333333333333333,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.5,
+ "ndcg@5": 0.5,
+ "ndcg@10": 0.5,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.3333333333333333
+ }
+ },
+ {
+ "id": "enforce-two-step",
+ "query": "How do I force everyone to use two-step login?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "network-security.md"
+ ],
+ "expected_text": "Require two-factor",
+ "chunks_returned": 50,
+ "unique_files_returned": 21,
+ "ranked_files": [
+ {
+ "filename": "single-sign-on.md",
+ "rank": 1,
+ "score": 4.196766,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 2,
+ "score": 4.1229253,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 3,
+ "score": 3.2927356,
+ "snippet_found": true
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 4,
+ "score": 3.2802827,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 5,
+ "score": 3.2128882,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 6,
+ "score": 3.0439441,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 7,
+ "score": 2.7201033,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 8,
+ "score": 2.644678,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 9,
+ "score": 2.5467582,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 10,
+ "score": 2.535389,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 11,
+ "score": 2.4999475,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 12,
+ "score": 2.4157667,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 13,
+ "score": 2.4001741,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 14,
+ "score": 2.3948617,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 15,
+ "score": 2.3878696,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 16,
+ "score": 2.282506,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 17,
+ "score": 2.2025251,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 18,
+ "score": 2.0695238,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 19,
+ "score": 1.9594917,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 20,
+ "score": 1.9291121,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 21,
+ "score": 1.796094,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "network-security.md": 3
+ },
+ "snippet_found": true,
+ "passage_rank": 9,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.3333333333333333,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.5,
+ "ndcg@5": 0.5,
+ "ndcg@10": 0.5,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 0.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.1111111111111111
+ }
+ },
+ {
+ "id": "encryption-at-rest",
+ "query": "Is stored data encrypted?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "network-security.md"
+ ],
+ "expected_text": "AES-256",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "network-security.md",
+ "rank": 1,
+ "score": 4.051931,
+ "snippet_found": true
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 2,
+ "score": 3.8924236,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 3,
+ "score": 3.4245567,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 4,
+ "score": 2.852664,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 5,
+ "score": 2.5582569,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 6,
+ "score": 2.4678826,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 7,
+ "score": 2.4120307,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 8,
+ "score": 2.2670894,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 9,
+ "score": 2.194287,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 10,
+ "score": 2.1178303,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 11,
+ "score": 1.8928216,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 12,
+ "score": 1.864538,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 13,
+ "score": 1.8635015,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 14,
+ "score": 1.7791486,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 15,
+ "score": 1.5821353,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 16,
+ "score": 1.5647495,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 17,
+ "score": 1.5531394,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 18,
+ "score": 1.5409894,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 19,
+ "score": 1.4597204,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "network-security.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "nothing-arriving",
+ "query": "My endpoint isn't getting any events, where do I start looking?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "troubleshooting.md"
+ ],
+ "expected_text": "Was the event accepted",
+ "chunks_returned": 50,
+ "unique_files_returned": 19,
+ "ranked_files": [
+ {
+ "filename": "troubleshooting.md",
+ "rank": 1,
+ "score": 5.6265793,
+ "snippet_found": true
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 2,
+ "score": 4.9406214,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 3,
+ "score": 4.242301,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 4,
+ "score": 4.122987,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 5,
+ "score": 4.0675335,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 6,
+ "score": 3.9629734,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 7,
+ "score": 3.9350436,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 8,
+ "score": 3.9290373,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 9,
+ "score": 3.8895578,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 10,
+ "score": 3.573028,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 11,
+ "score": 3.5166721,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 12,
+ "score": 3.334948,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 13,
+ "score": 3.2872982,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 14,
+ "score": 3.243579,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 15,
+ "score": 3.0642722,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 16,
+ "score": 3.0577633,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 17,
+ "score": 3.028301,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 18,
+ "score": 2.8070688,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 19,
+ "score": 2.774746,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "troubleshooting.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "lost-authenticator",
+ "query": "I lost my authenticator app, how do I get back in?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "troubleshooting.md"
+ ],
+ "expected_text": "second factor",
+ "chunks_returned": 50,
+ "unique_files_returned": 21,
+ "ranked_files": [
+ {
+ "filename": "troubleshooting.md",
+ "rank": 1,
+ "score": 5.154069,
+ "snippet_found": true
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 2,
+ "score": 4.039789,
+ "snippet_found": true
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 3,
+ "score": 3.4582567,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 4,
+ "score": 3.15412,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 5,
+ "score": 3.0289927,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 6,
+ "score": 3.0060077,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 7,
+ "score": 2.9968257,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 8,
+ "score": 2.9520168,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 9,
+ "score": 2.8144894,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 10,
+ "score": 2.6343243,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 11,
+ "score": 2.5093923,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 12,
+ "score": 2.4954455,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 13,
+ "score": 2.4666529,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 14,
+ "score": 2.3090854,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 15,
+ "score": 2.2876267,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 16,
+ "score": 2.262417,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 17,
+ "score": 2.1686492,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 18,
+ "score": 2.117509,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 19,
+ "score": 2.0579886,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 20,
+ "score": 1.9386961,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 21,
+ "score": 1.9223454,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "troubleshooting.md": 1
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 1.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ },
+ {
+ "id": "first-test-event",
+ "query": "What is the quickest way to send a first test event?",
+ "tags": [
+ "semantic"
+ ],
+ "expected_files": [
+ "getting-started.md"
+ ],
+ "expected_text": "202 Accepted",
+ "chunks_returned": 50,
+ "unique_files_returned": 18,
+ "ranked_files": [
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 1,
+ "score": 5.160555,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 2,
+ "score": 4.864584,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 3,
+ "score": 4.5930367,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 4,
+ "score": 4.4245358,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 5,
+ "score": 4.3252854,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 6,
+ "score": 4.2423105,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 7,
+ "score": 4.239122,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 8,
+ "score": 4.103107,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 9,
+ "score": 3.8884325,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 10,
+ "score": 3.7158773,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 11,
+ "score": 3.655969,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 12,
+ "score": 3.6182408,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 13,
+ "score": 3.407936,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 14,
+ "score": 3.300753,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 15,
+ "score": 3.2260537,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 16,
+ "score": 3.218966,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 17,
+ "score": 3.0701206,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 18,
+ "score": 3.029202,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "getting-started.md": 7
+ },
+ "snippet_found": false,
+ "passage_rank": null,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 0.0,
+ "recall@5": 0.0,
+ "recall@10": 0.0,
+ "reciprocal_rank": 0.0,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.0,
+ "ndcg@5": 0.0,
+ "ndcg@10": 0.0,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 0.0,
+ "passage_hit@10": 0.0,
+ "passage_rr": 0.0
+ }
+ },
+ {
+ "id": "source-vs-destination",
+ "query": "What is the difference between a source and a destination?",
+ "tags": [
+ "keyword"
+ ],
+ "expected_files": [
+ "getting-started.md"
+ ],
+ "expected_text": "Where events come from",
+ "chunks_returned": 50,
+ "unique_files_returned": 20,
+ "ranked_files": [
+ {
+ "filename": "cli-reference.md",
+ "rank": 1,
+ "score": 4.9485564,
+ "snippet_found": false
+ },
+ {
+ "filename": "getting-started.md",
+ "rank": 2,
+ "score": 4.9001284,
+ "snippet_found": true
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 3,
+ "score": 3.985413,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 4,
+ "score": 3.8785093,
+ "snippet_found": false
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 5,
+ "score": 3.8600652,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 6,
+ "score": 3.7427316,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 7,
+ "score": 3.673921,
+ "snippet_found": false
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 8,
+ "score": 3.6201513,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 9,
+ "score": 3.4754732,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 10,
+ "score": 3.3986077,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 11,
+ "score": 3.2834342,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 12,
+ "score": 3.2481108,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 13,
+ "score": 3.2231283,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 14,
+ "score": 3.2037265,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 15,
+ "score": 3.149214,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 16,
+ "score": 3.0159397,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 17,
+ "score": 2.9234078,
+ "snippet_found": false
+ },
+ {
+ "filename": "data-retention.md",
+ "rank": 18,
+ "score": 2.8746972,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 19,
+ "score": 2.7781506,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 20,
+ "score": 2.455618,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "getting-started.md": 2
+ },
+ "snippet_found": true,
+ "passage_rank": 4,
+ "metrics": {
+ "recall@1": 0.0,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 0.5,
+ "ndcg@1": 0.0,
+ "ndcg@3": 0.6309297535714575,
+ "ndcg@5": 0.6309297535714575,
+ "ndcg@10": 0.6309297535714575,
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 0.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 0.25
+ }
+ },
+ {
+ "id": "plain-http-destination",
+ "query": "Can a destination use a plain http URL?",
+ "tags": [
+ "keyword",
+ "multi-doc"
+ ],
+ "expected_files": [
+ "getting-started.md",
+ "error-codes.md"
+ ],
+ "expected_text": "localhost",
+ "chunks_returned": 50,
+ "unique_files_returned": 20,
+ "ranked_files": [
+ {
+ "filename": "getting-started.md",
+ "rank": 1,
+ "score": 6.999161,
+ "snippet_found": true
+ },
+ {
+ "filename": "error-codes.md",
+ "rank": 2,
+ "score": 6.419933,
+ "snippet_found": true
+ },
+ {
+ "filename": "network-security.md",
+ "rank": 3,
+ "score": 3.9198084,
+ "snippet_found": false
+ },
+ {
+ "filename": "cli-reference.md",
+ "rank": 4,
+ "score": 3.8565774,
+ "snippet_found": false
+ },
+ {
+ "filename": "dead-letter-queue.md",
+ "rank": 5,
+ "score": 3.5010269,
+ "snippet_found": false
+ },
+ {
+ "filename": "delivery-retries.md",
+ "rank": 6,
+ "score": 3.4314427,
+ "snippet_found": false
+ },
+ {
+ "filename": "event-filtering.md",
+ "rank": 7,
+ "score": 3.1986198,
+ "snippet_found": false
+ },
+ {
+ "filename": "transformations.md",
+ "rank": 8,
+ "score": 3.1823845,
+ "snippet_found": false
+ },
+ {
+ "filename": "signing-secrets.md",
+ "rank": 9,
+ "score": 3.1128054,
+ "snippet_found": false
+ },
+ {
+ "filename": "audit-log.md",
+ "rank": 10,
+ "score": 3.0919387,
+ "snippet_found": false
+ },
+ {
+ "filename": "configuration-reference.md",
+ "rank": 11,
+ "score": 2.918428,
+ "snippet_found": false
+ },
+ {
+ "filename": "monitoring-and-alerts.md",
+ "rank": 12,
+ "score": 2.7586346,
+ "snippet_found": false
+ },
+ {
+ "filename": "troubleshooting.md",
+ "rank": 13,
+ "score": 2.7350254,
+ "snippet_found": false
+ },
+ {
+ "filename": "regions.md",
+ "rank": 14,
+ "score": 2.7063363,
+ "snippet_found": false
+ },
+ {
+ "filename": "single-sign-on.md",
+ "rank": 15,
+ "score": 2.550699,
+ "snippet_found": false
+ },
+ {
+ "filename": "rate-limits.md",
+ "rank": 16,
+ "score": 2.3362465,
+ "snippet_found": false
+ },
+ {
+ "filename": "invoices-and-payments.md",
+ "rank": 17,
+ "score": 2.2635229,
+ "snippet_found": false
+ },
+ {
+ "filename": "roles-and-permissions.md",
+ "rank": 18,
+ "score": 2.2386634,
+ "snippet_found": false
+ },
+ {
+ "filename": "billing-plans.md",
+ "rank": 19,
+ "score": 2.2386358,
+ "snippet_found": false
+ },
+ {
+ "filename": "api-keys.md",
+ "rank": 20,
+ "score": 2.1976867,
+ "snippet_found": false
+ }
+ ],
+ "expected_ranks": {
+ "getting-started.md": 1,
+ "error-codes.md": 2
+ },
+ "snippet_found": true,
+ "passage_rank": 1,
+ "metrics": {
+ "recall@1": 0.5,
+ "recall@3": 1.0,
+ "recall@5": 1.0,
+ "recall@10": 1.0,
+ "reciprocal_rank": 1.0,
+ "ndcg@1": 1.0,
+ "ndcg@3": 1.0,
+ "ndcg@5": 1.0,
+ "ndcg@10": 1.0,
+ "passage_hit@1": 1.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_hit@10": 1.0,
+ "passage_rr": 1.0
+ }
+ }
+ ]
+}